ERF Class Reference

#include <ERF.H>

Inheritance diagram for ERF:

Collaboration diagram for ERF:

Classes
struct	Plot3DScratch

Public Member Functions
	ERF ()
	~ERF () override
void	ERF_shared ()
	ERF (ERF &&) noexcept=delete
ERF &	operator= (ERF &&other) noexcept=delete
	ERF (const ERF &other)=delete
ERF &	operator= (const ERF &other)=delete
void	Evolve ()
double	EvolveOneStep (double time, double dt_request)
void	ConfigureDriverAtmosToOceanCoupling (bool use_coupling_driver, bool use_two_way_coupling, bool use_state_contract)
void	SetDriverAtmosToOceanStateContract (bool use_state_contract)
void	GetOceanToAtmosSurfaceLayout (amrex::BoxArray &ba, amrex::DistributionMapping &dm)
void	GetOceanToAtmosCellCenterLonLat (const amrex::MultiFab *&lon_cc, const amrex::MultiFab *&lat_cc) const
void	GetLandMask (const amrex::iMultiFab *&lmask) const
bool	HasRadiation () const
bool	HasCloudWater () const
void	GetFillableAtmosToOceanLanes (bool state_contract, amrex::Vector< int > &can_fill) const
void	PackAtmosphericStates (amrex::Vector< amrex::MultiFab * > &states, double time, const amrex::Vector< const amrex::MultiFab * > &weight_mf, const amrex::Vector< const amrex::iMultiFab * > &index_mf, int max_stencil_size, const amrex::MultiFab *dst_mskr=nullptr, const amrex::MultiFab *dst_msku=nullptr, const amrex::MultiFab *dst_mskv=nullptr)
void	ApplyOceanSurfaceState (const amrex::Vector< amrex::MultiFab * > &state, double time, const amrex::iMultiFab *erf_coverage=nullptr)
void	ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override
void	read_box_for_refinement (std::string &ref_prefix, int &lev_for_box, amrex::RealBox &real_box)
void	update_box_for_refinement (std::string &ref_prefix, int &lev_for_box, amrex::RealBox &real_box, const double time)
void	HurricaneTracker (int lev, double time, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::TagBoxArray *tags=nullptr)
bool	FindInitialEye (int lev, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::Real &eye_x, amrex::Real &eye_y)
std::string	MakeVTKFilename (int nstep)
std::string	MakeVTKFilename_TrackerCircle (int nstep)
std::string	MakeVTKFilename_EyeTracker_xy (int nstep)
std::string	MakeFilename_EyeTracker_latlon (int nstep)
std::string	MakeFilename_EyeTracker_maxvel (int nstep)
std::string	MakeFilename_EyeTracker_minpressure (int nstep)
void	WriteVTKPolyline (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
void	WriteLinePlot (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
void	HurricaneEyeTracker (const SolverChoice &solverChoice)
void	HurricaneEyeTrackerInitial (const SolverChoice &sc, const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, const amrex::Real &hurricane_eye_latitude, const amrex::Real &hurricane_eye_longitude)
void	HurricaneEyeTrackerNotInitial (const SolverChoice &sc, const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, MoistureType moisture_type)
void	ComputeGlobalMinLocation (const SolverChoice &sc, const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, amrex::Real *d_val_min_ptr, int *d_i_min_ptr, int *d_j_min_ptr, amrex::Real &global_val_min, int &global_i_min, int &global_j_min)
void	HurricaneMaxVelTracker (const amrex::Geometry &geom, const amrex::MultiFab &mf_cc_vel, const double &time)
void	HurricaneMinPressureTracker (MoistureType moisture_type, const amrex::Geometry &geom, const amrex::MultiFab &mf_cons_var, const double &time)
void	HurricaneTrackerCircle ()
void	ReadStormTrackerRestart ()
void	InitData ()
void	InitData_pre ()
void	InitData_post ()
void	Interp2DArrays (int lev, const amrex::BoxArray &my_ba2d, const amrex::DistributionMapping &my_dm)
void	interp_mapfac_from_coarse (int lev)
void	interp_psfc_from_coarse (int lev)
void	interp_land_masks_from_coarse (int lev)
bool	has_own_init_file (int lev) const
void	WriteMyEBSurface ()
void	compute_divergence (int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::MultiFab const &mf_mx, amrex::MultiFab const &mf_my, amrex::MultiFab const &mf_vx, amrex::MultiFab const &mf_uy, amrex::MultiFab const &ax_in, amrex::MultiFab const &ay_in, amrex::MultiFab const &dJ_in, amrex::Geometry const &geom_at_lev)
void	project_initial_velocity (int lev, double time, double dt)
void	project_momenta (int lev, double l_time, double l_dt, amrex::Vector< amrex::MultiFab > &vars)
void	project_velocity_tb (int lev, double dt, amrex::Vector< amrex::MultiFab > &vars)
void	poisson_wall_dist (int lev)
void	make_subdomains (const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
	Group a list of boxes into subdomains based on adjacency. More...
void	solve_with_gmres (int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &az_sub, amrex::MultiFab &, amrex::MultiFab &znd_sub)
void	ImposeBCsOnPhi (int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
void	init_only (int lev, double time)
void	restart ()
void	check_state_for_nans (amrex::MultiFab const &S)
void	check_vels_for_nans (amrex::MultiFab const &xvel, amrex::MultiFab const &yvel, amrex::MultiFab const &zvel)
void	check_for_negative_theta (amrex::MultiFab &S)
void	check_for_low_temp (amrex::MultiFab &S)
void	compute_max_pressure_gradient_diagnostic (int lev)
void	compute_max_buoyancy_gradp_diagnostic (int lev)
void	check_mesh_type (int lev)
bool	writeNow (double cur_time, const int nstep, const int plot_int, const double plot_per, const double dt_0, double &last_file_time)
void	post_timestep (int nstep, double time, double dt_lev)
void	WriteAtIntermediateTime (int nstep, double time)
void	WriteAtFinalTime ()
void	sum_integrated_quantities (double time)
void	sum_derived_quantities (double time)
void	sum_energy_quantities (double time)
void	write_1D_profiles (double time)
void	write_1D_profiles_stag (double time)
amrex::Real	cloud_fraction (double time)
void	FillBdyCCVels (amrex::MultiFab &mf_cc_vel, amrex::Geometry &lev_geom)
void	sample_points (int lev, double time, amrex::IntVect cell, amrex::MultiFab &mf)
void	sample_lines (int lev, double time, amrex::IntVect cell, amrex::MultiFab &mf)
void	derive_diag_profiles (double time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
void	derive_diag_profiles_stag (double time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
void	derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
void	derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
void	derive_forcing_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_ttend, amrex::Gpu::HostVector< amrex::Real > &h_avg_qtend, amrex::Gpu::HostVector< amrex::Real > &h_avg_wsub, amrex::Gpu::HostVector< amrex::Real > &h_avg_thtend, amrex::Gpu::HostVector< amrex::Real > &h_avg_qhtend, amrex::Gpu::HostVector< amrex::Real > &h_avg_tvtend, amrex::Gpu::HostVector< amrex::Real > &h_avg_qvtend, amrex::Gpu::HostVector< amrex::Real > &h_avg_qcvtend, amrex::Gpu::HostVector< amrex::Real > &h_avg_tnudge, amrex::Gpu::HostVector< amrex::Real > &h_avg_qnudge, amrex::Gpu::HostVector< amrex::Real > &h_avg_unudge, amrex::Gpu::HostVector< amrex::Real > &h_avg_vnudge)
amrex::Real	volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, const amrex::MultiFab &dJ, const amrex::MultiFab &mfx, const amrex::MultiFab &mfy, bool finemask, bool local=true)
void	volWgtColumnSum (int lev, const amrex::MultiFab &mf, int comp, amrex::MultiFab &mf_2d, const amrex::MultiFab &dJ)
void	MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
void	RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
void	ClearLevel (int lev) override
void	MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
double	estTimeStep (int lev, long &dt_fast_ratio) const
void	advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, double dt, double time)
void	advance_microphysics (int lev, amrex::MultiFab &cons_in, const double &dt_advance, const int &iteration, const double &time)
void	advance_lsm (int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const double &time, const double &dt_advance)
void	advance_radiation (int lev, amrex::MultiFab &cons_in, const double &dt_advance)
	Advance radiation diagnostics and heating rates for one time step. More...
void	compute_native_shoc_tendencies (int lev, amrex::MultiFab *cons, amrex::MultiFab *xvel, amrex::MultiFab *yvel, amrex::MultiFab *zvel, amrex::Real *w_subsid, amrex::MultiFab *tau13, amrex::MultiFab *tau23, amrex::MultiFab *hfx3, amrex::MultiFab *qfx3, amrex::MultiFab *eddyDiffs, amrex::MultiFab *z_phys_nd, const double &dt_advance)
void	build_fine_mask (int lev, amrex::MultiFab &fine_mask)
void	MakeHorizontalAverages ()
void	MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)
void	derive_upwp (amrex::Vector< amrex::Real > &h_havg)
bool	Write3DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
void	Write2DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
void	ResetIntervalMeansAfter3DPlotfileBatch (int plotfiles_written, bool interval_diagnostic_consumed)
void	ResetIntervalMeans ()
void	WriteSubvolume (int isub, amrex::Vector< std::string > subvol_var_names)
void	WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, double time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
void	WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, double time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
void	FillPlot2DVars (int lev, const amrex::Vector< std::string > &plot_var_names, const amrex::Vector< std::string > &varnames, const amrex::Vector< plotfile2d::Plotfile2DOutputDescriptor > &output_descriptors, amrex::MultiFab &mf_dst, int ncomp_mf)
void	FillPlot3DVars (int lev, const amrex::Vector< std::string > &plot_var_names, Plot3DScratch &scratch, amrex::MultiFab &mf_dst, int ncomp_mf, amrex::Real tnew)
void	BuildPlot3DScratch (const amrex::Vector< std::string > &plot_var_names, Plot3DScratch &scratch, int max_lev=-1, bool sync_solution=true)
void	init_stations ()
void	resolve_station_positions ()
void	resolve_station_stencils ()
void	sample_stations (amrex::Real time)
void	flush_stations () const
void	erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
void	init_from_input_sounding (int lev)
void	init_immersed_forcing (int lev)
void	init_ibseb ()
void	ibseb_write_checkpoint (const std::string &checkpointname, int lev) const
void	ibseb_report (int nstep, amrex::Real time)
amrex::Real	ibseb_bulk_richardson_height (int lev, const amrex::MultiFab &cons, const amrex::MultiFab &xvel, const amrex::MultiFab &yvel)
void	ibseb_advance (int lev, amrex::Real time, amrex::Real dt_lev, const amrex::MultiFab &cons, const amrex::MultiFab &xvel, const amrex::MultiFab &yvel, const amrex::MultiFab &zvel)
void	input_sponge (int lev)
void	init_from_hse (int lev)
void	init_thin_body (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
void	FillForecastStateMultiFabs (const int lev, const std::string &filename, const std::unique_ptr< amrex::MultiFab > &z_phys_nd, amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
void	FillSurfaceStateMultiFabs (const int lev, const std::string &filename, amrex::Vector< amrex::MultiFab > &surface_state)
void	WeatherDataInterpolation (const int nlevs, const double time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)
void	SurfaceDataInterpolation (const int nlevs, const double time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)
void	create_background_state_for_ensemble (int lev, amrex::MultiFab &mf_cc_pert, amrex::MultiFab &cons_pert, amrex::MultiFab &xvel_pert, amrex::MultiFab &yvel_pert, amrex::MultiFab &zvel_pert)
void	create_random_perturbations (const int lev, amrex::MultiFab &mf_cc_pert)
void	apply_gaussian_smoothing_to_perturbations (const int lev, amrex::MultiFab &mf_cc_pert)
void	ComputeAndWriteEnsemblePerturbations (const int da_iter)
void	PerformDataAssimilation (int da_iter)
void	SetDirsForPlotfilesAndCheckpointsForDA (const int da_iter, const int ens_no)
void	GetEnsembleCheckpointName (int da_iter, int ens_no)
void	init_custom (int lev)
void	fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, double time)
void	MakeEBGeometry ()
void	make_eb_box ()
void	make_eb_regular ()
void	AverageDownTo (int crse_lev, int scomp, int ncomp, bool do_perturbational_and_momenta=true)
void	AverageDownMoistStateTo (int crse_lev)
	Conservation-preserving fine-to-coarse average of RhoTheta and the active moist conserved components using detJ/mfac weighting, reusing AverageDownTo with perturbational-state and momentum handling disabled. More...
void	WriteCheckpointFile () const
void	ReadCheckpointFile ()
void	ReadVelsOnlyFromCheckpointFile (int lev_to_fill, std::string &chkfile)
void	ReadCheckpointFileSurfaceLayer ()
void	fill_terrain_surface (int lev, amrex::MultiFab &z_phys, double elapsed_time)
void	init_zphys (int lev, double elapsed_time)
void	remake_zphys (int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
void	update_terrain_arrays (int lev)
void	writeJobInfo (const std::string &dir, erf_provenance::ArtifactType artifact_type, int artifact_step, double artifact_time_seconds) const

Static Public Member Functions
static bool	is_it_time_for_action (int nstep, double time, double dt, int action_interval, double action_per)
static void	writeBuildInfo (std::ostream &os)
static void	print_banner (MPI_Comm, std::ostream &)
static void	print_usage (MPI_Comm, std::ostream &)
static void	print_error (MPI_Comm, const std::string &msg)
static void	print_summary (std::ostream &)
static void	print_tpls (std::ostream &)

Public Attributes
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_track_xy
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_eye_track_xy
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_eye_track_latlon
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_maxvel_vs_time
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_minpressure_vs_time
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_tracker_circle
amrex::Vector< amrex::MultiFab >	weather_forecast_data_1
amrex::Vector< amrex::MultiFab >	weather_forecast_data_2
amrex::Vector< amrex::Vector< amrex::MultiFab > >	forecast_state_1
amrex::Vector< amrex::Vector< amrex::MultiFab > >	forecast_state_2
amrex::Vector< amrex::Vector< amrex::MultiFab > >	forecast_state_interp
amrex::Vector< amrex::MultiFab >	surface_state_1
amrex::Vector< amrex::MultiFab >	surface_state_2
amrex::Vector< amrex::MultiFab >	surface_state_interp
amrex::Vector< amrex::Vector< amrex::MultiFab > >	bckgnd_state
std::string	pp_prefix {"erf"}
std::set< std::string >	user_specified_inputs
bool	m_driver_has_atm2ocn_coupling = false
bool	m_driver_uses_two_way_coupling = false
bool	m_driver_atm2ocn_uses_state_contract = true
std::unique_ptr< amrex::MultiFab >	m_coupled_sst
std::unique_ptr< amrex::iMultiFab >	m_coupled_sst_valid
bool	m_warned_coupled_sst_declined = false

Private Member Functions
void	ReadParameters ()
void	ParameterSanityChecks ()
bool	has_surface_layer_inputs (const std::string &prefix) const
void	AverageDown ()
void	update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
void	Construct_ERFFillPatchers (int lev)
void	Define_ERFFillPatchers (int lev)
void	init1DArrays ()
void	init_bcs ()
void	init_phys_bcs (bool &read_prim_theta)
void	init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
void	turbPert_update (const int lev, const double dt)
void	turbPert_amplitude (const int lev)
void	turbPert_amplitude_w (const int lev)
void	initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
void	make_physbcs (int lev)
void	make_lsm_at_level (int lev, bool from_regrid[[maybe_unused]]=false)
void	initializeMicrophysics (const int &)
void	initializeEB ()
void	FillPatchCrseLevel (int lev, double time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
void	FillPatchFineLevel (int lev, double time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
void	FillIntermediatePatch (int lev, double time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
void	FillCoarsePatch (int lev, double time)
void	timeStep (int lev, double time, int iteration)
void	Advance (int lev, double time, double dt_lev, int iteration, int ncycle)
void	initHSE ()
	Initialize HSE. More...
void	initHSE (int lev)
void	interp_base_state_from_coarse (int lev)
void	initRayleigh_at_level (const int &lev)
	Initialize Rayleigh damping profiles at a level. More...
void	initSponge ()
	Initialize sponge profiles. More...
void	setRayleighRefFromSounding (bool restarting)
	Set Rayleigh mean profiles from input sounding. More...
void	setSpongeRefFromSounding (bool restarting)
	Set sponge mean profiles from input sounding. More...
void	ComputeDt (int step=-1, double cur_time_d=0.0)
std::string	PlotFileName (int lev) const
void	setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
void	canonicalizePlot3DVariables (amrex::Vector< std::string > &plot_var_names)
void	appendPlot3DVariables (const amrex::Vector< std::string > &plot_var_names, amrex::Vector< std::string > &a_plot_var_names)
void	setPlotVariables2D (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
void	appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
void	setSubVolVariables (const std::string &pp_subvol_var_names, amrex::Vector< std::string > &subvol_var_names)
void	init_Dirichlet_bc_data (const std::string input_file)
void	InitializeFromFile ()
void	InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)
void	post_update (amrex::MultiFab &state_mf, double time, const amrex::Geometry &geom)
void	fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, double time, const amrex::Geometry &geom)
void	define_column_kextent (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
void	init_geo_wind_profile (const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
void	refinement_criteria_setup ()
AMREX_FORCE_INLINE amrex::YAFluxRegister *	getAdvFluxReg (int lev)
AMREX_FORCE_INLINE std::ostream &	DataLog (int i)
AMREX_FORCE_INLINE std::ostream &	DerDataLog (int i)
AMREX_FORCE_INLINE int	NumDataLogs () noexcept
AMREX_FORCE_INLINE int	NumDerDataLogs () noexcept
AMREX_FORCE_INLINE std::ostream &	SamplePointLog (int i)
AMREX_FORCE_INLINE int	NumSamplePointLogs () noexcept
AMREX_FORCE_INLINE std::ostream &	SampleLineLog (int i)
AMREX_FORCE_INLINE int	NumSampleLineLogs () noexcept
amrex::IntVect &	SamplePoint (int i)
AMREX_FORCE_INLINE int	NumSamplePoints () noexcept
amrex::IntVect &	SampleLine (int i)
AMREX_FORCE_INLINE int	NumSampleLines () noexcept
void	setRecordDataInfo (int i, const std::string &filename)
void	setRecordDerDataInfo (int i, const std::string &filename)
void	setRecordEnergyDataInfo (int i, const std::string &filename)
void	setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
void	setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
std::string	DataLogName (int i) const noexcept
	The filename of the ith datalog file. More...
std::string	DerDataLogName (int i) const noexcept
std::string	SamplePointLogName (int i) const noexcept
	The filename of the ith sampleptlog file. More...
std::string	SampleLineLogName (int i) const noexcept
	The filename of the ith samplelinelog file. More...
eb_ const &	get_eb (int lev) const noexcept
amrex::EBFArrayBoxFactory const &	EBFactory (int lev) const noexcept

Static Private Member Functions
static amrex::Vector< std::string >	PlotFileVarNames (amrex::Vector< std::string > plot_var_names)
static void	GotoNextLine (std::istream &is)
static bool	grids_are_split_in_z (const amrex::BoxArray &ba)
static AMREX_FORCE_INLINE int	ComputeGhostCells (const SolverChoice &sc)
static double	getCPUTime ()
static int	nghost_eb_basic ()
static int	nghost_eb_volume ()
static int	nghost_eb_full ()

Private Attributes
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	lat_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	lon_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	sinPhi_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	cosPhi_m
InputSoundingData	input_sounding_data
InputSpongeData	input_sponge_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	xvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	yvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	zvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	th_bc_data
amrex::Vector< amrex::TableData< amrex::Real, 1 > >	r_plane_avg
amrex::Vector< amrex::TableData< amrex::Real, 1 > >	t_plane_avg
std::unique_ptr< ProblemBase >	prob = nullptr
amrex::Vector< int >	num_boxes_at_level
amrex::Vector< int >	num_files_at_level
amrex::Vector< amrex::Vector< amrex::Box > >	boxes_at_level
amrex::Vector< int >	istep
amrex::Vector< int >	nsubsteps
amrex::Vector< int >	rad_interp_from_coarse_pending
amrex::Vector< double >	t_new
amrex::Vector< double >	t_old
amrex::Vector< double >	dt
amrex::Vector< long >	dt_mri_ratio
amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_new
amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_old
amrex::Vector< amrex::Vector< amrex::MultiFab > >	gradp
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vel_t_avg
amrex::Vector< double >	t_avg_cnt
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	interval_means
amrex::Vector< double >	t_mean_cnt
int	mean_vars_time_reset_done {0}
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > >	mri_integrator_mem
amrex::Vector< amrex::MultiFab >	pp_inc
amrex::Vector< amrex::MultiFab >	lagged_delta_rt
amrex::Vector< amrex::MultiFab >	avg_xmom
amrex::Vector< amrex::MultiFab >	avg_ymom
amrex::Vector< amrex::MultiFab >	avg_zmom
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > >	physbcs_cons
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > >	physbcs_u
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > >	physbcs_v
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > >	physbcs_w
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > >	physbcs_base
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Theta_prim
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Qv_prim
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Qr_prim
amrex::Vector< amrex::MultiFab >	rU_old
amrex::Vector< amrex::MultiFab >	rU_new
amrex::Vector< amrex::MultiFab >	rV_old
amrex::Vector< amrex::MultiFab >	rV_new
amrex::Vector< amrex::MultiFab >	rW_old
amrex::Vector< amrex::MultiFab >	rW_new
amrex::Vector< amrex::MultiFab >	zmom_crse_rhs
std::unique_ptr< Microphysics >	micro
std::unique_ptr< CloudChamberBudget >	cloud_chamber_budget
erf_cloud_chamber::Config	cloud_chamber_config
amrex::Vector< amrex::Vector< amrex::MultiFab * > >	qmoist
LandSurface	lsm
amrex::Vector< std::string >	lsm_data_name
amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_data
amrex::Vector< std::string >	lsm_flux_name
amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_flux
LargeScaleForcingData	lsf
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	nudge_data
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	lsf_data
amrex::Vector< std::unique_ptr< IRadiation > >	rad
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	qheating_rates
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	rad_fluxes
TwoStreamRadiation	two_stream_rad
amrex::Vector< std::unique_ptr< ShocDriver > >	native_shoc_driver
bool	plot_rad = false
int	rad_datalog_int = -1
int	cf_width {0}
int	cf_set_width {0}
amrex::Vector< ERFFillPatcher >	FPr_c
amrex::Vector< ERFFillPatcher >	FPr_u
amrex::Vector< ERFFillPatcher >	FPr_v
amrex::Vector< ERFFillPatcher >	FPr_w
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	Tau
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	Tau_corr
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	eddyDiffs_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SmnSmn_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	sst_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	tsk_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	lmask_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	land_type_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	soil_type_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	urb_frac_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx1_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx2_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx3_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_diss_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx1_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx2_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx3_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q2fx3_lev
amrex::Vector< amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > >	Tau_EB
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	hfx3_EB
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	qfx3_EB
amrex::Vector< amrex::Vector< amrex::Real > >	zlevels_stag
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_cc
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_cc_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_t_rk
IBSEBParams	ibseb_params
amrex::Vector< std::unique_ptr< IBFaceSet > >	m_ibseb
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	terrain_blanking
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	terrain_blanking_xface
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	terrain_blanking_yface
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	terrain_blanking_zface
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	walldist
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	mapfac
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	fine_mask
amrex::Vector< amrex::Vector< amrex::Real > >	stretched_dz_h
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	stretched_dz_d
amrex::Vector< amrex::MultiFab >	base_state
amrex::Vector< amrex::MultiFab >	base_state_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Hwave
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Lwave
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Hwave_onegrid
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Lwave_onegrid
bool	finished_wave = false
amrex::Vector< amrex::YAFluxRegister * >	advflux_reg
amrex::Vector< amrex::BCRec >	domain_bcs_type
amrex::Gpu::DeviceVector< amrex::BCRec >	domain_bcs_type_d
amrex::Array< std::string, 2 *AMREX_SPACEDIM >	domain_bc_type
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max >	m_bc_extdir_vals
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max >	m_bc_neumann_vals
amrex::Array< amrex::Array< erf_wall_scalar_bc::WallScalarBCIntent, AMREX_SPACEDIM *2 >, NBCVAR_max >	m_wall_scalar_bc_intent {}
amrex::Array< bool, AMREX_SPACEDIM *2 >	m_bc_nonreflecting = {{false}}
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 >	phys_bc_type
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	xflux_imask
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	yflux_imask
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	zflux_imask
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_xforce
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_yforce
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_zforce
amrex::Vector< int >	last_subvol_step
amrex::Vector< double >	last_subvol_time
const int	datwidth = 14
const int	datprecision = 6
const int	timeprecision = 13
int	max_step = -1
bool	use_datetime = false
const std::string	datetime_format = "%Y-%m-%d %H:%M:%S"
std::string	restart_chkfile = ""
erf_provenance::ExecutionProvenance	execution_provenance
amrex::Vector< double >	fixed_dt
amrex::Vector< double >	fixed_fast_dt
int	regrid_int = -1
bool	regrid_level_0_on_restart = false
std::string	plot3d_file_1 {"plt_1_"}
std::string	plot3d_file_2 {"plt_2_"}
std::string	plot2d_file_1 {"plt2d_1_"}
std::string	plot2d_file_2 {"plt2d_2_"}
std::string	subvol_file {"subvol"}
bool	m_expand_plotvars_to_unif_rr = false
int	m_plot3d_int_1 = -1
int	m_plot3d_int_2 = -1
int	m_plot2d_int_1 = -1
int	m_plot2d_int_2 = -1
amrex::Vector< int >	m_subvol_int
amrex::Vector< double >	m_subvol_per
double	m_plot3d_per_1 = -1.0
double	m_plot3d_per_2 = -1.0
double	m_plot2d_per_1 = -1.0
double	m_plot2d_per_2 = -1.0
bool	m_plot_face_vels = false
bool	m_plot_face_terrain_blanking = false
bool	plot_lsm = false
int	profile_int = -1
bool	destag_profiles = true
std::string	check_file {"chk"}
int	m_check_int = -1
double	m_check_per = -1.0
amrex::Vector< std::string >	subvol3d_var_names
amrex::Vector< std::string >	plot3d_var_names_1
amrex::Vector< std::string >	plot3d_var_names_2
amrex::Vector< std::string >	plot2d_var_names_1
amrex::Vector< std::string >	plot2d_var_names_2
const amrex::Vector< std::string >	cons_names
const amrex::Vector< std::string >	derived_names
const amrex::Vector< std::string >	derived_subvol_names
TurbulentPerturbation	turbPert
int	file_name_digits = 5
bool	use_real_time_in_pltname = false
int	real_width {0}
bool	real_extrap_w {true}
bool	metgrid_debug_quiescent {false}
bool	metgrid_debug_isothermal {false}
bool	metgrid_debug_dry {false}
bool	metgrid_debug_psfc {false}
bool	metgrid_debug_msf {false}
bool	metgrid_interp_theta {false}
bool	metgrid_basic_linear {false}
bool	metgrid_use_below_sfc {true}
bool	metgrid_use_sfc {true}
bool	metgrid_retain_sfc {false}
amrex::Real	metgrid_proximity {amrex::Real(500.0)}
int	metgrid_order {2}
int	metgrid_force_sfc_k {6}
bool	write_erfbdy {false}
bool	use_erfbdy {false}
std::string	erfbdy_file {"erfbdy"}
int	nvars_erfbdy {0}
amrex::Vector< amrex::BoxArray >	ba1d
amrex::Vector< amrex::BoxArray >	ba2d
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	column_kextent
BaseStateParams	wrf_bsp
std::unique_ptr< amrex::MultiFab >	wrf_C1H
std::unique_ptr< amrex::MultiFab >	wrf_C2H
std::unique_ptr< amrex::MultiFab >	wrf_MUB
std::unique_ptr< amrex::MultiFab >	wrf_RDNW
std::unique_ptr< amrex::MultiFab >	wrf_PHB
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mf_PSFC
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	rhotheta_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	rhoqt_src
amrex::Vector< amrex::Vector< amrex::Real > >	h_w_subsid
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_w_subsid
amrex::Vector< amrex::Vector< amrex::Real > >	h_u_geos
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_u_geos
amrex::Vector< amrex::Vector< amrex::Real > >	h_v_geos
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_v_geos
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_rayleigh_ptrs
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_sponge_ptrs
amrex::Vector< amrex::Vector< amrex::Real > >	h_sinesq_ptrs
amrex::Vector< amrex::Vector< amrex::Real > >	h_sinesq_stag_ptrs
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_rayleigh_ptrs
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_sponge_ptrs
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_sinesq_ptrs
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_sinesq_stag_ptrs
amrex::Vector< amrex::Real >	h_havg_density
amrex::Vector< amrex::Real >	h_havg_temperature
amrex::Vector< amrex::Real >	h_havg_pressure
amrex::Vector< amrex::Real >	h_havg_qv
amrex::Vector< amrex::Real >	h_havg_qc
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_density
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_temperature
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_pressure
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qv
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qc
std::unique_ptr< WriteBndryPlanes >	m_w2d = nullptr
std::unique_ptr< ReadBndryPlanes >	m_r2d = nullptr
amrex::Vector< std::unique_ptr< SurfaceLayer > >	m_SurfaceLayer
amrex::Vector< std::unique_ptr< ForestDrag > >	m_forest_drag
amrex::Vector< amrex::Vector< amrex::BoxArray > >	subdomains
amrex::Vector< amrex::Real >	dz_min
int	line_sampling_interval = -1
int	plane_sampling_interval = -1
amrex::Real	line_sampling_per = -one
amrex::Real	plane_sampling_per = -one
std::unique_ptr< LineSampler >	line_sampler = nullptr
std::unique_ptr< PlaneSampler >	plane_sampler = nullptr
int	station_sampling_interval = -1
amrex::Real	station_sampling_per = -one
std::unique_ptr< StationSampler >	station_sampler = nullptr
amrex::Vector< std::string >	station_vars_3d
amrex::Vector< std::string >	station_vars_2d
amrex::Vector< std::unique_ptr< std::fstream > >	datalog
amrex::Vector< std::unique_ptr< std::fstream > >	der_datalog
amrex::Vector< std::unique_ptr< std::fstream > >	tot_e_datalog
amrex::Vector< std::string >	datalogname
amrex::Vector< std::string >	der_datalogname
amrex::Vector< std::string >	tot_e_datalogname
amrex::Vector< std::unique_ptr< std::fstream > >	sampleptlog
amrex::Vector< std::string >	sampleptlogname
amrex::Vector< amrex::IntVect >	samplepoint
amrex::Vector< std::unique_ptr< std::fstream > >	samplelinelog
amrex::Vector< std::string >	samplelinelogname
amrex::Vector< amrex::IntVect >	sampleline
amrex::Vector< std::unique_ptr< eb_ > >	eb

Static Private Attributes
static int	last_plot3d_file_step_1 = -1
static int	last_plot3d_file_step_2 = -1
static int	last_plot2d_file_step_1 = -1
static int	last_plot2d_file_step_2 = -1
static int	last_check_file_step = -1
static double	last_plot3d_file_time_1 = 0.0
static double	last_plot3d_file_time_2 = 0.0
static double	last_plot2d_file_time_1 = 0.0
static double	last_plot2d_file_time_2 = 0.0
static double	last_check_file_time = 0.0
static bool	plot_file_on_restart = true
static double	start_time = 0.0
static double	stop_time = std::numeric_limits<double>::max()
static amrex::Real	cfl = Real(0.8)
static amrex::Real	sub_cfl = one
static amrex::Real	init_shrink = one
static amrex::Real	change_max = Real(1.1)
static double	dt_max_initial = static_cast<double>(bogus_large_value)
static double	dt_max = 1.0e9
static int	fixed_mri_dt_ratio = 0
static SolverChoice	solverChoice
static int	verbose = 0
static int	mg_verbose = 0
static bool	use_fft = false
static int	check_for_nans = 0
static int	sum_interval = -1
static int	pert_interval = -1
static double	sum_per = -1.0
static PlotFileType	plotfile3d_type_1 = PlotFileType::None
static PlotFileType	plotfile3d_type_2 = PlotFileType::None
static PlotFileType	plotfile2d_type_1 = PlotFileType::None
static PlotFileType	plotfile2d_type_2 = PlotFileType::None
static StateInterpType	interpolation_type
static amrex::Vector< amrex::Vector< std::string > >	nc_init_file = {{""}}
static amrex::Vector< amrex::Vector< int > >	have_read_nc_init_file = {{0}}
static std::string	nc_bdy_file
static std::string	nc_low_file
static constexpr int	column_kextent_lo_sentinel = std::numeric_limits<int>::min()/2
static constexpr int	column_kextent_hi_sentinel = std::numeric_limits<int>::max()/2
static int	output_1d_column = 0
static int	column_interval = -1
static double	column_per = -1.0
static amrex::Real	column_loc_x = zero
static amrex::Real	column_loc_y = zero
static std::string	column_file_name = "column_data.nc"
static int	output_bndry_planes = 0
static int	bndry_output_planes_interval = -1
static double	bndry_output_planes_per = -1.0
static double	bndry_output_planes_start_time = 0.0
static int	input_bndry_planes = 0
static int	ng_dens_hse
static int	ng_pres_hse
static amrex::Vector< amrex::AMRErrorTag >	ref_tags
static amrex::Vector< std::string >	ref_tag_indicator_names
static double	startCPUTime = 0.0
static double	previousCPUTimeUsed = 0.0

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

ERF() [1/3]

ERF::ERF

(

)

{
    ERF_shared();
}

~ERF()

ERF::~ERF ( )

overridedefault

ERF() [2/3]

ERF::ERF ( ERF &&  )

deletenoexcept

ERF() [3/3]

ERF::ERF ( const ERF &  other )

delete

Member Function Documentation

Advance()

void ERF::Advance ( int  lev, double  time, double  dt_lev, int  iteration, int  ncycle  )

private

Function that advances the solution at one level for a single time step – this does some preliminaries then calls erf_advance

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	dt_lev	time step for this time advance

{
    BL_PROFILE("ERF::Advance()");
 
    // We must swap the pointers so the previous step's "new" is now this step's "old"
    std::swap(vars_old[lev], vars_new[lev]);
 
    // Surface energy balance on the building faces, with the state at the start of the step
    ibseb_advance(lev, time, dt_lev, vars_old[lev][Vars::cons],
                  vars_old[lev][Vars::xvel], vars_old[lev][Vars::yvel], vars_old[lev][Vars::zvel]);
 
    MultiFab& S_old = vars_old[lev][Vars::cons];
    MultiFab& S_new = vars_new[lev][Vars::cons];
 
    MultiFab& U_old = vars_old[lev][Vars::xvel];
    MultiFab& V_old = vars_old[lev][Vars::yvel];
    MultiFab& W_old = vars_old[lev][Vars::zvel];
 
    MultiFab& U_new = vars_new[lev][Vars::xvel];
    MultiFab& V_new = vars_new[lev][Vars::yvel];
    MultiFab& W_new = vars_new[lev][Vars::zvel];
 
    // We need to set these because otherwise in the first call to erf_advance we may
    //    read uninitialized data on ghost values in setting the bc's on the velocities
    U_new.setVal(bogus_large_value,U_new.nGrowVect());
    V_new.setVal(bogus_large_value,V_new.nGrowVect());
    W_new.setVal(bogus_large_value,W_new.nGrowVect());
 
    //
    // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
    // If lev == 0 we have already FillPatched this in ERF::TimeStep
    //
    if (lev > 0) {
        // Set ghost cells to bogus values so they aren't uninitialized
        W_old.setBndry(bogus_large_value);
        FillPatchFineLevel(lev, time, {&S_old, &U_old, &V_old, &W_old},
                           {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]},
                           base_state[lev], base_state[lev]);
    }
 
    //
    // So we must convert the fillpatched to momenta, including the ghost values
    //
    const MultiFab* c_vfrac = nullptr;
    if (solverChoice.terrain_type == TerrainType::EB) {
        c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
    }
 
    VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
                       V_old, rV_old[lev].nGrowVect(),
                       W_old, rW_old[lev].nGrowVect(),
                       S_old, rU_old[lev], rV_old[lev], rW_old[lev],
                       Geom(lev).Domain(),
                       domain_bcs_type, c_vfrac);
 
    // Update the inflow perturbation update time and amplitude
    if (solverChoice.use_perturbation(lev))
    {
        turbPert.calc_tpi_update(lev, dt_lev, U_old, V_old, S_old);
    }
 
    // If PerturbationType::Direct or CPM is selected, directly add the computed perturbation
    // on the conserved field
    if (solverChoice.use_direct_perturbation(lev))
    {
        if (solverChoice.use_wvel_perturbation(lev)) { // CPM_W
            auto m_ixtype = W_old.boxArray().ixType();
            for (MFIter mfi(W_old,TileNoZ()); mfi.isValid(); ++mfi) {
                Box bx  = mfi.tilebox();
                const Array4<Real> &cell_data  = W_old.array(mfi);
                const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
                turbPert.apply_tpi(lev, bx, -1, m_ixtype, cell_data, pert_cell);
            }
        } else {
            auto m_ixtype = S_old.boxArray().ixType(); // Conserved term
            for (MFIter mfi(S_old,TileNoZ()); mfi.isValid(); ++mfi) {
                Box bx  = mfi.tilebox();
                const Array4<Real> &cell_data  = S_old.array(mfi);
                const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
                turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cell_data, pert_cell);
            }
        }
    }
 
    // configure SurfaceLayer params if needed
    bool updated_prim = false;
    for (OrientationIter oit; oit; ++oit) {
        Orientation ori = oit();
        if (phys_bc_type[ori] == ERF_BC::surface_layer && m_SurfaceLayer[ori]) {
            if (!updated_prim) {
                // This only needs to be done once
                IntVect ng = Theta_prim[lev]->nGrowVect();
                MultiFab::Copy(  *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
                MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp     , 0, 1, ng);
                if (solverChoice.moisture_type != MoistureType::None) {
                    ng = Qv_prim[lev]->nGrowVect();
 
                    MultiFab::Copy(  *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
                    MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp  , 0, 1, ng);
 
                    if (solverChoice.moisture_indices.qr > -1) {
                        MultiFab::Copy(  *Qr_prim[lev], S_old, solverChoice.moisture_indices.qr, 0, 1, ng);
                        MultiFab::Divide(*Qr_prim[lev], S_old, Rho_comp  , 0, 1, ng);
                    } else {
                        Qr_prim[lev]->setVal(0);
                    }
                }
                updated_prim = true;
            }
            // NOTE: std::swap above causes the field ptrs to be out of date.
            //       Reassign the field ptrs for MAC avg computation.
            m_SurfaceLayer[ori]->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim, Qr_prim);
            m_SurfaceLayer[ori]->update_pblh(lev, vars_old, z_phys_cc[lev].get(),
                                             solverChoice.moisture_indices);
 
#ifdef ERF_USE_NETCDF
            double elapsed_time_since_start_low = time + (start_time - start_low_time);
#else
            double elapsed_time_since_start_low = time;
#endif
            m_SurfaceLayer[ori]->update_fluxes(lev, time, elapsed_time_since_start_low,
                                               S_old, z_phys_nd[lev], walldist[lev]);
        }
    }
 
#if defined(ERF_USE_WINDFARM)
    // **************************************************************************************
    // Update the windfarm sources
    // **************************************************************************************
    if (solverChoice.windfarm_type != WindFarmType::None) {
        advance_windfarm(Geom(lev), dt_lev, S_old,
                         U_old, V_old, W_old, vars_windfarm[lev],
                         Nturb[lev], SMark[lev], time);
    }
 
#endif
 
    // **************************************************************************************
    // Update the radiation sources with the "old" state
    // **************************************************************************************
    advance_radiation(lev, S_old, dt_lev);
 
    // **************************************************************************************
    // Update the "old" state using SHOC
    // **************************************************************************************
    if (solverChoice.turbChoice[lev].uses_shoc_family()) {
        // Get SFC fluxes from SurfaceLayer
        if (m_SurfaceLayer[Orientation::zlo()]) {
            Vector<const MultiFab*> mfs = {&S_old, &U_old, &V_old, &W_old};
            m_SurfaceLayer[Orientation::zlo()]->impose_SurfaceLayer_bcs(lev, mfs, Tau[lev],
                                                                        SFS_hfx1_lev[lev].get() , SFS_hfx2_lev[lev].get() , SFS_hfx3_lev[lev].get(),
                                                                        SFS_q1fx1_lev[lev].get(), SFS_q1fx2_lev[lev].get(), SFS_q1fx3_lev[lev].get(),
                                                                        z_phys_nd[lev].get());
        }
 
        // Apply SHOC before the dycore so it sees a coherent state.
        Real* w_sub = (solverChoice.custom_w_subsidence) ? d_w_subsid[lev].data() : nullptr;
        if (solverChoice.turbChoice[lev].uses_eamxx_shoc()) {
#ifdef ERF_USE_EAMXX_SHOC
            compute_shoc_tendencies(lev, &S_old, &U_old, &V_old, &W_old, w_sub,
                                    Tau[lev][TauType::tau13].get(), Tau[lev][TauType::tau23].get(),
                                    SFS_hfx3_lev[lev].get()       , SFS_q1fx3_lev[lev].get()      ,
                                    eddyDiffs_lev[lev].get()      , z_phys_nd[lev].get()          ,
                                    dt_lev);
#endif
        } else if (solverChoice.turbChoice[lev].uses_native_shoc()) {
            compute_native_shoc_tendencies(lev, &S_old, &U_old, &V_old, &W_old, w_sub,
                                           Tau[lev][TauType::tau13].get(), Tau[lev][TauType::tau23].get(),
                                           SFS_hfx3_lev[lev].get()       , SFS_q1fx3_lev[lev].get()      ,
                                           eddyDiffs_lev[lev].get()      , z_phys_nd[lev].get()          ,
                                           dt_lev);
 
            if (native_shoc_driver[lev]) {
                // Native SHOC updates the old-time state before the dycore reads it.
                // Re-fill the updated state, velocities, and momenta now so the
                // pre-dycore checks and strain calculation see coherent fields.
                Vector<MultiFab*> mfs_vel = {&S_old, &U_old, &V_old, &W_old};
                if (lev == 0) {
                    FillPatchCrseLevel(lev, time, mfs_vel, false);
                    VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
                                       V_old, rV_old[lev].nGrowVect(),
                                       W_old, rW_old[lev].nGrowVect(),
                                       S_old, rU_old[lev], rV_old[lev], rW_old[lev],
                                       Geom(lev).Domain(),
                                       domain_bcs_type, c_vfrac);
                } else {
                    Vector<MultiFab*> mfs_mom = {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]};
                    FillPatchFineLevel(lev, time, mfs_vel, mfs_mom,
                                       base_state[lev], base_state[lev],
                                       true, false);
                }
            }
        }
    }
 
    const BoxArray&            ba = S_old.boxArray();
    const DistributionMapping& dm = S_old.DistributionMap();
 
    int nvars = S_old.nComp();
 
    // Source array for conserved cell-centered quantities -- this will be filled
    //     in the call to make_sources in ERF_TI_slow_rhs_pre.H
    MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0);
 
    // Source arrays for momenta -- these will be filled
    //     in the call to make_mom_sources in ERF_TI_slow_rhs_pre.H
    BoxArray ba_x(ba); ba_x.surroundingNodes(0);
    MultiFab xmom_source(ba_x,dm,1,1); xmom_source.setVal(0);
 
    BoxArray ba_y(ba); ba_y.surroundingNodes(1);
    MultiFab ymom_source(ba_y,dm,1,1); ymom_source.setVal(0);
 
    BoxArray ba_z(ba); ba_z.surroundingNodes(2);
    MultiFab zmom_source(ba_z,dm,1,1); zmom_source.setVal(0);
    MultiFab    buoyancy(ba_z,dm,1,1); buoyancy.setVal(0);
 
    amrex::Vector<MultiFab> state_old;
    amrex::Vector<MultiFab> state_new;
 
    // **************************************************************************************
    // Here we define state_old and state_new which are to be advanced
    // **************************************************************************************
    // Initial solution
    // Note that "old" and "new" here are relative to each RK stage.
    state_old.push_back(MultiFab(S_old      , amrex::make_alias, 0, nvars)); // cons
    state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0,     1)); // xmom
    state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0,     1)); // ymom
    state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0,     1)); // zmom
 
    // Final solution
    // state_new at the end of the last RK stage holds the t^{n+1} data
    state_new.push_back(MultiFab(S_new      , amrex::make_alias, 0, nvars)); // cons
    state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0,     1)); // xmom
    state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0,     1)); // ymom
    state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0,     1)); // zmom
 
    // **************************************************************************************
    // Tests on the reasonableness of the solution before the dycore
    // **************************************************************************************
    // Test for NaNs after dycore
    if (check_for_nans > 1) {
        if (verbose > 1) {
            amrex::Print() << "Testing old state and vels for NaNs before dycore" << std::endl;
        }
        check_state_for_nans(S_old);
        check_vels_for_nans(rU_old[lev],rV_old[lev],rW_old[lev]);
    }
 
    // We only test on low temp if we have a moisture model because we are protecting against
    //    the test on low temp inside the moisture models
    if (solverChoice.moisture_type != MoistureType::None) {
        if (verbose > 1) {
            amrex::Print() << "Testing on low temperature before dycore" << std::endl;
        }
        check_for_low_temp(S_old);
    } else {
        if (verbose > 1) {
            amrex::Print() << "Testing on negative temperature before dycore" << std::endl;
        }
        check_for_negative_theta(S_old);
    }
 
    // **************************************************************************************
    // Update the dycore
    // **************************************************************************************
    advance_dycore(lev, state_old, state_new,
                   U_old, V_old, W_old,
                   U_new, V_new, W_new,
                   cc_source, xmom_source, ymom_source, zmom_source, buoyancy,
                   Geom(lev), dt_lev, time);
 
    // **************************************************************************************
    // Tests on the reasonableness of the solution after the dycore
    // **************************************************************************************
    // Test for NaNs after dycore
    if (check_for_nans > 0) {
        if (verbose > 1) {
            amrex::Print() << "Testing new state and vels for NaNs after dycore" << std::endl;
        }
        check_state_for_nans(S_new);
        check_vels_for_nans(rU_new[lev],rV_new[lev],rW_new[lev]);
    }
 
    // We only test on low temp if we have a moisture model because we are protecting against
    //    the test on low temp inside the moisture models
    if (solverChoice.moisture_type != MoistureType::None) {
        if (verbose > 1) {
            amrex::Print() << "Testing on low temperature after dycore" << std::endl;
        }
        check_for_low_temp(S_new);
    } else {
        // Otherwise we will test on negative (rhotheta) coming out of the dycore
        if (verbose > 1) {
            amrex::Print() << "Testing on negative temperature after dycore" << std::endl;
        }
        check_for_negative_theta(S_new);
    }
 
    // **************************************************************************************
    // Update the microphysics (moisture)
    // **************************************************************************************
    if (!solverChoice.moisture_tight_coupling)
    {
        // S_new ghost cells are stale after the dycore RK stages; refresh
        // them before microphysics (Lagrangian particle interpolation reads
        // the ghost region for cells along level boundaries).
        if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
            if (lev == 0) {
                FillPatchCrseLevel(lev, t_new[lev],
                                   {&S_new, &U_new, &V_new, &W_new},
                                   /*cons_only=*/true);
            } else {
                FillPatchFineLevel(lev, t_new[lev],
                                   {&S_new, &U_new, &V_new, &W_new},
                                   {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                                   base_state[lev], base_state[lev],
                                   /*fillset=*/true, /*cons_only=*/true);
            }
        }
 
        advance_microphysics(lev, S_new, dt_lev, iteration, time);
 
        // Test for NaNs after microphysics
        if (check_for_nans > 0) {
            amrex::Print() << "Testing new state for NaNs after advance_microphysics" << std::endl;
            check_state_for_nans(S_new);
        }
    }
 
    // **************************************************************************************
    // Update the land surface model
    // **************************************************************************************
    double time_at_end_of_step = time+dt_lev;
    advance_lsm(lev, S_new, U_new, V_new, time_at_end_of_step, dt_lev);
 
#ifdef ERF_USE_PARTICLES
    // **************************************************************************************
    // Update the particle positions
    // **************************************************************************************
   evolveTracers(lev, dt_lev, vars_new, z_phys_nd);
#endif
 
    // ***********************************************************************************************
    // Impose domain boundary conditions here so that in FillPatching the fine data we won't
    // need to re-fill these
    // ***********************************************************************************************
    if (lev < finest_level) {
         IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
         (*physbcs_cons[lev])(vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                              0,vars_new[lev][Vars::cons].nComp(),
                              vars_new[lev][Vars::cons].nGrowVect(),time,BCVars::cons_bc,true);
            (*physbcs_u[lev])(vars_new[lev][Vars::xvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                              ngvect_vels,time,BCVars::xvel_bc,true);
            (*physbcs_v[lev])(vars_new[lev][Vars::yvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                              ngvect_vels,time,BCVars::yvel_bc,true);
            (*physbcs_w[lev])(vars_new[lev][Vars::zvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                              ngvect_vels,time,BCVars::zvel_bc,true);
    }
 
    // **************************************************************************************
    // Register old and new coarse data if we are at a level less than the finest level
    // **************************************************************************************
    if (lev < finest_level) {
        if (cf_width > 0) {
            // We must fill the ghost cells of these so that the parallel copy works correctly
            state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
            FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
                                          {time, time+dt_lev});
        }
 
        if (cf_width >= 0) {
            // We must fill the ghost cells of these so that the parallel copy works correctly
            state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
            FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
                                          {time, time+dt_lev});
 
            state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
            FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
                                          {time, time+dt_lev});
 
            state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
            FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
                                          {time, time+dt_lev});
        }
 
            //
            // Now create a MultiFab that holds (S_new - S_old) / dt from the coarse level interpolated
            //     on to the coarse/fine boundary at the fine resolution
            //
            Interpolater* mapper_f = &face_cons_linear_interp;
 
            // PhysBCFunctNoOp null_bc;
            // MultiFab tempx(vars_new[lev+1][Vars::xvel].boxArray(),vars_new[lev+1][Vars::xvel].DistributionMap(),1,0);
            // tempx.setVal(0);
            // xmom_crse_rhs[lev+1].setVal(0);
            // FPr_u[lev].FillSet(tempx               , time       , null_bc, domain_bcs_type);
            // FPr_u[lev].FillSet(xmom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
            // MultiFab::Subtract(xmom_crse_rhs[lev+1],tempx,0,0,1,IntVect{0});
            // xmom_crse_rhs[lev+1].mult(one/dt_lev,0,1,0);
 
            // MultiFab tempy(vars_new[lev+1][Vars::yvel].boxArray(),vars_new[lev+1][Vars::yvel].DistributionMap(),1,0);
            // tempy.setVal(0);
            // ymom_crse_rhs[lev+1].setVal(0);
            // FPr_v[lev].FillSet(tempy               , time       , null_bc, domain_bcs_type);
            // FPr_v[lev].FillSet(ymom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
            // MultiFab::Subtract(ymom_crse_rhs[lev+1],tempy,0,0,1,IntVect{0});
            // ymom_crse_rhs[lev+1].mult(one/dt_lev,0,1,0);
 
            MultiFab temp_state(zmom_crse_rhs[lev+1].boxArray(),zmom_crse_rhs[lev+1].DistributionMap(),1,0);
            InterpFromCoarseLevel(temp_state,            IntVect{0}, IntVect{0}, state_old[IntVars::zmom], 0, 0, 1,
                                  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
            InterpFromCoarseLevel(zmom_crse_rhs[lev+1],  IntVect{0}, IntVect{0}, state_new[IntVars::zmom], 0, 0, 1,
                                  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
            MultiFab::Subtract(zmom_crse_rhs[lev+1],temp_state,0,0,1,IntVect{0});
 
            Real inv_dt = static_cast<Real>(one/dt_lev);
            zmom_crse_rhs[lev+1].mult(inv_dt,0,1,0);
    }
 
    // ***********************************************************************************************
    // Update the time averaged velocities if they are requested
    // ***********************************************************************************************
    if (solverChoice.time_avg_vel) {
        Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
    }
 
    // ***********************************************************************************************
    // Two-stream radiation, post-dycore call: reports the cached flux
    // diagnostics and advances the force-restore surface state by dt_lev.
    // No column sweep runs here; that happened in advance_radiation above.
    // ***********************************************************************************************
    if (solverChoice.rad_type == RadiationType::TwoStream) {
#ifdef ERF_USE_NETCDF
        const MultiFab* lat_ptr = lat_m[lev].get();
        const MultiFab* lon_ptr = lon_m[lev].get();
#else
        const MultiFab* lat_ptr = nullptr;
        const MultiFab* lon_ptr = nullptr;
#endif
        const MultiFab* t_surf = (m_SurfaceLayer[Orientation::zlo()])
                               ? m_SurfaceLayer[Orientation::zlo()]->get_t_surf(lev)
                               : nullptr;
        two_stream_rad.advance(lev, iteration, time + dt_lev, dt_lev, "post_dycore",
                               vars_old[lev][Vars::cons], z_phys_nd[lev].get(), geom[lev],
                               lsm, qheating_rates[lev].get(), rad_fluxes[lev].get(),
                               t_surf, lat_ptr, lon_ptr,
                               time + dt_lev + start_time, use_datetime);
    }
    if (solverChoice.compute_mean_vars) {
        // The interval window is shared by all AMR levels.  Reset it before
        // accumulating the first sample whose step starts at or beyond the
        // configured reset time.  The restart reader restores the flags when
        // that time has already passed, so this is a one-shot transition.
        if (solverChoice.mean_vars_reset_mode == "time" &&
            time >= static_cast<double>(solverChoice.mean_vars_reset_time)) {
            if (mean_vars_time_reset_done == 0) {
                ResetIntervalMeans();
                mean_vars_time_reset_done = 1;
            }
        }
 
        Accumulate_Interval_Means(dt_lev, t_mean_cnt[lev], interval_means[lev].get(),
                                  U_new, V_new, W_new, S_new);
    }
 
}

Here is the call graph for this function:

advance_dycore()

void ERF::advance_dycore	(	int	level,
		amrex::Vector< amrex::MultiFab > &	state_old,
		amrex::Vector< amrex::MultiFab > &	state_new,
		amrex::MultiFab &	xvel_old,
		amrex::MultiFab &	yvel_old,
		amrex::MultiFab &	zvel_old,
		amrex::MultiFab &	xvel_new,
		amrex::MultiFab &	yvel_new,
		amrex::MultiFab &	zvel_new,
		amrex::MultiFab &	source,
		amrex::MultiFab &	xmom_src,
		amrex::MultiFab &	ymom_src,
		amrex::MultiFab &	zmom_src,
		amrex::MultiFab &	buoyancy,
		amrex::Geometry	fine_geom,
		double	dt,
		double	time
	)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters

[in]	level	level of refinement (coarsest level is 0)
[in]	state_old	old-time conserved variables
[in]	state_new	new-time conserved variables
[in]	xvel_old	old-time x-component of velocity
[in]	yvel_old	old-time y-component of velocity
[in]	zvel_old	old-time z-component of velocity
[in]	xvel_new	new-time x-component of velocity
[in]	yvel_new	new-time y-component of velocity
[in]	zvel_new	new-time z-component of velocity
[in]	cc_src	source term for conserved variables
[in]	xmom_src	source term for x-momenta
[in]	ymom_src	source term for y-momenta
[in]	zmom_src	source term for z-momenta
[in]	fine_geom	container for geometry information at current level
[in]	dt_advance	time step for this time advance
[in]	old_time	old time for this time advance

{
    BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
 
    const Box& domain = fine_geom.Domain();
 
    if (cloud_chamber_budget) {
        cloud_chamber_budget->set_initial_state(state_old[IntVars::cons], fine_geom, 0, old_time);
    }
 
    DiffChoice dc    = solverChoice.diffChoice;
    TurbChoice tc    = solverChoice.turbChoice[level];
 
    MultiFab r_hse (base_state[level], make_alias, BaseState::r0_comp , 1);
    MultiFab p_hse (base_state[level], make_alias, BaseState::p0_comp , 1);
 
    // These pointers are used in the MRI utility functions
    MultiFab* r0  = &r_hse;
    MultiFab* p0  = &p_hse;
 
    MultiFab* rhotheta_src_ptr = solverChoice.custom_rhotheta_forcing ? rhotheta_src[level].get() : nullptr;
    MultiFab* rhoqt_src_ptr    = solverChoice.custom_moisture_forcing ? rhoqt_src[level].get()   : nullptr;
    Real* dptr_wbar_sub        = solverChoice.custom_w_subsidence     ? d_w_subsid[level].data()     : nullptr;
 
    // Turbulent Perturbation Pointer
    //Real* dptr_rhotheta_src = solverChoice.pert_type ? d_rhotheta_src[level].data() : nullptr;
 
    Vector<Real*> d_rayleigh_ptrs_at_lev;
    d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
    d_rayleigh_ptrs_at_lev[Rayleigh::ubar]     = solverChoice.dampingChoice.rayleigh_damp_U   ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
    d_rayleigh_ptrs_at_lev[Rayleigh::vbar]     = solverChoice.dampingChoice.rayleigh_damp_V   ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
    d_rayleigh_ptrs_at_lev[Rayleigh::wbar]     = solverChoice.dampingChoice.rayleigh_damp_W   ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
    d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.dampingChoice.rayleigh_damp_T   ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
 
    bool use_rayleigh =
       (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
        solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T);
    Real* d_sinesq_at_lev      = (use_rayleigh)  ? d_sinesq_ptrs[level].data() : nullptr;
    Real* d_sinesq_stag_at_lev = (use_rayleigh)  ? d_sinesq_stag_ptrs[level].data() : nullptr;
 
    Vector<Real*> d_sponge_ptrs_at_lev;
    if (SpongeChoice::sponge_type == SpongeType::Input_Sponge)
    {
        d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
        d_sponge_ptrs_at_lev[Sponge::ubar_sponge]  =  d_sponge_ptrs[level][Sponge::ubar_sponge].data();
        d_sponge_ptrs_at_lev[Sponge::vbar_sponge]  =  d_sponge_ptrs[level][Sponge::vbar_sponge].data();
    }
 
    bool l_use_terrain_fitted_coords = (solverChoice.mesh_type != MeshType::ConstantDz);
    bool l_use_kturb   = tc.use_kturb;
    bool l_use_diff    = ( (dc.molec_diff_type != MolecDiffType::None) ||
                           l_use_kturb );
 
    const bool use_SurfLayer = (m_SurfaceLayer[Orientation(Direction::z, Orientation::low)] != nullptr);
    const MultiFab* z_0      = (use_SurfLayer) ? m_SurfaceLayer[Orientation(Direction::z, Orientation::low)]->get_z0(level) : nullptr;
 
    const bool use_nudging = solverChoice.nudging_from_input_sounding;
    const bool has_moisture = (solverChoice.moisture_type != MoistureType::None);
    const bool use_lsf = solverChoice.large_scale_forcing;
 
    const BoxArray& ba            = state_old[IntVars::cons].boxArray();
    const BoxArray& ba_z          = zvel_old.boxArray();
    const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
 
    int num_prim = state_old[IntVars::cons].nComp() - 1;
 
    MultiFab    S_prim  (ba  , dm, num_prim,          state_old[IntVars::cons].nGrowVect());
    MultiFab  pi_stage  (ba  , dm,        1,          1);
    MultiFab fast_coeffs(ba_z, dm,        5,          0);
 
    MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
    MultiFab* SmnSmn    = SmnSmn_lev[level].get();
 
    // *****************************************************************************
    // Planar averages for subsidence terms, nudging, and immersed forcing
    // *****************************************************************************
    bool use_immersed_forcing = (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                                  solverChoice.buildings_type == BuildingsType::ImmersedForcing);
 
    Table1D<Real> dptr_r_plane, dptr_t_plane, dptr_qv_plane;
    TableData<Real, 1> r_plane_tab, t_plane_tab,  qv_plane_tab;
 
    Table1D<Real> dptr_u_plane, dptr_v_plane;
    TableData<Real, 1> u_plane_tab, v_plane_tab;
    if (use_nudging || use_immersed_forcing)
    {
        // Rho
        IntVect ng_c(state_old[IntVars::cons].nGrowVect()); ng_c[2] = 1;
        int ncomp = (solverChoice.moisture_type == MoistureType::None) ? 2 : RhoQ2_comp;
        MultiFab cons(state_old[IntVars::cons], make_alias, 0, ncomp);
 
        // Immersed forcing requires z-direction (ave_plane=2) averages to match persistent table sizing
        if (use_immersed_forcing) {
            AMREX_ALWAYS_ASSERT(solverChoice.ave_plane == 2);
        }
 
        PlaneAverage cons_ave(&cons, fine_geom, solverChoice.ave_plane, ng_c);
        cons_ave.compute_averages(ZDir(), cons_ave.field());
 
        int ncell = cons_ave.ncell_line();
 
        Gpu::HostVector<    Real> r_plane_h(ncell);
        Gpu::DeviceVector<  Real> r_plane_d(ncell);
        Gpu::HostVector<    Real> t_plane_h(ncell);
        Gpu::DeviceVector<  Real> t_plane_d(ncell);
 
        cons_ave.line_average(Rho_comp, r_plane_h);
        cons_ave.line_average(RhoTheta_comp, t_plane_h);
 
        Gpu::copy(Gpu::hostToDevice, r_plane_h.begin(), r_plane_h.end(), r_plane_d.begin());
        Gpu::copy(Gpu::hostToDevice, t_plane_h.begin(), t_plane_h.end(), t_plane_d.begin());
 
        Real* dptr_r = r_plane_d.data();
        Real* dptr_t = t_plane_d.data();
 
        Box tdomain  = domain; tdomain.grow(2,ng_c[2]);
        int offset = ng_c[2];
 
        // For immersed forcing without nudging, write directly to persistent storage
        // to avoid redundant allocation and kernel launch
        if (use_immersed_forcing && !use_nudging) {
            dptr_r_plane = r_plane_avg[level].table();
            dptr_t_plane = t_plane_avg[level].table();
        } else {
            r_plane_tab.resize({tdomain.smallEnd(2)}, {tdomain.bigEnd(2)});
            t_plane_tab.resize({tdomain.smallEnd(2)}, {tdomain.bigEnd(2)});
            dptr_r_plane = r_plane_tab.table();
            dptr_t_plane = t_plane_tab.table();
        }
 
        ParallelFor(ncell, [=] AMREX_GPU_DEVICE (int k) noexcept
        {
            dptr_r_plane(k-offset) = dptr_r[k];
            dptr_t_plane(k-offset) = dptr_t[k];
        });
 
        // U, V, and Qv velocity (only needed for nudging, not for immersed forcing)
        if (use_nudging) {
            if (solverChoice.moisture_type != MoistureType::None)
            {
                Gpu::HostVector<  Real> qv_plane_h(ncell), qc_plane_h(ncell);
                Gpu::DeviceVector<Real> qv_plane_d(ncell), qc_plane_d(ncell);
 
                // Water vapor
                cons_ave.line_average(RhoQ1_comp, qv_plane_h);
                Gpu::copy(Gpu::hostToDevice, qv_plane_h.begin(), qv_plane_h.end(), qv_plane_d.begin());
 
                Real* dptr_qv = qv_plane_d.data();
                qv_plane_tab.resize({tdomain.smallEnd(2)}, {tdomain.bigEnd(2)});
                dptr_qv_plane = qv_plane_tab.table();
                ParallelFor(ncell, [=] AMREX_GPU_DEVICE (int k) noexcept
                {
                    dptr_qv_plane(k-offset) = dptr_qv[k];
                });
            }
            // U, V velocity averages are only needed for the momentum nudging diagnostics below
            // (either sounding-based, gated by nudging_u, or LSF-based, gated by use_lsf)
            if (solverChoice.nudging_u || use_lsf) {
                IntVect ng_u = xvel_old.nGrowVect(); ng_u[2] = 1;
                IntVect ng_v = yvel_old.nGrowVect(); ng_v[2] = 1;
 
                PlaneAverage u_ave(&(xvel_old), fine_geom, solverChoice.ave_plane, ng_u);
                PlaneAverage v_ave(&(yvel_old), fine_geom, solverChoice.ave_plane, ng_v);
 
                u_ave.compute_averages(ZDir(), u_ave.field());
                v_ave.compute_averages(ZDir(), v_ave.field());
 
                int u_ncell = u_ave.ncell_line();
                int v_ncell = v_ave.ncell_line();
                Gpu::HostVector<    Real> u_plane_h(u_ncell), v_plane_h(v_ncell);
                Gpu::DeviceVector<  Real> u_plane_d(u_ncell), v_plane_d(v_ncell);
 
                u_ave.line_average(0, u_plane_h);
                v_ave.line_average(0, v_plane_h);
 
                Gpu::copy(Gpu::hostToDevice, u_plane_h.begin(), u_plane_h.end(), u_plane_d.begin());
                Gpu::copy(Gpu::hostToDevice, v_plane_h.begin(), v_plane_h.end(), v_plane_d.begin());
 
                Real* dptr_u = u_plane_d.data();
                Real* dptr_v = v_plane_d.data();
 
                Box udomain = domain; udomain.grow(2,ng_u[2]);
                Box vdomain = domain; vdomain.grow(2,ng_v[2]);
                u_plane_tab.resize({udomain.smallEnd(2)}, {udomain.bigEnd(2)});
                v_plane_tab.resize({vdomain.smallEnd(2)}, {vdomain.bigEnd(2)});
 
                int u_offset = ng_u[2];
                dptr_u_plane = u_plane_tab.table();
                ParallelFor(u_ncell, [=] AMREX_GPU_DEVICE (int k) noexcept
                {
                    dptr_u_plane(k-u_offset) = dptr_u[k];
                });
 
                int v_offset = ng_v[2];
                dptr_v_plane = v_plane_tab.table();
                ParallelFor(v_ncell, [=] AMREX_GPU_DEVICE (int k) noexcept
                {
                    dptr_v_plane(k-v_offset) = dptr_v[k];
                });
            }
        }
 
        // Store planar averages in persistent ERF member variables for immersed forcing
        // (only needed when nudging is also enabled; otherwise we wrote directly above)
        if (use_immersed_forcing && use_nudging) {
            Table1D<Real> r_avg_persistent = r_plane_avg[level].table();
            Table1D<Real> t_avg_persistent = t_plane_avg[level].table();
 
            // Copy from local computation to persistent storage
            // Both use the same Table1D indexing with offset, so direct copy
            ParallelFor(ncell, [=] AMREX_GPU_DEVICE (int k) noexcept
            {
                r_avg_persistent(k-offset) = dptr_r_plane(k-offset);
                t_avg_persistent(k-offset) = dptr_t_plane(k-offset);
            });
        }
    }
 
    if (use_lsf) {
        lsf_data[level]->setVal(0.0);
 
        int itime_curr = 0;
        int itime_next = 0;
        amrex::Real coeff_curr = 1.0;
        amrex::Real coeff_next = 0.0;
 
        lsf.get_forcing_time_coeffs(old_time, itime_curr, itime_next, coeff_curr, coeff_next);
 
        // ttend, qtend, wsub = lsf
        // ug0, vg0 = lsf - avg u,v
 
        const Real* theta_lsf_n   = lsf.t_int_lsf_d[itime_curr].dataPtr();
        const Real* theta_lsf_np1 = lsf.t_int_lsf_d[itime_next].dataPtr();
        const Real* qv_lsf_n   = lsf.q_int_lsf_d[itime_curr].dataPtr();
        const Real* qv_lsf_np1 = lsf.q_int_lsf_d[itime_next].dataPtr();
        const Real* w_lsf_n   = lsf.w_int_lsf_d[itime_curr].dataPtr();
        const Real* w_lsf_np1 = lsf.w_int_lsf_d[itime_next].dataPtr();
 
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        {
        for ( MFIter mfi(state_old[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
        {
            Box bx  = mfi.tilebox();
            const Array4<Real>& cell_data = state_old[IntVars::cons].array(mfi);
            const Array4<Real>& lsf_arr = lsf_data[level]->array(mfi);
            Real dzInv = fine_geom.InvCellSize(2);
            const int kmin = domain.smallEnd(2) + 1; // minimum k for vertical subsidence
            const int kmax = domain.bigEnd(2) - 1;   // maximum k for vertical subsidence
            const Array4<const Real>& z_cc_arr = (l_use_terrain_fitted_coords) ? z_phys_cc[level]->const_array(mfi) : Array4<Real>{};
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                amrex::Real ttend = coeff_curr*theta_lsf_n[k] + coeff_next*theta_lsf_np1[k];
                amrex::Real qtend = coeff_curr*qv_lsf_n[k] + coeff_next*qv_lsf_np1[k];
                amrex::Real wsub  = coeff_curr*w_lsf_n[k] + coeff_next*w_lsf_np1[k];
 
                lsf_arr(i, j, k, 0) = ttend;
                lsf_arr(i, j, k, 1) = qtend;
                lsf_arr(i, j, k, 2) = wsub;
 
                // horizontal tendencies
                lsf_arr(i, j, k, 3) = ttend;
                lsf_arr(i, j, k, 4) = qtend;
 
                if (k >= kmin && k <= kmax) {
                    int k1, k2;
                    amrex::Real rdz;
                    if (wsub >= 0.0)
                    {
                        k1 = k;
                        k2 = k-1;
                    } else {
                        k1 = k+1;
                        k2 = k;
                    }
                    rdz = (z_cc_arr) ? 1.0 / (z_cc_arr(i,j,k1) - z_cc_arr(i,j,k2)) : dzInv;
                    rdz *= wsub;
 
                    amrex::Real tvtend = -rdz * ( (cell_data(i, j, k1, RhoTheta_comp) / cell_data(i, j, k1, Rho_comp)) - (cell_data(i, j, k2, RhoTheta_comp) / cell_data(i, j, k2, Rho_comp)));
                    amrex::Real qvtend = zero, qctend = zero;
                    if (has_moisture) {
                        qvtend = -rdz * ( (cell_data(i, j, k1, RhoQ1_comp) / cell_data(i, j, k1, Rho_comp)) - (cell_data(i, j, k2, RhoQ1_comp) / cell_data(i, j, k2, Rho_comp)));
                        qctend = -rdz * ( (cell_data(i, j, k1, RhoQ2_comp) / cell_data(i, j, k1, Rho_comp)) - (cell_data(i, j, k2, RhoQ2_comp) / cell_data(i, j, k2, Rho_comp)));
                    }
 
                    lsf_arr(i, j, k, 0) += tvtend;
                    lsf_arr(i, j, k, 1) += qvtend;
 
                    lsf_arr(i, j, k, 5) = tvtend;
                    lsf_arr(i, j, k, 6) = qvtend;
                    lsf_arr(i, j, k, 7) = qctend;
                }
            });
 
            // subsidence
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                // directly apply tendencies for theta and qv
                cell_data(i, j, k, RhoTheta_comp) += cell_data(i, j, k, Rho_comp) * lsf_arr(i, j, k, 0) * dt_advance;
                if (has_moisture) {
                    cell_data(i, j, k, RhoQ1_comp) = max(Real(0), cell_data(i, j, k, RhoQ1_comp) + cell_data(i, j, k, Rho_comp) * lsf_arr(i, j, k, 1) * Real(dt_advance));
                    cell_data(i, j, k, RhoQ2_comp) += cell_data(i, j, k, Rho_comp) * lsf_arr(i, j, k, 7) * dt_advance;
                }
            });
        }
        }
    }
 
    if (use_nudging) {
        nudge_data[level]->setVal(0.0);
 
        int itime_n    = 0;
        int itime_np1  = 0;
        Real coeff_n   = Real(1.0);
        Real coeff_np1 = Real(0.0);
        Real tau_inv = Real(1.0) / input_sounding_data.tau_nudging;
 
        int n_sounding_times = input_sounding_data.input_sounding_time.size();
 
        for (int nt = 1; nt < n_sounding_times; nt++) {
            if (old_time > input_sounding_data.input_sounding_time[nt]) itime_n = nt;
        }
        if (itime_n == n_sounding_times-1) {
            itime_np1 = itime_n;
        } else {
            itime_np1 = itime_n+1;
            coeff_np1 = (old_time                                           - input_sounding_data.input_sounding_time[itime_n]) /
                        (input_sounding_data.input_sounding_time[itime_np1] - input_sounding_data.input_sounding_time[itime_n]);
            coeff_n   = Real(1.0) - coeff_np1;
        }
 
        const Real* theta_inp_sound_n   = input_sounding_data.theta_inp_sound_d[itime_n].dataPtr() + 1;
        const Real* theta_inp_sound_np1 = input_sounding_data.theta_inp_sound_d[itime_np1].dataPtr() + 1;
        const Real* qv_inp_sound_n   = input_sounding_data.qv_inp_sound_d[itime_n].dataPtr() + 1;
        const Real* qv_inp_sound_np1 = input_sounding_data.qv_inp_sound_d[itime_np1].dataPtr() + 1;
 
        const int n  = RhoTheta_comp;
        const int nq  = RhoQ1_comp;
 
        // lower and upper bounds to apply theta nudging
        const Real t_z1 = solverChoice.nudging_t_z1;
        const Real t_z2 = solverChoice.nudging_t_z2;
 
        // whether to nudge theta at all
        const bool nudge_theta   = solverChoice.nudging_t;
 
        // lower and upper bounds to apply qv nudging
        const Real q_z1 = solverChoice.nudging_q_z1;
        const Real q_z2 = solverChoice.nudging_q_z2;
 
        // whether to nudge qv at all
        const bool nudge_q = solverChoice.nudging_q;
 
        // lower and upper bounds to apply u,v nudging (sounding-based branch only; see below)
        const Real u_z1 = solverChoice.nudging_u_z1;
        const Real u_z2 = solverChoice.nudging_u_z2;
 
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        {
        for ( MFIter mfi(state_old[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
        {
            Box bx = mfi.tilebox();
            const Array4<Real>& cell_data  = state_old[IntVars::cons].array(mfi);
            const Array4<const Real>& z_cc_arr = (l_use_terrain_fitted_coords) ? z_phys_cc[level]->const_array(mfi) : Array4<Real>{};
            const Array4<Real>& nudge_arr = nudge_data[level]->array(mfi);
            Real zlo = fine_geom.ProbLo(2);
            Real dz = fine_geom.CellSize(2);
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                Real z = (z_cc_arr) ? z_cc_arr(i,j,k) : zlo + (k+0.5)*dz;
 
                // nudge_data = tnudge, qnudge, unudge, vnudge
 
                // Nudging for theta
                if (nudge_theta && z >= t_z1 && z <= t_z2) {
                    Real nudge = (coeff_n*theta_inp_sound_n[k] + coeff_np1*theta_inp_sound_np1[k]) - (dptr_t_plane(k)/dptr_r_plane(k));
                    nudge_arr(i, j, k, 0) = nudge * tau_inv;
                    //if (i == 0 && j == 0)
                    //    amrex::Print() << "   nudge i = " << i << " j = " << j << " k = " << k << ": theta_n = " << theta_inp_sound_n[k] << " theta_np1 = " << theta_inp_sound_np1[k] << " t / r plane(k) = " << dptr_t_plane(k) / dptr_r_plane(k) << ": nudge = " << nudge << " nudge*tau = " << nudge*tau_inv << " gamaz = " << gamaz << std::endl;
                    cell_data(i, j, k, n) += nudge * dptr_r_plane(k) * tau_inv * dt_advance;
                }
 
                // Nudging for qv
                if (has_moisture && nudge_q && z >= q_z1 && z <= q_z2) {
                    Real nudge = (coeff_n*qv_inp_sound_n[k] + coeff_np1*qv_inp_sound_np1[k]) - (dptr_qv_plane(k)/dptr_r_plane(k));
                    nudge_arr(i, j, k, 1) = nudge * tau_inv;
                    //if (i == 0 && j == 0)
                    //    amrex::Print() << "   nudge i = " << i << " j = " << j << " k = " << k << " z = " << z << ": q_n = " << qv_inp_sound_n[k] << " qv_np1 = " << qv_inp_sound_np1[k] << " q / r plane(k) = " << dptr_qv_plane(k) / dptr_r_plane(k) << ": nudge = " << nudge << " nudge*tau = " << nudge*tau_inv << " dptr_qv_plane(k) = " << dptr_qv_plane(k) << " dptr_r_plane(k) = " << dptr_r_plane(k) << std::endl;
                    cell_data(i, j, k, nq) += nudge * dptr_r_plane(k) * tau_inv * dt_advance;
                }
            });
 
            // Nudging for u and v
            // NOTE: if LSF is enabled, then the U,V nudging here uses the LSF values, not U and V from the input sounding
            if (solverChoice.nudging_u || use_lsf) {
                Real uv_coeff_n = coeff_n;
                Real uv_coeff_np1 = coeff_np1;
                Real tau = tau_inv;
                Real* u_nudge_n, *u_nudge_np1, *v_nudge_n, *v_nudge_np1;
                if (!use_lsf)
                {
                    u_nudge_n = input_sounding_data.U_inp_sound_d[itime_n].dataPtr() + 1;
                    u_nudge_np1 = input_sounding_data.U_inp_sound_d[itime_np1].dataPtr() + 1;
                    v_nudge_n  = input_sounding_data.V_inp_sound_d[itime_n].dataPtr() + 1;
                    v_nudge_np1 = input_sounding_data.V_inp_sound_d[itime_np1].dataPtr() + 1;
                } else {
                    int itime_curr = 0;
                    int itime_next = 0;
                    uv_coeff_n = 1.0;
                    uv_coeff_np1 = 0.0;
                    tau = 1.0 / lsf.tau_lsf; // only applies to u,v LSF nudging
 
                    lsf.get_forcing_time_coeffs(old_time, itime_curr, itime_next, uv_coeff_n, uv_coeff_np1);
                    u_nudge_n   = lsf.u_int_lsf_d[itime_curr].dataPtr();
                    u_nudge_np1 = lsf.u_int_lsf_d[itime_next].dataPtr();
                    v_nudge_n   = lsf.v_int_lsf_d[itime_curr].dataPtr();
                    v_nudge_np1 = lsf.v_int_lsf_d[itime_next].dataPtr();
                }
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    // height bound below applies only to sounding-based wind nudging, not LSF
                    Real z = (z_cc_arr) ? z_cc_arr(i,j,k) : zlo + (k+0.5)*dz;
                    if (use_lsf || (z >= u_z1 && z <= u_z2)) {
                        Real unudge = -(dptr_u_plane(k) - (uv_coeff_n*u_nudge_n[k] + uv_coeff_np1*u_nudge_np1[k]));
                        unudge *= tau;
 
                        Real vnudge = -(dptr_v_plane(k) - (uv_coeff_n*v_nudge_n[k] + uv_coeff_np1*v_nudge_np1[k]));
                        vnudge *= tau;
 
                        nudge_arr(i, j, k, 2) = unudge;
                        nudge_arr(i, j, k, 3) = vnudge;
                    }
                });
            }
        }
        }
    }
 
    // **************************************************************************************
    // Compute strain for use in slow RHS and Smagorinsky model
    // **************************************************************************************
    {
    BL_PROFILE("erf_advance_strain");
    if (l_use_diff) {
 
        const BCRec* bc_ptr_h = domain_bcs_type.data();
        const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
 
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
        {
            Box bxcc  = mfi.growntilebox(IntVect(1,1,0));
            Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
            Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
            Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
 
            if (bxcc.smallEnd(2) != domain.smallEnd(2)) {
                 bxcc.growLo(2,1);
                tbxxy.growLo(2,1);
                tbxxz.growLo(2,1);
                tbxyz.growLo(2,1);
            }
 
            if (bxcc.bigEnd(2) != domain.bigEnd(2)) {
                 bxcc.growHi(2,1);
                tbxxy.growHi(2,1);
                tbxxz.growHi(2,1);
                tbxyz.growHi(2,1);
            }
 
            const Array4<const Real> & u = xvel_old.array(mfi);
            const Array4<const Real> & v = yvel_old.array(mfi);
            const Array4<const Real> & w = zvel_old.array(mfi);
 
            Array4<Real> tau11 = Tau[level][TauType::tau11].get()->array(mfi);
            Array4<Real> tau22 = Tau[level][TauType::tau22].get()->array(mfi);
            Array4<Real> tau33 = Tau[level][TauType::tau33].get()->array(mfi);
            Array4<Real> tau12 = Tau[level][TauType::tau12].get()->array(mfi);
            Array4<Real> tau13 = Tau[level][TauType::tau13].get()->array(mfi);
            Array4<Real> tau23 = Tau[level][TauType::tau23].get()->array(mfi);
 
            Array4<Real> tau21  = (Tau[level][TauType::tau21]) ? Tau[level][TauType::tau21].get()->array(mfi) : Array4<Real>{};
            Array4<Real> tau31  = (Tau[level][TauType::tau31]) ? Tau[level][TauType::tau31].get()->array(mfi) : Array4<Real>{};
            Array4<Real> tau32  = (Tau[level][TauType::tau32]) ? Tau[level][TauType::tau32].get()->array(mfi) : Array4<Real>{};
 
            const Array4<const Real>& z_nd = z_phys_nd[level]->const_array(mfi);
 
            const Array4<const Real> mf_mx = mapfac[level][MapFacType::m_x]->const_array(mfi);
            const Array4<const Real> mf_ux = mapfac[level][MapFacType::u_x]->const_array(mfi);
            const Array4<const Real> mf_vx = mapfac[level][MapFacType::v_x]->const_array(mfi);
            const Array4<const Real> mf_my = mapfac[level][MapFacType::m_y]->const_array(mfi);
            const Array4<const Real> mf_uy = mapfac[level][MapFacType::u_y]->const_array(mfi);
            const Array4<const Real> mf_vy = mapfac[level][MapFacType::v_y]->const_array(mfi);
 
            // We update Tau_corr[level] in erf_make_tau_terms, not here
            Array4<Real> no_tau_corr_update_here{};
 
            if (solverChoice.mesh_type == MeshType::StretchedDz) {
                ComputeStrain_S(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                u, v, w,
                                tau11, tau22, tau33,
                                tau12, tau21,
                                tau13, tau31,
                                tau23, tau32,
                                stretched_dz_d[level], dxInv,
                                mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
                                no_tau_corr_update_here, no_tau_corr_update_here);
            } else if (l_use_terrain_fitted_coords) {
                ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                u, v, w,
                                tau11, tau22, tau33,
                                tau12, tau21,
                                tau13, tau31,
                                tau23, tau32,
                                z_nd, detJ_cc[level]->const_array(mfi), dxInv,
                                mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
                                no_tau_corr_update_here, no_tau_corr_update_here);
            } else {
                if (solverChoice.terrain_type == TerrainType::EB) {
                    ComputeStrain_EB(mfi, bxcc, tbxxy, tbxxz, tbxyz, domain,
                                    u, v, w,
                                    tau11, tau22, tau33,
                                    tau12, tau13, tau23,
                                    dxInv,
                                    bc_ptr_h,
                                    get_eb(level),
                                    no_tau_corr_update_here, no_tau_corr_update_here);
                } else {
                    ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                    u, v, w,
                                    tau11, tau22, tau33,
                                    tau12, tau13, tau23,
                                    dxInv,
                                    mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
                                    no_tau_corr_update_here, no_tau_corr_update_here);
                }
            }
        } // mfi
    } // l_use_diff
    } // profile
 
#include "ERF_TI_utils.H"
 
    // Additional SFS quantities, calculated once per timestep
    MultiFab* Hfx1  = SFS_hfx1_lev[level].get();
    MultiFab* Hfx2  = SFS_hfx2_lev[level].get();
    MultiFab* Hfx3  = SFS_hfx3_lev[level].get();
    MultiFab* Q1fx1 = SFS_q1fx1_lev[level].get();
    MultiFab* Q1fx2 = SFS_q1fx2_lev[level].get();
    MultiFab* Q1fx3 = SFS_q1fx3_lev[level].get();
    MultiFab* Q2fx3 = SFS_q2fx3_lev[level].get();
    MultiFab* Diss  = SFS_diss_lev[level].get();
 
    MultiFab* Hfx3_EB = nullptr;
    if (solverChoice.terrain_type == TerrainType::EB) {
        Hfx3_EB = hfx3_EB[level].get();
    }
 
    // *************************************************************************
    // Calculate cell-centered eddy viscosity & diffusivities
    //
    // Notes -- we fill all the data in ghost cells before calling this so
    //    that we can fill the eddy viscosity in the ghost regions and
    //    not have to call a boundary filler on this data itself
    //
    // LES - updates both horizontal and vertical eddy viscosity components
    // PBL - only updates vertical eddy viscosity components so horizontal
    //       components come from the LES model or are left as zero.
    // *************************************************************************
    if (l_use_kturb)
    {
        // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
        bool l_use_moisture = ( solverChoice.moisture_type != MoistureType::None );
        const BCRec* bc_ptr_h = domain_bcs_type.data();
        ComputeTurbulentViscosity(dt_advance, xvel_old, yvel_old,Tau[level],
                                  state_old[IntVars::cons],
                                  *walldist[level].get(),
                                  *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
                                  fine_geom, mapfac[level],
                                  z_phys_nd[level], z_phys_cc[level], solverChoice,
                                  m_SurfaceLayer[Orientation(Direction::z, Orientation::low)], z_0, l_use_terrain_fitted_coords,
                                  l_use_moisture, level,
                                  bc_ptr_h,
                                  get_eb(level),
                                  false, // vert_only
                                  qheating_rates[level].get(),
                                  (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                                   solverChoice.buildings_type == BuildingsType::ImmersedForcing) ?
                                      terrain_blanking[level].get() : nullptr);
 
        // Zero turbulent mixing in fully immersed cells
        MultiFab* terrain_blank = (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                                   solverChoice.buildings_type == BuildingsType::ImmersedForcing) ?
            terrain_blanking[level].get() : nullptr;
 
        if (terrain_blank) {
            for (MFIter mfi(*eddyDiffs); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.growntilebox(eddyDiffs->nGrowVect());
                auto const& t_blank_arr = terrain_blank->const_array(mfi);
                auto const& eddy_arr = eddyDiffs->array(mfi);
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    // Zero eddy diffusivities in fully immersed cells (sanitizes NaN/Inf)
                    if (t_blank_arr(i,j,k) == one) {
                        for (int n = 0; n < EddyDiff::NumDiffs; ++n) {
                            eddy_arr(i,j,k,n) = zero;
                        }
                    }
                });
            }
        }
    }
 
    // ***********************************************************************************************
    // Update user-defined source terms -- these are defined once per time step (not per RK stage)
    // ***********************************************************************************************
    if (solverChoice.custom_rhotheta_forcing) {
        prob->update_rhotheta_sources(old_time,
                                      rhotheta_src_ptr,
                                      fine_geom, z_phys_cc[level]);
    }
 
    if (solverChoice.custom_moisture_forcing) {
        prob->update_rhoqt_sources(old_time,
                                   rhoqt_src_ptr,
                                   fine_geom, z_phys_cc[level]);
    }
 
    if (solverChoice.custom_geostrophic_profile) {
        prob->update_geostrophic_profile(old_time,
                                   h_u_geos[level], d_u_geos[level],
                                   h_v_geos[level], d_v_geos[level],
                                   fine_geom, z_phys_cc[level]);
    }
 
    if (solverChoice.custom_w_subsidence) {
        prob->update_w_subsidence(old_time,
                                  h_w_subsid[level], d_w_subsid[level],base_state[level],
                                  fine_geom, z_phys_nd[level]);
    }
 
    // ***********************************************************************************************
    // Convert old velocity available on faces to old momentum on faces to be used in time integration
    // ***********************************************************************************************
    MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
 
    //
    // This is an optimization since we won't need more than one ghost
    // cell of momentum in the integrator if not using numerical diffusion
    //
    IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
    IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
    IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
 
    const MultiFab* c_vfrac = nullptr;
    if (solverChoice.terrain_type == TerrainType::EB) {
        c_vfrac = &((get_eb(level).get_const_factory())->getVolFrac());
    }
 
    VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
                       state_old[IntVars::xmom],
                       state_old[IntVars::ymom],
                       state_old[IntVars::zmom],
                       domain, domain_bcs_type, c_vfrac);
 
    MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
    MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
    MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
 
    bool fast_only = false;
    bool vel_and_mom_synced = true;
 
    apply_bcs(state_old, old_time,
              state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
              fast_only, vel_and_mom_synced);
 
    cons_to_prim(state_old[IntVars::cons], S_prim, state_old[IntVars::cons].nGrow());
 
    make_pi_stage(state_old[IntVars::cons]);
 
    // ***********************************************************************************************
    // Define a new MultiFab that holds q_total and fill it by summing the moisture components --
    //      to be used in buoyancy calculation and as part of the inertial weighting in the
    // ***********************************************************************************************
 
    const bool l_eb_terrain = (solverChoice.terrain_type == TerrainType::EB);
    MultiFab qt(grids[level], dmap[level], 1, (l_eb_terrain) ? 2 : 1);
    qt.setVal(0);
 
#include "ERF_TI_no_substep_fun.H"
#include "ERF_TI_substep_fun.H"
#include "ERF_TI_slow_rhs_pre.H"
#include "ERF_TI_slow_rhs_post.H"
 
    // ***************************************************************************************
    // Setup the integrator and integrate for a single timestep
    // **************************************************************************************
    MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
 
    // Define rhs and 'post update' utility function that is called after calculating
    // any state data (e.g. at RK stages or at the end of a timestep)
    mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
    mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
 
    mri_integrator.set_acoustic_substepping(acoustic_substepping_fun);
    mri_integrator.set_slow_fast_timestep_ratio(fixed_mri_dt_ratio > 0 ? fixed_mri_dt_ratio : dt_mri_ratio[level]);
    mri_integrator.set_no_substep(no_substep_fun);
 
    mri_integrator.advance(state_old, state_new, old_time, dt_advance);
 
    if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
}

Here is the call graph for this function:

advance_lsm()

void ERF::advance_lsm	(	int	lev,
		amrex::MultiFab &	cons_in,
		amrex::MultiFab &	xvel_in,
		amrex::MultiFab &	yvel_in,
		const double &	time,
		const double &	dt_advance
	)

{
    if (solverChoice.lsm_type != LandSurfaceType::None) {
        if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
            // Typed surface-precip accumulation sources from the active microphysics
            // scheme: borrowed const views of the scheme-native cumulative accumulators
            // plus their native->kg/m^2 (== water-equivalent mm) conversion factors.
            // Noah-MP forms the water-equivalent interval precip (RAINBL / MP_RAINNC /
            // MP_SNOW / MP_GRAUP / SR) from these, so schemes with differing accumulator
            // semantics or units (e.g. SAM's density-scaled snow/graupel) are handled
            // correctly. Empty when moisture/precip is off -> land model runs precip-free.
            const SurfacePrecipAccumulationSources precip_sources =
                micro ? micro->Get_Surface_Precip_Accumulation_Ptrs(lev)
                      : SurfacePrecipAccumulationSources{};
            lsm.Advance(lev, cons_in, xvel_in, yvel_in,
                        SFS_hfx3_lev[lev].get(), SFS_q1fx3_lev[lev].get(),
                        precip_sources,
                        time, dt_advance, istep[0], lsm.Get_LSM_Update_Status(0));
        } else {
            lsm.Advance(lev, dt_advance);
        }
    }
}

Here is the call graph for this function:

advance_microphysics()

void ERF::advance_microphysics	(	int	lev,
		amrex::MultiFab &	cons_in,
		const double &	dt_advance,
		const int &	iteration,
		const double &	time
	)

{
    if (solverChoice.moisture_type != MoistureType::None) {
        if (lev > 0) {
            MultiFab& U_new = vars_new[lev][Vars::xvel];
            MultiFab& V_new = vars_new[lev][Vars::yvel];
            MultiFab& W_new = vars_new[lev][Vars::zvel];
            FillPatchFineLevel(lev, time + dt_advance,
                               {&cons, &U_new, &V_new, &W_new},
                               {&cons, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                               base_state[lev], base_state[lev]);
        } else {
            cons.FillBoundary(geom[lev].periodicity());
        }
        const auto micro_state_before = cloud_chamber_budget ?
            cloud_chamber_budget->state_integrals(cons, geom[lev]) :
            std::array<Real, CloudChamberBudget::NumScalars>{};
        const amrex::MultiFab* base = solverChoice.anelastic[lev] ? &base_state[lev] : nullptr;
        micro->Update_Micro_Vars_Lev(lev, cons, base);
        micro->Advance(lev, static_cast<Real>(dt_advance), iteration,
                       static_cast<Real>(time), solverChoice, vars_new, z_phys_nd, phys_bc_type);
        micro->Update_State_Vars_Lev(lev, cons, *z_phys_nd[lev]);
        if (cloud_chamber_budget) {
            cloud_chamber_budget->record_internal_source(
                micro_state_before, cloud_chamber_budget->state_integrals(cons, geom[lev]));
        }
 
        // Sync cons[lev-1] covered cells with the moist state just written
        // to cons[lev].  Without this, the next sub-cycle's FillPatchFineLevel
        // for level lev would pull stale (latent-heat-less) values from
        // cons[lev-1]'s coarse cells via cell-conservative interpolation,
        // causing an artificial outward heat/q flux across the lev/(lev-1)
        // boundary that drains the bubble interior.
        if (lev > 0 && solverChoice.coupling_type == CouplingType::TwoWay &&
            Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
            AverageDownMoistStateTo(lev - 1);
        }
    }
}

Here is the call graph for this function:

advance_radiation()

void ERF::advance_radiation	(	int	lev,
		amrex::MultiFab &	cons_in,
		const double &	dt_advance
	)

Advance radiation diagnostics and heating rates for one time step.

Temporal semantics

This function is called exactly once per ERF::Advance() invocation, after the SurfaceLayer and LSM updates and before the dycore slow and fast substeps. It operates on the old state (t^n) at the beginning of the slow step.

• RRTMGP / Simple path (erf.radiation_model = RRTMGP or Simple): a full spectral model with its own time-centering and source-term semantics. Produces qheating_rates[lev].
• Two-stream path (erf.radiation_model = TwoStream): a shortwave and longwave model that computes heating rates from the old-state atmosphere (t^n) with clear-sky and cloudy column algorithms. The heating rates go into qheating_rates[lev], a 2-component MultiFab holding shortwave and longwave.

Source-term application

The computed qheating_rates are injected into the RhoTheta source term in ERF_MakeSources.cpp only while the slow RHS is being built (is_slow_step is true), which ensures:

1. Radiation tendencies are applied once per slow step, not per substep.
2. The tendencies represent the old-state atmosphere throughout all fast substeps of the current slow step.
3. There is no temporal aliasing from repeated calls to advance_radiation() within a slow step, since there is only one call per slow step.

Key contracts

• Radiation heating is an old-state forcing. The qheating_rates computed here are the radiative heating of the old-state atmosphere (t^n), applied as a source term while the slow RHS is built. That gives one radiative increment per slow step, consistent with the old state across every fast substep. Radiation does not adapt to the state within a slow step.
• The two radiation paths are mutually exclusive. RRTMGP and two-stream never both run in one simulation; the if/else below selects one. Both produce qheating_rates in the same 2-component (SW, LW) format, and the source-term gate in ERF_MakeSources.cpp tests both, so exactly one matches in any given simulation.

Parameters

[in]	lev	Level of refinement (coarsest level is 0)
[in,out]	cons	Conservative quantities (Rho, RhoTheta, RhoQ*, RhoRE)
[in]	dt_advance	Time step for this slow-step stage [seconds]

{
    BL_PROFILE("ERF::advance_radiation()");
 
    // Fill this level's radiation fields by interpolation from its parent.
    //
    // Two situations need this and need exactly the same work done:
    //
    //   * a nested patch -- a fine level that does not span the full column, for which
    //     RRTMGP cannot be run at all because it needs complete atmospheric columns; and
    //   * the first step of a level built by interp_atmos_from_coarse, whose atmospheric
    //     state came from FillCoarsePatch and is not yet thermodynamically consistent.
    //
    // NOTE: the coarse MultiFabs are handed to InterpFromCoarseLevel directly, with no
    //       intermediate ghosted temporary.  That is only correct because each of them
    //       carries -- and has filled -- the ghost cells this overload reads.  It builds a
    //       PhysBCFunctUseCoarseGhost, whose constructor derives the coarse halo the
    //       interpolation stencil needs and then asserts
    //
    //           cghost = min(cmf.nGrowVect(), src_ghost);
    //           AMREX_ALWAYS_ASSERT(cghost.allGE(src_ghost_outside_domain));
    //
    //       and the ParallelCopy that fills the coarse patch runs with send_ghost = cghost,
    //       i.e. it reads the coarse source's GHOST region, not only its valid region.  A
    //       coarse source with too few ghost cells therefore aborts on that assert, and one
    //       whose halo was never filled silently feeds garbage to the interpolation.  This
    //       is why qheating_rates and rad_fluxes are both defined with a (1,1,1) halo in
    //       ERF_MakeNewArrays.cpp, zeroed there, and FillBoundary'd below.
    auto interp_rad_from_coarse = [&] ()
    {
        // Ensure the parent level's ghost cells are filled before interpolation.  This is
        // needed even when radiation did not run this step, especially with two-way coupling
        // where the grid structure may have changed.  A nested patch is exempt: it never
        // runs radiation itself, so its halo is already whatever its own pass through this
        // lambda interpolated into it.
        if (!rad[lev-1]->is_nested_patch()) {
            qheating_rates[lev-1]->FillBoundary(geom[lev-1].periodicity());
            if (rad_fluxes[lev-1]) {
                // The whole halo, as for qheating_rates -- the z ghosts matter too, since
                // grids split in z (amr.no_box_split_dir = -1) put interior z interfaces in
                // them -- but with z forced non-periodic.  The z-ghost at khi+1 holds the
                // top-of-atmosphere interface, physical data rather than a halo (see below);
                // it lies outside the domain, so no ordinary exchange reaches it, but a
                // z-periodic one would overwrite it with the bottom of the column.
                const IntVect& per = geom[lev-1].periodicity().intVect();
                rad_fluxes[lev-1]->FillBoundary(Periodicity(IntVect(per[0],per[1],0)));
            }
        }
 
        InterpFromCoarseLevel(*qheating_rates[lev], qheating_rates[lev]->nGrowVect(),
                              IntVect(0,0,0),
                              *qheating_rates[lev-1], 0, 0, 2,
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
 
        // Radiation fluxes (needed for plotfiles and diagnostics)
        if (rad_fluxes[lev] && rad_fluxes[lev-1]) {
            const int nc = rad_fluxes[lev]->nComp();
 
            InterpFromCoarseLevel(*rad_fluxes[lev], rad_fluxes[lev]->nGrowVect(),
                                  IntVect(0,0,0),
                                  *rad_fluxes[lev-1], 0, 0, nc,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &cell_cons_interp,
                                  domain_bcs_type, BCVars::cons_bc);
 
            // The interpolation above cannot carry the top-of-atmosphere interface.
            // RRTMGP's layout puts the TOA fluxes in the z-ghost cell at k = khi+1 (see
            // ERF_MakeNewArrays.cpp), which is physical data rather than a halo.  On the
            // coarse side that cell does now fall inside the ghost region the ParallelCopy
            // reads, so it serves as the stencil's neighbor above the top layer -- the right
            // value for that role, being the next interface up in the same sequence -- but
            // it is only ever read there.  On the fine side, a level that spans the full
            // column has its TOA outside the fine domain, and the call above is asked for no
            // ghost cells outside the domain, so nothing is written there at all.  A nested
            // patch does not reach the model top, so its top plane is an ordinary interior
            // interface that the call above has already filled.
            //
            // Move both planes into valid index space -- a single layer at the coarse domain
            // top -- so the ordinary machinery can interpolate them horizontally, then put
            // the result back in the fine level's ghost cell.
            if (!rad[lev]->is_nested_patch()) {
                const int khi_c = geom[lev-1].Domain().bigEnd(2);
                const int khi_f = geom[lev  ].Domain().bigEnd(2);
 
                // Flatten a level's grids onto the single layer at k = khi_c, keeping only
                // the boxes that reach the top of their own domain.  A box that stops short
                // of it carries no TOA plane at all: flattening it would put a duplicate box
                // in the BoxArray, for which FillBoundary is ill-defined, and would ask the
                // copy below to read a z index outside the fab.  Every box spans the column
                // under the default amr.no_box_split_dir = 2, so this filter keeps all of
                // them there; it matters only for amr.no_box_split_dir = -1.  idx_out maps a
                // slab box back to the box it came from, since dropping boxes breaks the
                // one-to-one correspondence an MFIter would otherwise rely on.
                auto top_slab = [&] (const MultiFab& mf_in, int khi_in, BoxArray& ba_out,
                                     DistributionMapping& dm_out, Vector<int>& idx_out)
                {
                    const BoxArray&            ba_in = mf_in.boxArray();
                    const DistributionMapping& dm_in = mf_in.DistributionMap();
                    BoxList bl;
                    Vector<int> pmap;
                    for (int i = 0, n = int(ba_in.size()); i < n; ++i) {
                        if (ba_in[i].bigEnd(2) != khi_in) { continue; }
                        Box b = ba_in[i];
                        b.setSmall(2, khi_c); b.setBig(2, khi_c);
                        bl.push_back(b);
                        pmap.push_back(dm_in[i]);
                        idx_out.push_back(i);
                    }
                    ba_out = BoxArray(std::move(bl));
                    dm_out = DistributionMapping(std::move(pmap));
                };
 
                BoxArray            ba_toa_c, ba_toa_f;
                DistributionMapping dm_toa_c, dm_toa_f;
                Vector<int>         idx_toa_c, idx_toa_f;
                top_slab(*rad_fluxes[lev-1], khi_c, ba_toa_c, dm_toa_c, idx_toa_c);
                top_slab(*rad_fluxes[lev  ], khi_f, ba_toa_f, dm_toa_f, idx_toa_f);
 
                // toa_crse is the coarse source of an InterpFromCoarseLevel, so it must carry
                // the halo that stencil reads -- (1,1,0), the z ratio being one -- and that
                // halo must be defined.  Unlike rad_fluxes it is freshly allocated here and
                // the loop below writes valid boxes only, so zero it first: FillBoundary can
                // define only the ghosts backed by another box's valid data, and the ones
                // outside the domain are left to the setVal.
                MultiFab toa_crse(ba_toa_c, dm_toa_c, nc, IntVect(1,1,0));
                MultiFab toa_fine(ba_toa_f, dm_toa_f, nc, 0);
                toa_crse.setVal(Real(0.0));
 
                for (MFIter mfi(toa_crse); mfi.isValid(); ++mfi) {
                    const Box& dbx = mfi.validbox();
                    Box sbx(dbx); sbx.shift(2, 1);   // the coarse TOA, one above the top layer
                    toa_crse[mfi].template copy<RunOn::Device>(
                        (*rad_fluxes[lev-1])[idx_toa_c[mfi.index()]], sbx, 0, dbx, 0, nc);
                }
                // Unconditional, unlike the exchange at the top of this lambda: toa_crse is a
                // new MultiFab every step, so its halo is undefined here whatever the parent
                // level is -- being a nested patch says nothing about it.
                toa_crse.FillBoundary(geom[lev-1].periodicity());
 
                // Both planes live at the same z index, so the ratio in z is one and the
                // interpolation is purely horizontal.
                IntVect rr2d(refRatio(lev-1)[0], refRatio(lev-1)[1], 1);
                InterpFromCoarseLevel(toa_fine, IntVect(0,0,0),
                                      IntVect(0,0,0),
                                      toa_crse, 0, 0, nc,
                                      geom[lev-1], geom[lev],
                                      rr2d, &cell_cons_interp,
                                      domain_bcs_type, BCVars::cons_bc);
 
                for (MFIter mfi(toa_fine); mfi.isValid(); ++mfi) {
                    const Box& sbx = mfi.validbox();
                    Box dbx(sbx); dbx.shift(2, khi_f + 1 - khi_c);
                    (*rad_fluxes[lev])[idx_toa_f[mfi.index()]].template copy<RunOn::Device>(
                        toa_fine[mfi], sbx, 0, dbx, 0, nc);
                }
            }
        }
 
        // LSM radiation output fields (surface fluxes needed by NoahMP)
        if (solverChoice.lsm_type != LandSurfaceType::None) {
            Vector<std::string> lsm_output_names = rad[lev]->get_lsm_output_varnames();
 
            for (int i = 0; i < lsm_output_names.size(); ++i) {
                int varIdx_fine   = lsm.Get_DataIdx(lev  , lsm_output_names[i]);
                int varIdx_coarse = lsm.Get_DataIdx(lev-1, lsm_output_names[i]);
                if (varIdx_fine >= 0 && varIdx_coarse >= 0) {
                    MultiFab* lsm_fine   = lsm.Get_Data_Ptr(lev  , varIdx_fine);
                    MultiFab* lsm_coarse = lsm.Get_Data_Ptr(lev-1, varIdx_coarse);
                    if (lsm_fine && lsm_coarse && lsm_coarse->nComp() > 0) {
                        // LSM data are 2D surface fields: use 2D refinement ratio and pc_interp
                        // to avoid computing z-slopes from uninitialized ghost cells
                        IntVect rr2d(refRatio(lev-1)[0], refRatio(lev-1)[1], 1);
                        InterpFromCoarseLevel(*lsm_fine, IntVect(0,0,0),
                                              IntVect(0,0,0),
                                              *lsm_coarse, 0, 0, lsm_coarse->nComp(),
                                              geom[lev-1], geom[lev],
                                              rr2d, &pc_interp,
                                              domain_bcs_type, BCVars::cons_bc);
                    }
                }
            }
        }
    };
 
    if (solverChoice.rad_uses_interface()) {
        BL_PROFILE_VAR("ERF::advance_radiation():RRTMGP", rrtmgp_region);
 
        // On the first step of a level that has just been built by interp_atmos_from_coarse,
        // interpolate the heating rates and radiation fluxes from the parent instead of
        // computing them.  That level's atmospheric state came from FillCoarsePatch, which
        // interpolates rho, theta and qv independently and so leaves them thermodynamically
        // inconsistent; running RRTMGP on it produces NaNs (see MakeNewLevelFromCoarse).
        // Interpolating also gives the LSM the radiation fields it needs for that step.
        //
        // The flag is set by whichever routine built the level and is cleared here as soon as
        // it has been acted on, so exactly one step is skipped per level creation.  Levels
        // that read a full state of their own never have it set.
        if (lev > 0 &&
            lev < static_cast<int>(rad_interp_from_coarse_pending.size()) &&
            rad_interp_from_coarse_pending[lev]) {
            amrex::Print() << "Interpolating radiation heating rates and fluxes from level " << lev-1
                           << " to level " << lev << " on the first step after that level was built\n";
            interp_rad_from_coarse();
            rad_interp_from_coarse_pending[lev] = 0;
            return;
        }
 
#ifdef ERF_USE_NETCDF
        MultiFab *lat_ptr = lat_m[lev].get();
        MultiFab *lon_ptr = lon_m[lev].get();
#else
        MultiFab *lat_ptr = nullptr;
        MultiFab *lon_ptr = nullptr;
#endif
        // T surf from SurfaceLayer if we have it
        MultiFab* t_surf = (m_SurfaceLayer[Orientation(Direction::z, Orientation::low)]) ? m_SurfaceLayer[Orientation(Direction::z, Orientation::low)]->get_t_surf(lev) : nullptr;
 
        // RRTMGP inputs names and pointers
        Vector<std::string> lsm_input_names = rad[lev]->get_lsm_input_varnames();
        Vector<MultiFab*> lsm_input_ptrs(lsm_input_names.size(),nullptr);
        for (int i(0); i<lsm_input_ptrs.size(); ++i) {
            int varIdx = lsm.Get_DataIdx(lev,lsm_input_names[i]);
            if (varIdx >= 0) { lsm_input_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx); }
        }
 
        // RRTMGP output names and pointers
        Vector<std::string> lsm_output_names = rad[lev]->get_lsm_output_varnames();
        Vector<MultiFab*> lsm_output_ptrs(lsm_output_names.size(),nullptr);
        for (int i(0); i<lsm_output_ptrs.size(); ++i) {
            int varIdx = lsm.Get_DataIdx(lev,lsm_output_names[i]);
            if (varIdx >= 0) { lsm_output_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx); }
        }
 
        // Force radiation update to sync with lsm?
        bool lsm_updated = (lev==0 && max_level>0) ? lsm.Get_LSM_Update_Status(lev) : false;
 
        // Enter radiation class driver
        double time_for_rad = t_old[lev] + start_time;
        rad[lev]->Run(lev, istep[lev], time_for_rad, dt_advance,
                      cons.boxArray(), geom[lev], &(cons),
                      lmask_lev[lev][0].get(), t_surf,
                      lsm_input_ptrs, lsm_output_ptrs,
                      qheating_rates[lev].get(), rad_fluxes[lev].get(),
                      z_phys_nd[lev].get()     , lat_ptr, lon_ptr,
                      lsm_updated);
 
        // Fill ghost cells after radiation computes (needed for interpolation to finer levels)
        // This should be fast since it only fills this level's own ghost cells
        if (solverChoice.rad_type != RadiationType::None && !rad[lev]->is_nested_patch()) {
            qheating_rates[lev]->FillBoundary(geom[lev].periodicity());
        }
 
        // For nested patches (fine levels that don't reach model top), radiation
        // was skipped. Interpolate the radiation fields from the parent level.
        if (lev > 0 && rad[lev]->is_nested_patch()) {
            interp_rad_from_coarse();
        }
    }
    // Two-stream radiation driver, a separate path from the IRadiation
    // models above; erf.radiation_model selects exactly one of them.
    //
    // - The call happens exactly once per slow step (from ERF::Advance).
    // - The heating rates computed here are old-state based (t^n).
    // - They are injected into the RhoTheta source only on is_slow_step
    //   (see ERF_MakeSources.cpp), so there is no duplicate forcing.
    // - istep[lev] is the CSV row index, t_old[lev] the time logged with it,
    //   and dt_advance the step size (used by the surface-energy-balance
    //   update, which runs at the post-dycore call).
    // - The sun, the site and the surface temperature come from the same
    //   sources RRTMGP uses: start_time + t for the calendar, the lat_m/lon_m
    //   fields of a WRF or metgrid grid, and the surface layer's temperature.
    else if (solverChoice.rad_type == RadiationType::TwoStream) {
#ifdef ERF_USE_NETCDF
        const MultiFab* lat_ptr = lat_m[lev].get();
        const MultiFab* lon_ptr = lon_m[lev].get();
#else
        const MultiFab* lat_ptr = nullptr;
        const MultiFab* lon_ptr = nullptr;
#endif
        const MultiFab* t_surf = (m_SurfaceLayer[Orientation::zlo()])
                               ? m_SurfaceLayer[Orientation::zlo()]->get_t_surf(lev)
                               : nullptr;
        two_stream_rad.advance(lev, istep[lev], t_old[lev], dt_advance, "pre_dycore",
                               vars_old[lev][Vars::cons], z_phys_nd[lev].get(), geom[lev],
                               lsm, qheating_rates[lev].get(), rad_fluxes[lev].get(),
                               t_surf, lat_ptr, lon_ptr,
                               t_old[lev] + start_time, use_datetime);
    }
}

Here is the call graph for this function:

appendPlot3DVariables()

void ERF::appendPlot3DVariables ( const amrex::Vector< std::string > &  plot_var_names, amrex::Vector< std::string > &  a_plot_var_names  )

private

{
    Vector<std::string> tmp_plot_names(0);
#ifdef ERF_USE_PARTICLES
    Vector<std::string> particle_mesh_plot_names;
    particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
    if (particle_mesh_plot_names.size() > 0) {
        static bool first_call = true;
        if (first_call) {
            Print() << "ParticleData: the following additional Eulerian variables are available to plot:\n";
            for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
                Print() << "    " << particle_mesh_plot_names[i] << "\n";
            }
            first_call = false;
        }
        for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
            std::string tmp(particle_mesh_plot_names[i]);
            if (containerHasElement(plot_var_names, tmp) &&
                !containerHasElement(tmp_plot_names, tmp)) {
                tmp_plot_names.push_back(tmp);
            }
        }
    }
#endif
 
    {
        Vector<std::string> microphysics_plot_names;
        micro->GetPlotVarNames(microphysics_plot_names);
        if (microphysics_plot_names.size() > 0) {
            static bool first_call = true;
            if (first_call) {
                Print() << getEnumNameString(solverChoice.moisture_type)
                        << ": the following additional variables are available to plot:\n";
                for (int i = 0; i < microphysics_plot_names.size(); i++) {
                    Print() << "    " << microphysics_plot_names[i] << "\n";
                }
                first_call = false;
            }
            for (auto& plot_name : microphysics_plot_names) {
                if (containerHasElement(plot_var_names, plot_name) &&
                    !containerHasElement(tmp_plot_names, plot_name)) {
                    tmp_plot_names.push_back(plot_name);
                }
            }
        }
    }
 
    for (int i = 0; i < tmp_plot_names.size(); i++) {
        if (!containerHasElement(a_plot_var_names, tmp_plot_names[i])) {
            a_plot_var_names.push_back(tmp_plot_names[i]);
        }
    }
}

Here is the call graph for this function:

appendPlotVariables()

void ERF::appendPlotVariables ( const std::string &  pp_plot_var_names, amrex::Vector< std::string > &  plot_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    Vector<std::string> plot_var_names(0);
    if (pp.contains(pp_plot_var_names.c_str())) {
        std::string nm;
        int nPltVars = pp.countval(pp_plot_var_names.c_str());
        for (int i = 0; i < nPltVars; i++) {
            pp.get(pp_plot_var_names.c_str(), nm, i);
            // Add the named variable to our list of plot variables
            // if it is not already in the list
            if (!containerHasElement(plot_var_names, nm)) {
                plot_var_names.push_back(nm);
            }
        }
    }
 
    appendPlot3DVariables(plot_var_names, a_plot_var_names);
 
    // Finally, check to see if we found all the requested variables
    for (const auto& plot_name : plot_var_names) {
        if (!containerHasElement(a_plot_var_names, plot_name)) {
             if (amrex::ParallelDescriptor::IOProcessor()) {
                 Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
             }
        }
    }
}

Here is the call graph for this function:

apply_gaussian_smoothing_to_perturbations()

void ERF::apply_gaussian_smoothing_to_perturbations	(	const int	lev,
		amrex::MultiFab &	mf_cc_pert
	)

Apply 3D isotropic/anisotropic Gaussian smoothing to cell-centered perturbations.

Parameters

lev	Integer specifying the current level
mf_cc_pert	MultiFab containing perturbations to smooth in place

{
    const Geometry& gm = geom[lev];
    const Real dx = gm.CellSize(0);
    const Real dy = gm.CellSize(1);
    const Real dz = gm.CellSize(2);
 
    const Real sigma = solverChoice.ens_pert_correlated_radius;
 
    const int rx = static_cast<int>(std::ceil(3.0 * sigma / dx));
    const int ry = static_cast<int>(std::ceil(3.0 * sigma / dy));
    const int rz = static_cast<int>(std::ceil(3.0 * sigma / dz));
 
    const int ncomp = mf_cc_pert.nComp();
 
    // 1. Build 1D Gaussian weight vectors on host
    auto compute_1d_weights = [](int r, Real d, Real sig) {
        Vector<Real> w(2*r + 1);
        Real sum = 0.0;
        for (int m = -r; m <= r; ++m) {
            Real val = std::exp(-(m * m * d * d) / (2.0 * sig * sig));
            w[m + r] = val;
            sum += val;
        }
        for (auto& v : w) v /= sum;
        return w;
    };
 
    Vector<Real> wx_h = compute_1d_weights(rx, dx, sigma);
    Vector<Real> wy_h = compute_1d_weights(ry, dy, sigma);
    Vector<Real> wz_h = compute_1d_weights(rz, dz, sigma);
 
    Gpu::DeviceVector<Real> wx_d(wx_h.size()), wy_d(wy_h.size()), wz_d(wz_h.size());
    Gpu::copy(Gpu::hostToDevice, wx_h.begin(), wx_h.end(), wx_d.begin());
    Gpu::copy(Gpu::hostToDevice, wy_h.begin(), wy_h.end(), wy_d.begin());
    Gpu::copy(Gpu::hostToDevice, wz_h.begin(), wz_h.end(), wz_d.begin());
 
    Real const* wx = wx_d.data();
    Real const* wy = wy_d.data();
    Real const* wz = wz_d.data();
 
    // Temp MultiFab for separable intermediate states
    IntVect ngrow(AMREX_D_DECL(rx, ry, rz));
    MultiFab mf_tmp(mf_cc_pert.boxArray(), mf_cc_pert.DistributionMap(), ncomp, ngrow);
 
    // --- Pass 1: X Direction ---
    mf_tmp.ParallelCopy(mf_cc_pert, 0, 0, ncomp, IntVect(0), ngrow, gm.periodicity());
    for (MFIter mfi(mf_cc_pert, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        auto const& in  = mf_tmp.const_array(mfi);
        auto const& out = mf_cc_pert.array(mfi);
        ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept {
            Real sum = 0.0;
            for (int m = -rx; m <= rx; ++m) {
                sum += wx[m + rx] * in(i + m, j, k, n);
            }
            out(i, j, k, n) = sum;
        });
    }
 
    // --- Pass 2: Y Direction ---
    mf_tmp.ParallelCopy(mf_cc_pert, 0, 0, ncomp, IntVect(0), ngrow, gm.periodicity());
    for (MFIter mfi(mf_cc_pert, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        auto const& in  = mf_tmp.const_array(mfi);
        auto const& out = mf_cc_pert.array(mfi);
        ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept {
            Real sum = 0.0;
            for (int nn = -ry; nn <= ry; ++nn) {
                sum += wy[nn + ry] * in(i, j + nn, k, n);
            }
            out(i, j, k, n) = sum;
        });
    }
 
    // --- Pass 3: Z Direction ---
    mf_tmp.ParallelCopy(mf_cc_pert, 0, 0, ncomp, IntVect(0), ngrow, gm.periodicity());
    for (MFIter mfi(mf_cc_pert, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        auto const& in  = mf_tmp.const_array(mfi);
        auto const& out = mf_cc_pert.array(mfi);
        ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept {
            Real sum = 0.0;
            for (int p = -rz; p <= rz; ++p) {
                sum += wz[p + rz] * in(i, j, k + p, n);
            }
            out(i, j, k, n) = sum;
        });
    }
 
    NormalizeMultiFabRMS_PerComponent(mf_cc_pert);
}

Here is the call graph for this function:

ApplyOceanSurfaceState()

void ERF::ApplyOceanSurfaceState	(	const amrex::Vector< amrex::MultiFab * > &	state,
		double	time,
		const amrex::iMultiFab *	erf_coverage = nullptr
	)

{
    // The gate is "is coupled SST configured"
    if (!solverChoice.use_coupled_sst) {
        if (!m_warned_coupled_sst_declined) {
            m_warned_coupled_sst_declined = true;
            amrex::Print() << "ERF::ApplyOceanSurfaceState was called but "
                           << "erf.use_coupled_sst is not set; the ocean SST is "
                           << "being discarded. Set erf.use_coupled_sst = 1."
                           << std::endl;
        }
        return;
    }
 
    if (state.empty() || state[0] == nullptr) { return; }
 
    if (m_coupled_sst == nullptr) {
        amrex::BoxArray ba;
        amrex::DistributionMapping dm;
        GetOceanToAtmosSurfaceLayout(ba, dm);
        // No ghost cells: SurfaceLayer::fill_tsurf_with_coupled_sst clamps its sample
        // index into the valid box, exactly as get_lsm_tsurf does for the LSM
        // arrays, so ghosts here would never be read. t_surf's own ghosts are
        // filled by the FillBoundary at the end of update_fluxes.
        m_coupled_sst       = std::make_unique<amrex::MultiFab> (ba, dm, 1, 0);
        m_coupled_sst_valid = std::make_unique<amrex::iMultiFab>(ba, dm, 1, 0);
    }
 
    // Zero coverage first: a cell the driver does not write this call must not
    // inherit the previous call's coverage flag.
    m_coupled_sst_valid->setVal(0);
    m_coupled_sst->ParallelCopy(*state[0], 0, 0, 1);
    if (erf_coverage != nullptr) {
        m_coupled_sst_valid->ParallelCopy(*erf_coverage, 0, 0, 1);
    }
 
    // Coupled SST is only consumed by the bottom z face.
    if (m_SurfaceLayer[amrex::Orientation::zlo()]) {
        m_SurfaceLayer[amrex::Orientation::zlo()]->update_coupled_sst_ptr(
            0, m_coupled_sst.get(), m_coupled_sst_valid.get());
    }
 
    // Report over the covered cells only. The old whole-array min/max was
    // dominated by the remap's zero fill wherever the ocean grid did not reach,
    // so the range check could not distinguish "cold ocean" from "no ocean".
    constexpr amrex::Real cov_min_sentinel = amrex::Real( 1.e30);
    constexpr amrex::Real cov_max_sentinel = amrex::Real(-1.e30);
 
    amrex::ReduceOps<amrex::ReduceOpSum, amrex::ReduceOpMin, amrex::ReduceOpMax> reduce_ops;
    amrex::ReduceData<amrex::Long, amrex::Real, amrex::Real> reduce_data(reduce_ops);
    using ReduceTuple = typename decltype(reduce_data)::Type;
 
    for (amrex::MFIter mfi(*m_coupled_sst); mfi.isValid(); ++mfi) {
        const amrex::Box& bx = mfi.validbox();
        const auto sst_arr   = m_coupled_sst->const_array(mfi);
        const auto valid_arr = m_coupled_sst_valid->const_array(mfi);
        reduce_ops.eval(bx, reduce_data,
            [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple
            {
                const bool covered = (valid_arr(i,j,k) != 0);
                return { covered ? amrex::Long(1) : amrex::Long(0),
                         covered ? sst_arr(i,j,k) : cov_min_sentinel,
                         covered ? sst_arr(i,j,k) : cov_max_sentinel };
            });
    }
 
    auto reduced = reduce_data.value(reduce_ops);
    amrex::Long n_valid = amrex::get<0>(reduced);
    amrex::Real cov_min = amrex::get<1>(reduced);
    amrex::Real cov_max = amrex::get<2>(reduced);
 
    amrex::ParallelDescriptor::ReduceLongSum(n_valid);
    amrex::ParallelDescriptor::ReduceRealMin(cov_min);
    amrex::ParallelDescriptor::ReduceRealMax(cov_max);
 
    const amrex::Long n_total = m_coupled_sst->boxArray().numPts();
 
    if (n_valid == 0) {
        amrex::Print() << "Coupled SST apply at t=" << time
                       << " s: no ERF cell has an ocean donor; the lower-boundary "
                       << "SST/TSK data stands everywhere." << std::endl;
        return;
    }
 
    amrex::Print() << "Coupled SST apply at t=" << time
                   << " s: covered " << n_valid << " of " << n_total
                   << " surface cells, min/max over covered = "
                   << cov_min << " / " << cov_max << " K" << std::endl;
 
    if (cov_min < amrex::Real(260.0) || cov_max > amrex::Real(320.0)) {
        amrex::Warning("Coupled SST is outside the expected [260, 320] K range");
    }
}

Here is the call graph for this function:

AverageDown()

void ERF::AverageDown ( )

private

{
    AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
 
    int src_comp, num_comp;
    for (int lev = finest_level-1; lev >= 0; --lev)
    {
        // If anelastic we don't average down rho because rho == rho0.
        if (solverChoice.anelastic[lev]) {
            src_comp = 1;
        } else {
            src_comp = 0;
        }
        num_comp = vars_new[0][Vars::cons].nComp() - src_comp;
        AverageDownTo(lev,src_comp,num_comp);
    }
}

Here is the call graph for this function:

AverageDownMoistStateTo()

void ERF::AverageDownMoistStateTo

(

int

crse_lev

)

Conservation-preserving fine-to-coarse average of RhoTheta and the active moist conserved components using detJ/mfac weighting, reusing AverageDownTo with perturbational-state and momentum handling disabled.

{
    AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
    AMREX_ALWAYS_ASSERT(crse_lev >= 0 && crse_lev < finest_level);
 
    AverageDownTo(crse_lev, RhoTheta_comp, 1, false);
 
    const int n_moist = (micro) ? micro->Get_Qstate_Size() : 0;
    if (n_moist > 0) {
        AverageDownTo(crse_lev, RhoQ1_comp, n_moist, false);
    }
}

Here is the call graph for this function:

AverageDownTo()

void ERF::AverageDownTo	(	int	crse_lev,
		int	scomp,
		int	ncomp,
		bool	do_perturbational_and_momenta = true
	)

{
    if (do_perturbational_and_momenta) {
        if (solverChoice.anelastic[crse_lev]) {
            AMREX_ALWAYS_ASSERT(scomp == 1);
        } else {
            AMREX_ALWAYS_ASSERT(scomp == 0);
        }
        AMREX_ALWAYS_ASSERT(ncomp == vars_new[crse_lev][Vars::cons].nComp() - scomp);
    } else {
        AMREX_ALWAYS_ASSERT(scomp >= 0);
        AMREX_ALWAYS_ASSERT(ncomp > 0);
        AMREX_ALWAYS_ASSERT(scomp + ncomp <= vars_new[crse_lev][Vars::cons].nComp());
    }
    AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
 
    // ******************************************************************************************
    // First do cell-centered quantities
    // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
    // m is the map scale factor at cell centers
    // Here we multiply (rho S) by detJ and divide (rho S) by m^2 before average down
    // ******************************************************************************************
    for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
      for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
        const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
            });
        } else {
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
            });
        }
      } // mfi
    } // lev
 
    int fine_lev = crse_lev+1;
 
    if (do_perturbational_and_momenta && interpolation_type == StateInterpType::Perturbational) {
        // Make the fine rho and (rho theta) be perturbational
        MultiFab::Divide(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
                         Rho_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                           BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                           BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
 
        // Make the crse rho and (rho theta) be perturbational
        MultiFab::Divide(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                         Rho_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                           BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                           BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
    }
 
    if (SolverChoice::terrain_type != TerrainType::EB) {
        average_down(vars_new[crse_lev+1][Vars::cons],vars_new[crse_lev  ][Vars::cons],
                    scomp, ncomp, refRatio(crse_lev));
    } else {
        // const auto dx = geom[fine_lev].CellSize();
        // Setting cell_vol to the exact value may cause round-off errors in volume average.
        // const Real cell_vol = dx[0]*dx[1]*dx[2];
        constexpr Real cell_vol = one;
        const BoxArray& ba = vars_new[fine_lev][IntVars::cons].boxArray();
        const DistributionMapping& dm = vars_new[fine_lev][IntVars::cons].DistributionMap();
        MultiFab vol_fine(ba, dm, 1, 0);
        vol_fine.setVal(cell_vol);
        EB_average_down(vars_new[fine_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                    vol_fine, *detJ_cc[fine_lev],
                    scomp, ncomp, refRatio(crse_lev));
    }
 
    if (do_perturbational_and_momenta && interpolation_type == StateInterpType::Perturbational) {
        // Restore the fine data to what it was
        MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                      BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                      BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Multiply(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
                           Rho_comp,RhoTheta_comp,1,IntVect{0});
 
        // Make the crse data be full values not perturbational
        MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                      BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                      BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Multiply(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                           Rho_comp,RhoTheta_comp,1,IntVect{0});
    }
 
    vars_new[crse_lev][Vars::cons].FillBoundary(geom[crse_lev].periodicity());
 
    // ******************************************************************************************
    // Here we multiply (rho S) by m^2 and divide by detJ after average down
    // ******************************************************************************************
    for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
      for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        const Array4<      Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
        const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
            });
        } else { // MeshType::ConstantDz
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
            });
        }
      } // mfi
    } // lev
 
    // Fill EB covered cells by old values
    // (This won't be needed because EB_average_down copies the covered value.)
    if (SolverChoice::terrain_type == TerrainType::EB) {
        for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
            for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<      Real> cons_new = vars_new[lev][Vars::cons].array(mfi);
                const Array4<const Real> cons_old = vars_old[lev][Vars::cons].array(mfi);
                const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
                ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                {
                    if (detJ_arr(i,j,k) == zero) {
                        cons_new(i,j,k,scomp+n) = cons_old(i,j,k,scomp+n);
                    }
                });
            } // mfi
        } // lev
    }
 
    if (!do_perturbational_and_momenta) return;
 
    // ******************************************************************************************
    // Now average down momenta.
    // Note that vars_new holds velocities not momenta, but we want to do conservative
    //    averaging so we first convert to momentum, then average down, then convert
    //    back to velocities -- only on the valid region
    // ******************************************************************************************
    for (int lev = crse_lev; lev <= crse_lev+1; lev++)
    {
        // FillBoundary for density so we can go back and forth between velocity and momentum
        vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
 
        const MultiFab* c_vfrac = nullptr;
        if (SolverChoice::terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
        }
 
        VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
                        vars_new[lev][Vars::yvel], IntVect(0,0,0),
                        vars_new[lev][Vars::zvel], IntVect(0,0,0),
                        vars_new[lev][Vars::cons],
                            rU_new[lev],
                            rV_new[lev],
                            rW_new[lev],
                        Geom(lev).Domain(),
                        domain_bcs_type,
                        c_vfrac);
    }
 
    if (SolverChoice::terrain_type != TerrainType::EB) {
        average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
        average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
        average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
    } else {
        EB_average_down_faces({&rU_new[crse_lev+1], &rV_new[crse_lev+1], &rW_new[crse_lev+1]},
                            {&rU_new[crse_lev], &rV_new[crse_lev], &rW_new[crse_lev]},
                            refRatio(crse_lev), 0);
    }
 
    for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
 
        const MultiFab* c_vfrac = nullptr;
        if (SolverChoice::terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
        }
 
        MomentumToVelocity(vars_new[lev][Vars::xvel],
                        vars_new[lev][Vars::yvel],
                        vars_new[lev][Vars::zvel],
                        vars_new[lev][Vars::cons],
                            rU_new[lev],
                            rV_new[lev],
                            rW_new[lev],
                        Geom(lev).Domain(),
                        domain_bcs_type,
                        c_vfrac);
    }
}

Here is the call graph for this function:

build_fine_mask()

void ERF::build_fine_mask	(	int	lev,
		amrex::MultiFab &	fine_mask
	)

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters

level

Fine level index which masks underlying coarser data

{
    // Mask for zeroing covered cells
    AMREX_ASSERT(level > 0);
 
    BoxArray cba            = grids[level-1];
    DistributionMapping cdm = dmap[level-1];
 
    BoxArray fba            = grids[level];
 
    iMultiFab ifine_mask_lev = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
 
    const auto  fma =  fine_mask_lev.arrays();
    const auto ifma = ifine_mask_lev.arrays();
    ParallelFor(fine_mask_lev, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
    {
       fma[bno](i,j,k) = ifma[bno](i,j,k);
    });
}

Here is the call graph for this function:

BuildPlot3DScratch()

void ERF::BuildPlot3DScratch	(	const amrex::Vector< std::string > &	plot_var_names,
		Plot3DScratch &	scratch,
		int	max_lev = -1,
		bool	sync_solution = true
	)

{
    auto& mf_cc_vel = scratch.mf_cc_vel;
    auto& mf_cc_tau = scratch.mf_cc_tau;
    auto& mf_cc_fx  = scratch.mf_cc_fx;
 
    // The scratch is indexed by level whatever is built, so that a level the
    // caller does not read is simply an empty MultiFab rather than a shift.
    mf_cc_vel.resize(finest_level+1);
    mf_cc_tau.resize(finest_level+1);
    mf_cc_fx.resize(finest_level+1);
 
    const int top_lev = (max_lev < 0) ? finest_level : std::min(max_lev, finest_level);
 
    // Lagrangian microphysics with AMR (TwoWay): synchronize the
    // microphysics-owned storage before rebuilding the active moisture state
    // used by the plotfile diagnostics.  This modifies the coarse solution, so
    // a caller that runs at every step rather than at plotfile cadence asks not
    // to do it: turning a diagnostic on must not change the answer.
    if (sync_solution
        && Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian
        && solverChoice.coupling_type == CouplingType::TwoWay
        && finest_level >= 1) {
        micro->AverageDownMicroVars(finest_level);
        for (int flev = finest_level-1; flev >= 0; --flev) {
            AverageDownMoistStateTo(flev);
        }
    }
 
    // We Fillpatch here because some of the derived quantities require derivatives
    //     which require ghost cells to be filled.  We do not need to call FillPatcher
    //     because we don't need to set interior fine points.
    // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
 
    // Level 0 FillPatch
    FillPatchCrseLevel(0, t_new[0], {&vars_new[0][Vars::cons], &vars_new[0][Vars::xvel],
                       &vars_new[0][Vars::yvel], &vars_new[0][Vars::zvel]});
 
    for (int lev = 1; lev <= top_lev; ++lev) {
        bool fillset = false;
        FillPatchFineLevel(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
                           &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
                          {&vars_new[lev][Vars::cons], &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                           base_state[lev], base_state[lev], fillset);
    }
 
    // Get qmoist pointers if using moisture
    for (int lev = 0; lev <= finest_level; ++lev) {
        for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
            qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
        }
    }
 
    // Array of MultiFabs for cell-centered velocity
 
    if (containerHasElement(plot_var_names, "x_velocity"    ) ||
        containerHasElement(plot_var_names, "y_velocity"    ) ||
        containerHasElement(plot_var_names, "z_velocity"    ) ||
        containerHasElement(plot_var_names, "magvel"        ) ||
        containerHasElement(plot_var_names, "helicity"      ) ||
        containerHasElement(plot_var_names, "local_helicity") ||
        containerHasElement(plot_var_names, "vort_stretching") ||
        containerHasElement(plot_var_names, "vorticity_x"   ) ||
        containerHasElement(plot_var_names, "vorticity_y"   ) ||
        containerHasElement(plot_var_names, "vorticity_z"   ) ) {
 
        for (int lev = 0; lev <= top_lev; ++lev) {
            mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
            mf_cc_vel[lev].setVal(bogus_large_value);
            average_face_to_cellcenter(mf_cc_vel[lev],0,
                                       Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                                &vars_new[lev][Vars::yvel],
                                                                &vars_new[lev][Vars::zvel]}, 1);
        } // lev
    } // if (vel or vort)
 
 
    if (containerHasElement(plot_var_names, "Tau11" ) ||
        containerHasElement(plot_var_names, "Tau12" ) ||
        containerHasElement(plot_var_names, "Tau13" ) ||
        containerHasElement(plot_var_names, "Tau21" ) ||
        containerHasElement(plot_var_names, "Tau22" ) ||
        containerHasElement(plot_var_names, "Tau23" ) ||
        containerHasElement(plot_var_names, "Tau31" ) ||
        containerHasElement(plot_var_names, "Tau32" ) ||
        containerHasElement(plot_var_names, "Tau33" )) {
 
        for (int lev = 0; lev <= top_lev; ++lev) {
            mf_cc_tau[lev].define(grids[lev], dmap[lev], 9, IntVect(1,1,1));
            mf_cc_tau[lev].setVal(bogus_large_value);
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf_cc_tau[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // todo - clean this up
                const Box& bxcc= mfi.tilebox();
 
                const Array4<Real>& tau12 = (Tau[lev][TauType::tau12]) ? Tau[lev][TauType::tau12]->array(mfi) : Array4<Real>{};
                const Array4<Real>& tau21 = (Tau[lev][TauType::tau21]) ? Tau[lev][TauType::tau21]->array(mfi) : Array4<Real>{};
 
                const Array4<Real>& tau13 = (Tau[lev][TauType::tau13]) ? Tau[lev][TauType::tau13]->array(mfi) : Array4<Real>{};
                const Array4<Real>& tau31 = (Tau[lev][TauType::tau31]) ? Tau[lev][TauType::tau31]->array(mfi) : Array4<Real>{};
 
                const Array4<Real>& tau23 = (Tau[lev][TauType::tau23]) ? Tau[lev][TauType::tau23]->array(mfi) : Array4<Real>{};
                const Array4<Real>& tau32 = (Tau[lev][TauType::tau32]) ? Tau[lev][TauType::tau32]->array(mfi) : Array4<Real>{};
 
                const Array4<Real>& tau_cc   = mf_cc_tau[lev].array(mfi);
                ParallelFor(bxcc, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    tau_cc(i, j, k, TauType::tau12) = (tau12) ? fourth * (tau12(i, j, k) + tau12(i, j+1, k) + tau12(i+1,j+1,k) + tau12(i+1, j, k)) : zero;
                    tau_cc(i, j, k, TauType::tau21) = (tau21) ? fourth * (tau21(i, j, k) + tau21(i, j+1, k) + tau21(i+1,j+1,k) + tau21(i+1, j, k)) : zero;
                    tau_cc(i, j, k, TauType::tau13) = (tau13) ? fourth * (tau13(i, j, k) + tau13(i, j, k+1) + tau13(i+1,j,k+1) + tau13(i+1, j, k)) : zero;
                    tau_cc(i, j, k, TauType::tau31) = (tau31) ? fourth * (tau31(i, j, k) + tau31(i, j, k+1) + tau31(i+1,j,k+1) + tau31(i+1, j, k)) : zero;
                    tau_cc(i, j, k, TauType::tau23) = (tau23) ? fourth * (tau23(i, j, k) + tau23(i, j, k+1) + tau23(i,j+1,k+1) + tau23(i, j+1, k)) : zero;
                    tau_cc(i, j, k, TauType::tau32) = (tau32) ? fourth * (tau32(i, j, k) + tau32(i, j, k+1) + tau32(i,j+1,k+1) + tau32(i, j+1, k)) : zero;
                });
            }
        }
    }
 
    if (containerHasElement(plot_var_names, "hfx1" ) ||
        containerHasElement(plot_var_names, "hfx2" ) ||
        containerHasElement(plot_var_names, "hfx3" ) ||
        containerHasElement(plot_var_names, "q1fx1" ) ||
        containerHasElement(plot_var_names, "q1fx2" ) ||
        containerHasElement(plot_var_names, "q1fx3" ) ||
        containerHasElement(plot_var_names, "q2fx3" ))
    {
        for (int lev = 0; lev <= top_lev; ++lev) {
            mf_cc_fx[lev].define(grids[lev], dmap[lev], 7, IntVect(1,1,1));
            mf_cc_fx[lev].setVal(bogus_large_value);
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf_cc_fx[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bxcc= mfi.tilebox();
 
                const Array4<Real>& hfx1 = (SFS_hfx1_lev[lev]) ? SFS_hfx1_lev[lev]->array(mfi) : Array4<Real>{};
                const Array4<Real>& hfx2 = (SFS_hfx2_lev[lev]) ? SFS_hfx2_lev[lev]->array(mfi) : Array4<Real>{};
                const Array4<Real>& hfx3 = (SFS_hfx3_lev[lev]) ? SFS_hfx3_lev[lev]->array(mfi) : Array4<Real>{};
                const Array4<Real>& q1fx1 = (SFS_q1fx1_lev[lev]) ? SFS_q1fx1_lev[lev]->array(mfi) : Array4<Real>{};
                const Array4<Real>& q1fx2 = (SFS_q1fx2_lev[lev]) ? SFS_q1fx2_lev[lev]->array(mfi) : Array4<Real>{};
                const Array4<Real>& q1fx3 = (SFS_q1fx3_lev[lev]) ? SFS_q1fx3_lev[lev]->array(mfi) : Array4<Real>{};
                const Array4<Real>& q2fx3 = (SFS_q2fx3_lev[lev]) ? SFS_q2fx3_lev[lev]->array(mfi) : Array4<Real>{};
 
                const Array4<Real>& tau_fx   = mf_cc_fx[lev].array(mfi);
                ParallelFor(bxcc, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    tau_fx(i, j, k, 0) = (hfx1) ? 0.5 * (hfx1(i, j, k) + hfx1(i+1, j  , k)) : 0.0;
                    tau_fx(i, j, k, 1) = (hfx2) ? 0.5 * (hfx2(i, j, k) + hfx2(i  , j+1, k)) : 0.0;
                    tau_fx(i, j, k, 2) = (hfx3) ? 0.5 * (hfx3(i, j, k) + hfx3(i  , j  , k+1)) : 0.0;
                    tau_fx(i, j, k, 3) = (q1fx1) ? 0.5 * (q1fx1(i, j, k) + q1fx1(i+1, j  , k)) : 0.0;
                    tau_fx(i, j, k, 4) = (q1fx2) ? 0.5 * (q1fx2(i, j, k) + q1fx2(i  , j+1, k)) : 0.0;
                    tau_fx(i, j, k, 5) = (q1fx3) ? 0.5 * (q1fx3(i, j, k) + q1fx3(i  , j  , k+1)) : 0.0;
                    tau_fx(i, j, k, 6) = (q2fx3) ? 0.5 * (q2fx3(i, j, k) + q2fx3(i  , j  , k+1)) : 0.0;
                });
            }
        }
    }
 
    // We need ghost cells if computing vorticity
    if ( containerHasElement(plot_var_names, "vorticity_x")||
         containerHasElement(plot_var_names, "vorticity_y") ||
         containerHasElement(plot_var_names, "vorticity_z") )
    {
        amrex::Interpolater* mapper = &cell_cons_interp;
        FillBdyCCVels(mf_cc_vel[0],geom[0]);
        for (int lev = 1; lev <= top_lev; ++lev)
        {
            Real new_time = static_cast<Real>(t_new[lev]);
            Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
            Vector<Real> ftime    = {new_time,new_time};
            Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
            Vector<Real> ctime    = {new_time,new_time};
 
            // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mf_cc_vel[lev], mf_cc_vel[lev].nGrowVect(), IntVect(0,0,0),
                               static_cast<Real>(t_new[lev]), cmf, ctime, fmf, ftime,
                               0, 0, mf_cc_vel[lev].nComp(), geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BaseBCVars::rho0_bc_comp);
            FillBdyCCVels(mf_cc_vel[lev],geom[lev]);
        } // lev
    } // if (vort)
}

Here is the call graph for this function:

canonicalizePlot3DVariables()

void ERF::canonicalizePlot3DVariables ( amrex::Vector< std::string > &  plot_var_names )

private

{
    // Get state variables in the same order as we define them,
    // since they may be in any order in the input list.
    Vector<std::string> tmp_plot_names;
    erf_plotfile::Plot3DSelectionCapabilities capabilities;
    // This is called before vars_new is allocated, so the microphysics interface
    // is the source of truth for the conserved-state layout.
    //
    // NOTE: the allocated width bounds the dry names and the non-water species
    //       that sit above the moist window, but *not* the moist components
    //       themselves: those are selected from solverChoice.moisture_indices,
    //       because a scheme may allocate moist components it never integrates
    //       (see the MoistureComponentIndices class comment) and those must not
    //       reach a plotfile.
    erf_plotfile::plot3d_set_state_capabilities(capabilities,
                                                solverChoice.moisture_indices,
                                                micro->Get_Qstate_Moist_Size(),
                                                micro->Get_Qstate_Size());
    capabilities.time_average_storage = solverChoice.time_avg_vel;
    capabilities.interval_mean_storage = solverChoice.compute_mean_vars;
    // qsrc_sw / qsrc_lw are available whenever qheating_rates is allocated,
    // i.e. for any erf.radiation_model other than None.
    capabilities.radiation_heating_storage =
        erf_plotfile::radiation_heating_storage_available(solverChoice.rad_type);
    capabilities.eddy_diffusivity_storage = true;
    capabilities.dissipation_storage = true;
    capabilities.wall_distance_storage = true;
    for (int lev = 0; lev <= max_level; ++lev) {
        capabilities.eddy_diffusivity_storage =
            capabilities.eddy_diffusivity_storage && solverChoice.turbChoice[lev].use_kturb;
        capabilities.dissipation_storage =
            capabilities.dissipation_storage &&
            (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None ||
             solverChoice.turbChoice[lev].use_kturb);
        capabilities.wall_distance_storage =
            capabilities.wall_distance_storage &&
            solverChoice.turbChoice[lev].rans_type != RANSType::None;
    }
 
    for (int i = 0; i < cons_names.size(); ++i) {
        if (containerHasElement(plot_var_names, cons_names[i])) {
            if (erf_plotfile::plot3d_fixed_variable_available(cons_names[i], capabilities)) {
                tmp_plot_names.push_back(cons_names[i]);
            }
        }
    }
 
    // check for velocity since it's not in cons_names
    // if we are asked for any velocity component, we will need them all
    if (containerHasElement(plot_var_names, "x_velocity") ||
        containerHasElement(plot_var_names, "y_velocity") ||
        containerHasElement(plot_var_names, "z_velocity")) {
        tmp_plot_names.push_back("x_velocity");
        tmp_plot_names.push_back("y_velocity");
        tmp_plot_names.push_back("z_velocity");
    }
 
    //
    // If the model we are running doesn't have the variable listed in the inputs file,
    //     just ignore it rather than aborting
    //
#ifdef ERF_USE_WINDFARM
    // NOTE: these mirror the conditions guarding the Nturb/SMark fills in Write3DPlotFile;
    //       if they diverge, the names and the filled components will not line up.
    const bool wf_is_AD  = (solverChoice.windfarm_type == WindFarmType::SimpleAD ||
                            solverChoice.windfarm_type == WindFarmType::GeneralAD);
    const bool wf_active = (solverChoice.windfarm_type == WindFarmType::Fitch ||
                            solverChoice.windfarm_type == WindFarmType::EWP   || wf_is_AD);
#endif
 
    for (int i = 0; i < derived_names.size(); ++i) {
        if ( containerHasElement(plot_var_names, derived_names[i]) ) {
            bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing || solverChoice.buildings_type == BuildingsType::ImmersedForcing ) ||
                               (derived_names[i] != "terrain_IB_mask") );
            ok_to_add     &= ( ibseb_params.enable ||
                               (derived_names[i] != "ibseb_nfaces" && derived_names[i] != "ibseb_tskin" &&
                                derived_names[i] != "ibseb_sw_abs" && derived_names[i] != "ibseb_shadow" &&
                                derived_names[i] != "ibseb_lw_net" && derived_names[i] != "ibseb_f_sky" &&
                                derived_names[i] != "ibseb_H" && derived_names[i] != "ibseb_G") );
            ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                               (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                               (derived_names[i] != "detJ") );
            ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                               (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                               (derived_names[i] != "z_phys") );
            ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                               (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                               (derived_names[i] != "h_xi" && derived_names[i] != "h_eta" &&
                                derived_names[i] != "h_zeta") );
#ifdef ERF_USE_WINDFARM
            // NOTE: the windfarm names must be added here, in derived_names order, since
            //       that is where Write3DPlotFile fills them (see ERF.H "MUST MATCH THE ORDER").
            ok_to_add     &= ( wf_active ||
                               (derived_names[i] != "num_turb" && derived_names[i] != "SMark0") );
            ok_to_add     &= ( wf_is_AD  || (derived_names[i] != "SMark1") );
#endif
            if (ok_to_add)
            {
                if (erf_plotfile::plot3d_fixed_variable_available(derived_names[i], capabilities)) {
                    tmp_plot_names.push_back(derived_names[i]);
                }
            } // use_terrain?
        } // hasElement
    }
 
#ifdef ERF_USE_PARTICLES
    Vector<std::string> configured_particle_names;
    for (const auto& name : particleData.getNamesUnalloc()) {
        configured_particle_names.push_back(name);
    }
    for (const auto& name : particleData.getNames()) {
        if (!containerHasElement(configured_particle_names, name)) {
            configured_particle_names.push_back(name);
        }
    }
    for (const auto& name : erf_plotfile::plot3d_selected_particle_count_names(
             plot_var_names, configured_particle_names)) {
        tmp_plot_names.push_back(name + "_count");
    }
#endif
 
    plot_var_names = tmp_plot_names;
}

Here is the call graph for this function:

check_for_low_temp()

void ERF::check_for_low_temp

(

amrex::MultiFab &

S

)

{
    // *****************************************************************************
    // Test for low temp (low is defined as beyond the microphysics range of validity)
    // *****************************************************************************
    //
    // This value is defined in erf_dtesati in Source/Utils/ERF_MicrophysicsUtils.H
    Real t_low = Real(273.16) - Real(85.);
    //
    for (MFIter mfi(S); mfi.isValid(); ++mfi)
    {
        Box bx = mfi.tilebox();
        const Array4<Real> &s_arr  = S.array(mfi);
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            const Real rho      = s_arr(i, j, k, Rho_comp);
            const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
            const Real qv       = s_arr(i, j, k, RhoQ1_comp) / rho;
 
            Real temp = getTgivenRandRTh(rho, rhotheta, qv);
 
            if (temp < t_low) {
#ifdef AMREX_USE_GPU
                AMREX_DEVICE_PRINTF("Temperature too low in cell: %d %d %d %e \n", i,j,k,temp);
#else
                printf("Temperature too low in cell: %d %d %d \n", i,j,k);
                printf("Based on temp / rhotheta / rho / qv %e %e %e %e \n", temp,rhotheta,rho,qv);
#endif
                Abort();
            }
        });
    }
}

Here is the call graph for this function:

check_for_negative_theta()

void ERF::check_for_negative_theta

(

amrex::MultiFab &

S

)

{
    // *****************************************************************************
    // Test for negative (rho theta)
    // *****************************************************************************
    for (MFIter mfi(S); mfi.isValid(); ++mfi)
    {
        Box bx = mfi.tilebox();
        const Array4<Real> &s_arr  = S.array(mfi);
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            const Real rho      = s_arr(i, j, k, Rho_comp);
            const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
 
            if (rho <= zero) {
#ifdef AMREX_USE_GPU
                AMREX_DEVICE_PRINTF("Rho is negative at %d %d %d %e \n", i,j,k,rho);
#else
                printf("Rho is negative at %d %d %d %e \n", i,j,k,rho);
                Abort("Bad rho in check_for_negative_theta");
#endif
            }
 
            if (rhotheta <= zero) {
#ifdef AMREX_USE_GPU
                AMREX_DEVICE_PRINTF("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
#else
                printf("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
                Abort("Bad theta in check_for_negative_theta");
#endif
            }
 
            });
    } // mfi
}

Here is the call graph for this function:

check_mesh_type()

void ERF::check_mesh_type

(

int

lev

)

{
   if (SolverChoice::mesh_type == MeshType::VariableDz) {
       MultiFab z_slab(convert(ba2d[lev],IntVect(1,1,1)),dmap[lev],1,0);
       int klo = geom[lev].Domain().smallEnd(2);
       for (MFIter mfi(z_slab); mfi.isValid(); ++mfi) {
           Box nbx = mfi.tilebox();
           Array4<Real const> const& z_arr      = z_phys_nd[lev]->const_array(mfi);
           Array4<Real      > const& z_slab_arr = z_slab.array(mfi);
           ParallelFor(nbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
           {
               z_slab_arr(i,j,k) = z_arr(i,j,klo);
           });
       }
       Real z_min = z_slab.min(0);
       Real z_max = z_slab.max(0);
 
       auto dz = geom[lev].CellSize()[2];
       if (z_max - z_min < Real(1.e-8) * dz) {
           SolverChoice::set_mesh_type(MeshType::StretchedDz);
           if (verbose > 0) {
               amrex::Print() << "Resetting mesh type to StretchedDz since terrain is flat" << std::endl;
           }
       }
   }
}

Here is the call graph for this function:

check_state_for_nans()

void ERF::check_state_for_nans

(

amrex::MultiFab const &

S

)

{
    amrex::Gpu::DeviceScalar<int> d_found(0);
 
    // comp, i, j, k
    amrex::Gpu::DeviceVector<int> d_info(4, -1);
 
    for (MFIter mfi(S,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        auto const& s_arr = S.const_array(mfi);
 
        const int ncomp = S.nComp();
 
        int* found = d_found.dataPtr();
        int* info  = d_info.dataPtr();
 
        ParallelFor(bx,
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            // Somebody already found a NaN
            if (*found) return;
 
            for (int n = 0; n < ncomp; ++n)
            {
                Real val = s_arr(i,j,k,n);
 
                if (val != val) // NaN test
                {
                    // Only one thread wins
                    if (amrex::Gpu::Atomic::CAS(found,0,1) == 0)
                    {
                        info[0] = n;
                        info[1] = i;
                        info[2] = j;
                        info[3] = k;
                    }
                    return;
                }
            }
        });
    }
 
    amrex::Gpu::streamSynchronize();
 
    if (d_found.dataValue())
    {
        amrex::Vector<int> h_info(4);
        amrex::Print() << "Found flag = " << d_found.dataValue() << "\n";
 
        amrex::Gpu::copy(
            amrex::Gpu::deviceToHost,
            d_info.begin(),
            d_info.end(),
            h_info.begin());
 
        std::cout << "NaN found in component " << h_info[0]
            << " at (i,j,k) = ("
            << h_info[1] << ", "
            << h_info[2] << ", "
            << h_info[3] << ")\n";
 
        amrex::Abort("NaN detected in state");
    }
}

Here is the call graph for this function:

check_vels_for_nans()

void ERF::check_vels_for_nans	(	amrex::MultiFab const &	xvel,
		amrex::MultiFab const &	yvel,
		amrex::MultiFab const &	zvel
	)

{
    //
    // Test at the end of every full timestep whether the solution data contains NaNs
    //
    bool any_have_nans = false;
    if (xvel.contains_nan(0,1,0))
    {
        amrex::Print() << "x-velocity contains NaNs " << '\n';
        any_have_nans = true;
    }
    if (yvel.contains_nan(0,1,0))
    {
        amrex::Print() << "y-velocity contains NaNs" << '\n';
        any_have_nans = true;
    }
    if (zvel.contains_nan(0,1,0))
    {
        amrex::Print() << "z-velocity contains NaNs" << '\n';
        any_have_nans = true;
    }
    if (any_have_nans) {
        exit(0);
    }
}

ClearLevel()

void ERF::ClearLevel ( int  lev )

override

{
    for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
        vars_new[lev][var_idx].clear();
        vars_old[lev][var_idx].clear();
    }
 
    base_state[lev].clear();
 
    rU_new[lev].clear();
    rU_old[lev].clear();
    rV_new[lev].clear();
    rV_old[lev].clear();
    rW_new[lev].clear();
    rW_old[lev].clear();
 
    if (lev > 0) {
        zmom_crse_rhs[lev].clear();
    }
 
    if ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) {
        pp_inc[lev].clear();
    }
    if (solverChoice.anelastic[lev] == 0) {
        lagged_delta_rt[lev].clear();
    }
    avg_xmom[lev].clear();
    avg_ymom[lev].clear();
    avg_zmom[lev].clear();
 
    // Clears the integrator memory
    mri_integrator_mem[lev].reset();
 
    // Clears the map of the vertical extent of the grid column over each (i,j)
    column_kextent[lev].reset();
 
    // Clears the physical boundary condition routines
    physbcs_cons[lev].reset();
    physbcs_u[lev].reset();
    physbcs_v[lev].reset();
    physbcs_w[lev].reset();
    physbcs_base[lev].reset();
 
    // Clears the flux register array (only allocated for TwoWay coupling)
    if (advflux_reg[lev]) {
        advflux_reg[lev]->reset();
    }
 
    // Clears the 2D arrays
    if (sst_lev[lev][0]) {
        for (int n = 0; n < sst_lev[lev].size(); n++) {
            sst_lev[lev][n].reset();
        }
    }
    if (tsk_lev[lev][0]) {
        for (int n = 0; n < tsk_lev[lev].size(); n++) {
            tsk_lev[lev][n].reset();
        }
    }
    if (lat_m[lev]) {
        lat_m[lev].reset();
    }
    if (lon_m[lev]) {
        lon_m[lev].reset();
    }
    if (sinPhi_m[lev]) {
        sinPhi_m[lev].reset();
    }
    if (cosPhi_m[lev]) {
        cosPhi_m[lev].reset();
    }
 
#ifdef ERF_USE_FFT
    // Clear any FFT solvers built at this level
    if (m_3D_poisson.size() > lev) {
        for (int n = 0; n < m_3D_poisson[lev].size(); n++) {
            m_3D_poisson[lev][n].reset();
        }
    }
    if (m_2D_poisson.size() > lev) {
        for (int n = 0; n < m_2D_poisson[lev].size(); n++) {
            m_2D_poisson[lev][n].reset();
        }
    }
#endif
 
    // This level is going away, so nothing is pending for its radiation.  Whichever routine
    // builds the level next is responsible for setting this again.
    if (lev < static_cast<int>(rad_interp_from_coarse_pending.size())) {
        rad_interp_from_coarse_pending[lev] = 0;
    }
}

cloud_fraction()

Real ERF::cloud_fraction

(

double

time

)

{
    BL_PROFILE("ERF::cloud_fraction()");
 
    int lev = 0;
    // This holds all of qc
    MultiFab qc(vars_new[lev][Vars::cons],make_alias,RhoQ2_comp,1);
 
    int direction = 2; // z-direction
    Box const& domain = geom[lev].Domain();
 
    auto const& qc_arr = qc.const_arrays();
 
    // qc_2d is an BaseFab<int> holding the max value over the column
    auto qc_2d = ReduceToPlane<ReduceOpMax,int>(direction, domain, qc,
         [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) -> int
         {
             if (qc_arr[box_no](i,j,k) > 0) {
                 return 1;
             } else {
                 return 0;
             }
         });
 
    auto* p = qc_2d.dataPtr();
 
    Long numpts = qc_2d.numPts();
 
    AMREX_ASSERT(numpts < Long(std::numeric_limits<int>::max));
 
#if 1
    if (ParallelDescriptor::UseGpuAwareMpi()) {
        ParallelDescriptor::ReduceIntMax(p,static_cast<int>(numpts));
    } else {
        Gpu::PinnedVector<int> hv(numpts);
        Gpu::copyAsync(Gpu::deviceToHost, p, p+numpts, hv.data());
        Gpu::streamSynchronize();
        ParallelDescriptor::ReduceIntMax(hv.data(),static_cast<int>(numpts));
        Gpu::copyAsync(Gpu::hostToDevice, hv.data(), hv.data()+numpts, p);
        // We must synchronize before hv goes out of scope and its pinned memory
        //    is returned to the arena, since the copy above may still be reading it
        Gpu::streamSynchronize();
    }
 
    // Sum over component 0
    Long num_cloudy = qc_2d.template sum<RunOn::Device>(0);
 
#else
    //
    // We need this if we allow domain decomposition in the vertical
    //    but for now we leave it commented out
    //
    Long num_cloudy = Reduce::Sum<Long>(numpts,
         [=] AMREX_GPU_DEVICE (Long i) -> Long {
             if (p[i] == 1) {
                 return 1;
             } else {
                 return 0;
             }
         });
    ParallelDescriptor::ReduceLongSum(num_cloudy);
#endif
 
    Real num_total = Real(qc_2d.box().d_numPts());
 
    Real cloud_frac = num_cloudy / num_total;
 
    return cloud_frac;
}

Referenced by PackAtmosphericStates().

Here is the caller graph for this function:

compute_divergence()

void ERF::compute_divergence	(	int	lev,
		amrex::MultiFab &	rhs,
		amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM >	rho0_u_const,
		amrex::MultiFab const &	mf_mx,
		amrex::MultiFab const &	mf_my,
		amrex::MultiFab const &	mf_vx,
		amrex::MultiFab const &	mf_uy,
		amrex::MultiFab const &	ax_in,
		amrex::MultiFab const &	ay_in,
		amrex::MultiFab const &	dJ_in,
		amrex::Geometry const &	geom_at_lev
	)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level zero

Parameters

lev	Level index for the projection data
rhs	Cell-centered divergence field to be filled
rho0_u_const	Face-centered momentum components to differentiate
geom_at_lev	Geometry for the level being projected

{
    BL_PROFILE("ERF::compute_divergence()");
 
    auto dxInv = geom_at_lev.InvCellSizeArray();
 
    // ****************************************************************************
    // Compute divergence which will form RHS
    // Note that we replace "rho0w" with the contravariant momentum, Omega
    // ****************************************************************************
    if (solverChoice.terrain_type == TerrainType::EB)
    {
        bool already_on_centroids = true;
        EB_computeDivergence(rhs, rho0_u_const, geom_at_lev, already_on_centroids);
    }
    else if (SolverChoice::mesh_type == MeshType::ConstantDz)
    {
        computeDivergence(rhs, rho0_u_const, geom_at_lev);
    }
    else
    {
        for ( MFIter mfi(rhs,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            Box bx = mfi.tilebox();
            const Array4<Real const>& rho0u_arr = rho0_u_const[0]->const_array(mfi);
            const Array4<Real const>& rho0v_arr = rho0_u_const[1]->const_array(mfi);
            const Array4<Real const>& rho0w_arr = rho0_u_const[2]->const_array(mfi);
            const Array4<Real      >&   rhs_arr = rhs.array(mfi);
 
            const Array4<Real const>&      mf_mx_arr = mf_mx.const_array(mfi);
            const Array4<Real const>&      mf_my_arr = mf_my.const_array(mfi);
            const Array4<Real const>&      mf_vx_arr = mf_vx.const_array(mfi);
            const Array4<Real const>&      mf_uy_arr = mf_uy.const_array(mfi);
 
            if (SolverChoice::mesh_type == MeshType::StretchedDz)
            {
                Real* stretched_dz_d_ptr = stretched_dz_d[lev].data();
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    Real inv_dz = one/stretched_dz_d_ptr[k];
                    Real mfsq   = mf_mx_arr(i,j,0) * mf_my_arr(i,j,0);
                    rhs_arr(i,j,k) = (  (rho0u_arr(i+1,j  ,k  )/mf_uy_arr(i+1,j,0) - rho0u_arr(i,j,k)/mf_uy_arr(i,j,0)) * dxInv[0]
                                       +(rho0v_arr(i  ,j+1,k  )/mf_vx_arr(i,j+1,0) - rho0v_arr(i,j,k)/mf_vx_arr(i,j,0)) * dxInv[1]
                                       +(rho0w_arr(i  ,j  ,k+1)/mfsq           - rho0w_arr(i,j,k)/mfsq        ) * inv_dz  ) * mfsq;
                });
            }
            else
            {
                //
                // Note we compute the divergence using "rho0w" == Omega
                //
                const Array4<Real const>& ax_arr = ax_in.const_array(mfi);
                const Array4<Real const>& ay_arr = ay_in.const_array(mfi);
                const Array4<Real const>& dJ_arr = dJ_in.const_array(mfi);
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    Real mfsq = mf_mx_arr(i,j,0) * mf_my_arr(i,j,0);
                    rhs_arr(i,j,k) = ( ( ax_arr(i+1,j,k)*rho0u_arr(i+1,j,k)/mf_uy_arr(i+1,j,0)
                                        -ax_arr(i  ,j,k)*rho0u_arr(i  ,j,k)/mf_uy_arr(i  ,j,0)  ) * dxInv[0]
                                     + ( ay_arr(i,j+1,k)*rho0v_arr(i,j+1,k)/mf_vx_arr(i,j+1,0)
                                        -ay_arr(i,j  ,k)*rho0v_arr(i,j  ,k)/mf_vx_arr(i,j  ,0)  ) * dxInv[1]
                                      +(                 rho0w_arr(i,j,k+1)/mfsq
                                        -                rho0w_arr(i,j,k  )/mfsq            ) * dxInv[2] ) * mfsq / dJ_arr(i,j,k);
                });
            }
        } // mfi
    }
}

Here is the call graph for this function:

compute_max_buoyancy_gradp_diagnostic()

void ERF::compute_max_buoyancy_gradp_diagnostic

(

int

lev

)

{
    // When anelastic the pressure gradient comes from the projection, not from
    //    a perturbational pressure, so this diagnostic doesn't apply
    if (solverChoice.anelastic[lev]) return;
 
    auto& lev_new = vars_new[lev];
 
    const BoxArray&            ba = lev_new[Vars::cons].boxArray();
    const DistributionMapping& dm = lev_new[Vars::cons].DistributionMap();
 
    // *******************************************************************************
    // Compute the gradient of the perturbational pressure exactly as the dycore does
    // *******************************************************************************
 
    Vector<MultiFab> lgradp;  lgradp.resize(AMREX_SPACEDIM);
    lgradp[GpVars::gpx].define(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, 0);
    lgradp[GpVars::gpx].setVal(0.);
    lgradp[GpVars::gpy].define(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, 0);
    lgradp[GpVars::gpy].setVal(0.);
    lgradp[GpVars::gpz].define(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, 0);
    lgradp[GpVars::gpz].setVal(0.);
 
    MultiFab p0(base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    make_gradp_pert(lev, solverChoice, geom[lev], lev_new, p0,
                    *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
                    get_eb(lev), lgradp);
 
    // *******************************************************************************
    // Compute the buoyancy term exactly as the dycore does
    // *******************************************************************************
 
    MultiFab qt(ba, dm, 1, 1);
    qt.setVal(0.);
    int n_qstate_into_total = micro->Get_Qstate_Moist_Size() - micro->Get_Qstate_Moist_NumConc_Size();
    if (solverChoice.moisture_type != MoistureType::None) {
        make_qt(lev_new[Vars::cons], qt, n_qstate_into_total);
    }
 
    //
    // NOTE: we must fill one ghost cell of S_prim here because make_buoyancy
    //       reads cell_prim(i,j,k-1) at the lower z face of every box
    //
    MultiFab S_prim(ba, dm, lev_new[Vars::cons].nComp()-1, 1);
    cons_to_prim(lev_new[Vars::cons], S_prim, 1);
 
    // Initialize to zero because buoyancy is not defined on the faces at the top and bottom of the domain
    MultiFab buoyancy(lgradp[GpVars::gpz].boxArray(), lgradp[GpVars::gpz].DistributionMap(), 1, 0);
    buoyancy.setVal(0.);
 
    make_buoyancy(lev, lev_new, S_prim, qt, buoyancy, geom[lev], solverChoice, base_state[lev],
                  n_qstate_into_total, get_eb(lev), solverChoice.anelastic[lev]);
 
    // *******************************************************************************
    // Now form the combined term that appears in the z-momentum equation:
    //      -d(p - p0)/dz + buoyancy
    // If the initial state is in HSE and the buoyancy is computed from (rho - rho0)
    //      (i.e. buoyancy_type = 1) then this is identically zero
    // *******************************************************************************
 
    MultiFab combined(lgradp[GpVars::gpz].boxArray(), lgradp[GpVars::gpz].DistributionMap(), 1, 0);
    combined.setVal(0.);
 
    MultiFab::Copy    (combined, buoyancy          , 0, 0, 1, 0);
    MultiFab::Subtract(combined, lgradp[GpVars::gpz], 0, 0, 1, 0);
 
    // Don't include the top and bottom faces of the domain, where neither term is defined
    Box zface_domain = surroundingNodes(geom[lev].Domain(), 2);
    int klo = zface_domain.smallEnd(2);
    int khi = zface_domain.bigEnd(2);
    zface_domain.growLo(2,-1);
    zface_domain.growHi(2,-1);
 
    int comp = 0;
    Real min_val = combined.min(zface_domain,comp);
    Real max_val = combined.max(zface_domain,comp);
 
    Print() << " " << std::endl;
    if (max_val != zero || min_val != zero) {
        IntVect min_loc = combined.minIndex(comp);
        IntVect max_loc = combined.maxIndex(comp);
        Print() << "Min/max value of -dp'/dz + buoyancy are " << min_val << " " << max_val;
        if (min_loc[2] != klo && min_loc[2] != khi) Print() << " with min at face " << min_loc;
        if (max_loc[2] != klo && max_loc[2] != khi) Print() << " with max at face " << max_loc;
        Print() << std::endl;
    } else {
        Print() << "Min/max value of -dp'/dz + buoyancy are zero " << std::endl;
    }
    Print() << " " << std::endl;
}

Here is the call graph for this function:

compute_max_pressure_gradient_diagnostic()

void ERF::compute_max_pressure_gradient_diagnostic

(

int

lev

)

{
    // We don't require HSE when anelastic because the pressure gradient
    //    is computed from the Poisson solve
    if (solverChoice.anelastic[lev]) return;
 
    auto& lev_new = vars_new[lev];
 
    int ng = (solverChoice.terrain_type == TerrainType::EB) ? 3 : 1;
 
    const Real grav = solverChoice.gravity;
 
    Vector<MultiFab> gradp_temp;  gradp_temp.resize(AMREX_SPACEDIM);
    gradp_temp[0].define(vars_new[lev][Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, 0);
    gradp_temp[0].setVal(0.);
    gradp_temp[1].define(vars_new[lev][Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, 0);
    gradp_temp[1].setVal(0.);
    gradp_temp[2].define(vars_new[lev][Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, 0);
    gradp_temp[2].setVal(0.);
 
    int comp = 0;
 
    // The lateral boundary faces are included here.  The base state in the ghost cells
    // outside them is built at the height the mesh puts those cells at, rather than copied
    // outward at constant index, so the gradient across a lateral domain face is a gradient
    // of the reference atmosphere like any other and belongs in these extrema.
    Box xface_domain = surroundingNodes(geom[lev].Domain(), 0);
    Box yface_domain = surroundingNodes(geom[lev].Domain(), 1);
 
 
    // Use this region to take max/min of gpz without including top and bottom faces
    Box zface_domain = surroundingNodes(geom[lev].Domain(), 2);
    int klo = zface_domain.smallEnd(2);
    int khi = zface_domain.bigEnd(2);
 
    zface_domain.growLo(2,-1);
    zface_domain.growHi(2,-1);
 
    // *******************************************************************************
    // First check that base state satisfies EOS
    // *******************************************************************************
 
    Print() << " " << std::endl;
 
    MultiFab  r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
    MultiFab  p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
    MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp , 1);
    MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
 
    MultiFab dp(p_hse.boxArray(), p_hse.DistributionMap(), 1, 0);
 
    // Initialize to zero in case of EB covered cells
    dp.setVal(0.);
 
    for (MFIter mfi(dp); mfi.isValid(); ++mfi) {
        Box bx = mfi.validbox();
        auto const  rhse_arr  =  r_hse.const_array(mfi);
        auto const  phse_arr  =  p_hse.const_array(mfi);
        auto const qvhse_arr  = qv_hse.const_array(mfi);
        auto const thhse_arr  = th_hse.const_array(mfi);
        auto       dpeos_arr  = dp.array(mfi);
 
        if (solverChoice.terrain_type != TerrainType::EB) {
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                Real rhotheta = rhse_arr(i,j,k) * thhse_arr(i,j,k);
                dpeos_arr(i,j,k) = std::abs(getPgivenRTh(rhotheta, qvhse_arr(i,j,k)) - phse_arr(i,j,k));
            });
        } else {
            Array4<const Real> volfrac = (get_eb(lev).get_const_factory())->getVolFrac().const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                if (volfrac(i,j,k) > zero) {
                    Real rhotheta = rhse_arr(i,j,k) * thhse_arr(i,j,k);
                    dpeos_arr(i,j,k) = std::abs(getPgivenRTh(rhotheta, qvhse_arr(i,j,k)) - phse_arr(i,j,k));
                }
            });
        }
    }
    // This compares two pressures, so the tolerance has to be scaled to the
    // pressure rather than fixed in Pa.  The base state is O(1e5) Pa, where one
    // single-precision ulp is already ~1e-2 Pa, so the absolute 1e-8 Pa bound
    // used here previously could never be met in single precision even for a
    // perfectly consistent base state.  The double-precision coefficient below
    // reproduces that 1e-8 bound at 1e5 Pa, so the check is unchanged there.
#ifdef AMREX_USE_FLOAT
    const Real eos_rel_tol = Real(1.e-5);
#else
    const Real eos_rel_tol = Real(1.e-13);
#endif
    const Real p_hse_max = p_hse.max(0);
    const Real eos_tol = eos_rel_tol * p_hse_max;
 
    Real max_diff = dp.max(0);
    if (max_diff > eos_tol) {
        IntVect max_loc = dp.maxIndex(0);
        Print() << "Max value of |p_hse - p_eos| is " << max_diff << std::endl;
        Print() << " with max in cell " << max_loc << std::endl;
        Abort("Base state violates EOS ");
    } else {
        Print() << "Max value of |p_hse - p_eos| is " << max_diff
                << ", less than the tolerance " << eos_tol << std::endl;
    }
 
    // *******************************************************************************
    // Now compute pressure gradients for base state pressure
    // *******************************************************************************
 
    compute_gradp(p_hse, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
                  get_eb(lev), gradp_temp, solverChoice);
 
    Real min_gpx = gradp_temp[0].min(xface_domain,comp);
    Real max_gpx = gradp_temp[0].max(xface_domain,comp);
    if (max_gpx != zero || min_gpx != zero) {
        Print() << "Min/max value of dp0/dx            are " << min_gpx << " " << max_gpx << std::endl;
        IntVect min_loc = gradp_temp[0].minIndex(comp);
        IntVect max_loc = gradp_temp[0].maxIndex(comp);
        amrex::Print() << " with min at face " << min_loc;
        amrex::Print() << " with max at face " << max_loc;
        Print() << std::endl;
    } else {
        Print() << "Min/max value of dp0/dx            are zero " << std::endl;
    }
 
    Real min_gpy = gradp_temp[1].min(yface_domain,comp);
    Real max_gpy = gradp_temp[1].max(yface_domain,comp);
    if (max_gpy != zero || min_gpy != zero) {
        Print() << "Min/max value of dp0/dy            are " << min_gpy << " " << max_gpy << std::endl;
        IntVect min_loc = gradp_temp[1].minIndex(comp);
        IntVect max_loc = gradp_temp[1].maxIndex(comp);
        amrex::Print() << " with min at face " << min_loc;
        amrex::Print() << " with max at face " << max_loc;
        Print() << std::endl;
    } else {
        Print() << "Min/max value of dp0/dy            are zero " << std::endl;
    }
 
    if (solverChoice.terrain_type != TerrainType::EB) {
        for (MFIter mfi(gradp_temp[2]); mfi.isValid(); ++mfi) {
            Box bx = mfi.validbox();
            if (bx.bigEnd(2)   == khi) bx.growHi(2,-1);
            if (bx.smallEnd(2) == 0  ) bx.growLo(2,-1);
            auto        gpz_arr  = gradp_temp[2].array(mfi);
            auto const  rhse_arr  =  r_hse.const_array(mfi);
            auto const qvhse_arr  = qv_hse.const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                gpz_arr(i,j,k) += grav * myhalf * ( rhse_arr(i,j,k  ) * (one + qvhse_arr(i,j,k  ))
                                                   +rhse_arr(i,j,k-1) * (one + qvhse_arr(i,j,k-1)) );
            });
        }
    // EB case: check HSE only for uncovered cells
    } else {
        for (MFIter mfi(gradp_temp[2]); mfi.isValid(); ++mfi) {
            Box bx = mfi.validbox();
            if (bx.bigEnd(2)   == khi) bx.growHi(2,-1);
            if (bx.smallEnd(2) == 0  ) bx.growLo(2,-1);
            auto        gpz_arr  = gradp_temp[2].array(mfi);
            auto const  rhse_arr  =  r_hse.const_array(mfi);
            auto const qvhse_arr  = qv_hse.const_array(mfi);
            Array4<const Real> w_volfrac = (get_eb(lev).get_w_const_factory())->getVolFrac().const_array(mfi);
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                if (w_volfrac(i,j,k) > zero) {
                    gpz_arr(i,j,k) += grav * myhalf * ( rhse_arr(i,j,k  ) * (one + qvhse_arr(i,j,k  ))
                                                       +rhse_arr(i,j,k-1) * (one + qvhse_arr(i,j,k-1)) );
                }
            });
        }
    }
 
    // The residual tested below, dp0/dz + rho0*|g|, is a cancellation between two
    // terms of size rho0*|g|.  The smallest value it can take is therefore set by
    // the roundoff in p0 spread over one vertical cell, eps*|p0|/dz.  That floor
    // is grid dependent -- in single precision it is ~1e-4 Pa/m at dz = 100 m but
    // ~3e-4 Pa/m at dz = 40 m -- so no fixed tolerance can work across the range
    // of meshes the tests cover.  Use the larger of the historical fixed value and
    // that floor.  In double the floor is O(1e-12), so the fixed value always wins
    // and this check keeps exactly its previous behavior.
    const Real tol_fixed = (solverChoice.terrain_type == TerrainType::EB) ? Real(1.e-4)
                                                                         : Real(1.e-8);
 
    const Real dz_for_tol = (lev < static_cast<int>(dz_min.size()) && dz_min[lev] > Real(0))
                          ? dz_min[lev] : geom[lev].CellSize(2);
 
    const Real tol_roundoff = Real(16) * std::numeric_limits<Real>::epsilon()
                            * p_hse_max / dz_for_tol;
 
    const Real tol = std::max(tol_fixed, tol_roundoff);
 
    Real min_gpz = gradp_temp[2].min(zface_domain,comp);
    Real max_gpz = gradp_temp[2].max(zface_domain,comp);
 
    if (std::abs(max_gpz) > tol || std::abs(min_gpz) > tol) {
        IntVect min_loc = gradp_temp[2].minIndex(comp);
        IntVect max_loc = gradp_temp[2].maxIndex(comp);
        Print() << "Min/max value of dp0/dz + rho0*|g| are " << min_gpz << " " << max_gpz;
        if (min_loc[2] != klo && min_loc[2] != khi) amrex::Print() << " with min at face " << min_loc;
        if (max_loc[2] != klo && max_loc[2] != khi) amrex::Print() << " with max at face " << max_loc;
        amrex::Abort("Base state is too far out of HSE");
    } else {
        Print() << "Min/max value of dp0/dz + rho0*|g| are less than " << tol << std::endl;
    }
    Print() << " " << std::endl;
 
    if (!solverChoice.anelastic[lev]) {
 
        // *******************************************************************************
        // Now compute for full (moist) pressure
        // *******************************************************************************
 
        MultiFab p(p_hse.boxArray(), p_hse.DistributionMap(), 1, ng);
        MultiFab rho(lev_new[Vars::cons], make_alias, Rho_comp , 1);
 
        if (solverChoice.moisture_type != MoistureType::None) {
 
            for (MFIter mfi(rho); mfi.isValid(); ++mfi)
            {
                Box gbx = mfi.tilebox();
                gbx.grow(IntVect(ng,ng,ng));
                if (gbx.smallEnd(2) < 0) gbx.setSmall(2,0);
 
                const Array4<const Real>& cell_data = lev_new[Vars::cons].array(mfi);
                const Array4<const Real>&  r_arr = rho.array(mfi);
                const Array4<      Real>& pp_arr = p.array(mfi);
                ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    Real qv_for_p = cell_data(i,j,k,RhoQ1_comp)/r_arr(i,j,k);
                    pp_arr(i,j,k) = getPgivenRTh(cell_data(i,j,k,RhoTheta_comp),qv_for_p);
                });
            }
            compute_gradp(p, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
                          get_eb(lev), gradp_temp, solverChoice);
 
            min_gpx = gradp_temp[0].min(xface_domain,comp);
            max_gpx = gradp_temp[0].max(xface_domain,comp);
            if (max_gpx != zero || min_gpx != zero) {
                Print() << "Min/Max value of x-gradient of full (moist) pressure are " << min_gpx << " " << max_gpx;
                IntVect min_loc = gradp_temp[0].minIndex(comp);
                IntVect max_loc = gradp_temp[0].maxIndex(comp);
                amrex::Print() << " with min at face " << min_loc;
                amrex::Print() << " with max at face " << max_loc;
                Print() << std::endl;
            } else {
                Print() << "Min/max value of x-gradient of full (moist) pressure are zero " << std::endl;
            }
 
            min_gpy = gradp_temp[1].min(yface_domain,comp);
            max_gpy = gradp_temp[1].max(yface_domain,comp);
            if (max_gpy != zero || min_gpy != zero) {
                Print() << "Min/Max value of y-gradient of full (moist) pressure are " << min_gpy << " " << max_gpy;
                IntVect min_loc = gradp_temp[1].minIndex(comp);
                IntVect max_loc = gradp_temp[1].maxIndex(comp);
                amrex::Print() << " with min at face " << min_loc;
                amrex::Print() << " with max at face " << max_loc;
                Print() << std::endl;
            } else {
                Print() << "Min/max value of y-gradient of full (moist) pressure are zero " << std::endl;
            }
 
            MultiFab qt(rho.boxArray(), rho.DistributionMap(), 1, 1);
            int n_qstate_into_total = micro->Get_Qstate_Moist_Size() - micro->Get_Qstate_Moist_NumConc_Size();
            make_qt(lev_new[Vars::cons], qt, n_qstate_into_total);
 
            for (MFIter mfi(gradp_temp[2]); mfi.isValid(); ++mfi)
            {
                Box bx = mfi.validbox();
                if (bx.bigEnd(2)   == khi) bx.growHi(2,-1);
                if (bx.smallEnd(2) == 0  ) bx.growLo(2,-1);
                auto      gpz_arr   = gradp_temp[2].array(mfi);
                auto const  r_arr   = rho.const_array(mfi);
                auto const qt_arr   =  qt.const_array(mfi);
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    gpz_arr(i,j,k) += grav * myhalf * (r_arr(i,j,k  )*(one+qt_arr(i,j,k  )) +
                                                       r_arr(i,j,k-1)*(one+qt_arr(i,j,k-1)) );
                });
            }
 
            min_gpz = gradp_temp[2].min(zface_domain,comp);
            max_gpz = gradp_temp[2].max(zface_domain,comp);
            if (max_gpz != zero || min_gpz != zero) {
                IntVect min_loc = gradp_temp[2].minIndex(comp);
                IntVect max_loc = gradp_temp[2].maxIndex(comp);
                Print() << "Min/max value of moist dp/dz + rho_m*|g|             are " << min_gpz << " " << max_gpz;
                if (min_loc[2] != klo && min_loc[2] != khi) amrex::Print() << " with min at face " << min_loc;
                if (max_loc[2] != klo && max_loc[2] != khi) amrex::Print() << " with max at face " << max_loc;
            } else {
                Print() << "Min/max value of moist dp/dz + rho_m*|g|  are zero " << std::endl;
            }
            Print() << " " << std::endl;
        } // if moist
    } // if !anelastic
}

Here is the call graph for this function:

compute_native_shoc_tendencies()

void ERF::compute_native_shoc_tendencies	(	int	lev,
		amrex::MultiFab *	cons,
		amrex::MultiFab *	xvel,
		amrex::MultiFab *	yvel,
		amrex::MultiFab *	zvel,
		amrex::Real *	w_subsid,
		amrex::MultiFab *	tau13,
		amrex::MultiFab *	tau23,
		amrex::MultiFab *	hfx3,
		amrex::MultiFab *	qfx3,
		amrex::MultiFab *	eddyDiffs,
		amrex::MultiFab *	z_phys_nd,
		const double &	dt_advance
	)

{
    AMREX_ALWAYS_ASSERT(native_shoc_driver[lev]);
    native_shoc_driver[lev]->advance(*cons, *xvel, *yvel, *zvel,
                                     tau13, tau23, hfx3, qfx3, eddyDiffs,
                                     *z_phys_nd_in, Geom(lev), dt_advance);
}

Here is the call graph for this function:

ComputeAndWriteEnsemblePerturbations()

void ERF::ComputeAndWriteEnsemblePerturbations

(

const int

da_iter

)

ComputeDt()

void ERF::ComputeDt ( int  step = -1, double  cur_time_d = 0.0  )

private

Function that calls estTimeStep for each level

{
    Vector<double> dt_tmp(finest_level+1);
 
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
    }
 
    ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
 
    double dt_0 = dt_tmp[0];
    int n_factor = 1;
    for (int lev = 0; lev <= finest_level; ++lev) {
        dt_tmp[lev] = amrex::min(dt_tmp[lev], static_cast<double>(change_max*dt[lev]));
        n_factor *= nsubsteps[lev];
        dt_0 = std::min(dt_0, static_cast<double>(n_factor*dt_tmp[lev]));
 
    }
    // Limit level 0 time step if requested
    if (step == 0) {
        dt_0 *= init_shrink;
        if (verbose && init_shrink != one) {
            Print() << "Timestep 0: shrink level 0 initial dt by " << init_shrink << std::endl;
        }
    }
    //
    // Limit dt by the value of stop_time.
    // Recall that stop_time is total time, but t_new is elapsed time,
    //     so we must add start_time to t_new
    //
    const double eps = 1.e-3*dt_0;
    if (cur_time_d + dt_0 > (stop_time - start_time) - eps) {
        dt_0 = (stop_time - start_time) - cur_time_d;
    }
 
    dt[0] = dt_0;
    for (int lev = 1; lev <= finest_level; ++lev) {
        dt[lev] = dt[lev-1] / nsubsteps[lev];
    }
}

Referenced by EvolveOneStep().

Here is the caller graph for this function:

ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells ( const SolverChoice &  sc )

inlinestaticprivate

    {
        // We never have fewer than 2 ghost cells
        int ngrow = 2;
 
        // This sets the minimum to 3 if using num_diff but we might still need more
        if (sc.use_num_diff) {
            ngrow = 3;
        }
 
        // With EB we always have 4 so no need to do further tests
        if (sc.terrain_type == TerrainType::EB) {
            ngrow = 4;
            return ngrow;
        }
 
        if (
             (sc.advChoice.dycore_horiz_adv_type    == AdvType::Centered_6th)
          || (sc.advChoice.dycore_vert_adv_type     == AdvType::Centered_6th)
          || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
          || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
          || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Centered_6th)
          || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Centered_6th) )
        { ngrow = 3; }
        else if (
             (sc.advChoice.dycore_horiz_adv_type    == AdvType::Upwind_5th)
          || (sc.advChoice.dycore_vert_adv_type     == AdvType::Upwind_5th)
          || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
          || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
          || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Upwind_5th)
          || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Upwind_5th) )
        { ngrow = 3; }
        else if (
              (sc.advChoice.dycore_horiz_adv_type    == AdvType::Weno_5)
           || (sc.advChoice.dycore_vert_adv_type     == AdvType::Weno_5)
           || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5)
           || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_5)
           || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_5)
           || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_5) )
        { ngrow = 3; }
        else if (
              (sc.advChoice.dycore_horiz_adv_type    == AdvType::Weno_5Z)
           || (sc.advChoice.dycore_vert_adv_type     == AdvType::Weno_5Z)
           || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5Z)
           || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_5Z)
           || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_5Z)
           || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_5Z) )
        { ngrow = 3; }
        else if (
              (sc.advChoice.dycore_horiz_adv_type    == AdvType::Weno_7)
           || (sc.advChoice.dycore_vert_adv_type     == AdvType::Weno_7)
           || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_7)
           || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_7)
           || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_7)
           || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_7) )
        { ngrow = 3; }
        else if (
              (sc.advChoice.dycore_horiz_adv_type    == AdvType::Weno_7Z)
           || (sc.advChoice.dycore_vert_adv_type     == AdvType::Weno_7Z)
           || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_7Z)
           || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_7Z)
           || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_7Z)
           || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_7Z) )
        { ngrow = 4; }
 
        return ngrow;
    }

ComputeGlobalMinLocation()

void ERF::ComputeGlobalMinLocation	(	const SolverChoice &	sc,
		const amrex::Geometry &	geom,
		const amrex::Vector< amrex::MultiFab > &	S_data,
		amrex::Real *	d_val_min_ptr,
		int *	d_i_min_ptr,
		int *	d_j_min_ptr,
		amrex::Real &	global_val_min,
		int &	global_i_min,
		int &	global_j_min
	)

Compute the global minimum and its location across all ranks.

Parameters

[in]	sc	Solver choices
[in]	lev_geom	Geometry of the current level
[in]	S_data	Conservative state data
[in]	d_val_min_ptr	Device pointer to local minimum value
[in]	d_i_min_ptr	Device pointer to local minimum i-index
[in]	d_j_min_ptr	Device pointer to local minimum j-index
[out]	global_val_min	Global minimum value
[out]	global_i_min	Global minimum i-index
[out]	global_j_min	Global minimum j-index

{
    Real h_val_min;
    int h_i_min, h_j_min;
 
    Gpu::copy(Gpu::deviceToHost, d_val_min_ptr, d_val_min_ptr + 1, &h_val_min);
    Gpu::copy(Gpu::deviceToHost, d_i_min_ptr, d_i_min_ptr + 1, &h_i_min);
    Gpu::copy(Gpu::deviceToHost, d_j_min_ptr, d_j_min_ptr + 1, &h_j_min);
    Gpu::synchronize();
 
    Real local_val_min = h_val_min;
    int local_i_min = h_i_min;
    int local_j_min = h_j_min;
 
    int rank = ParallelDescriptor::MyProc();
 
    // NOTE: reduce through the amrex wrappers rather than a hard-coded
    //       MPI_DOUBLE_INT MINLOC. The latter is a type mismatch whenever
    //       amrex::Real is float (ERF_PRECISION=SINGLE), in which case MPI
    //       reads and writes 8 bytes of a 4-byte member. Reducing the value
    //       and then taking the smallest rank that attains it reproduces
    //       MPI_MINLOC's lowest-rank tie-break without naming an MPI type.
    global_val_min = local_val_min;
    ParallelDescriptor::ReduceRealMin(global_val_min);
 
    int owner_rank = (local_val_min == global_val_min) ? rank : ParallelDescriptor::NProcs();
    ParallelDescriptor::ReduceIntMin(owner_rank);
    AMREX_ALWAYS_ASSERT(owner_rank < ParallelDescriptor::NProcs());
 
    // Broadcast the indices from the rank that owns the minimum
    global_i_min = local_i_min;
    global_j_min = local_j_min;
 
    ParallelDescriptor::Bcast(&global_i_min, 1, owner_rank);
    ParallelDescriptor::Bcast(&global_j_min, 1, owner_rank);
 
    if (rank == 0) {
        Print() << "Global minimum distance to hurricane eye (k=0): "
                       << global_val_min << " at (i,j) = ("
                       << global_i_min << ", " << global_j_min << ")\n";
    }
 
    Gpu::DeviceScalar<Real> d_eye_lat(zero), d_eye_lon(zero);
 
    Real* d_eye_lat_ptr = d_eye_lat.dataPtr();
    Real* d_eye_lon_ptr = d_eye_lon.dataPtr();
 
    int levc = finest_level;
    // On owner_rank, compute eye_lat and eye_lon
    if (sc.init_type == InitType::WRFInput and rank == owner_rank) {
        for (MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
            const Box& box = mfi.validbox();
            FArrayBox& fab_lat = (*(lat_m[levc]))[mfi];
            FArrayBox& fab_lon = (*(lon_m[levc]))[mfi];
            const Array4<Real>& lat_arr = fab_lat.array();
            const Array4<Real>& lon_arr = fab_lon.array();
 
            if (box.smallEnd()[2] == 0) {
                Box bx2d = makeSlab(box,2,0);
                ParallelFor(bx2d, [=] AMREX_GPU_DEVICE(int i, int j, int ) {
                    if (i == global_i_min && j == global_j_min) {
                        *d_eye_lat_ptr = lat_arr(i,j,0);
                        *d_eye_lon_ptr = lon_arr(i,j,0);
                    }
                });
            }
        }
    }
 
    if (sc.init_type == InitType::HindCast and rank == owner_rank) {
        // On owner_rank, compute eye_lat and eye_lon
        if (rank == owner_rank) {
            for (amrex::MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
                const amrex::Box& box = mfi.validbox();
                const auto& mf_latlon = forecast_state_interp[levc][4];
                const auto latlon_arr = mf_latlon.array(mfi);
                amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    if (i == global_i_min && j == global_j_min && k == 0) {
                        *d_eye_lat_ptr = latlon_arr(i,j,k,0);
                        *d_eye_lon_ptr = latlon_arr(i,j,k,1);
                    }
                });
            }
        }
    }
 
    Real eye_lat = d_eye_lat.dataValue();
    Real eye_lon = d_eye_lon.dataValue();
 
    // Synchronize to ensure the owner has computed values
    Gpu::synchronize();
 
    ParallelDescriptor::Bcast(&eye_lat, 1, owner_rank);
    ParallelDescriptor::Bcast(&eye_lon, 1, owner_rank);
 
    const auto dx = lev_geom.CellSizeArray();
    const auto prob_lo = lev_geom.ProbLoArray();
 
    Real eye_x =  prob_lo[0] + (global_i_min+myhalf)*dx[0];
    Real eye_y =  prob_lo[1] + (global_j_min+myhalf)*dx[1];
 
    hurricane_eye_track_xy.push_back({eye_x, eye_y});
    hurricane_eye_track_latlon.push_back({eye_lon, eye_lat});
}

Here is the call graph for this function:

ConfigureDriverAtmosToOceanCoupling()

void ERF::ConfigureDriverAtmosToOceanCoupling	(	bool	use_coupling_driver,
		bool	use_two_way_coupling,
		bool	use_state_contract
	)

{
    m_driver_has_atm2ocn_coupling = use_coupling_driver;
    m_driver_uses_two_way_coupling = use_two_way_coupling;
    m_driver_atm2ocn_uses_state_contract = use_state_contract;
}

Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int  lev )

private

{
    auto& fine_new = vars_new[lev];
    auto& crse_new = vars_new[lev-1];
    auto& ba_fine  = fine_new[Vars::cons].boxArray();
    auto& ba_crse  = crse_new[Vars::cons].boxArray();
    auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
    auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
 
    int ncomp = vars_new[lev][Vars::cons].nComp();
 
    FPr_c.emplace_back(ba_fine, dm_fine, geom[lev]  ,
                       ba_crse, dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
    FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                       convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                       convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                       convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
}

create_background_state_for_ensemble()

void ERF::create_background_state_for_ensemble	(	int	lev,
		amrex::MultiFab &	mf_cc_pert,
		amrex::MultiFab &	cons_pert,
		amrex::MultiFab &	xvel_pert,
		amrex::MultiFab &	yvel_pert,
		amrex::MultiFab &	zvel_pert
	)

Build ensemble background perturbation MultiFabs from custom coarse data.

Parameters

lev	Integer specifying the current level
mf_cc_pert	Cell-centered perturbations to apply to the background
cons_pert	Conserved-state perturbation MultiFab to fill
xvel_pert	x-face velocity perturbation MultiFab to fill
yvel_pert	y-face velocity perturbation MultiFab to fill
zvel_pert	z-face velocity perturbation MultiFab to fill

{
 
    ignore_unused(lev);
    int nx_crse, ny_crse, nz_crse, ng_crse, ncomp_crse;
    Vector<Vector<Real>> data_crse;
    std::array<Real,3> problo_ext, probhi_ext;
 
    Vector<Real> data_rho, data_theta, data_xvel, data_yvel, data_zvel;
    Vector<Real> data_qv, data_qc, data_qrain;
 
    ReadCustomDataFile(solverChoice.coarse_bckgnd_data_file,
                       nx_crse, ny_crse, nz_crse, ng_crse, ncomp_crse,
                       problo_ext, probhi_ext,
                       data_rho, data_theta,
                       data_xvel, data_yvel, data_zvel,
                       data_qv, data_qc, data_qrain);
 
    Geometry& geom_fine = geom[0];
    // Create a cell-centered multifab on the fine mesh - ie. something with the same boxarray,
    // distributed mapping, nGrow, but with 5 components
    MultiFab mf_cc_fine;
    const MultiFab& src = vars_new[0][0];
    int ncomp = 8;
    mf_cc_fine.define(src.boxArray(), src.DistributionMap(),
                      ncomp, src.nGrow());
 
    InterpolateToFineMF(data_rho, data_theta,
                        data_xvel, data_yvel, data_zvel,
                        data_qv, data_qc, data_qrain,
                        nx_crse, ny_crse, nz_crse,
                        problo_ext, probhi_ext,
                        mf_cc_fine,
                        geom_fine);
 
 
    Vector<std::string> varnames = {"density","theta", "x_velocity","y_velocity","z_velocity", "qv", "qc", "qrain"};
    WriteSingleLevelPlotfile("1_plt_final_interp", mf_cc_fine, varnames, geom_fine, zero, 0);
 
    ApplyNeumannBCs(geom_fine, mf_cc_fine);
     // Add pertubrations stored in the "pert" variables in the function arguments
    // (multiplied by the corresponding amplitude)
    AddPertToBckgnd(mf_cc_fine, mf_cc_pert, solverChoice.ens_pert_amplitude);
    ApplyNeumannBCs(geom_fine, mf_cc_fine);
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
    int n_qstate_moist = 0;
    if (use_moisture) {
        n_qstate_moist = micro->Get_Qstate_Moist_Size();
    }
 
    MakeFinalMultiFabs(mf_cc_fine, cons_pert, xvel_pert, yvel_pert, zvel_pert, n_qstate_moist);
}

Here is the call graph for this function:

create_random_perturbations()

void ERF::create_random_perturbations	(	const int	lev,
		amrex::MultiFab &	mf_cc_pert
	)

Create a cell-centered MultiFab of random perturbations for one AMR level.

Parameters

lev	Integer specifying the current level
mf_cc_pert	MultiFab filled with random perturbation components

{
    const MultiFab& src = vars_new[lev][Vars::cons];
 
    int ncomp = 8;
    mf_cc_pert.define(src.boxArray(), src.DistributionMap(),
                      ncomp, src.nGrow());
 
    // Loop over cell-centered boxes
    for (MFIter mfi(mf_cc_pert, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
 
        auto const& pert_arr = mf_cc_pert.array(mfi);
 
        // Loop over all 5 components
        amrex::ParallelForRNG(bx, ncomp,
        [=] AMREX_GPU_DEVICE (int i, int j, int k, int n,
                             const amrex::RandomEngine& engine) noexcept
        {
            pert_arr(i,j,k,n) = amrex::Real(2.0) * amrex::Random(engine) - amrex::Real(1.0);
        });
    }
}

Here is the call graph for this function:

DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int  i )

inlineprivate

    {
        return *datalog[i];
    }

DataLogName()

std::string ERF::DataLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith datalog file.

{ return datalogname[i]; }

define_column_kextent()

void ERF::define_column_kextent ( int  lev, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

private

Build the map of the vertical extent of the grid column over each (i,j).

The implicit vertical diffusion solves invert one tridiagonal system per column, so they are only well posed if each column lives in a single box. We enforce that here, then record each column's [klo,khi] on a z-slab with a one-cell halo in x and y; a box can then read the vertical extent of the column on the other side of any of its faces, which is what the staggered (u,v) solves need to agree with their neighbors.

Parameters

[in]	lev	level of refinement
[in]	ba	BoxArray at this level
[in]	dm	DistributionMapping at this level

{
    if (grids_are_split_in_z(ba))
    {
        bool implicit_var   = (solverChoice.implicit_thermal_diffusion ||
                               solverChoice.implicit_momentum_diffusion);
        bool implicit_stage = false;
        if (lev < solverChoice.vert_implicit_fac.size()) {
            for (int nrk(0); nrk < solverChoice.vert_implicit_fac[lev].size(); ++nrk) {
                if (solverChoice.vert_implicit_fac[lev][nrk] > zero) { implicit_stage = true; }
            }
        }
        if (implicit_var && implicit_stage) {
            Abort("The grids at level " + std::to_string(lev) + " are decomposed in the vertical, "
                  "which cannot be combined with implicit vertical diffusion: the solve inverts one "
                  "tridiagonal system per column, and a column split across boxes would instead be "
                  "solved piecewise with spurious internal boundaries, giving an answer that depends "
                  "on the grid decomposition.  Either set erf.vert_implicit_fac = 0 0 0 (or turn off "
                  "erf.implicit_thermal_diffusion and erf.implicit_momentum_diffusion), or choose "
                  "grids that are not split in z.");
        }
    }
 
    column_kextent[lev] = std::make_unique<iMultiFab>(ba2d[lev], dm, 2, IntVect(1,1,0));
 
    // Columns that no box covers -- outside the level, or outside the domain -- keep these
    // sentinels, which drop out of the max/min that defines the solve range below.
    column_kextent[lev]->setVal(column_kextent_lo_sentinel, 0, 1, IntVect(1,1,0));
    column_kextent[lev]->setVal(column_kextent_hi_sentinel, 1, 1, IntVect(1,1,0));
 
    // NOTE: no ParallelFor here.  This is a private member function, and nvcc does not allow
    //       an extended (__device__) lambda inside a member function with private or protected
    //       access.  Each box contributes a single (klo,khi) pair over its whole footprint,
    //       so BaseFab::setVal fills it on the device without needing a lambda at all.
    for (MFIter mfi(*column_kextent[lev]); mfi.isValid(); ++mfi)
    {
        const Box& vbx  = mfi.validbox();
        const Box& bx3d = ba[mfi.index()];
        IArrayBox& kext_fab = (*column_kextent[lev])[mfi];
        kext_fab.setVal<RunOn::Device>(bx3d.smallEnd(2), vbx, 0, 1);
        kext_fab.setVal<RunOn::Device>(bx3d.bigEnd(2)  , vbx, 1, 1);
    }
 
    column_kextent[lev]->FillBoundary(geom[lev].periodicity());
}

Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int  lev )

private

{
    auto& fine_new = vars_new[lev];
    auto& crse_new = vars_new[lev-1];
    auto& ba_fine  = fine_new[Vars::cons].boxArray();
    auto& ba_crse  = crse_new[Vars::cons].boxArray();
    auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
    auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
 
    int ncomp = fine_new[Vars::cons].nComp();
 
    FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev]  ,
                        ba_crse, dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
    FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
}

DerDataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DerDataLog ( int  i )

inlineprivate

    {
        return *der_datalog[i];
    }

DerDataLogName()

std::string ERF::DerDataLogName ( int  i ) const

inlineprivatenoexcept

{ return der_datalogname[i]; }

derive_diag_profiles()

void ERF::derive_diag_profiles	(	double	time,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Kmv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Khv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qi,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qg,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wthv
	)

Computes the profiles for diagnostic quantities.

Parameters

time	Current simulation time
h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_Kmv	Profile for vertical turbulent viscosity on Host
h_avg_Khv	Profile for vertical scalar diffusivity on Host
h_avg_qv	Profile for water vapor on Host
h_avg_qc	Profile for cloud water on Host
h_avg_qr	Profile for rain water on Host
h_avg_wqv	Profile for vertical velocity * water vapor on Host
h_avg_wqc	Profile for vertical velocity * cloud water on Host
h_avg_wqr	Profile for vertical velocity * rain water on Host
h_avg_qi	Profile for cloud ice on Host
h_avg_qs	Profile for snow on Host
h_avg_qg	Profile for graupel on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_vth	Profile for y-velocity * potential temperature on Host
h_avg_wth	Profile for z-velocity * potential temperature on Host
h_avg_thth	Profile for potential temperature squared on Host The definition also carries u_i*u_i velocity triple-product accumulators.
h_avg_ku	Profile for resolved x-momentum kinetic-energy flux on Host
h_avg_kv	Profile for resolved y-momentum kinetic-energy flux on Host
h_avg_kw	Profile for resolved z-momentum kinetic-energy flux on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host
h_avg_wthv	Profile for vertical velocity * virtual potential temperature on Host

{
    // We assume that this is always called at level 0
    int lev = 0;
 
    bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
    bool l_use_KE    = solverChoice.turbChoice[lev].use_tke;
    // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, uw, vv, vw, ww, uth, vth, wth,
    //                  0      1     2    3    4   5   6   7   8   9  10   11   12   13
    //                thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr,
    //                  14     15     16     17 18  19  20  21  22  23  24   25   26   27
    //                  qi, qs, qg, wthv
    //                  28  29  30    31
    MultiFab mf_out(grids[lev], dmap[lev], 32, 0);
 
    MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
 
    MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
    MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
    MultiFab  w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
 
    average_face_to_cellcenter(mf_vels,0,
        Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
 
    int zdir = 2;
    auto domain = geom[0].Domain();
 
    // Sum in the horizontal plane
    h_avg_u  = sumToLine(mf_vels ,0,1,domain,zdir);
    h_avg_v  = sumToLine(mf_vels ,1,1,domain,zdir);
    h_avg_w  = sumToLine(mf_vels ,2,1,domain,zdir);
 
    int hu_size =  h_avg_u.size();
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < hu_size; ++k) {
        h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
    }
 
    Gpu::DeviceVector<Real> d_avg_u(hu_size, zero);
    Gpu::DeviceVector<Real> d_avg_v(hu_size, zero);
    Gpu::DeviceVector<Real> d_avg_w(hu_size, zero);
 
#if 0
    auto* avg_u_ptr = d_avg_u.data();
    auto* avg_v_ptr = d_avg_v.data();
    auto* avg_w_ptr = d_avg_w.data();
#endif
 
    Gpu::copy(Gpu::hostToDevice, h_avg_u.begin(), h_avg_u.end(), d_avg_u.begin());
    Gpu::copy(Gpu::hostToDevice, h_avg_v.begin(), h_avg_v.end(), d_avg_v.begin());
    Gpu::copy(Gpu::hostToDevice, h_avg_w.begin(), h_avg_w.end(), d_avg_w.begin());
 
    int nvars = vars_new[lev][Vars::cons].nComp();
    MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
 
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
    const MultiFab* eta_src = nullptr;
    const bool have_native_shoc_diagnostics =
        solverChoice.turbChoice[lev].uses_native_shoc() &&
        native_shoc_driver[lev] &&
        native_shoc_driver[lev]->has_native_diagnostics();
    if (l_use_kturb) {
        if (have_native_shoc_diagnostics) {
            eta_src = &native_shoc_driver[lev]->native_diagnostics();
        } else
        {
            eta_src = eddyDiffs_lev[lev].get();
        }
    }
 
    for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
        const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
        const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
        const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
        const Array4<Real>& cons_arr = mf_cons.array(mfi);
        const Array4<Real>&   p0_arr = p_hse.array(mfi);
        const Array4<const Real>& eta_arr = (eta_src) ? eta_src->const_array(mfi) :
                                                        Array4<const Real>{};
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 1) = theta;
            Real ksgs = zero;
            if (l_use_KE) {
                ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
            }
            fab_arr(i, j, k, 2) = ksgs;
#if 1
            if (l_use_kturb) {
                fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
                fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
            } else {
                fab_arr(i, j, k, 3) = zero;
                fab_arr(i, j, k, 4) = zero;
            }
#else
            // Here we hijack the "Kturb" variable name to print out the resolved kinetic energy
            Real upert = u_cc_arr(i,j,k) - avg_u_ptr[k];
            Real vpert = v_cc_arr(i,j,k) - avg_v_ptr[k];
            Real wpert = w_cc_arr(i,j,k) - avg_w_ptr[k];
            fab_arr(i, j, k, 3) = myhalf * (upert*upert + vpert*vpert + wpert*wpert);
#endif
            fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
            fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
            fab_arr(i, j, k, 7) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // u*w
            fab_arr(i, j, k, 8) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
            fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // v*w
            fab_arr(i, j, k,10) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // w*w
            fab_arr(i, j, k,11) = u_cc_arr(i,j,k) * theta; // u*th
            fab_arr(i, j, k,12) = v_cc_arr(i,j,k) * theta; // v*th
            fab_arr(i, j, k,13) = w_cc_arr(i,j,k) * theta; // w*th
            fab_arr(i, j, k,14) = theta * theta;           // th*th
 
            // if the number of fields is changed above, then be sure to update
            // the following def!
            Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,8) + fab_arr(i,j,k,10);
            fab_arr(i, j, k,15) = uiui * u_cc_arr(i,j,k);   // (ui*ui)*u
            fab_arr(i, j, k,16) = uiui * v_cc_arr(i,j,k);   // (ui*ui)*v
            fab_arr(i, j, k,17) = uiui * w_cc_arr(i,j,k);   // (ui*ui)*w
 
            if (!use_moisture) {
                Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,18) = p;                       // p
                fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k);     // p*w
                fab_arr(i, j, k,22) = zero;  // qv
                fab_arr(i, j, k,23) = zero;  // qc
                fab_arr(i, j, k,24) = zero;  // qr
                fab_arr(i, j, k,25) = zero;  // w*qv
                fab_arr(i, j, k,26) = zero;  // w*qc
                fab_arr(i, j, k,27) = zero;  // w*qr
                fab_arr(i, j, k,28) = zero;  // qi
                fab_arr(i, j, k,29) = zero;  // qs
                fab_arr(i, j, k,30) = zero;  // qg
                fab_arr(i, j, k,31) = zero;  // w*thv
            }
        });
    } // mfi
 
    if (use_moisture)
    {
        for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& fab_arr  = mf_out.array(mfi);
            const Array4<Real>& cons_arr = mf_cons.array(mfi);
            const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
            const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
            const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
            const Array4<Real>&   p0_arr = p_hse.array(mfi);
 
            // The moisture map is the authority on which species this scheme
            // carries; a slot the scheme allocates but never integrates must not
            // be reported as a profile of data.
            int rhoqv_comp = solverChoice.moisture_indices.qv;
            int rhoqc_comp = solverChoice.moisture_indices.qc;
            int rhoqr_comp = solverChoice.moisture_indices.qr;
            int rhoqi_comp = solverChoice.moisture_indices.qi;
            int rhoqs_comp = solverChoice.moisture_indices.qs;
            int rhoqg_comp = solverChoice.moisture_indices.qg;
            AMREX_ALWAYS_ASSERT( (rhoqv_comp > -1) && (rhoqc_comp > -1) );
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                Real qv = cons_arr(i,j,k,rhoqv_comp) / cons_arr(i,j,k,Rho_comp);
                Real qc = cons_arr(i,j,k,rhoqc_comp) / cons_arr(i,j,k,Rho_comp);
                Real qr = (rhoqr_comp > -1) ?  cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
                                               zero;
                Real p  = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
 
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,18) = p;                       // p
                fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k);     // p*w
                fab_arr(i, j, k,22) = qv;  // qv
                fab_arr(i, j, k,23) = qc;  // qc
                fab_arr(i, j, k,24) = qr;  // qr
                fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * qv;  // w*qv
                fab_arr(i, j, k,26) = w_cc_arr(i,j,k) * qc;  // w*qc
                fab_arr(i, j, k,27) = w_cc_arr(i,j,k) * qr;  // w*qr
                fab_arr(i, j, k,28) = (rhoqi_comp > -1) ?
                    cons_arr(i,j,k,rhoqi_comp) / cons_arr(i,j,k,Rho_comp) : zero;  // qi
                fab_arr(i, j, k,29) = (rhoqs_comp > -1) ?
                    cons_arr(i,j,k,rhoqs_comp) / cons_arr(i,j,k,Rho_comp) : zero;  // qs
                fab_arr(i, j, k,30) = (rhoqg_comp > -1) ?
                    cons_arr(i,j,k,rhoqg_comp) / cons_arr(i,j,k,Rho_comp) : zero;  // qg
                Real ql    = qc + qr;
                Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
                Real thv   = theta * (one + epsv*qv - ql);
                fab_arr(i, j, k,31) = w_cc_arr(i,j,k) * thv; // w*thv
            });
        } // mfi
    } // use_moisture
 
    h_avg_rho   = sumToLine(mf_out, 0,1,domain,zdir);
    h_avg_th    = sumToLine(mf_out, 1,1,domain,zdir);
    h_avg_ksgs  = sumToLine(mf_out, 2,1,domain,zdir);
    h_avg_Kmv   = sumToLine(mf_out, 3,1,domain,zdir);
    h_avg_Khv   = sumToLine(mf_out, 4,1,domain,zdir);
    h_avg_uu    = sumToLine(mf_out, 5,1,domain,zdir);
    h_avg_uv    = sumToLine(mf_out, 6,1,domain,zdir);
    h_avg_uw    = sumToLine(mf_out, 7,1,domain,zdir);
    h_avg_vv    = sumToLine(mf_out, 8,1,domain,zdir);
    h_avg_vw    = sumToLine(mf_out, 9,1,domain,zdir);
    h_avg_ww    = sumToLine(mf_out,10,1,domain,zdir);
    h_avg_uth   = sumToLine(mf_out,11,1,domain,zdir);
    h_avg_vth   = sumToLine(mf_out,12,1,domain,zdir);
    h_avg_wth   = sumToLine(mf_out,13,1,domain,zdir);
    h_avg_thth  = sumToLine(mf_out,14,1,domain,zdir);
    h_avg_uiuiu = sumToLine(mf_out,15,1,domain,zdir);
    h_avg_uiuiv = sumToLine(mf_out,16,1,domain,zdir);
    h_avg_uiuiw = sumToLine(mf_out,17,1,domain,zdir);
    h_avg_p     = sumToLine(mf_out,18,1,domain,zdir);
    h_avg_pu    = sumToLine(mf_out,19,1,domain,zdir);
    h_avg_pv    = sumToLine(mf_out,20,1,domain,zdir);
    h_avg_pw    = sumToLine(mf_out,21,1,domain,zdir);
    h_avg_qv    = sumToLine(mf_out,22,1,domain,zdir);
    h_avg_qc    = sumToLine(mf_out,23,1,domain,zdir);
    h_avg_qr    = sumToLine(mf_out,24,1,domain,zdir);
    h_avg_wqv   = sumToLine(mf_out,25,1,domain,zdir);
    h_avg_wqc   = sumToLine(mf_out,26,1,domain,zdir);
    h_avg_wqr   = sumToLine(mf_out,27,1,domain,zdir);
    h_avg_qi    = sumToLine(mf_out,28,1,domain,zdir);
    h_avg_qs    = sumToLine(mf_out,29,1,domain,zdir);
    h_avg_qg    = sumToLine(mf_out,30,1,domain,zdir);
    h_avg_wthv  = sumToLine(mf_out,31,1,domain,zdir);
 
    // Divide by the total number of cells we are averaging over
    int h_avg_u_size = static_cast<int>(h_avg_u.size());
    for (int k = 0; k < h_avg_u_size; ++k) {
        h_avg_rho[k] /= area_z;
        h_avg_ksgs[k]  /= area_z;
        h_avg_Kmv[k]   /= area_z;
        h_avg_Khv[k]   /= area_z;
        h_avg_th[k]    /= area_z;
        h_avg_thth[k]  /= area_z;
        h_avg_uu[k]    /= area_z;
        h_avg_uv[k]    /= area_z;
        h_avg_uw[k]    /= area_z;
        h_avg_vv[k]    /= area_z;
        h_avg_vw[k]    /= area_z;
        h_avg_ww[k]    /= area_z;
        h_avg_uth[k]   /= area_z;
        h_avg_vth[k]   /= area_z;
        h_avg_wth[k]   /= area_z;
        h_avg_uiuiu[k] /= area_z;
        h_avg_uiuiv[k] /= area_z;
        h_avg_uiuiw[k] /= area_z;
        h_avg_p[k]     /= area_z;
        h_avg_pu[k]    /= area_z;
        h_avg_pv[k]    /= area_z;
        h_avg_pw[k]    /= area_z;
        h_avg_qv[k]    /= area_z;
        h_avg_qc[k]    /= area_z;
        h_avg_qr[k]    /= area_z;
        h_avg_wqv[k]   /= area_z;
        h_avg_wqc[k]   /= area_z;
        h_avg_wqr[k]   /= area_z;
        h_avg_qi[k]    /= area_z;
        h_avg_qs[k]    /= area_z;
        h_avg_qg[k]    /= area_z;
        h_avg_wthv[k]  /= area_z;
    }
 
#if 0
    // Here we print the integrated total kinetic energy as computed in the 1D profile above
    Real sum = zero;
    Real dz = geom[0].ProbHi(2) / static_cast<Real>(h_avg_u_size);
    for (int k = 0; k < h_avg_u_size; ++k) {
        sum += h_avg_kturb[k] * h_avg_rho[k] * dz;
    }
    amrex::Print() << "ITKE " << time << " " << sum << " using " << h_avg_u_size << " " << dz << std::endl;
#endif
}

Here is the call graph for this function:

derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag	(	double	time,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Kmv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Khv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qi,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qg,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wthv
	)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters

time	Current simulation time
h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_Kmv	Profile for vertical turbulent viscosity on Host
h_avg_Khv	Profile for vertical scalar diffusivity on Host
h_avg_qv	Profile for water vapor on Host
h_avg_qc	Profile for cloud water on Host
h_avg_qr	Profile for rain water on Host
h_avg_wqv	Profile for vertical velocity * water vapor on Host
h_avg_wqc	Profile for vertical velocity * cloud water on Host
h_avg_wqr	Profile for vertical velocity * rain water on Host
h_avg_qi	Profile for cloud ice on Host
h_avg_qs	Profile for snow on Host
h_avg_qg	Profile for graupel on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_vth	Profile for y-velocity * potential temperature on Host
h_avg_wth	Profile for z-velocity * potential temperature on Host
h_avg_thth	Profile for potential temperature squared on Host The definition also carries u_i*u_i velocity triple-product accumulators.
h_avg_ku	Profile for resolved x-momentum kinetic-energy flux on Host
h_avg_kv	Profile for resolved y-momentum kinetic-energy flux on Host
h_avg_kw	Profile for resolved z-momentum kinetic-energy flux on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host
h_avg_wthv	Profile for vertical velocity * virtual potential temperature on Host

{
    // We assume that this is always called at level 0
    int lev = 0;
 
    bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
    bool l_use_KE    = solverChoice.turbChoice[lev].use_tke;
    // Note: "uiui" == u_i*u_i = u*u + v*v + w*w
    // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, vv, uth, vth,
    //       indices:   0      1     2    3    4   5   6   7    8    9
    //                thth, uiuiu, uiuiv, p, pu, pv, qv, qc, qr, qi, qs, qg
    //                  10     11     12 13  14  15  16  17  18  19  20  21
    MultiFab mf_out(grids[lev], dmap[lev], 22, 0);
 
    // This will hold uw, vw, ww, wth, uiuiw, pw, wqv, wqc, wqr, wthv
    //       indices:  0   1   2    3      4   5    6    7    8     9
    MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 10, 0);
 
    // This is only used to average u and v; w is not averaged to cell centers
    MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
 
    MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
    MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
    MultiFab  w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers (staggered)
 
    int zdir = 2;
    auto domain = geom[0].Domain();
    Box stag_domain = domain;
    stag_domain.convert(IntVect(0,0,1));
 
    int nvars = vars_new[lev][Vars::cons].nComp();
    MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
 
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
    const MultiFab* eta_src = nullptr;
    const bool have_native_shoc_diagnostics =
        solverChoice.turbChoice[lev].uses_native_shoc() &&
        native_shoc_driver[lev] &&
        native_shoc_driver[lev]->has_native_diagnostics();
    if (l_use_kturb) {
        if (have_native_shoc_diagnostics) {
            eta_src = &native_shoc_driver[lev]->native_diagnostics();
        } else
        {
            eta_src = eddyDiffs_lev[lev].get();
        }
    }
 
    for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
        const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
        const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
        const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
        const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
        const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
        const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
        const Array4<Real>& cons_arr = mf_cons.array(mfi);
        const Array4<Real>&   p0_arr = p_hse.array(mfi);
        const Array4<const Real>& eta_arr = (eta_src) ? eta_src->const_array(mfi) :
                                                        Array4<const Real>{};
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            u_cc_arr(i,j,k) = myhalf * (u_arr(i,j,k) + u_arr(i+1,j  ,k));
            v_cc_arr(i,j,k) = myhalf * (v_arr(i,j,k) + v_arr(i  ,j+1,k));
 
            Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 1) = theta;
            Real ksgs = zero;
            if (l_use_KE) {
                ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
            }
            fab_arr(i, j, k, 2) = ksgs;
            if (l_use_kturb) {
                fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
                fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
            } else {
                fab_arr(i, j, k, 3) = zero;
                fab_arr(i, j, k, 4) = zero;
            }
            fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
            fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
            fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
            fab_arr(i, j, k, 8) = u_cc_arr(i,j,k) * theta;             // u*th
            fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * theta;             // v*th
            fab_arr(i, j, k,10) = theta * theta;                       // th*th
 
            Real wcc = myhalf * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
 
            // if the number of fields is changed above, then be sure to update
            // the following def!
            Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,7) + wcc*wcc;
            fab_arr(i, j, k,11) = uiui * u_cc_arr(i,j,k);           // (ui*ui)*u
            fab_arr(i, j, k,12) = uiui * v_cc_arr(i,j,k);           // (ui*ui)*v
 
            if (!use_moisture) {
                Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,13) = p;                       // p
                fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,16) = zero;  // qv
                fab_arr(i, j, k,17) = zero;  // qc
                fab_arr(i, j, k,18) = zero;  // qr
                fab_arr(i, j, k,19) = zero;  // qi
                fab_arr(i, j, k,20) = zero;  // qs
                fab_arr(i, j, k,21) = zero;  // qg
            }
        });
 
        const Box& zbx = mfi.tilebox(IntVect(0,0,1));
        ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            // average to z faces (first to cell centers, then in z)
            Real uface = fourth * ( u_arr(i  ,j,k) + u_arr(i  ,j,k-1)
                                + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
            Real vface = fourth * ( v_arr(i,j  ,k) + v_arr(i,j  ,k-1)
                                + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
            Real theta0 = cons_arr(i,j,k  ,RhoTheta_comp) / cons_arr(i,j,k  ,Rho_comp);
            Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
            Real thface = myhalf*(theta0 + theta1);
            fab_arr_stag(i,j,k,0) =           uface * w_fc_arr(i,j,k); // u*w
            fab_arr_stag(i,j,k,1) =           vface * w_fc_arr(i,j,k); // v*w
            fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
            fab_arr_stag(i,j,k,3) =          thface * w_fc_arr(i,j,k); // th*w
            Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
            fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
            if (!use_moisture) {
                Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp)) - p0_arr(i,j,k  );
                Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
                Real pface = myhalf * (p0 + p1);
                fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k);       // p*w
                fab_arr_stag(i,j,k,6) = zero;  // w*qv
                fab_arr_stag(i,j,k,7) = zero;  // w*qc
                fab_arr_stag(i,j,k,8) = zero;  // w*qr
                fab_arr_stag(i,j,k,9) = zero;  // w*thv
            }
        });
 
    } // mfi
 
    if (use_moisture)
    {
        for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& fab_arr  = mf_out.array(mfi);
            const Array4<Real>& fab_arr_stag  = mf_out_stag.array(mfi);
            const Array4<Real>& cons_arr = mf_cons.array(mfi);
            const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
            const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
            const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
            const Array4<Real>&   p0_arr = p_hse.array(mfi);
 
            // The moisture map is the authority on which species this scheme
            // carries; a slot the scheme allocates but never integrates must not
            // be reported as a profile of data.
            int rhoqv_comp = solverChoice.moisture_indices.qv;
            int rhoqc_comp = solverChoice.moisture_indices.qc;
            int rhoqr_comp = solverChoice.moisture_indices.qr;
            int rhoqi_comp = solverChoice.moisture_indices.qi;
            int rhoqs_comp = solverChoice.moisture_indices.qs;
            int rhoqg_comp = solverChoice.moisture_indices.qg;
            AMREX_ALWAYS_ASSERT( (rhoqv_comp > -1) && (rhoqc_comp > -1) );
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                Real qv = cons_arr(i,j,k,rhoqv_comp) / cons_arr(i,j,k,Rho_comp);
                Real qc = cons_arr(i,j,k,rhoqc_comp) / cons_arr(i,j,k,Rho_comp);
                Real qr = (rhoqr_comp > -1) ?  cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
                                               zero;
                Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
 
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,13) = p;                       // p
                fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,16) = qv;  // qv
                fab_arr(i, j, k,17) = qc;  // qc
                fab_arr(i, j, k,18) = qr;  // qr
                fab_arr(i, j, k,19) = (rhoqi_comp > -1) ?
                    cons_arr(i,j,k,rhoqi_comp) / cons_arr(i,j,k,Rho_comp) : zero;  // qi
                fab_arr(i, j, k,20) = (rhoqs_comp > -1) ?
                    cons_arr(i,j,k,rhoqs_comp) / cons_arr(i,j,k,Rho_comp) : zero;  // qs
                fab_arr(i, j, k,21) = (rhoqg_comp > -1) ?
                    cons_arr(i,j,k,rhoqg_comp) / cons_arr(i,j,k,Rho_comp) : zero;  // qg
            });
 
            const Box& zbx = mfi.tilebox(IntVect(0,0,1));
            ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                Real qv0 = cons_arr(i,j,k  ,rhoqv_comp) / cons_arr(i,j,k  ,Rho_comp);
                Real qv1 = cons_arr(i,j,k-1,rhoqv_comp) / cons_arr(i,j,k-1,Rho_comp);
                Real qc0 = cons_arr(i,j,k  ,rhoqc_comp) / cons_arr(i,j,k  ,Rho_comp);
                Real qc1 = cons_arr(i,j,k-1,rhoqc_comp) / cons_arr(i,j,k-1,Rho_comp);
                Real qr0 = (rhoqr_comp > -1) ? cons_arr(i,j,k  ,rhoqr_comp) / cons_arr(i,j,k  ,Rho_comp) :
                                               zero;
                Real qr1 = (rhoqr_comp > -1) ? cons_arr(i,j,k-1,rhoqr_comp) / cons_arr(i,j,k-1,Rho_comp) :
                                               zero;
                Real qvface = myhalf * (qv0 + qv1);
                Real qcface = myhalf * (qc0 + qc1);
                Real qrface = myhalf * (qr0 + qr1);
 
                Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp), qv0) - p0_arr(i,j,k  );
                Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv1) - p0_arr(i,j,k-1);
                Real pface = myhalf * (p0 + p1);
 
                Real theta0 = cons_arr(i,j,k  ,RhoTheta_comp) / cons_arr(i,j,k  ,Rho_comp);
                Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
                Real thface = myhalf*(theta0 + theta1);
                Real ql = qcface + qrface;
                Real thv = thface * (one + epsv*qvface - ql);
 
                fab_arr_stag(i,j,k,5) = pface  * w_fc_arr(i,j,k); // p*w
                fab_arr_stag(i,j,k,6) = qvface * w_fc_arr(i,j,k); // w*qv
                fab_arr_stag(i,j,k,7) = qcface * w_fc_arr(i,j,k); // w*qc
                fab_arr_stag(i,j,k,8) = qrface * w_fc_arr(i,j,k); // w*qr
                fab_arr_stag(i,j,k,9) = thv    * w_fc_arr(i,j,k); // w*thv
            });
        } // mfi
    } // use_moisture
 
    // Sum in the horizontal plane
    h_avg_u = sumToLine(u_cc,0,1,     domain,zdir);
    h_avg_v = sumToLine(v_cc,0,1,     domain,zdir);
    h_avg_w = sumToLine(w_fc,0,1,stag_domain,zdir);
 
    h_avg_rho   = sumToLine(mf_out, 0,1,domain,zdir);
    h_avg_th    = sumToLine(mf_out, 1,1,domain,zdir);
    h_avg_ksgs  = sumToLine(mf_out, 2,1,domain,zdir);
    h_avg_Kmv   = sumToLine(mf_out, 3,1,domain,zdir);
    h_avg_Khv   = sumToLine(mf_out, 4,1,domain,zdir);
    h_avg_uu    = sumToLine(mf_out, 5,1,domain,zdir);
    h_avg_uv    = sumToLine(mf_out, 6,1,domain,zdir);
    h_avg_vv    = sumToLine(mf_out, 7,1,domain,zdir);
    h_avg_uth   = sumToLine(mf_out, 8,1,domain,zdir);
    h_avg_vth   = sumToLine(mf_out, 9,1,domain,zdir);
    h_avg_thth  = sumToLine(mf_out,10,1,domain,zdir);
    h_avg_uiuiu = sumToLine(mf_out,11,1,domain,zdir);
    h_avg_uiuiv = sumToLine(mf_out,12,1,domain,zdir);
    h_avg_p     = sumToLine(mf_out,13,1,domain,zdir);
    h_avg_pu    = sumToLine(mf_out,14,1,domain,zdir);
    h_avg_pv    = sumToLine(mf_out,15,1,domain,zdir);
    h_avg_qv    = sumToLine(mf_out,16,1,domain,zdir);
    h_avg_qc    = sumToLine(mf_out,17,1,domain,zdir);
    h_avg_qr    = sumToLine(mf_out,18,1,domain,zdir);
    h_avg_qi    = sumToLine(mf_out,19,1,domain,zdir);
    h_avg_qs    = sumToLine(mf_out,20,1,domain,zdir);
    h_avg_qg    = sumToLine(mf_out,21,1,domain,zdir);
 
    h_avg_uw    = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
    h_avg_vw    = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
    h_avg_ww    = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
    h_avg_wth   = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
    h_avg_uiuiw = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
    h_avg_pw    = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
    h_avg_wqv   = sumToLine(mf_out_stag,6,1,stag_domain,zdir);
    h_avg_wqc   = sumToLine(mf_out_stag,7,1,stag_domain,zdir);
    h_avg_wqr   = sumToLine(mf_out_stag,8,1,stag_domain,zdir);
    h_avg_wthv  = sumToLine(mf_out_stag,9,1,stag_domain,zdir);
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
    for (int k = 0; k < unstag_size; ++k) {
        h_avg_u[k]     /= area_z;
        h_avg_v[k]     /= area_z;
        h_avg_rho[k]   /= area_z;
        h_avg_ksgs[k]  /= area_z;
        h_avg_Kmv[k]   /= area_z;
        h_avg_Khv[k]   /= area_z;
        h_avg_th[k]    /= area_z;
        h_avg_thth[k]  /= area_z;
        h_avg_uu[k]    /= area_z;
        h_avg_uv[k]    /= area_z;
        h_avg_vv[k]    /= area_z;
        h_avg_uth[k]   /= area_z;
        h_avg_vth[k]   /= area_z;
        h_avg_uiuiu[k] /= area_z;
        h_avg_uiuiv[k] /= area_z;
        h_avg_p[k]     /= area_z;
        h_avg_pu[k]    /= area_z;
        h_avg_pv[k]    /= area_z;
        h_avg_qv[k]    /= area_z;
        h_avg_qc[k]    /= area_z;
        h_avg_qr[k]    /= area_z;
        h_avg_qi[k]    /= area_z;
        h_avg_qs[k]    /= area_z;
        h_avg_qg[k]    /= area_z;
    }
 
    for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
        h_avg_w[k]     /= area_z;
        h_avg_uw[k]    /= area_z;
        h_avg_vw[k]    /= area_z;
        h_avg_ww[k]    /= area_z;
        h_avg_wth[k]   /= area_z;
        h_avg_uiuiw[k] /= area_z;
        h_avg_pw[k]    /= area_z;
        h_avg_wqv[k]   /= area_z;
        h_avg_wqc[k]   /= area_z;
        h_avg_wqr[k]   /= area_z;
        h_avg_wthv[k]  /= area_z;
    }
}

Here is the call graph for this function:

derive_forcing_profiles_stag()

void ERF::derive_forcing_profiles_stag	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_ttend,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qtend,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wsub,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thtend,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qhtend,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tvtend,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qvtend,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qcvtend,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tnudge,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qnudge,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_unudge,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vnudge
	)

{
    // We assume that this is always called at level 0
    int lev = 0;
    int zdir = 2;
    auto domain = geom[0].Domain();
    int nz = domain.length(2);
 
    // Sum in the horizontal plane
    if (solverChoice.large_scale_forcing) {
        h_avg_ttend   = sumToLine(*lsf_data[lev], 0,1,domain,zdir);
        h_avg_qtend   = sumToLine(*lsf_data[lev], 1,1,domain,zdir);
        h_avg_wsub    = sumToLine(*lsf_data[lev], 2,1,domain,zdir);
 
        // horizontal and vertical tendencies:
        h_avg_thtend   = sumToLine(*lsf_data[lev], 3,1,domain,zdir);
        h_avg_qhtend   = sumToLine(*lsf_data[lev], 4,1,domain,zdir);
        h_avg_tvtend   = sumToLine(*lsf_data[lev], 5,1,domain,zdir);
        h_avg_qvtend   = sumToLine(*lsf_data[lev], 6,1,domain,zdir);
        h_avg_qcvtend  = sumToLine(*lsf_data[lev], 7,1,domain,zdir);
    } else {
        h_avg_ttend = Gpu::HostVector<Real>(nz, zero);
        h_avg_qtend = Gpu::HostVector<Real>(nz, zero);
        h_avg_wsub  = Gpu::HostVector<Real>(nz, zero);
        h_avg_thtend = Gpu::HostVector<Real>(nz, zero);
        h_avg_qhtend = Gpu::HostVector<Real>(nz, zero);
        h_avg_tvtend = Gpu::HostVector<Real>(nz, zero);
        h_avg_qvtend = Gpu::HostVector<Real>(nz, zero);
        h_avg_qcvtend = Gpu::HostVector<Real>(nz, zero);
    }
 
    if (solverChoice.nudging_from_input_sounding) {
        h_avg_tnudge   = sumToLine(*nudge_data[lev], 0,1,domain,zdir);
        h_avg_qnudge   = sumToLine(*nudge_data[lev], 1,1,domain,zdir);
        h_avg_unudge   = sumToLine(*nudge_data[lev], 2,1,domain,zdir);
        h_avg_vnudge   = sumToLine(*nudge_data[lev], 3,1,domain,zdir);
    } else {
        h_avg_tnudge = Gpu::HostVector<Real>(nz, zero);
        h_avg_qnudge = Gpu::HostVector<Real>(nz, zero);
        h_avg_unudge = Gpu::HostVector<Real>(nz, zero);
        h_avg_vnudge = Gpu::HostVector<Real>(nz, zero);
    }
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < nz; ++k) {
        if (solverChoice.large_scale_forcing) {
            h_avg_ttend[k]     /= area_z;
            h_avg_qtend[k]     /= area_z;
            h_avg_wsub[k]      /= area_z;
 
            h_avg_thtend[k]     /= area_z;
            h_avg_qhtend[k]     /= area_z;
            h_avg_tvtend[k]     /= area_z;
            h_avg_qvtend[k]     /= area_z;
            h_avg_qcvtend[k]    /= area_z;
        }
 
        if (solverChoice.nudging_from_input_sounding) {
            h_avg_tnudge[k]    /= area_z;
            h_avg_qnudge[k]    /= area_z;
            h_avg_unudge[k]    /= area_z;
            h_avg_vnudge[k]    /= area_z;
        }
    }
}

derive_stress_profiles()

void ERF::derive_stress_profiles	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q1fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q2fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

{
    int lev = 0;
 
    // This will hold the stress tensor components
    MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
 
    MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
 
    bool l_use_moist   = ( solverChoice.moisture_type != MoistureType::None );
 
    for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
 
        const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
 
        // NOTE: These are from the last RK stage...
        const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
        const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
        const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
        const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
        const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
        const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
 
        // These should be re-calculated during ERF_slow_rhs_post
        // -- just vertical SFS kinematic heat flux for now
        //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
        //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
        const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
        const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
            fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 1) = ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
                                / (   rho_arr(i,j  ,k) +   rho_arr(i+1,j  ,k)
                                  +   rho_arr(i,j+1,k) +   rho_arr(i+1,j+1,k) );
            fab_arr(i, j, k, 2) = ( tau13_arr(i,j,k  ) + tau13_arr(i+1,j,k  )
                                  + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) )
                                / (   rho_arr(i,j,k  ) +   rho_arr(i+1,j,k  )
                                  +   rho_arr(i,j,k+1) +   rho_arr(i+1,j,k+1) );
            fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 4) = ( tau23_arr(i,j,k  ) + tau23_arr(i,j+1,k  )
                                  + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) )
                                / (   rho_arr(i,j,k  ) +   rho_arr(i,j+1,k  )
                                  +   rho_arr(i,j,k+1) +   rho_arr(i,j+1,k+1) );
            fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 6) = myhalf * ( hfx3_arr(i,j,k) + hfx3_arr(i,j,k+1) ) / rho_arr(i,j,k);
            fab_arr(i, j, k, 7) = (l_use_moist) ? myhalf * ( q1fx3_arr(i,j,k) + q1fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : zero;
            fab_arr(i, j, k, 8) = (l_use_moist) ? myhalf * ( q2fx3_arr(i,j,k) + q2fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : zero;
            fab_arr(i, j, k, 9) =  diss_arr(i,j,k) / rho_arr(i,j,k);
        });
    }
 
    int zdir = 2;
    auto domain = geom[0].Domain();
 
    h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
    h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
    h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
    h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
    h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
    h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
    h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
    h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
    h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
    h_avg_diss  = sumToLine(mf_out,9,1,domain,zdir);
 
    int ht_size =  h_avg_tau11.size();
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < ht_size; ++k) {
        h_avg_tau11[k] /= area_z;
        h_avg_tau12[k] /= area_z;
        h_avg_tau13[k] /= area_z;
        h_avg_tau22[k] /= area_z;
        h_avg_tau23[k] /= area_z;
        h_avg_tau33[k] /= area_z;
        h_avg_hfx3[k] /= area_z;
        h_avg_q1fx3[k] /= area_z;
        h_avg_q2fx3[k] /= area_z;
        h_avg_diss[k] /= area_z;
    }
}

Here is the call graph for this function:

derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q1fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q2fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

{
    int lev = 0;
 
    // This will hold the stress tensor components
    MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
 
    // This will hold Tau13 and Tau23
    MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 5, 0);
 
    MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
 
    bool l_use_moist   = ( solverChoice.moisture_type != MoistureType::None );
 
    for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
        const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
 
        const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
 
        // NOTE: These are from the last RK stage...
        const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
        const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
        const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
        const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
        const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
        const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
 
        // These should be re-calculated during ERF_slow_rhs_post
        // -- just vertical SFS kinematic heat flux for now
        //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
        //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
        const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
        const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
            fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 1) = ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
                                / (   rho_arr(i,j  ,k) +   rho_arr(i+1,j  ,k)
                                  +   rho_arr(i,j+1,k) +   rho_arr(i+1,j+1,k) );
            fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 9) =  diss_arr(i,j,k) / rho_arr(i,j,k);
        });
 
        const Box& zbx = mfi.tilebox(IntVect(0,0,1));
        ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            Real rho_face = myhalf * (rho_arr(i,j,k-1) + rho_arr(i,j,k));
            // average from edge to face center
            fab_arr_stag(i,j,k,0) = myhalf*(tau13_arr(i,j,k) + tau13_arr(i+1,j  ,k)) / rho_face;
            fab_arr_stag(i,j,k,1) = myhalf*(tau23_arr(i,j,k) + tau23_arr(i  ,j+1,k)) / rho_face;
 
            fab_arr_stag(i,j,k,2) = hfx3_arr(i,j,k) / rho_face;
            fab_arr_stag(i,j,k,3) = (l_use_moist) ? q1fx3_arr(i,j,k) / rho_face : zero;
            fab_arr_stag(i,j,k,4) = (l_use_moist) ? q2fx3_arr(i,j,k) / rho_face : zero;
        });
    }
 
    int zdir = 2;
    auto domain = geom[0].Domain();
    Box stag_domain = domain;
    stag_domain.convert(IntVect(0,0,1));
 
    h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
    h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
//  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
    h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
//  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
    h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
//  h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
//  h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
//  h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
    h_avg_diss  = sumToLine(mf_out,9,1,domain,zdir);
 
    h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
    h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
    h_avg_hfx3  = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
    h_avg_q1fx3 = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
    h_avg_q2fx3 = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
 
    int ht_size =  h_avg_tau11.size(); // _un_staggered
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < ht_size; ++k) {
        h_avg_tau11[k] /= area_z;
        h_avg_tau12[k] /= area_z;
        h_avg_tau13[k] /= area_z;
        h_avg_tau22[k] /= area_z;
        h_avg_tau23[k] /= area_z;
        h_avg_tau33[k] /= area_z;
        h_avg_hfx3[k] /= area_z;
        h_avg_q1fx3[k] /= area_z;
        h_avg_q2fx3[k] /= area_z;
        h_avg_diss[k] /= area_z;
    }
    // staggered heights
    h_avg_tau13[ht_size] /= area_z;
    h_avg_tau23[ht_size] /= area_z;
    h_avg_hfx3[ht_size]  /= area_z;
    h_avg_q1fx3[ht_size] /= area_z;
    h_avg_q2fx3[ht_size] /= area_z;
}

Here is the call graph for this function:

derive_upwp()

void ERF::derive_upwp

(

amrex::Vector< amrex::Real > &

h_havg

)

EBFactory()

amrex::EBFArrayBoxFactory const& ERF::EBFactory ( int  lev ) const

inlineprivatenoexcept

                                       {
        AMREX_ASSERT(lev >= 0 && lev < eb.size() && eb[lev] != nullptr);
        AMREX_ASSERT(eb[lev]->get_const_factory() != nullptr);
        return *(eb[lev]->get_const_factory());
    }

erf_enforce_hse()

void ERF::erf_enforce_hse	(	int	lev,
		amrex::MultiFab &	dens,
		amrex::MultiFab &	pres,
		amrex::MultiFab &	pi,
		amrex::MultiFab &	th,
		amrex::MultiFab &	qv,
		std::unique_ptr< amrex::MultiFab > &	z_cc
	)

Enforces hydrostatic equilibrium when using terrain.

Parameters

[in]	lev	Integer specifying the current level
[out]	dens	MultiFab storing base state density
[out]	pres	MultiFab storing base state pressure
[out]	pi	MultiFab storing base state Exner function
[out]	th	MultiFab storing base state potential temperature
[out]	qv	MultiFab storing base state water vapor mixing ratio
[in]	z_cc	Pointer to MultiFab storing cell centered z-coordinates

{
    Real l_gravity = solverChoice.gravity;
    bool l_use_terrain = (solverChoice.mesh_type != MeshType::ConstantDz);
 
    const auto geomdata = geom[lev].data();
    const Real dz = geomdata.CellSize(2);
 
    //
    // A box stacked on another box of this level continues from the pressure (and density) that
    //    box reached; see erf_init_dens_hse_dry.  The boxes are integrated in bands of equal lowest
    //    index, bottom up, and before each band the cells just below it are filled from the bands
    //    already done.  With a single band nothing is filled.
    //
    const Vector<int> bands = column_bands(dens.boxArray());
 
    for (const int klo_band : bands)
    {
        if (klo_band != bands[0]) {
            fill_below_band(dens, 0, 1, klo_band, IntVect(1,1,0), geom[lev]);
            fill_below_band(pres, 0, 1, klo_band, IntVect(1,1,0), geom[lev]);
        }
 
        for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
        {
            // Create a flat box with same horizontal extent but only one cell in vertical
            const Box& tbz = mfi.nodaltilebox(2);
            int klo = tbz.smallEnd(2);
            int khi = tbz.bigEnd(2);
 
            if (klo != klo_band) { continue; }
 
            // Note we only grow by 1 because that is how big z_cc is.
            Box b2d = tbz; // Copy constructor
            b2d.grow(0,1);
            b2d.grow(1,1);
            b2d.setRange(2,0);
 
            // Intersect this box with the domain
            Box zdomain = convert(geom[lev].Domain(),tbz.ixType());
            b2d &= zdomain;
 
            // We integrate to the first cell (and below) by using rho in this cell
            // If gravity == 0 this is constant pressure
            // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
            // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
            // Note ng_pres_hse = 1
 
           // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
           // Note that gravity is positive
 
            Array4<Real>  rho_arr = dens.array(mfi);
            Array4<Real> pres_arr = pres.array(mfi);
            Array4<Real>   pi_arr =   pi.array(mfi);
            Array4<Real>   th_arr = theta.array(mfi);
            Array4<Real> zcc_arr;
            if (l_use_terrain) {
               zcc_arr = z_cc->array(mfi);
            }
 
            const Real rdOcp = solverChoice.rdOcp;
 
            ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
            {
                // Set value at surface from Newton iteration for rho
                if (klo == 0)
                {
                    // Physical height of the terrain at cell center
                    Real hz;
                    if (l_use_terrain) {
                        hz = zcc_arr(i,j,klo);
                    } else {
                        hz = myhalf*dz;
                    }
 
                    pres_arr(i,j,klo) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
                      pi_arr(i,j,klo) = getExnergivenP(pres_arr(i,j,klo), rdOcp);
                      th_arr(i,j,klo) = getRhoThetagivenP(pres_arr(i,j,klo)) / rho_arr(i,j,klo);
 
                    //
                    // Set ghost cell with dz and rho at boundary
                    // (We will set the rest of the ghost cells in the boundary condition routine)
                    //
                    pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
                      pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
                      th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
 
                } else {
 
                    // If klo > 0 we use the value of pres_arr(i,j,klo-1): the pressure the box of this
                    //    level below reached or, where there is no such box, the one interpolated from
                    //    the coarser level.
                    Real dz_loc;
                    if (l_use_terrain) {
                        dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
                    } else {
                        dz_loc = dz;
                    }
 
                    Real dens_interp = myhalf*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
                    pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
 
                    pi_arr(i,j,klo  ) = getExnergivenP(pres_arr(i,j,klo  ), rdOcp);
                    th_arr(i,j,klo  ) = getRhoThetagivenP(pres_arr(i,j,klo  )) / rho_arr(i,j,klo  );
 
                    pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
                    th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
                }
 
                Real dens_interp;
                if (l_use_terrain) {
                    for (int k = klo+1; k <= khi; k++) {
                        Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
                        dens_interp = myhalf*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                        pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
                        pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                        th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
                    }
                } else {
                    for (int k = klo+1; k <= khi; k++) {
                        dens_interp = myhalf*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                        pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
                        pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                        th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
                    }
                }
            });
 
        } // mfi
    } // band
 
     dens.FillBoundary(geom[lev].periodicity());
     pres.FillBoundary(geom[lev].periodicity());
       pi.FillBoundary(geom[lev].periodicity());
    theta.FillBoundary(geom[lev].periodicity());
       qv.FillBoundary(geom[lev].periodicity());
}

Here is the call graph for this function:

ERF_shared()

void ERF::ERF_shared

(

)

{
    // Seeding lives here so every constructor gets it
    int fix_random_seed = 0;
    long random_seed = -1;
    {
        ParmParse pp("erf");
        pp.queryAdd("fix_random_seed", fix_random_seed);
        pp.queryAdd("random_seed", random_seed);
    }
    // Note that the value of 1024UL is not significant -- the point here is just to set the
    // same seed for all MPI processes for the purpose of regression testing
    if (fix_random_seed) {
        Print() << "Fixing the random seed" << std::endl;
        InitRandom(1024UL, ParallelDescriptor::NProcs(), 1024UL);
    } else if (random_seed >= 0) {
        // User-supplied seed: vary the random sampling per run (e.g. across ensemble
        // realizations), still offset by rank so the ranks draw independent streams.
        Print() << "Using user random seed " << random_seed << std::endl;
        auto s = static_cast<unsigned long>(random_seed);
        InitRandom(s + static_cast<unsigned long>(ParallelDescriptor::MyProc()) + 1UL,
                   ParallelDescriptor::NProcs(), s * 1234567UL + 12345UL);
    }
 
    if (ParallelDescriptor::IOProcessor()) {
        const char* erf_hash = buildInfoGetGitHash(1);
        const char* amrex_hash = buildInfoGetGitHash(2);
        const char* buildgithash = buildInfoGetBuildGitHash();
        const char* buildgitname = buildInfoGetBuildGitName();
 
        if (strlen(erf_hash) > 0) {
          Print() << "\n"
                         << "ERF git hash: " << erf_hash << "\n";
        }
        if (strlen(amrex_hash) > 0) {
          Print() << "AMReX git hash: " << amrex_hash << "\n";
        }
        if (strlen(buildgithash) > 0) {
          Print() << buildgitname << " git hash: " << buildgithash << "\n";
        }
 
        Print() << "\n";
    }
 
    int nlevs_max = max_level + 1;
 
#ifdef ERF_USE_WINDFARM
    Nturb.resize(nlevs_max);
    vars_windfarm.resize(nlevs_max);
    SMark.resize(nlevs_max);
#endif
 
    qheating_rates.resize(nlevs_max);
    rad_fluxes.resize(nlevs_max);
    two_stream_rad.resize(nlevs_max);
 
    // NOTE: size lsm before readparams (chooses the model at all levels)
    lsm.ReSize(nlevs_max);
    lsm_data.resize(nlevs_max);
    lsm_flux.resize(nlevs_max);
 
    nudge_data.resize(nlevs_max);
    lsf_data.resize(nlevs_max);
 
    rhotheta_src.resize(nlevs_max);
    rhoqt_src.resize(nlevs_max);
 
    // NOTE: size canopy model before readparams (if file exists, we construct)
    m_forest_drag.resize(nlevs_max);
    for (int lev = 0; lev <= max_level; ++lev) { m_forest_drag[lev] = nullptr;}
 
    // Surface layer object for each possible face
    m_SurfaceLayer.resize(AMREX_SPACEDIM*2);
 
    ReadParameters();
    // Create one invocation identity after inputs are available and before
    // InitData can read restart metadata or write an output on restart.
    execution_provenance = erf_provenance::initialize_execution_provenance();
    initializeMicrophysics(nlevs_max);
 
#ifdef ERF_USE_WINDFARM
    initializeWindFarm(nlevs_max);
#endif
 
#ifdef ERF_USE_EAMXX_SHOC
    eamxx_shoc_interface.resize(nlevs_max);
    for (int lev = 0; lev <= max_level; ++lev) {
        if (solverChoice.turbChoice[lev].uses_eamxx_shoc()) {
            eamxx_shoc_interface[lev] = std::make_unique<SHOCInterface>(lev, solverChoice);
        }
    }
#endif
 
    native_shoc_driver.resize(nlevs_max);
    for (int lev = 0; lev <= max_level; ++lev) {
        if (solverChoice.turbChoice[lev].uses_native_shoc()) {
            native_shoc_driver[lev] = std::make_unique<ShocDriver>(lev, solverChoice);
        }
    }
 
    rad.resize(nlevs_max);
    for (int lev = 0; lev <= max_level; ++lev) {
        if (solverChoice.rad_type == RadiationType::RRTMGP) {
#ifdef ERF_USE_RRTMGP
            rad[lev] = std::make_unique<Radiation>(lev, solverChoice);
            // pass radiation datalog frequency to model - RRTMGP needs to know when to save data for profiles
            rad[lev]->setDataLogFrequency(rad_datalog_int);
#endif
        } else if (solverChoice.rad_type == RadiationType::Simple) {
            rad[lev] = std::make_unique<RadiationSimple>(lev, solverChoice);
            rad[lev]->setDataLogFrequency(rad_datalog_int);
        } else if (solverChoice.rad_type == RadiationType::TwoStream) {
            // Runs through TwoStreamRadiation (two_stream_rad), not IRadiation.
        } else if (solverChoice.rad_type != RadiationType::None) {
            Abort("Don't know this radiation model!");
        }
    }
    // NOTE: these must come after initializeMicrophysics() -- the conserved-state
    //       layout they select against is owned by the microphysics interface.
    const std::string& pv3d_1 = "plot_vars_1"  ; setPlotVariables(pv3d_1,plot3d_var_names_1);
    const std::string& pv3d_2 = "plot_vars_2"  ; setPlotVariables(pv3d_2,plot3d_var_names_2);
    setSubVolVariables("subvol_sampling_vars",subvol3d_var_names);
 
    // This is only used when we have mesh_type == MeshType::StretchedDz
    stretched_dz_h.resize(nlevs_max);
    stretched_dz_d.resize(nlevs_max);
 
    // Initialize staggered vertical levels for grid stretching or terrain, and
    // to simplify Rayleigh damping layer calculations.
    zlevels_stag.resize(max_level+1);
    init_zlevels(zlevels_stag,
                 stretched_dz_h,
                 stretched_dz_d,
                 geom,
                 refRatio(),
                 solverChoice.grid_stretching_ratio,
                 solverChoice.zsurf,
                 solverChoice.dz0);
 
    if (SolverChoice::mesh_type == MeshType::StretchedDz ||
        SolverChoice::mesh_type == MeshType::VariableDz) {
        int nz = geom[0].Domain().length(2) + 1; // staggered
        if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > Real(1.0e-4)) {
            Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
                << " does not match highest requested z level " << zlevels_stag[0][nz-1]
                << std::endl;
        }
        if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > Real(1.0e-4)) {
            Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
                << " does not match lowest requested level " << zlevels_stag[0][0]
                << std::endl;
        }
 
        // Redefine the problem domain here?
    }
 
    // Get lo/hi indices for massflux calc
    if ((solverChoice.const_massflux_u != 0) || (solverChoice.const_massflux_v != 0)) {
        if (solverChoice.mesh_type == MeshType::ConstantDz) {
            const bool zlo_unset = (solverChoice.const_massflux_layer_lo == amrex::Real(-bogus_large_value));
            const bool zhi_unset = (solverChoice.const_massflux_layer_hi == amrex::Real( bogus_large_value));
            const Real massflux_zlo = solverChoice.const_massflux_layer_lo - geom[0].ProbLo(2);
            const Real massflux_zhi = solverChoice.const_massflux_layer_hi - geom[0].ProbLo(2);
            const Real dz = geom[0].CellSize(2);
            if (zlo_unset) {
                solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
            } else {
                solverChoice.massflux_klo = static_cast<int>(std::ceil(massflux_zlo / dz - myhalf));
            }
            if (zhi_unset) {
                solverChoice.massflux_khi = geom[0].Domain().bigEnd(2);
            } else {
                solverChoice.massflux_khi = static_cast<int>(std::floor(massflux_zhi / dz - myhalf));
            }
        } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
            const Real massflux_zlo = solverChoice.const_massflux_layer_lo;
            const Real massflux_zhi = solverChoice.const_massflux_layer_hi;
            solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
            solverChoice.massflux_khi = geom[0].Domain().bigEnd(2) + 1;
            for (int k=0; k <= geom[0].Domain().bigEnd(2)+1; ++k) {
                if (zlevels_stag[0][k] <= massflux_zlo) solverChoice.massflux_klo = k;
                if (zlevels_stag[0][k] <= massflux_zhi) solverChoice.massflux_khi = k;
            }
        } else { // solverChoice.mesh_type == MeshType::VariableDz
            Error("Const massflux with variable dz not supported -- planar averages are on k rather than constant-z planes");
        }
 
        Print() << "Constant mass flux based on k in ["
            << solverChoice.massflux_klo << ", " << solverChoice.massflux_khi << "]" << std::endl;
    }
 
#ifdef ERF_REMORA_FORCE_PROBINIT_LINK
    extern void erf_probinit_link_anchor_func () noexcept;
    erf_probinit_link_anchor_func();
#endif
    prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
 
    ParmParse pp_erf("erf");
    std::string prob_name;
    pp_erf.queryAdd("prob_name", prob_name);
    const std::string prob_name_ci = amrex::toLower(prob_name);
    if (prob_name_ci == "cloud chamber" || prob_name_ci == "cloudchamber") {
        cloud_chamber_config = erf_cloud_chamber::parse_config(
            geom[0].ProbLo(), geom[0].ProbHi());
        const amrex::GpuArray<Real, AMREX_SPACEDIM> cloud_chamber_dx = {
            geom[0].CellSize(0), geom[0].CellSize(1), geom[0].CellSize(2)};
        erf_cloud_chamber::validate_wall_roughness_geometry(
            cloud_chamber_config.wall_boundary(), cloud_chamber_dx);
    }
    {
        int budget_interval = 0;
        pp_erf.queryAdd("cloud_chamber_budget_interval", budget_interval);
        if (budget_interval > 0) {
            if (!cloud_chamber_config.active ||
                !cloud_chamber_config.physical_initialization ||
                (solverChoice.moisture_type != MoistureType::None &&
                 solverChoice.moisture_type != MoistureType::SatAdj)) {
                Error("Cloud Chamber: cloud_chamber_budget_interval requires physical_temperature_rh with no moisture or SatAdj");
            }
            cloud_chamber_budget = std::make_unique<CloudChamberBudget>(budget_interval);
        }
    }
 
    // Geometry on all levels has been defined already.
 
    // No valid BoxArray and DistributionMapping have been defined.
    // But the arrays for them have been resized.
 
    t_new.resize(nlevs_max, zero);
    t_old.resize(nlevs_max, -bogus_large_value);
    dt.resize(nlevs_max, std::min(static_cast<double>(bogus_large_value),dt_max_initial));
    dt_mri_ratio.resize(nlevs_max, 1);
 
    vars_new.resize(nlevs_max);
    vars_old.resize(nlevs_max);
    gradp.resize(nlevs_max);
 
    // We resize this regardless in order to pass it without error
    pp_inc.resize(nlevs_max);
 
    // Used in the fast substepping only
    lagged_delta_rt.resize(nlevs_max);
    avg_xmom.resize(nlevs_max);
    avg_ymom.resize(nlevs_max);
    avg_zmom.resize(nlevs_max);
 
    rU_new.resize(nlevs_max);
    rV_new.resize(nlevs_max);
    rW_new.resize(nlevs_max);
 
    rU_old.resize(nlevs_max);
    rV_old.resize(nlevs_max);
    rW_old.resize(nlevs_max);
 
    // xmom_crse_rhs.resize(nlevs_max);
    // ymom_crse_rhs.resize(nlevs_max);
    zmom_crse_rhs.resize(nlevs_max);
 
    for (int lev = 0; lev < nlevs_max; ++lev) {
        vars_new[lev].resize(Vars::NumTypes);
        vars_old[lev].resize(Vars::NumTypes);
        gradp[lev].resize(AMREX_SPACEDIM);
    }
 
    // Time integrator
    mri_integrator_mem.resize(nlevs_max);
 
    // Physical boundary conditions
    physbcs_cons.resize(nlevs_max);
    physbcs_u.resize(nlevs_max);
    physbcs_v.resize(nlevs_max);
    physbcs_w.resize(nlevs_max);
    physbcs_base.resize(nlevs_max);
 
    // Planes to hold Dirichlet values at boundaries
    xvel_bc_data.resize(nlevs_max);
    yvel_bc_data.resize(nlevs_max);
    zvel_bc_data.resize(nlevs_max);
    th_bc_data.resize(nlevs_max);
 
    advflux_reg.resize(nlevs_max);
 
    // Stresses
    Tau.resize(nlevs_max);
    Tau_corr.resize(nlevs_max);
    SFS_hfx1_lev.resize(nlevs_max);  SFS_hfx2_lev.resize(nlevs_max);  SFS_hfx3_lev.resize(nlevs_max);
    SFS_diss_lev.resize(nlevs_max);
    SFS_q1fx1_lev.resize(nlevs_max); SFS_q1fx2_lev.resize(nlevs_max); SFS_q1fx3_lev.resize(nlevs_max);
    SFS_q2fx3_lev.resize(nlevs_max);
    eddyDiffs_lev.resize(nlevs_max);
    SmnSmn_lev.resize(nlevs_max);
    Tau_EB.resize(nlevs_max);
    hfx3_EB.resize(nlevs_max);
 
    // Sea surface temps
    sst_lev.resize(nlevs_max);
    tsk_lev.resize(nlevs_max);
    lmask_lev.resize(nlevs_max);
 
    // Land and soil grid type and urban fractions
    land_type_lev.resize(nlevs_max);
    soil_type_lev.resize(nlevs_max);
    urb_frac_lev.resize(nlevs_max);
 
    // Metric terms
    z_phys_nd.resize(nlevs_max);
    z_phys_cc.resize(nlevs_max);
    detJ_cc.resize(nlevs_max);
    ax.resize(nlevs_max);
    ay.resize(nlevs_max);
    az.resize(nlevs_max);
 
    z_phys_nd_new.resize(nlevs_max);
    detJ_cc_new.resize(nlevs_max);
 
    z_phys_nd_src.resize(nlevs_max);
    z_phys_cc_src.resize(nlevs_max);
    detJ_cc_src.resize(nlevs_max);
    ax_src.resize(nlevs_max);
    ay_src.resize(nlevs_max);
    az_src.resize(nlevs_max);
 
    z_t_rk.resize(nlevs_max);
 
    terrain_blanking.resize(nlevs_max);
    terrain_blanking_xface.resize(nlevs_max);
    terrain_blanking_yface.resize(nlevs_max);
    terrain_blanking_zface.resize(nlevs_max);
 
    // Wall distance
    walldist.resize(nlevs_max);
 
    // BoxArrays to make MultiFabs needed to convert WRFBdy data
    ba1d.resize(nlevs_max);
    ba2d.resize(nlevs_max);
    column_kextent.resize(nlevs_max);
 
    // MultiFabs needed to convert WRFBdy data
    mf_PSFC.resize(nlevs_max);
 
    // Map factors
    mapfac.resize(nlevs_max);
 
    // Fine mask
    fine_mask.resize(nlevs_max);
 
    // Thin immersed body
    xflux_imask.resize(nlevs_max);
    yflux_imask.resize(nlevs_max);
    zflux_imask.resize(nlevs_max);
    //overset_imask.resize(nlevs_max);
    thin_xforce.resize(nlevs_max);
    thin_yforce.resize(nlevs_max);
    thin_zforce.resize(nlevs_max);
 
    // Base state
    base_state.resize(nlevs_max);
    base_state_new.resize(nlevs_max);
 
    // Wave coupling data
    Hwave.resize(nlevs_max);
    Lwave.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev)
    {
        Hwave[lev] = nullptr;
        Lwave[lev] = nullptr;
    }
    Hwave_onegrid.resize(nlevs_max);
    Lwave_onegrid.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev)
    {
        Hwave_onegrid[lev] = nullptr;
        Lwave_onegrid[lev] = nullptr;
    }
 
    // Theta prim for MOST
    Theta_prim.resize(nlevs_max);
 
    // Qv prim for MOST
    Qv_prim.resize(nlevs_max);
 
    // Qr prim for MOST
    Qr_prim.resize(nlevs_max);
 
    // Time averaged velocity field
    vel_t_avg.resize(nlevs_max);
    t_avg_cnt.resize(nlevs_max);
 
    // Interval mean variables
    interval_means.resize(nlevs_max);
    t_mean_cnt.resize(nlevs_max);
    mean_vars_time_reset_done = 0;
 
    // Size lat long arrays and default to null pointers
    lat_m.resize(nlevs_max);
    lon_m.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev) {
        lat_m[lev] = nullptr;
        lon_m[lev] = nullptr;
    }
 
    // Variable coriolis
    sinPhi_m.resize(nlevs_max);
    cosPhi_m.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev) {
        sinPhi_m[lev] = nullptr;
        cosPhi_m[lev] = nullptr;
    }
 
    // Rayleigh damping
    h_rayleigh_ptrs.resize(nlevs_max);
    d_rayleigh_ptrs.resize(nlevs_max);
    h_sinesq_ptrs.resize(nlevs_max);
    d_sinesq_ptrs.resize(nlevs_max);
    h_sinesq_stag_ptrs.resize(nlevs_max);
    d_sinesq_stag_ptrs.resize(nlevs_max);
 
    // Planar averages for immersed forcing
    r_plane_avg.resize(nlevs_max);
    t_plane_avg.resize(nlevs_max);
 
    // Initialize tagging criteria for mesh refinement
    refinement_criteria_setup();
 
    for (int lev = 0; lev < max_level; ++lev)
    {
       Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
          ref_ratio[lev][0]  << " " << ref_ratio[lev][1]  <<  " " << ref_ratio[lev][2] << std::endl;
    }
 
    // We will create each of these in MakeNewLevelFromScratch
    eb.resize(max_level+1);
    for (int lev = 0; lev < max_level + 1; lev++){
        eb[lev] = std::make_unique<eb_>();
    }
 
    // Construct EB geometry based on terrain and buildings configuration
    initializeEB();
 
    forecast_state_1.resize(nlevs_max);
    forecast_state_2.resize(nlevs_max);
    forecast_state_interp.resize(nlevs_max);
 
    surface_state_1.resize(nlevs_max);
    surface_state_2.resize(nlevs_max);
    surface_state_interp.resize(nlevs_max);
}

Here is the call graph for this function:

ErrorEst()

void ERF::ErrorEst ( int  lev, amrex::TagBoxArray &  tags, amrex::Real  time, int  ngrow  )

override

Function to tag cells for refinement – this overrides the pure virtual function in AmrCore

Parameters

[in]	levc	level of refinement at which we tag cells (0 is coarsest level)
[out]	tags	array of tagged cells
[in]	time	current time
[in]	ngrow	number of ghost cells (not used here)

{
    const int clearval = TagBox::CLEAR;
    const int   tagval = TagBox::SET;
 
#ifdef ERF_USE_NETCDF
    if ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) {
        int ratio;
        Box subdomain;
 
        // This is the number of boxes that may have already been defined in the refinement_criteria_setup routine.
        // If nb == 0 then no boxes have been specified in the inputs file, and we will use the boxes given in wrfinput_d*
        // If nb >  0 then    boxes have been specified in the inputs file, and we will use the specified boxes as long
        //    as we can ensure that they are contained inside the boxes given in wrfinput_d*
        int nb_prespecified = num_boxes_at_level[levc+1];
 
        if (!nc_init_file[levc+1].empty())
        {
            double levc_start_time = read_start_time_from_wrfinput(levc  , nc_init_file[levc  ][0]);
            if (solverChoice.init_type == InitType::WRFInput) {
                amrex::Print() << " WRFInput       time at level " << levc << " is " << levc_start_time << std::endl;
            } else if (solverChoice.init_type == InitType::Metgrid) {
                amrex::Print() << " met_em         time at level " << levc << " is " << levc_start_time << std::endl;
            }
 
            for (int isub = 0; isub < nc_init_file[levc+1].size(); isub++) {
                if (!have_read_nc_init_file[levc+1][isub])
                {
                    double levf_start_time = read_start_time_from_wrfinput(levc+1, nc_init_file[levc+1][isub]);
                    if (solverChoice.init_type == InitType::WRFInput) {
                        amrex::Print() << " WRFInput start_time at level " << levc+1 << " is " << levf_start_time << std::endl;
                    } else if (solverChoice.init_type == InitType::Metgrid) {
                        amrex::Print() << " met_em   start time at level " << levc+1 << " is " << levf_start_time << std::endl;
                    }
 
                    // We assume there is only one subdomain at levc; otherwise we don't know
                    //     which one is the parent of the fine region we are trying to create
                    AMREX_ALWAYS_ASSERT(subdomains[levc].size() == 1);
 
                    if ( levf_start_time <= (levc_start_time + t_new[levc]) ) {
                        if (solverChoice.init_type == InitType::WRFInput) {
                            amrex::Print() << " WRFInput file to read: " << nc_init_file[levc+1][isub] << std::endl;
                            subdomain = read_subdomain_from_wrfinput(levc, nc_init_file[levc+1][isub], ratio);
                            amrex::Print() << " WRFInput subdomain " << isub << " at level " << levc+1 << " is " << subdomain << std::endl;
                        } else if (solverChoice.init_type == InitType::Metgrid) {
                            amrex::Print() << "met_em file to read: " << nc_init_file[levc+1][0] << std::endl;
                            const Box& domain = geom[levc].Domain();
                            int klo = domain.smallEnd(2);
                            int khi = domain.bigEnd(2);
                            subdomain = read_subdomain_from_metgrid(levc, nc_init_file[levc+1][0], ratio, klo, khi);
                            amrex::Print() << " met_em subdomain at level " << levc+1 << " is " << subdomain << std::endl;
                        }
 
                        if ( (ratio != ref_ratio[levc][0]) || (ratio != ref_ratio[levc][1]) ) {
                            amrex::Print() << "File " << nc_init_file[levc+1][0] << " has refinement ratio = " << ratio << std::endl;
                            amrex::Print() << "The inputs file has refinement ratio = " << ref_ratio[levc] << std::endl;
                            amrex::Abort("These must be the same in the horizontal -- please edit your inputs file and try again.");
                        }
 
                        if (solverChoice.init_type == InitType::WRFInput) {
                            //
                            // A WRF nest is refined in the horizontal only: wrfinput_d02 carries
                            // exactly the eta levels of wrfinput_d01.  The z extent just read is
                            // therefore already in the PARENT's vertical index space, so only the
                            // horizontal directions are coarsened here; any vertical refinement is
                            // applied when the box is refined below.  Clip that extent to the
                            // parent's domain in case the file carries more layers than this run
                            // was given cells in z.
                            //
                            subdomain.setSmall(2, amrex::max(subdomain.smallEnd(2),
                                                             geom[levc].Domain().smallEnd(2)));
                            subdomain.setBig  (2, amrex::min(subdomain.bigEnd(2),
                                                             geom[levc].Domain().bigEnd(2)));
                            subdomain.coarsen(IntVect(ref_ratio[levc][0],ref_ratio[levc][1],1));
                        } else {
                            subdomain.coarsen(ref_ratio[levc]);
                        }
 
                        // Recall we asserted that there is only one box at level levc
                        Box coarser_level(subdomains[levc][0].minimalBox());
                        subdomain.shift(coarser_level.smallEnd());
 
                        if (verbose > 0) {
                            amrex::Print() << " Crse version of subdomain available for tagging is" << subdomain << std::endl;
                        }
 
                        Box new_fine(subdomain);
                        if (solverChoice.init_type == InitType::WRFInput) {
                            // The horizontal ratio comes from the file (and was just checked
                            // against the inputs file); the vertical one is whatever the inputs
                            // file asked for, since the file itself has nothing to say about it.
                            new_fine.refine(IntVect(ratio,ratio,ref_ratio[levc][2]));
                        } else if (solverChoice.init_type == InitType::Metgrid) {
                            new_fine.refine(ref_ratio[levc]);
                        }
                        if (nb_prespecified == 0) {
                            num_boxes_at_level[levc+1] += 1;
                            boxes_at_level[levc+1].push_back(new_fine);
                        } else {
                            if (!new_fine.contains(boxes_at_level[levc+1][isub])) {
                                amrex::Print() << "\n";
                                amrex::Print() << "Box available in wrfinputs file             " << new_fine << std::endl;
                                amrex::Print() << "Box requested for refinement in inputs file " << boxes_at_level[levc+1][isub] << std::endl;
                                amrex::Abort("Specified boxes must be contained within boxes specified in wrfinput at this level");
                            }
                        }
 
                        Box coarsened_bx(boxes_at_level[levc+1][isub]); coarsened_bx.coarsen(ref_ratio[levc]);
 
                        for (MFIter mfi(tags); mfi.isValid(); ++mfi)
                        {
                            auto tag_arr = tags.array(mfi);  // Get device-accessible array
 
                            Box bx = mfi.validbox() & coarsened_bx;
 
                            if (!bx.isEmpty()) {
                                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                                    tag_arr(i,j,k) = TagBox::SET;
                                });
                            }
                        }
                    } // time is right
                } else {
                    // Re-tag this region
                    for (MFIter mfi(tags); mfi.isValid(); ++mfi)
                    {
                        auto tag_arr = tags.array(mfi);  // Get device-accessible array
 
                        Box existing_bx_coarsened(boxes_at_level[levc+1][isub]);
                        existing_bx_coarsened.coarsen(ref_ratio[levc]);
 
                        Box bx = mfi.validbox(); bx &= existing_bx_coarsened;
 
                        if (!bx.isEmpty()) {
                            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                                tag_arr(i,j,k) = TagBox::SET;
                            });
                        }
                    }
                } // has file been read?
            } // isub
            return;
        } // file not empty
    }
#endif
 
    //
    // Make sure the ghost cells of the level we are tagging at are filled
    //    in case we take differences that require them
    // NOTE: We are Fillpatching only the cell-centered variables here
    //
    MultiFab& S_new = vars_new[levc][Vars::cons];
    MultiFab& U_new = vars_new[levc][Vars::xvel];
    MultiFab& V_new = vars_new[levc][Vars::yvel];
    MultiFab& W_new = vars_new[levc][Vars::zvel];
    //
    if (levc == 0) {
        FillPatchCrseLevel(levc, time, {&S_new, &U_new, &V_new, &W_new});
    } else {
        FillPatchFineLevel(levc, time, {&S_new, &U_new, &V_new, &W_new},
                           {&S_new, &rU_new[levc], &rV_new[levc], &rW_new[levc]},
                           base_state[levc], base_state[levc],
                           false, true);
    }
 
    for (int t=0; t < ref_tags.size(); ++t)
    {
        //
        // This mf must have ghost cells because we may take differences between adjacent values
        //
        std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 1);
        mf->setVal(0.0);
 
        RealBox real_box = ref_tags[t].GetInfo().m_realbox;
        if (real_box.ok()) {
            // Use the indicator name stored when this tag was created; the tags do not
            // correspond one-to-one with the entries of erf.refinement_indicators.
            int lev_for_box;
            std::string ref_prefix = pp_prefix + "." + ref_tag_indicator_names[t];
            update_box_for_refinement(ref_prefix, lev_for_box, real_box, time);
            ref_tags[t].GetInfo().SetRealBox(real_box);
        }
 
        // This allows dynamic refinement based on the value of the density
        if (ref_tags[t].Field() == "density")
        {
            MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,1);
 
        // Refinement based on a moisture mixing ratio (qv, qc, qi, qr, qs, qg).
        // The map from name to RhoQ component depends on the moisture model and
        // is held in solverChoice.moisture_indices.
        } else if ( (ref_tags[t].Field() == "qv") ||
                    (ref_tags[t].Field() == "qc") ||
                    (ref_tags[t].Field() == "qi") ||
                    (ref_tags[t].Field() == "qr") ||
                    (ref_tags[t].Field() == "qs") ||
                    (ref_tags[t].Field() == "qg") )
        {
            const auto& mi = solverChoice.moisture_indices;
            int qcomp = -1;
            if      (ref_tags[t].Field() == "qv") { qcomp = mi.qv; }
            else if (ref_tags[t].Field() == "qc") { qcomp = mi.qc; }
            else if (ref_tags[t].Field() == "qi") { qcomp = mi.qi; }
            else if (ref_tags[t].Field() == "qr") { qcomp = mi.qr; }
            else if (ref_tags[t].Field() == "qs") { qcomp = mi.qs; }
            else if (ref_tags[t].Field() == "qg") { qcomp = mi.qg; }
            AMREX_ALWAYS_ASSERT(qcomp >= 0);
            MultiFab::Copy(  *mf, vars_new[levc][Vars::cons], qcomp,    0, 1, 1);
            MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
 
        // qt = total condensed water mixing ratio (qc + qi + qr + qs + qg).
        // Excludes qv by convention here.
        } else if (ref_tags[t].Field() == "qt") {
            const auto& mi = solverChoice.moisture_indices;
            const int idx_qc = mi.qc, idx_qi = mi.qi, idx_qr = mi.qr,
                      idx_qs = mi.qs, idx_qg = mi.qg;
            AMREX_ALWAYS_ASSERT(idx_qc >= 0 || idx_qi >= 0 || idx_qr >= 0 ||
                                idx_qs >= 0 || idx_qg >= 0);
            mf->setVal(0.0);
            for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.growntilebox();
                auto qt_arr   = mf->array(mfi);
                auto cons_arr = vars_new[levc][Vars::cons].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    const Real rho_inv = Real(1.0) / cons_arr(i,j,k,Rho_comp);
                    Real q = Real(0.0);
                    if (idx_qc >= 0) { q += cons_arr(i,j,k,idx_qc) * rho_inv; }
                    if (idx_qi >= 0) { q += cons_arr(i,j,k,idx_qi) * rho_inv; }
                    if (idx_qr >= 0) { q += cons_arr(i,j,k,idx_qr) * rho_inv; }
                    if (idx_qs >= 0) { q += cons_arr(i,j,k,idx_qs) * rho_inv; }
                    if (idx_qg >= 0) { q += cons_arr(i,j,k,idx_qg) * rho_inv; }
                    qt_arr(i,j,k) = q;
                });
            }
 
        // This allows dynamic refinement based on the value of the z-component of vorticity
        } else if (ref_tags[t].Field() == "vorticity" ) {
            MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
            average_face_to_cellcenter(mf_cc_vel,0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new}, 1);
 
            for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                auto& dfab = (*mf)[mfi];
                auto& sfab = mf_cc_vel[mfi];
                auto& zfab = (*z_phys_cc[levc])[mfi];
                derived::erf_dervortz(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
            }
 
        // This allows dynamic refinement based on the value of the scalar/theta
        } else if ( (ref_tags[t].Field() == "scalar"  ) ||
                    (ref_tags[t].Field() == "theta"   ) )
        {
            for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                auto& dfab = (*mf)[mfi];
                auto& sfab = vars_new[levc][Vars::cons][mfi];
                auto& zfab = (*z_phys_cc[levc])[mfi];
                if (ref_tags[t].Field() == "scalar") {
                    derived::erf_derscalar(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
                } else if (ref_tags[t].Field() == "theta") {
                    derived::erf_dertheta(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
                }
            } // mfi
        // This allows dynamic refinement based on the value of updraft helicity
        } else if (ref_tags[t].Field() == "helicity")
        {
            MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
            average_face_to_cellcenter(mf_cc_vel,0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new}, 1);
 
            // NOTE: helicity is a vertical integral so we must not tile in z
            for (MFIter mfi(*mf, TileNoZ()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                auto& dfab = (*mf)[mfi];
                auto& sfab = mf_cc_vel[mfi];
                auto& zfab = (*z_phys_cc[levc])[mfi];
 
                derived::erf_derhelicity(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
            }
        } else if (ref_tags[t].Field() == "max_reflectivity")
        {
            // NOTE: this tests the data layout the reflectivity kernel assumes rather
            //       than enumerating moisture models, so it picks up any scheme that
            //       carries rain/snow/graupel in RhoQ4/RhoQ5/RhoQ6
            if (solverChoice.moisture_indices.has_reflectivity_species()) {
                // NOTE: this takes a max over the column so we must not tile in z
                for (MFIter mfi(*mf, TileNoZ()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    auto& dfab = (*mf)[mfi];
                    auto& sfab = vars_new[levc][Vars::cons][mfi];
                    auto& zfab = (*z_phys_cc[levc])[mfi];
 
                    derived::erf_dermaxreflectivity(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
                }
            } else {
                Abort("Max reflectivity requires a microphysics model carrying rain, snow and graupel.");
            }
        // This allows dynamic refinement based on the terrain blanking
        } else if ( (SolverChoice::terrain_type == TerrainType::ImmersedForcing) &&
                    (ref_tags[t].Field() == "terrain_blanking") )
        {
            MultiFab::Copy(*mf,*terrain_blanking[levc],0,0,1,1);
        }
        else if (ref_tags[t].Field() == "velmag")
        {
            ParmParse pp(pp_prefix);
            Vector<std::string> refinement_indicators;
            pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
            Real velmag_threshold;
            bool is_hurricane_tracker = false;
            for (int i=0; i<refinement_indicators.size(); ++i)
            {
                if (refinement_indicators[i]=="hurricane_tracker") {
                    is_hurricane_tracker = true;
                    std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
                    ParmParse ppr(ref_prefix);
                    ppr.get("value_greater", velmag_threshold);
                    break;
                }
            }
 
            Vector<MultiFab> mf_cc_vel(1);
            mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
            average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
 
            if (is_hurricane_tracker) {
                HurricaneTracker(levc, time, mf_cc_vel[0], velmag_threshold, &tags);
            } else {
                for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    auto& dfab = (*mf)[mfi];
                    auto& sfab = mf_cc_vel[0][mfi];
                    auto& zfab = (*z_phys_cc[levc])[mfi];
                    derived::erf_dermagvel(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
                }
            }
 
#ifdef ERF_USE_PARTICLES
        } else {
            //
            // Particle-derived refinement.  Two forms of `field_name` are
            // supported:
            //   <species>_count          : particle count per cell
            //   <species>_<mesh_var>     : Eulerian mesh variable provided
            //                              by the species' `meshPlotVarNames()`
            //                              (e.g. `super_droplets_moisture_mass_density`,
            //                              `tracer_particles_mass_density`).
            //
            // In both cases the field is deposited at every level in
            // [levc, finest_level] and averaged down level-by-level so that
            // a signal present only on a finer level still triggers
            // refinement at the coarser level being tagged.  Without this,
            // particles localised at level 1 would not register at level 0
            // and the fine grid would disappear at the next regrid.
            //
            const auto& particles_namelist( particleData.getNames() );
            mf->setVal(0.0);
            for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
            {
                auto* pc = particleData[particles_namelist[i]];
                const std::string& sp_name = particles_namelist[i];
                const std::string& field   = ref_tags[t].Field();
 
                const std::string count_str = sp_name + "_count";
                const std::string prefix    = sp_name + "_";
                std::string mesh_var;
                if (field != count_str
                    && field.size() > prefix.size()
                    && field.compare(0, prefix.size(), prefix) == 0)
                {
                    const std::string suffix = field.substr(prefix.size());
                    for (const auto& v : pc->meshPlotVarNames()) {
                        if (v == suffix) { mesh_var = v; break; }
                    }
                }
                if (field != count_str && mesh_var.empty()) { continue; }
 
                pc->resizeData();
                const int pc_nlevs = static_cast<int>(pc->GetParticles().size());
 
                // Deposit at each level into per-level MultiFabs.
                Vector<MultiFab> per_lev(finest_level+1);
                for (int lev = levc; lev <= finest_level; lev++) {
                    per_lev[lev].define(grids[lev], dmap[lev], 1, 0);
                    per_lev[lev].setVal(0);
                    if (field == count_str) {
                        if (lev < pc_nlevs) {
                            pc->IncrementWithTotal(per_lev[lev], lev);
                        }
                    } else {
                        pc->computeMeshVar(mesh_var, per_lev[lev],
                                           *z_phys_nd[lev], lev);
                    }
                }
 
                // Average down level-by-level from finest to levc.  This
                // avoids multi-level coarsening (e.g. L2->L0 with ratio
                // (4,1,4)) which can fail when fine-level boxes are not
                // aligned to the composite refinement ratio.
                for (int lev = finest_level; lev > levc; lev--) {
                    MultiFab temp_crse(grids[lev-1], dmap[lev-1], 1, 0);
                    temp_crse.setVal(0);
                    average_down(per_lev[lev], temp_crse,
                                 0, 1, ref_ratio[lev-1]);
                    MultiFab::Add(per_lev[lev-1], temp_crse, 0, 0, 1, 0);
                }
 
                MultiFab::Copy(*mf, per_lev[levc], 0, 0, 1, 0);
            }
#endif
        }
 
        ref_tags[t](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
    } // loop over t
 
    // ********************************************************************************************
    // Refinement based on 2d distance from the "eye" which is defined here as the (x,y) location of
    //    the integrated qv
    // ********************************************************************************************
    ParmParse pp(pp_prefix);
    Vector<std::string> refinement_indicators;
    pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
    for (int i=0; i<refinement_indicators.size(); ++i)
    {
        if ( (refinement_indicators[i]=="storm_tracker") && (solverChoice.moisture_type != MoistureType::None) )
        {
            std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
            ParmParse ppr(ref_prefix);
 
            double ref_start_time = -1.0;
            ppr.queryAdd("start_time",ref_start_time);
 
            if (time >= ref_start_time) {
 
                Real max_radius = -one;
                ppr.get("max_radius", max_radius);
 
                // Create the volume-weighted sum of (rho qv) in each column
                MultiFab mf_qv_int(ba2d[levc], dmap[levc], 1, 0); mf_qv_int.setVal(0.);
 
                // Define the 2D MultiFab holding the column-integrated (rho qv)
                volWgtColumnSum(levc, S_new, RhoQ1_comp, mf_qv_int, *detJ_cc[levc]);
 
                // Find the max value in the domain
                IntVect eye = mf_qv_int.maxIndex(0);
 
                const auto dx      = geom[levc].CellSizeArray();
                const auto prob_lo = geom[levc].ProbLoArray();
 
                Real eye_x = prob_lo[0] + (eye[0] + myhalf) * dx[0];
                Real eye_y = prob_lo[1] + (eye[1] + myhalf) * dx[1];
 
                tag_on_distance_from_eye(geom[levc], &tags, eye_x, eye_y, max_radius);
            }
        }
    }
}

Here is the call graph for this function:

estTimeStep()

double ERF::estTimeStep	(	int	level,
		long &	dt_fast_ratio
	)		const

Function that calls estTimeStep for each level

Parameters

[in]	level	level of refinement (coarsest level i 0)
[out]	dt_fast_ratio	ratio of slow to fast time step

{
    BL_PROFILE("ERF::estTimeStep()");
 
    // Terrain aware (T) and terrain unaware (N) time step estimates.
    double estdt_comp_T = bogus_large_value;
    double estdt_comp_N = bogus_large_value;
    double estdt_lowM_T = bogus_large_value;
    double estdt_lowM_N = bogus_large_value;
 
    // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation
    const int nxc = geom[0].Domain().length(0);
    const int nyc = geom[0].Domain().length(1);
 
    auto const dxinv = geom[level].InvCellSizeArray();
    auto dxinv_EB = dxinv; dxinv_EB[2] = one / dz_min[level];
 
    MultiFab const& S_new = vars_new[level][Vars::cons];
 
    // Keep the thermodynamic samples alongside the cell-centered velocity so
    // the wall-rate reduction can call the same pointwise MOST evaluator as
    // production wall transfer without allocating a second global temporary.
    MultiFab ccvel_N(grids[level],dmap[level],7,0);
    MultiFab ccvel_T(grids[level],dmap[level],3,0);
 
    int klo = geom[level].Domain().smallEnd(2);
    int khi = geom[level].Domain().bigEnd(2);
    MultiFab omega(convert(grids[level],IntVect(0,0,1)),dmap[level],1,0);
    for (MFIter mfi(vars_new[level][Vars::zvel]); mfi.isValid(); ++mfi)
    {
        Box vbx = mfi.validbox();
 
        const Array4<      Real>& omega_arr = omega.array(mfi);
        const Array4<const Real>& u_arr     = vars_new[level][IntVars::xmom].const_array(mfi);
        const Array4<const Real>& v_arr     = vars_new[level][IntVars::ymom].const_array(mfi);
        const Array4<const Real>& w_arr     = vars_new[level][IntVars::zmom].const_array(mfi);
 
        const Array4<const Real>& z_nd_arr  = z_phys_nd[level]->const_array(mfi);
 
        const Array4<const Real>& mf_ux     = mapfac[level][MapFacType::u_x]->const_array(mfi);
        const Array4<const Real>& mf_vy     = mapfac[level][MapFacType::v_y]->const_array(mfi);
 
        ParallelFor(vbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            if (k==klo || k==(khi+1)) {
                omega_arr(i,j,k) = zero;
            } else {
                omega_arr(i,j,k) = OmegaFromW(i,j,k,w_arr(i,j,k),
                                              u_arr,v_arr,mf_ux,mf_vy,
                                              z_nd_arr,dxinv);
            }
        });
    }
 
    average_face_to_cellcenter(ccvel_T,0,
                               Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
                                                        &vars_new[level][Vars::yvel],
                                                        &omega});
    average_face_to_cellcenter(ccvel_N,0,
                               Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
                                                        &vars_new[level][Vars::yvel],
                                                        &vars_new[level][Vars::zvel],});
 
    const bool chamber_cloudy = cloud_chamber_config.active &&
        cloud_chamber_config.cloudy;
    const Real rdOcp = solverChoice.rdOcp;
    const MultiFab& chamber_base_state = base_state[level];
    for (MFIter mfi(S_new); mfi.isValid(); ++mfi) {
        const Array4<const Real> state = S_new.const_array(mfi);
        const Array4<const Real> base = chamber_base_state.const_array(mfi);
        const Array4<Real> velocity = ccvel_N.array(mfi);
        const Box bx = mfi.validbox();
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            const Real rho = state(i,j,k,Rho_comp);
            velocity(i,j,k,3) = state(i,j,k,RhoTheta_comp) / rho;
            velocity(i,j,k,4) = chamber_cloudy ?
                state(i,j,k,RhoQ1_comp) / rho : Real(0.0);
            velocity(i,j,k,5) = base(i,j,k,BaseState::p0_comp);
            velocity(i,j,k,6) = rdOcp;
        });
    }
 
    bool l_substepping = (solverChoice.substepping_type[level] == SubsteppingType::Implicit);
    int  l_anelastic   = solverChoice.anelastic[level];
 
    bool l_comp_substepping_diag = (verbose && l_substepping && !l_anelastic && solverChoice.substepping_diag);
 
    Real estdt_comp_inv_N, estdt_comp_inv_T;
    Real estdt_lowM_inv_N, estdt_lowM_inv_T;
    Real estdt_vert_comp_inv, estdt_vert_lowM_inv;
 
    const MultiFab& z_nd_mf = *z_phys_nd[level];
 
    if (l_substepping && (nxc==1) && (nyc==1)) {
        // SCM -- should not depend on dx or dy; force minimum number of substeps
        estdt_comp_inv_T = std::numeric_limits<Real>::min();
        estdt_comp_inv_N = estdt_comp_inv_T;
    }
    else if (solverChoice.terrain_type == TerrainType::EB)
    {
        const eb_& eb_lev = get_eb(level);
        const MultiFab& detJ = (eb_lev.get_const_factory())->getVolFrac();
 
        estdt_comp_inv_N = ReduceMax(S_new, ccvel_N, detJ, 0,
        [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                   Array4<Real const> const& s,
                                   Array4<Real const> const& u,
                                   Array4<Real const> const& vf) -> Real
        {
           Real new_comp_dt = -bogus_large_value;
           amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
           {
               if (vf(i,j,k) > zero)
               {
                   const Real rho      = s(i, j, k, Rho_comp);
                   const Real rhotheta = s(i, j, k, RhoTheta_comp);
 
                   // NOTE: even when moisture is present,
                   //       we only use the partial pressure of the dry air
                   //       to compute the soundspeed
                   Real pressure = getPgivenRTh(rhotheta);
                   Real c = std::sqrt(Gamma * pressure / rho);
 
                   // If we are doing implicit acoustic substepping, then the z-direction does not contribute
                   //    to the computation of the time step
                   if (l_substepping) {
                       if ((nxc > 1) && (nyc==1)) {
                           // 2-D in x-z
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv_EB[0]), new_comp_dt);
                       } else if ((nyc > 1) && (nxc==1)) {
                           // 2-D in y-z
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv_EB[1]), new_comp_dt);
                       } else {
                           // 3-D
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv_EB[0]),
                                                    ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv_EB[1]), new_comp_dt);
                       }
 
                   // If we are not doing implicit acoustic substepping, then the z-direction contributes
                   //    to the computation of the time step
                   } else {
                       if (nxc > 1 && nyc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv_EB[0]),
                                                    ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv_EB[1]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dxinv_EB[2]), new_comp_dt);
                       } else if (nxc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv_EB[0]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dxinv_EB[2]), new_comp_dt);
                       } else if (nyc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv_EB[1]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dxinv_EB[2]), new_comp_dt);
                       } else {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dxinv_EB[2]), new_comp_dt);
                       }
 
                   }
               }
           });
           return new_comp_dt;
       });
 
        // The metric terms do not exist for EB, so the terrain aware
        // estimate is identical to the terrain unaware estimate
        estdt_comp_inv_T = estdt_comp_inv_N;
 
    } else {
        // One pass over the data returning both the terrain aware (T) estimate,
        // and the terrain unaware (N) estimate
        ReduceOps<ReduceOpMax,ReduceOpMax> reduce_op;
        ReduceData<Real,Real>              reduce_data(reduce_op);
 
#ifdef AMREX_USE_OMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
 
            const Array4<const Real>& s    = S_new.const_array(mfi);
            const Array4<const Real>& u_T  = ccvel_T.const_array(mfi);
            const Array4<const Real>& u_N  = ccvel_N.const_array(mfi);
            const Array4<const Real>& z_nd = z_nd_mf.const_array(mfi);
 
            reduce_op.eval(bx, reduce_data,
            [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept -> GpuTuple<Real,Real>
            {
                const Real rho      = s(i,j,k,Rho_comp);
                const Real rhotheta = s(i,j,k,RhoTheta_comp);
 
                // NOTE: even when moisture is present,
                //       we only use the partial pressure of the dry air
                //       to compute the soundspeed
                Real pressure = getPgivenRTh(rhotheta);
                Real c = std::sqrt(Gamma * pressure / rho);
 
                Real inv_dt_T = Compute_InvDt_Compressible(i,j,k, c,
                                    u_T(i,j,k,0), u_T(i,j,k,1), u_T(i,j,k,2),
                                    z_nd, dxinv, true, l_substepping, nxc, nyc);
 
                Real inv_dt_N = Compute_InvDt_Compressible(i,j,k, c,
                                    u_N(i,j,k,0), u_N(i,j,k,1), u_N(i,j,k,2),
                                    z_nd, dxinv, false, l_substepping, nxc, nyc);
 
                return {inv_dt_T, inv_dt_N};
            });
        }
 
        GpuTuple<Real,Real> hv = reduce_data.value(reduce_op);
        estdt_comp_inv_T       = amrex::get<0>(hv);
        estdt_comp_inv_N       = amrex::get<1>(hv);
    } // not EB
 
    {
        Real comp_inv[2] = {estdt_comp_inv_T, estdt_comp_inv_N};
        ParallelDescriptor::ReduceRealMax(comp_inv,2);
        estdt_comp_inv_T = comp_inv[0];
        estdt_comp_inv_N = comp_inv[1];
    }
 
    // Globally empty level -> ReduceMax = lowest(); treat level as non-constraining.
    estdt_comp_T = (estdt_comp_inv_T > zero) ? (cfl / estdt_comp_inv_T) : bogus_large_value;
    estdt_comp_N = (estdt_comp_inv_N > zero) ? (cfl / estdt_comp_inv_N) : bogus_large_value;
 
    //
    // Anelastic (low Mach) estimate -- purely advective, again terrain aware
    // and terrain unaware
    //
    if (solverChoice.terrain_type == TerrainType::EB)
    {
        estdt_lowM_inv_N = ReduceMax(ccvel_N, z_nd_mf, 0,
        [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                   Array4<Real const> const& u,
                                   Array4<Real const> const& z_nd) -> Real
        {
            Real new_lm_dt = -bogus_large_value;
            Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
            {
                Real inv_dt_lowM_N = Compute_InvDt_Anelastic(i,j,k,
                                             u(i,j,k,0), u(i,j,k,1), u(i,j,k,2),
                                             z_nd, dxinv_EB, false);
                new_lm_dt = amrex::max(inv_dt_lowM_N, new_lm_dt);
            });
            return new_lm_dt;
        });
 
        // The metric terms do not exist for EB
        estdt_lowM_inv_T = estdt_lowM_inv_N;
 
    } else {
 
        ReduceOps<ReduceOpMax,ReduceOpMax> reduce_op;
        ReduceData<Real,Real>              reduce_data(reduce_op);
 
#ifdef AMREX_USE_OMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        for (MFIter mfi(ccvel_T,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
 
            const Array4<const Real>& u_T  = ccvel_T.const_array(mfi);
            const Array4<const Real>& u_N  = ccvel_N.const_array(mfi);
            const Array4<const Real>& z_nd = z_nd_mf.const_array(mfi);
 
            reduce_op.eval(bx, reduce_data,
            [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept -> GpuTuple<Real,Real>
            {
                Real inv_dt_T = Compute_InvDt_Anelastic(i,j,k,
                                    u_T(i,j,k,0), u_T(i,j,k,1), u_T(i,j,k,2),
                                    z_nd, dxinv, true);
 
                Real inv_dt_N = Compute_InvDt_Anelastic(i,j,k,
                                    u_N(i,j,k,0), u_N(i,j,k,1), u_N(i,j,k,2),
                                    z_nd, dxinv, false);
 
                return {inv_dt_T, inv_dt_N};
            });
        }
 
        GpuTuple<Real,Real> hv = reduce_data.value(reduce_op);
        estdt_lowM_inv_T       = amrex::get<0>(hv);
        estdt_lowM_inv_N       = amrex::get<1>(hv);
    }
 
    {
        Real lowM_inv[2] = {estdt_lowM_inv_T, estdt_lowM_inv_N};
        ParallelDescriptor::ReduceRealMax(lowM_inv,2);
        estdt_lowM_inv_T = lowM_inv[0];
        estdt_lowM_inv_N = lowM_inv[1];
    }
 
    if (estdt_lowM_inv_T > zero) { estdt_lowM_T = cfl / estdt_lowM_inv_T; }
    if (estdt_lowM_inv_N > zero) { estdt_lowM_N = cfl / estdt_lowM_inv_N; }
 
     Real max_wall_rate = Real(0.0);
     Real estdt_wall = bogus_large_value;
     if (cloud_chamber_config.active &&
         cloud_chamber_config.physical_initialization &&
         cloud_chamber_config.has_wall_rate_channel()) {
         const auto walls = cloud_chamber_config.wall_boundary();
         const Box domain = geom[level].Domain();
         max_wall_rate = ReduceMax(ccvel_N, 0,
         [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                    Array4<Real const> const& velocity) -> Real
         {
             Real rate = Real(0.0);
             amrex::Loop(b, [=,&rate] (int i, int j, int k) noexcept
             {
                 // A free staggered component can receive tangential traction
                 // from every active perpendicular wall at an edge/corner.
                 // Fixed and neutral coefficients use the exact local
                 // velocity-parallel Jacobian row-sum factor.  MOST evaluates
                 // its current coefficients once per wall/cell and uses that
                 // state as a frozen-coefficient local rate estimate; it is
                 // not a nonlinear MOST Jacobian bound.
                 Real momentum_rate = Real(0.0);
                 amrex::GpuArray<Real, AMREX_SPACEDIM> low_momentum_rates{};
                 amrex::GpuArray<Real, AMREX_SPACEDIM> high_momentum_rates{};
                 // Evaluate each encountered wall/cell state once.  The
                 // component row sums below only compose these retained
                 // per-face rates; they must not repeat the MOST solve.
                 for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
                     const bool low = (dir == 0 ? i == domain.smallEnd(0) :
                                       (dir == 1 ? j == domain.smallEnd(1) :
                                                    k == domain.smallEnd(2)));
                     const bool high = (dir == 0 ? i == domain.bigEnd(0) :
                                        (dir == 1 ? j == domain.bigEnd(1) :
                                                     k == domain.bigEnd(2)));
                     if (low) {
                         const auto& wall = walls[2*dir];
                         if (erf_cloud_chamber_wall_flux::
                             wall_rate_requires_tangential_speed(wall)) {
                             const Real U_t =
                                 erf_cloud_chamber_wall_flux::
                                 tangential_speed_cell_centered(
                                     dir, i, j, k, velocity, wall);
                             const auto runtime =
                                 erf_cloud_chamber_wall_flux::most_wall_coefficients(
                                     wall, velocity(i,j,k,3), velocity(i,j,k,4),
                                     velocity(i,j,k,5), velocity(i,j,k,6), U_t,
                                     Real(0.5) / dxinv[dir],
                                     dir == 2 ? 1 : 0);
                             rate = amrex::max(rate,
                                 erf_cloud_chamber_wall_flux::wall_rate_for_face(
                                     wall, U_t, dxinv[dir], runtime));
                             low_momentum_rates[dir] =
                                 erf_cloud_chamber_wall_flux::momentum_rate_for_face(
                                     wall, U_t, dxinv[dir], runtime);
                         }
                     }
                     if (high) {
                         const auto& wall = walls[2*dir+1];
                         if (erf_cloud_chamber_wall_flux::
                             wall_rate_requires_tangential_speed(wall)) {
                             const Real U_t =
                                 erf_cloud_chamber_wall_flux::
                                 tangential_speed_cell_centered(
                                     dir, i, j, k, velocity, wall);
                             const auto runtime =
                                 erf_cloud_chamber_wall_flux::most_wall_coefficients(
                                     wall, velocity(i,j,k,3), velocity(i,j,k,4),
                                     velocity(i,j,k,5), velocity(i,j,k,6), U_t,
                                     Real(0.5) / dxinv[dir],
                                     dir == 2 ? -1 : 0);
                             rate = amrex::max(rate,
                                 erf_cloud_chamber_wall_flux::wall_rate_for_face(
                                     wall, U_t, dxinv[dir], runtime));
                             high_momentum_rates[dir] =
                                 erf_cloud_chamber_wall_flux::momentum_rate_for_face(
                                     wall, U_t, dxinv[dir], runtime);
                         }
                     }
                 }
                 for (int component = 0; component < AMREX_SPACEDIM; ++component) {
                     const Real component_rate =
                         erf_cloud_chamber_wall_flux::momentum_row_sum_rate(
                             component, low_momentum_rates, high_momentum_rates);
                     momentum_rate = amrex::max(momentum_rate, component_rate);
                 }
                 rate = amrex::max(rate, momentum_rate);
             });
             return rate;
         });
         ParallelDescriptor::ReduceRealMax(max_wall_rate);
         if (max_wall_rate > Real(0.0)) {
             estdt_wall = erf_cloud_chamber_wall_flux::wall_dt_from_max_rate(max_wall_rate);
         }
     }
     // Wall-rate kernels and reductions use amrex::Real, while ERF's host
     // timestep estimates are double even in ERF_PRECISION=SINGLE builds.
     const double estdt_wall_host = static_cast<double>(estdt_wall);
     estdt_comp_T = std::min(estdt_comp_T, estdt_wall_host);
     estdt_comp_N = std::min(estdt_comp_N, estdt_wall_host);
     estdt_lowM_T = std::min(estdt_lowM_T, estdt_wall_host);
     estdt_lowM_N = std::min(estdt_lowM_N, estdt_wall_host);
 
     const double fixed_dt_level = static_cast<double>(fixed_dt[level]);
     erf_cloud_chamber_wall_dt_guard::enforce_fixed_dt_limit(
         level, fixed_dt_level, estdt_wall_host, max_wall_rate);
 
     // Additional vertical diagnostics
     if (l_comp_substepping_diag) {
         estdt_vert_comp_inv = ReduceMax(S_new, ccvel_T, z_nd_mf, 0,
         [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                    Array4<Real const> const& s,
                                    Array4<Real const> const& u,
                                    Array4<Real const> const& z_nd) -> Real
         {
             Real new_comp_dt = -bogus_large_value;
             amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
             {
                 {
                     const Real rho      = s(i, j, k, Rho_comp);
                     const Real rhotheta = s(i, j, k, RhoTheta_comp);
 
                     // NOTE: even when moisture is present,
                     //       we only use the partial pressure of the dry air
                     //       to compute the soundspeed
                     Real pressure = getPgivenRTh(rhotheta);
                     Real c = std::sqrt(Gamma * pressure / rho);
 
                     Real h_zeta  = Compute_h_zeta_AtCellCenter(i,j,k,dxinv,z_nd);
                     Real idz_loc = dxinv[2] / h_zeta;
 
                     // Look at z-direction only
                     new_comp_dt = amrex::max((amrex::Math::abs(u(i,j,k,2)) + c) * idz_loc, new_comp_dt);
                 }
             });
             return new_comp_dt;
         });
 
         estdt_vert_lowM_inv = ReduceMax(ccvel_T, z_nd_mf, 0,
         [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                    Array4<Real const> const& u,
                                    Array4<Real const> const& z_nd) -> Real
         {
             Real new_lowM_dt = -bogus_large_value;
             amrex::Loop(b, [=,&new_lowM_dt] (int i, int j, int k) noexcept
             {
                 Real h_zeta  = Compute_h_zeta_AtCellCenter(i,j,k,dxinv,z_nd);
                 Real idz_loc = dxinv[2] / h_zeta;
                 new_lowM_dt = amrex::max((amrex::Math::abs(u(i,j,k,2))) * idz_loc, new_lowM_dt);
             });
             return new_lowM_dt;
         });
 
         ParallelDescriptor::ReduceRealMax(estdt_vert_comp_inv);
         ParallelDescriptor::ReduceRealMax(estdt_vert_lowM_inv);
     }
 
     if (verbose) {
         // Terrain aware   (T): includes h_xi, h_eta and h_zeta
         // Terrain unaware (N): no metric terms, dxinv[2] used for the vertical spacing
         if (fixed_dt[level] <= zero) {
             Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " <<
               one/dxinv[0] << " " << one/dxinv[1] << " " << dz_min[level] << std::endl;
             Print() << "Compressible dt at level " << level << ":  "
                     << estdt_comp_T << " (terrain aware)  "
                     << estdt_comp_N << " (terrain unaware)" << std::endl;
             if (estdt_lowM_inv_T > 0.0_rt) {
                 Print() << "Anelastic   dt at level " << level << ":  "
                         << estdt_lowM_T << " (terrain aware)  "
                         << estdt_lowM_N << " (terrain unaware)" << std::endl;
             } else {
                 Print() << "Anelastic dt at level " << level << ": undefined " << std::endl;
             }
         }
 
         if (fixed_dt[level] > zero) {
             Print() << "Based on cfl of one " << std::endl;
             Print() << "Compressible dt at level " << level << " would be:  "
                     << estdt_comp_T/cfl << " (terrain aware)  "
                     << estdt_comp_N/cfl << " (terrain unaware)" << std::endl;
             if (estdt_lowM_inv_T > zero) {
                 Print() << "Anelastic    dt at level " << level << " would be:  "
                         << estdt_lowM_T/cfl << " (terrain aware)  "
                         << estdt_lowM_N/cfl << " (terrain unaware)" << std::endl;
             } else {
                 Print() << "Anelastic    dt at level " << level << " would be undefined " << std::endl;
             }
             Print() << "Fixed dt at level " << level << "       is:  " << fixed_dt[level] << std::endl;
             if (fixed_fast_dt[level] > zero) {
                 Print() << "Fixed fast dt at level " << level << "       is:  " << fixed_fast_dt[level] << std::endl;
             }
         }
     }
 
     if (solverChoice.substepping_type[level] != SubsteppingType::None) {
         if (fixed_dt[level] > zero && fixed_fast_dt[level] > zero) {
             dt_fast_ratio = static_cast<long>( fixed_dt[level] / fixed_fast_dt[level] );
             if (dt_fast_ratio < 1) {
                 Abort("Invalid fixed_fast_dt: must be <= fixed_dt so mri_dt_ratio >= 1");
             }
         } else if (fixed_dt[level] > zero) {
             // Max CFL_c = one for substeps by default, but we enforce a min of 4 substeps
         auto dt_sub_max = (estdt_comp_T * (third/cfl) * sub_cfl);
             dt_fast_ratio = static_cast<long>( std::max(fixed_dt[level]/static_cast<double>(dt_sub_max), 4.0) );
         } else {
             // auto dt_sub_max = (estdt_comp_T/cfl * sub_cfl);
             // dt_fast_ratio = static_cast<long>( std::max(estdt_comp_T/dt_sub_max,Real(4.)) );
         dt_fast_ratio = static_cast<long>( std::max((cfl/third) / sub_cfl, Real(4.)) );
         }
 
         // Force time step ratio to be an even value
         if (solverChoice.force_stage1_single_substep) {
             if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
         } else {
             if ( dt_fast_ratio%6 != 0) {
                 Print() << "mri_dt_ratio = " << dt_fast_ratio
                         << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
                 dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/Real(6.0)) * 6);
             }
         }
 
         if (verbose) {
             Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
         }
     } // if substepping
 
     // Print out some extra diagnostics -- dt calcs are repeated so as to not
     // disrupt the overall code flow...
     if (l_comp_substepping_diag) {
         double dt_diag = (fixed_dt[level] > zero) ? fixed_dt[level] : static_cast<double>(estdt_comp_T);
         int  ns      = (fixed_mri_dt_ratio > zero) ? fixed_mri_dt_ratio : dt_fast_ratio;
 
         // horizontal acoustic CFL must be < 1 (fully explicit)
         // vertical   acoustic CFL may  be > 1
         Print() << "effective horiz,vert acoustic CFL with " << ns << " substeps : "
            << (dt_diag / ns) * estdt_comp_inv_T << " "
            << (dt_diag / ns) * estdt_vert_comp_inv << std::endl;
 
         // vertical advective CFL should be < 1, otherwise w-damping may be needed
         Print() << "effective vert advective CFL : "
            << dt_diag * estdt_vert_lowM_inv << std::endl;
     }
 
     if (fixed_dt[level] > zero) {
         return fixed_dt[level];
     } else {
         // Anelastic (substepping is not allowed)
         if (l_anelastic) {
 
            // Make sure that timestep is less than the dt_max
            estdt_lowM_T = std::min(estdt_lowM_T, dt_max);
 
            // On the first timestep enforce dt_max_initial
            if (istep[level] == 0) {
                return std::min(dt_max_initial, estdt_lowM_T);
            } else {
                return estdt_lowM_T;
            }
 
 
         // Compressible with or without substepping
         } else {
             return estdt_comp_T;
         }
     }
}

Here is the call graph for this function:

Evolve()

void ERF::Evolve

(

)

{
    BL_PROFILE_VAR("ERF::Evolve()", evolve);
 
    //
    // cur_time = t_new is elapsed time, not total time
    // stop_time is total time
    // Tracked in double to avoid float32 drift over many timesteps in single-precision builds.
    double cur_time = static_cast<double>(t_new[0]);
 
    // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
    //      for finer levels (with or without subcycling)
    for (int step = istep[0]; (step < max_step) && (start_time+cur_time < stop_time); ++step)
    {
        if (use_datetime) {
            Print() << "\n" << getTimestamp(start_time+cur_time, datetime_format)
                    << " (" << cur_time << " s elapsed)" << std::endl;
        }
        Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
 
        ComputeDt(step, cur_time);
 
        // Make sure we have read enough of the boundary plane data to make it through this timestep
        if (input_bndry_planes)
        {
            m_r2d->read_input_files(cur_time+start_time,dt[0],m_bc_extdir_vals);
        }
 
#ifdef ERF_USE_PARTICLES
        // We call this every time step with the knowledge that the particles may be
        //    initialized at a later time than the simulation start time.
        // The ParticleContainer carries a "start time" so the initialization will happen
        //    only when a) time > start_time, and b) particles have not yet been initialized
        initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,cur_time);
#endif
 
        if(solverChoice.init_type == InitType::HindCast and
          solverChoice.hindcast_lateral_forcing) {
            for(int lev=0;lev<finest_level+1;lev++){
                WeatherDataInterpolation(lev,cur_time,z_phys_nd,false);
            }
        }
 
        if(solverChoice.init_type == InitType::HindCast and
          solverChoice.hindcast_surface_bcs) {
            for(int lev=0;lev<finest_level+1;lev++){
                SurfaceDataInterpolation(lev,cur_time,z_phys_nd,false);
            }
        }
 
        auto dEvolveTime0 = amrex::second();
 
        int iteration = 1;
        timeStep(0, cur_time, iteration);
 
        cur_time += static_cast<double>(dt[0]);
        t_new[0] = cur_time;
 
        Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
                << " DT = " << dt[0]  << std::endl;
 
        if (check_for_nans > 0) {
            amrex::Print() << "Testing new state and vels for NaNs at end of timestep" << std::endl;
            for (int lev = 0; lev <= finest_level; ++lev) {
                check_state_for_nans(vars_new[lev][IntVars::cons]);
                check_vels_for_nans(vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],vars_new[lev][Vars::zvel]);
            }
        }
 
        if (verbose > 0)
        {
            auto dEvolveTime = amrex::second() - dEvolveTime0;
            ParallelDescriptor::ReduceRealMax(dEvolveTime,ParallelDescriptor::IOProcessorNumber());
            amrex::Print() << "Timestep time = " << dEvolveTime << " seconds." << '\n';
        }
 
        post_timestep(step, cur_time, dt[0]);
 
        WriteAtIntermediateTime(step, cur_time);
 
#ifdef AMREX_MEM_PROFILING
        {
            std::ostringstream ss;
            ss << "[STEP " << step+1 << "]";
            MemProfiler::report(ss.str());
        }
#endif
 
        if (start_time+cur_time >= stop_time - 1.e-6*dt[0]) break;
    }
 
    WriteAtFinalTime();
 
    flush_stations();
 
    BL_PROFILE_VAR_STOP(evolve);
}

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

EvolveOneStep()

double ERF::EvolveOneStep	(	double	time,
		double	dt_request
	)

{
    double cur_time = t_new[0];
    const int step = istep[0];
 
    if (start_time + cur_time >= stop_time) {
        return 0.0;
    }
 
    ComputeDt(step);
 
    int iteration = 1;
    timeStep(0, cur_time, iteration);
    cur_time += dt[0];
 
    post_timestep(step, cur_time, dt[0]);
 
    // ****************************************************************************************
    // Write plotfiles at intermediate times
    // ****************************************************************************************
    WriteAtIntermediateTime(step, cur_time);
 
    return dt[0];
}

Here is the call graph for this function:

fill_from_bndryregs()

void ERF::fill_from_bndryregs	(	const amrex::Vector< amrex::MultiFab * > &	mfs,
		double	time
	)

Impose boundary conditions using data read in as BndryRegisters from a previous ERF run

Parameters

[in,out]	mfs	Vector of MultiFabs to be filled
[in]	time	time at which the data should be filled

{
    //
    // We now assume that if we read in on one face, we read in on all faces
    //
    AMREX_ALWAYS_ASSERT(m_r2d);
 
    int lev = 0;
    const Box& domain = geom[lev].Domain();
 
    const auto& dom_lo = lbound(domain);
    const auto& dom_hi = ubound(domain);
 
    // Boundary-plane files are indexed by absolute simulation time.
    Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time + start_time);
 
    const BCRec* bc_ptr = domain_bcs_type_d.data();
 
    // xlo: ori = 0
    // ylo: ori = 1
    // zlo: ori = 2
    // xhi: ori = 3
    // yhi: ori = 4
    // zhi: ori = 5
    const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
    const auto& bdatylo = (*bndry_data[1])[lev].const_array();
    const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
    const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
 
    int bccomp;
 
    for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
    {
        MultiFab& mf = *mfs[var_idx];
        const int icomp = 0;
        const int ncomp = mf.nComp();
 
        if (var_idx == Vars::xvel) {
           bccomp = BCVars::xvel_bc;
        } else if (var_idx == Vars::yvel) {
           bccomp = BCVars::yvel_bc;
        } else if (var_idx == Vars::zvel) {
           bccomp = BCVars::zvel_bc;
        } else  if (var_idx == Vars::cons) {
           bccomp = BCVars::cons_bc;
        }
 
#ifdef AMREX_USE_OMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        for (MFIter mfi(mf); mfi.isValid(); ++mfi)
        {
            const Array4<Real>& dest_arr = mf.array(mfi);
            Box bx = mfi.growntilebox();
 
            // x-faces
            {
            Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
            if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
 
            Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
 
            ParallelFor(
                bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].lo(0) == ERFBCType::ext_dir_ingested) {
                        int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
                    }
                },
                bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].hi(0) == ERFBCType::ext_dir_ingested) {
                        int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
                    }
                }
            );
            } // x-faces
 
            // y-faces
            {
            Box bx_ylo(bx); bx_ylo.setBig  (1,dom_lo.y-1);
            if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
 
            Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
 
            ParallelFor(
               bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].lo(1) == ERFBCType::ext_dir_ingested) {
                        int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
                    }
                },
                bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].hi(1) == ERFBCType::ext_dir_ingested) {
                        int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
                    }
                }
            );
            } // y-faces
        } // mf
    } // var_idx
}

Here is the call graph for this function:

fill_rhs()

void ERF::fill_rhs ( amrex::MultiFab &  rhs_mf, const amrex::MultiFab &  state_mf, double  time, const amrex::Geometry &  geom  )

private

fill_terrain_surface()

void ERF::fill_terrain_surface	(	int	lev,
		amrex::MultiFab &	z_phys,
		double	elapsed_time
	)

{
    int ngrow = ComputeGhostCells(solverChoice) + 2;
    Box bx(surroundingNodes(Geom(lev).Domain())); bx.grow(ngrow);
    FArrayBox terrain_fab(makeSlab(bx,2,0),1);
 
    //
    // If we are using fitted mesh then we use the surface as defined above
    // If we are not using fitted mesh but are using z_levels, we still need z_phys (for now)
    //    but we need to use a flat terrain for the mesh itself (the EB data has already been made
    //    from the correct terrain)
    //
    if (solverChoice.terrain_type != TerrainType::StaticFittedMesh &&
        solverChoice.terrain_type != TerrainType::MovingFittedMesh) {
            terrain_fab.template setVal<RunOn::Device>(zero);
    } else {
        //
        // Fill the values of the terrain height at k=0 only
        //
        prob->init_terrain_surface(geom[lev],terrain_fab,elapsed_time);
    }
 
    for (MFIter mfi(z_phys,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        Box isect = terrain_fab.box() & z_phys[mfi].box();
        if (!isect.isEmpty()) {
            z_phys[mfi].template copy<RunOn::Device>(terrain_fab,isect,0,isect,0,1);
        }
    }
}

FillBdyCCVels()

void ERF::FillBdyCCVels	(	amrex::MultiFab &	mf_cc_vel,
		amrex::Geometry &	lev_geom
	)

Fill boundary ghost cells for cell-centered velocities.

Parameters

[in,out]	mf_cc_vel	cell-centered velocity MultiFab to be filled
[in]	lev_geom	geometry for the level being filled

{
    // Impose bc's at domain boundaries
    Box domain(lev_geom.Domain());
 
    int ihi = domain.bigEnd(0);
    int jhi = domain.bigEnd(1);
    int khi = domain.bigEnd(2);
 
    // Impose periodicity first
    mf_cc_vel.FillBoundary(lev_geom.periodicity());
 
    int jper = (lev_geom.isPeriodic(1));
    int kper = (lev_geom.isPeriodic(2));
 
    for (MFIter mfi(mf_cc_vel, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& vel_arr = mf_cc_vel.array(mfi);
 
        if (!lev_geom.isPeriodic(0)) {
            // Low-x side
            if (bx.smallEnd(0) <= domain.smallEnd(0)) {
                Real multn = ( (phys_bc_type[0] == ERF_BC::slip_wall   ) ||
                               (phys_bc_type[0] == ERF_BC::no_slip_wall) ||
                               (phys_bc_type[0] == ERF_BC::symmetry    ) ) ? -one : one;
                Real multt = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -one : one;
                Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
                ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                {
                    vel_arr(-1,j,k,0) = multn*vel_arr(0,j,k,0); // u
                    vel_arr(-1,j,k,1) = multt*vel_arr(0,j,k,1); // v
                    vel_arr(-1,j,k,2) = multt*vel_arr(0,j,k,2); // w
                });
            }
 
            // High-x side
            if (bx.bigEnd(0) >= domain.bigEnd(0)) {
                Real multn = ( (phys_bc_type[3] == ERF_BC::slip_wall   ) ||
                               (phys_bc_type[3] == ERF_BC::no_slip_wall) ||
                               (phys_bc_type[3] == ERF_BC::symmetry    ) ) ? -one : one;
                Real multt = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -one : one;
                Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
                ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                {
                    vel_arr(ihi+1,j,k,0) = multn*vel_arr(ihi,j,k,0); // u
                    vel_arr(ihi+1,j,k,1) = multt*vel_arr(ihi,j,k,1); // v
                    vel_arr(ihi+1,j,k,2) = multt*vel_arr(ihi,j,k,2); // w
                });
            }
        } // !periodic
 
        if (!lev_geom.isPeriodic(1)) {
            // Low-y side
            if (bx.smallEnd(1) <= domain.smallEnd(1)) {
                Real multn = ( (phys_bc_type[1] == ERF_BC::slip_wall   ) ||
                               (phys_bc_type[1] == ERF_BC::no_slip_wall) ||
                               (phys_bc_type[1] == ERF_BC::symmetry    ) ) ? -one : one;
                Real multt = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -one : one;
                Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
                ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int  , int k) noexcept
                {
                    vel_arr(i,-1,k,0) = multt*vel_arr(i,0,k,0); // u
                    vel_arr(i,-1,k,1) = multn*vel_arr(i,0,k,1); // u
                    vel_arr(i,-1,k,2) = multt*vel_arr(i,0,k,2); // w
                });
            }
 
            // High-y side
            if (bx.bigEnd(1) >= domain.bigEnd(1)) {
                Real multn = ( (phys_bc_type[4] == ERF_BC::slip_wall   ) ||
                               (phys_bc_type[4] == ERF_BC::no_slip_wall) ||
                               (phys_bc_type[4] == ERF_BC::symmetry    ) ) ? -one : one;
                Real multt = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -one : one;
                Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
                ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
                {
                    vel_arr(i,jhi+1,k,0) = multt*vel_arr(i,jhi,k,0); // u
                    vel_arr(i,jhi+1,k,1) = multn*vel_arr(i,jhi,k,1); // v
                    vel_arr(i,jhi+1,k,2) = multt*vel_arr(i,jhi,k,2); // w
                });
            }
        } // !periodic
 
        if (!lev_geom.isPeriodic(2)) {
            // Low-z side
            if (bx.smallEnd(2) <= domain.smallEnd(2)) {
                Real multn = ( (phys_bc_type[2] == ERF_BC::slip_wall   ) ||
                               (phys_bc_type[2] == ERF_BC::no_slip_wall) ||
                               (phys_bc_type[2] == ERF_BC::symmetry    ) ) ? -one : one;
                Real multt = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -one : one;
                Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
                ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                {
                    vel_arr(i,j,-1,0) = multt*vel_arr(i,j,0,0); // u
                    vel_arr(i,j,-1,1) = multt*vel_arr(i,j,0,1); // v
                    vel_arr(i,j,-1,2) = multn*vel_arr(i,j,0,2); // w
                });
            }
 
            // High-z side
            if (bx.bigEnd(2) >= domain.bigEnd(2)) {
                Real multn = ( (phys_bc_type[5] == ERF_BC::slip_wall   ) ||
                               (phys_bc_type[5] == ERF_BC::no_slip_wall) ||
                               (phys_bc_type[5] == ERF_BC::symmetry    ) ) ? -one : one;
                Real multt = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -one : one;
                Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
                ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                {
                    vel_arr(i,j,khi+1,0) = multt*vel_arr(i,j,khi,0); // u
                    vel_arr(i,j,khi+1,1) = multt*vel_arr(i,j,khi,1); // v
                    vel_arr(i,j,khi+1,2) = multn*vel_arr(i,j,khi,2); // w
                });
            }
        } // !periodic
    } // MFIter
 
    // Impose periodicity again
    mf_cc_vel.FillBoundary(lev_geom.periodicity());
}

Here is the call graph for this function:

FillCoarsePatch()

void ERF::FillCoarsePatch ( int  lev, double  time  )

private

Fill valid and ghost data. This version fills an entire MultiFab by interpolating from the coarser level – this is used only when a new level of refinement is being created during a run (i.e not at initialization) This will never be used with static refinement.

Parameters

[in]	lev	level of refinement at which to fill the data
[in]	time	time at which the data should be filled

{
    BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
    AMREX_ASSERT(lev > 0);
 
    //
    //****************************************************************************************************************
    // First fill velocities and density at the COARSE level so we can convert velocity to momenta at the COARSE level
    //****************************************************************************************************************
    //
    bool cons_only = false;
    if (lev == 1) {
        FillPatchCrseLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
                           &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
                           cons_only);
    } else {
        FillPatchFineLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
                           &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
                          {&vars_new[lev-1][Vars::cons],
                           &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
                           base_state[lev-1], base_state[lev-1],
                           false, cons_only);
    }
 
    //
    // ************************************************
    // Convert velocity to momentum at the COARSE level
    // ************************************************
    //
    const MultiFab* c_vfrac = nullptr;
    if (solverChoice.terrain_type == TerrainType::EB) {
        c_vfrac = &((get_eb(lev-1).get_const_factory())->getVolFrac());
    }
 
    VelocityToMomentum(vars_new[lev-1][Vars::xvel], IntVect{0},
                       vars_new[lev-1][Vars::yvel], IntVect{0},
                       vars_new[lev-1][Vars::zvel], IntVect{0},
                       vars_new[lev-1][Vars::cons],
                         rU_new[lev-1],
                         rV_new[lev-1],
                         rW_new[lev-1],
                       Geom(lev-1).Domain(),
                       domain_bcs_type, c_vfrac);
 
    // Fill ghost cells of coarse momentum before interpolation to fine level.
    // VelocityToMomentum above fills only valid cells (IntVect{0} grow).  On restart
    // from a non-AMR checkpoint, init_stuff initialises rU/rV/rW_new[lev-1] with large
    // sentinel values for ALL cells including ghost cells; the checkpoint read then
    // overwrites only valid cells.  InterpFromCoarseLevel (see comments below) ASSUMES
    // ghost cells at lev-1 are already filled and uses them in its stencil near periodic
    // boundaries.  Without this FillBoundary, those sentinel ghost cells contaminate the
    // fine-level interpolation, producing unphysical velocities that blow up WENO5.
    rU_new[lev-1].FillBoundary(geom[lev-1].periodicity());
    rV_new[lev-1].FillBoundary(geom[lev-1].periodicity());
    rW_new[lev-1].FillBoundary(geom[lev-1].periodicity());
 
    //
    // *****************************************************************
    // Interpolate all cell-centered variables from coarse to fine level
    // *****************************************************************
    //
    Interpolater* mapper_c = &cell_cons_interp;
    Interpolater* mapper_f = &face_cons_linear_interp;
 
    //
    //************************************************************************************************
    // Interpolate cell-centered data from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
    // ************************************************************************************************
    IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
    int      ncomp_cons = vars_new[lev][Vars::cons].nComp();
 
    InterpFromCoarseLevel(vars_new[lev  ][Vars::cons], ngvect_cons, IntVect(0,0,0),
                          vars_new[lev-1][Vars::cons], 0, 0, ncomp_cons,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_c, domain_bcs_type, BCVars::cons_bc);
 
    // ***************************************************************************
    // Physical bc's for cell centered variables at domain boundary
    // ***************************************************************************
    (*physbcs_cons[lev])(vars_new[lev][Vars::cons],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                         0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc,true);
 
    //
    //************************************************************************************************
    // Interpolate x-momentum from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
    // ************************************************************************************************
    //
    InterpFromCoarseLevel(rU_new[lev], IntVect{0}, IntVect{0}, rU_new[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::xvel_bc);
 
    //
    //************************************************************************************************
    // Interpolate y-momentum from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
    // ************************************************************************************************
    //
    InterpFromCoarseLevel(rV_new[lev], IntVect{0}, IntVect{0}, rV_new[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::yvel_bc);
 
    //************************************************************************************************
    // Interpolate z-momentum from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
    // ************************************************************************************************
    InterpFromCoarseLevel(rW_new[lev],  IntVect{0}, IntVect{0}, rW_new[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::zvel_bc);
    //
    // *********************************************************
    // After interpolation of momentum, convert back to velocity
    // *********************************************************
    //
    for (int which_lev = lev-1; which_lev <= lev; which_lev++)
    {
        c_vfrac = nullptr;
        if (solverChoice.terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(which_lev).get_const_factory())->getVolFrac());
        }
 
        MomentumToVelocity(vars_new[which_lev][Vars::xvel],
                           vars_new[which_lev][Vars::yvel],
                           vars_new[which_lev][Vars::zvel],
                           vars_new[which_lev][Vars::cons],
                             rU_new[which_lev],
                             rV_new[which_lev],
                             rW_new[which_lev],
                           Geom(which_lev).Domain(),
                           domain_bcs_type, c_vfrac);
    }
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
 
    (*physbcs_u[lev])(vars_new[lev][Vars::xvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                      ngvect_vels,time,BCVars::xvel_bc,true);
    (*physbcs_v[lev])(vars_new[lev][Vars::yvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                      ngvect_vels,time,BCVars::yvel_bc,true);
    (*physbcs_w[lev])(vars_new[lev][Vars::zvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                      ngvect_vels,time,BCVars::zvel_bc,true);
 
    // ***************************************************************************
    // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
    // ***************************************************************************
}

Here is the call graph for this function:

FillForecastStateMultiFabs()

void ERF::FillForecastStateMultiFabs	(	const int	lev,
		const std::string &	filename,
		const std::unique_ptr< amrex::MultiFab > &	z_phys_nd,
		amrex::Vector< amrex::Vector< amrex::MultiFab >> &	forecast_state
	)

Fill forecast state MultiFabs by interpolating from binary forecast files.

Parameters

[in]	lev	Level index.
[in]	filename	Path to the binary forecast data file.
[in]	a_z_phys_nd	Nodal physical height field.
[out]	forecast_state	MultiFabs to be filled with interpolated forecast data.

{
 
    Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
    Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
 
    ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
                       xvec_h, yvec_h, zvec_h, rho_h,
                       uvel_h, vvel_h, wvel_h,
                       theta_h, qv_h, qc_h, qr_h);
 
    Real zmax = *std::max_element(zvec_h.begin(), zvec_h.end());
 
    const auto prob_lo_erf  = geom[lev].ProbLoArray();
    const auto prob_hi_erf  = geom[lev].ProbHiArray();
    const auto dx_erf       = geom[lev].CellSizeArray();
 
    if (prob_hi_erf[2] >= zmax) {
        Abort("ERROR: the maximum z of the domain (" + std::to_string(prob_hi_erf[2]) +
        ") should be less than the maximum z in the forecast data (" + std::to_string(zmax) +
        "). Change geometry.prob_hi[2] in the inputs to be less than " + std::to_string(zmax) + "."
        );
    }
 
    if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
        amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
    }
    if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
        amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
    }
    if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
        amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
    }
    if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
        amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
    }
 
 
    int nx = static_cast<int>(xvec_h.size());
    int ny = static_cast<int>(yvec_h.size());
    int nz = static_cast<int>(zvec_h.size());
 
    amrex::Real dxvec = (xvec_h[nx-1]-xvec_h[0])/(nx-1);
    amrex::Real dyvec = (yvec_h[ny-1]-yvec_h[0])/(ny-1);
 
    amrex::Gpu::DeviceVector<Real> latvec_d(nx*ny), lonvec_d(nx*ny), zvec_d(nz);
    amrex::Gpu::DeviceVector<Real> xvec_d(nx*ny*nz), yvec_d(nx*ny*nz);
    amrex::Gpu::DeviceVector<Real> rho_d(nx*ny*nz), uvel_d(nx*ny*nz), vvel_d(nx*ny*nz), wvel_d(nx*ny*nz),
                                   theta_d(nx*ny*nz), qv_d(nx*ny*nz), qc_d(nx*ny*nz), qr_d(nx*ny*nz);
 
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, latvec_h.begin(), latvec_h.end(), latvec_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, lonvec_h.begin(), lonvec_h.end(), lonvec_d.begin());
 
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, xvec_h.begin(), xvec_h.end(), xvec_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, yvec_h.begin(), yvec_h.end(), yvec_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, zvec_h.begin(), zvec_h.end(), zvec_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, rho_h.begin(), rho_h.end(), rho_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, theta_h.begin(), theta_h.end(), theta_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, uvel_h.begin(), uvel_h.end(), uvel_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, vvel_h.begin(), vvel_h.end(), vvel_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, wvel_h.begin(), wvel_h.end(), wvel_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qv_h.begin(), qv_h.end(), qv_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qc_h.begin(), qc_h.end(), qc_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qr_h.begin(), qr_h.end(), qr_d.begin());
 
    amrex::Gpu::streamSynchronize();
 
    Real* latvec_d_ptr = latvec_d.data();
    Real* lonvec_d_ptr = lonvec_d.data();
    Real* xvec_d_ptr = xvec_d.data();
    Real* yvec_d_ptr = yvec_d.data();
    Real* zvec_d_ptr = zvec_d.data();
    Real* rho_d_ptr   = rho_d.data();
    Real* uvel_d_ptr  = uvel_d.data();
    Real* vvel_d_ptr  = vvel_d.data();
    Real* wvel_d_ptr  = wvel_d.data();
    Real* theta_d_ptr = theta_d.data();
    Real* qv_d_ptr = qv_d.data();
    Real* qc_d_ptr = qc_d.data();
    Real* qr_d_ptr = qr_d.data();
 
    MultiFab& erf_mf_cons   = forecast_state[lev][Vars::cons];
    MultiFab& erf_mf_xvel   = forecast_state[lev][Vars::xvel];
    MultiFab& erf_mf_yvel   = forecast_state[lev][Vars::yvel];
    MultiFab& erf_mf_zvel   = forecast_state[lev][Vars::zvel];
    MultiFab& erf_mf_latlon = forecast_state[lev][4];
 
    erf_mf_cons.setVal(0.0);
    erf_mf_xvel.setVal(0.0);
    erf_mf_yvel.setVal(0.0);
    erf_mf_zvel.setVal(0.0);
    erf_mf_latlon.setVal(0.0);
 
    // a_z_phys_nd must live on the SAME BoxArray/DistributionMap as the forecast state:
    // fabPtr(mfi) resolves by local index, so a stale (pre-regrid) z_phys_nd silently
    // returns the wrong FAB rather than failing.
    if (a_z_phys_nd) {
        AMREX_ALWAYS_ASSERT(amrex::BoxArray(a_z_phys_nd->boxArray()).convert(amrex::IndexType::TheCellType())
                            == erf_mf_cons.boxArray());
 
        AMREX_ALWAYS_ASSERT(a_z_phys_nd->DistributionMap() ==
                            erf_mf_cons.DistributionMap());
    }
 
    // Interpolate the data on to the ERF mesh
 
     for (MFIter mfi(erf_mf_cons); mfi.isValid(); ++mfi) {
        const auto z_arr    = (a_z_phys_nd) ? a_z_phys_nd->const_array(mfi) :
                                            Array4<const Real> {};
        const Array4<Real> &fine_cons_arr = erf_mf_cons.array(mfi);
        const Array4<Real> &fine_xvel_arr = erf_mf_xvel.array(mfi);
        const Array4<Real> &fine_yvel_arr = erf_mf_yvel.array(mfi);
        const Array4<Real> &fine_zvel_arr = erf_mf_zvel.array(mfi);
        const Array4<Real> &fine_latlon_arr = erf_mf_latlon.array(mfi);
 
 
        const Box& gbx = mfi.growntilebox(); // tilebox + ghost cells
 
        const Box &gtbx = mfi.tilebox(IntVect(1,0,0));
        const Box &gtby = mfi.tilebox(IntVect(0,1,0));
        const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
        const auto prob_lo  = geom[lev].ProbLoArray();
        const auto dx       = geom[lev].CellSizeArray();
       //const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
 
        ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
            // Geometry (note we must include these here to get the data on device)
            const Real x        = prob_lo[0] + (i + myhalf) * dx[0];
            const Real y        = prob_lo[1] + (j + myhalf) * dx[1];
            //const Real z        = prob_lo[2] + (k + myhalf) * dx[2];
            const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/two;
 
            // First interpolate where the weather data is available from
            Real tmp_rho, tmp_theta, tmp_qv, tmp_qc, tmp_qr, tmp_lat, tmp_lon;
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   rho_d_ptr, tmp_rho);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   theta_d_ptr, tmp_theta);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   qv_d_ptr, tmp_qv);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   qc_d_ptr, tmp_qc);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   qr_d_ptr, tmp_qr);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, 1,
                                   x, y, zero,
                                   latvec_d_ptr, tmp_lat);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, 1,
                                   x, y, zero,
                                   lonvec_d_ptr, tmp_lon);
 
            fine_cons_arr(i,j,k,Rho_comp) = tmp_rho;
            fine_latlon_arr(i,j,k,0) = tmp_lat;
            fine_latlon_arr(i,j,k,1) = tmp_lon;
        });
 
        ParallelFor(gtbx, gtby, gtbz,
        [=] AMREX_GPU_DEVICE(int i, int j, int k) {
             // Physical location of the fine node
            Real x = prob_lo_erf[0] + i       * dx_erf[0];
            Real y = prob_lo_erf[1] + (j+myhalf) * dx_erf[1];
            //Real z = prob_lo_erf[2] + (k+myhalf) * dx_erf[2];
            const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/two;
 
            Real tmp_uvel;
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   uvel_d_ptr, tmp_uvel);
 
            fine_xvel_arr(i, j, k, 0) = tmp_uvel;
        },
        [=] AMREX_GPU_DEVICE(int i, int j, int k) {
             // Physical location of the fine node
            Real x = prob_lo_erf[0] + (i+myhalf) * dx_erf[0];
            Real y = prob_lo_erf[1] + j       * dx_erf[1];
            //Real z = prob_lo_erf[2] + (k+myhalf) * dx_erf[2];
            const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/two;
 
            Real tmp_vvel;
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   vvel_d_ptr, tmp_vvel);
 
            fine_yvel_arr(i, j, k, 0) = tmp_vvel;
        },
        [=] AMREX_GPU_DEVICE(int i, int j, int k) {
             // Physical location of the fine node
            Real x = prob_lo_erf[0] + (i+myhalf) * dx_erf[0];
            Real y = prob_lo_erf[1] + (j+myhalf) * dx_erf[1];
            Real z = prob_lo_erf[2] + k       * dx_erf[2];
            //const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/two;
 
            Real tmp_wvel;
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   wvel_d_ptr, tmp_wvel);
 
            fine_zvel_arr(i, j, k, 0) = tmp_wvel;
        });
    }
 
    /*Vector<std::string> varnames = {
    "rho", "uvel", "vvel", "wvel", "theta", "qv", "qc", "qr"
    }; // Customize variable names
 
     Vector<std::string> varnames_cons = {
    "rho", "rhotheta", "ke", "sc", "rhoqv", "rhoqc", "rhoqr"
    }; // Customize variable names
 
    Vector<std::string> varnames_plot_mf = {
    "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
    }; // Customize variable names
 
    const double time = 0.0;
 
    std::string pltname = "plt_interp";
 
    MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
                     10, 0);
 
    plot_mf.setVal(0.0);
 
    for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
        const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
        const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
        const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
        const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
        const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
        const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
 
        const Box& bx = mfi.validbox();
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
            plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
            plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
            plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
            plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
            plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
 
            plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/two;
            plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/two;
            plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/two;
 
            plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
            plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
        });
    }
 
 
    WriteSingleLevelPlotfile(
            pltname,
            plot_mf,
            varnames_plot_mf,
            geom[0],
            time,
            0 // level
        );*/
}

Here is the call graph for this function:

FillIntermediatePatch()

void ERF::FillIntermediatePatch ( int  lev, double  time, const amrex::Vector< amrex::MultiFab * > &  mfs_vel, const amrex::Vector< amrex::MultiFab * > &  mfs_mom, int  ng_cons, int  ng_vel, bool  cons_only, int  icomp_cons, int  ncomp_cons  )

private

Fill valid and ghost data This version fills mfs in valid regions with the values in "mfs" when it is passed in; it is used only to compute ghost values for intermediate stages of a time integrator.

Parameters

[in]	lev	level of refinement at which to fill the data
[in]	time_d	time at which the data should be filled
[in,out]	mfs_vel	Vector of MultiFabs to be filled containing, in order: cons, xvel, yvel, and zvel
[in,out]	mfs_mom	Vector of MultiFabs to be filled containing, in order: cons, xmom, ymom, and zmom
[in]	ng_cons	number of ghost cells to be filled for conserved variables
[in]	ng_vel	number of ghost cells to be filled for velocity components
[in]	cons_only	if true then only fill conserved variables
[in]	icomp_cons	starting component for conserved variables
[in]	ncomp_cons	number of components for conserved variables

{
    BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
    Interpolater* mapper;
 
    Real time = static_cast<Real>(time_d);
 
    PhysBCFunctNoOp null_bc;
 
    //
    // ***************************************************************************
    // The first thing we do is interpolate the momenta on the "valid" faces of
    // the fine grids (where the interface is coarse/fine not fine/fine) -- this
    // will not be over-written by interpolation below because the FillPatch
    // operators see these as valid faces.  But we must have these interpolated
    // values in the fine data before we call FillPatchTwoLevels.
    //
    // Also -- note that we might be filling values by interpolation at physical boundaries
    //         here but that's ok because we will overwrite those values when we impose
    //         the physical bc's below
    // ***************************************************************************
    if (lev>0) {
        if (cf_set_width > 0) {
            // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
            FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
        }
        if ( !cons_only && (cf_set_width >= 0) ) {
            FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
            FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
            FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
        }
    }
 
    // amrex::Print() << "LEVEL " << lev << " CONS ONLY   " << cons_only <<
    //                   " ICOMP NCOMP " << icomp_cons << " " << ncomp_cons << " NGHOST " << ng_cons << std::endl;
 
    if (!cons_only) {
        AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
        AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
    }
 
    // Enforce no penetration for thin immersed body
    if (!cons_only) {
        // Enforce no penetration for thin immersed body
        if (xflux_imask[lev]) {
            ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
        }
        if (yflux_imask[lev]) {
            ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
        }
        if (zflux_imask[lev]) {
            ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
        }
    }
 
    //
    // We now start working on conserved quantities + VELOCITY
    //
    if (lev == 0)
    {
        // We don't do anything here because we will call the physbcs routines below,
        // which calls FillBoundary and fills other domain boundary conditions
        // Physical boundaries will be filled below
 
        if (!cons_only)
        {
            // ***************************************************************************
            // We always come in to this call with updated momenta but we need to create updated velocity
            //    in order to impose the rest of the bc's
            // ***************************************************************************
            const MultiFab* c_vfrac = nullptr;
            if (solverChoice.terrain_type == TerrainType::EB) {
                c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
            }
 
            // This only fills VALID region of velocity
            MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(), domain_bcs_type, c_vfrac);
        }
    }
    else
    {
        //
        // We must fill a temporary then copy it back so we don't double add/subtract
        //
        MultiFab mf(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
                    mfs_vel[Vars::cons]->nComp()   ,mfs_vel[Vars::cons]->nGrowVect());
        //
        // Set all components to bogus_large_value, then copy just the density from *mfs_vel[Vars::cons]
        //
        mf.setVal(bogus_large_value);
        MultiFab::Copy(mf,*mfs_vel[Vars::cons],Rho_comp,Rho_comp,1,mf.nGrowVect());
 
        Vector<MultiFab*> fmf = {mfs_vel[Vars::cons],mfs_vel[Vars::cons]};
        Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
        Vector<Real> ctime    = {static_cast<Real>(t_old[lev-1]), static_cast<Real>(t_new[lev-1])};
        Vector<Real> ftime    = {static_cast<Real>(time), static_cast<Real>(time)};
 
        if (interpolation_type == StateInterpType::Perturbational)
        {
            if (icomp_cons+ncomp_cons > 1)
            {
                // Divide (rho theta) by rho to get theta
                MultiFab::Divide(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
 
                // Subtract theta_0 from theta
                MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
 
                if (!amrex::almostEqual(time,ctime[1])) {
                    MultiFab::Divide(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                                     Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                    MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                       BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                }
                if (!amrex::almostEqual(time,ctime[0])) {
                    MultiFab::Divide(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                                     Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                    MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                       BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                }
            }
 
            // Subtract rho_0 from rho before we interpolate -- note we only subtract
            //    on valid region of mf since the ghost cells will be filled below
            if (icomp_cons == 0)
            {
                MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
 
                if (!amrex::almostEqual(time,ctime[1])) {
                    MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                       BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                }
                if (!amrex::almostEqual(time,ctime[0])) {
                    MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                       BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                }
            }
        } // interpolation_type == StateInterpType::Perturbational
 
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        mapper = &cell_cons_interp;
        FillPatchTwoLevels(mf, IntVect{ng_cons}, IntVect(0,0,0),
                           time, cmf, ctime, fmf, ftime,
                           icomp_cons, icomp_cons, ncomp_cons, geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper, domain_bcs_type,
                           icomp_cons);
 
        if (interpolation_type == StateInterpType::Perturbational)
        {
            if (icomp_cons == 0)
            {
                // Restore the coarse values to what they were
                if (!amrex::almostEqual(time,ctime[1])) {
                    MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                }
                if (!amrex::almostEqual(time,ctime[0])) {
                    MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                }
 
                // Set values in the cells outside the domain boundary so that we can do the Add
                //     without worrying about uninitialized values outside the domain -- these
                //     will be filled in the physbcs call
                mf.setDomainBndry(bogus_large_value,Rho_comp,1,geom[lev]);
 
                // Add rho_0 back to rho after we interpolate -- on all the valid + ghost region
                MultiFab::Add(mf, base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{ng_cons});
            }
 
            if (icomp_cons+ncomp_cons > 1)
            {
                // Add theta_0 to theta
                if (!amrex::almostEqual(time,ctime[1])) {
                    MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                    MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                                       Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                }
                if (!amrex::almostEqual(time,ctime[0])) {
                    MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                    MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                                       Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                }
 
                // Multiply theta by rho to get (rho theta)
                MultiFab::Multiply(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
 
                // Add theta_0 to theta
                MultiFab::Add(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
 
                // Add theta_0 back to theta
                MultiFab::Add(mf,base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{ng_cons});
 
                // Multiply (theta) by rho to get (rho theta)
                MultiFab::Multiply(mf,mf,Rho_comp,RhoTheta_comp,1,IntVect{ng_cons});
            }
        } // interpolation_type == StateInterpType::Perturbational
 
        // Impose physical bc's on fine data (note time and 0 are not used)
        // Note that we do this after the FillPatch because imposing physical bc's on fine ghost
        //      cells that need to be filled from coarse requires that we have done the interpolation first
        bool do_fb = true; bool do_terrain_adjustment = false;
        (*physbcs_cons[lev])(mf,*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                             icomp_cons,ncomp_cons,IntVect{ng_cons},time,BCVars::cons_bc,
                             do_fb, do_terrain_adjustment);
 
        // Make sure to only copy back the components we worked on
        MultiFab::Copy(*mfs_vel[Vars::cons],mf,icomp_cons,icomp_cons,ncomp_cons,IntVect{ng_cons});
 
        // *****************************************************************************************
 
        if (!cons_only)
        {
            // ***************************************************************************
            // We always come in to this call with updated momenta but we need to create updated velocity
            //    in order to impose the rest of the bc's
            // ***************************************************************************
            const MultiFab* c_vfrac = nullptr;
            if (solverChoice.terrain_type == TerrainType::EB) {
                c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
            }
 
            // This only fills VALID region of velocity
            MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(), domain_bcs_type, c_vfrac);
 
            mapper = &face_cons_linear_interp;
 
            //
            // NOTE: All interpolation here happens on velocities not momenta;
            //       note we only do the interpolation and FillBoundary here,
            //       physical bc's are imposed later
            //
            // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
            //       unlike the FillSet calls above which filled momenta on the coarse/fine bdy
            //
 
            MultiFab& mfu = *mfs_vel[Vars::xvel];
 
            fmf = {&mfu,&mfu};
            cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mfu, IntVect{ng_vel}, IntVect(0,0,0),
                               time, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::xvel_bc);
 
            // *****************************************************************************************
 
            MultiFab& mfv = *mfs_vel[Vars::yvel];
 
            fmf = {&mfv,&mfv};
            cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mfv, IntVect{ng_vel}, IntVect(0,0,0),
                               time, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::yvel_bc);
 
            // *****************************************************************************************
 
            MultiFab& mfw = *mfs_vel[Vars::zvel];
 
            fmf = {&mfw,&mfw};
            cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mfw, IntVect{ng_vel}, IntVect(0,0,0),
                               time, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::zvel_bc);
        } // !cons_only
    } // lev > 0
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
    IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
 
    bool do_fb = true;
 
#ifdef ERF_USE_NETCDF
    if (solverChoice.use_real_bcs && (lev==0)) {
        fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
        do_fb = false;
    }
#endif
 
    if (m_r2d && !solverChoice.use_real_bcs) { fill_from_bndryregs(mfs_vel,time); }
 
    // We call this even if use_real_bcs is true because these will fill the vertical bcs
    (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                         icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
    if (!cons_only) {
        (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::xvel_bc, do_fb);
        (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::yvel_bc, do_fb);
        (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::zvel_bc, do_fb);
    }
    // ***************************************************************************
 
    // We always come in to this call with momenta so we need to leave with momenta!
    // We need to make sure we convert back on all ghost cells/faces because this is
    // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
    if (!cons_only)
    {
        IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
        IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
        IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
 
        const MultiFab* c_vfrac = nullptr;
        if (solverChoice.terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
        }
 
        VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
                           *mfs_vel[Vars::yvel], ngv,
                           *mfs_vel[Vars::zvel], ngw,
                           *mfs_vel[Vars::cons],
                           *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                           Geom(lev).Domain(),
                           domain_bcs_type, c_vfrac);
    }
 
    // NOTE: There are not FillBoundary calls here for the following reasons:
    // Removal of the FillBoundary (FB) calls has bee completed for the following reasons:
    //
    // 1. physbc_cons is called before VelocityToMomentum and a FB is completed in that functor.
    //    Therefore, the conserved CC vars have their inter-rank ghost cells filled and then their
    //    domain ghost cells filled from the BC operations. We should not call FB on this MF again.
    //
    // 2. physbc_u/v/w is also called before VelocityToMomentum and a FB is completed those functors.
    //    Furthermore, VelocityToMomentum operates on a growntilebox so we exit that routine with momentum
    //    filled everywhere---i.e., physbc_u/v/w fills velocity ghost cells (inter-rank and domain)
    //    and then V2M does the conversion to momenta everywhere; so there is again no need to do a FB on momenta.
}

Here is the call graph for this function:

FillPatchCrseLevel()

void ERF::FillPatchCrseLevel ( int  lev, double  time, const amrex::Vector< amrex::MultiFab * > &  mfs_vel, bool  cons_only = false  )

private

Fill valid and ghost data on the coarse level with state data at the given time.

Parameters

[in]	lev	coarse level to fill
[in]	time_d	time at which the data should be filled
[in,out]	mfs_vel	Vector of MultiFabs to be filled containing cons, xvel, yvel, and zvel
[in]	cons_only	whether to fill only conserved variables

{
    BL_PROFILE_VAR("ERF::FillPatchCrseLevel()",ERF_FillPatchCrseLevel);
 
    AMREX_ALWAYS_ASSERT(lev == 0);
 
    Real time = static_cast<Real>(time_d);
 
    IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
    IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
 
    Vector<Real> ftime    = {static_cast<Real>(t_old[lev]), static_cast<Real>(t_new[lev])};
 
    //
    // Below we call FillPatchSingleLevel which does NOT fill ghost cells outside the domain
    //
 
    Vector<MultiFab*> fmf;
    Vector<MultiFab*> fmf_u;
    Vector<MultiFab*> fmf_v;
    Vector<MultiFab*> fmf_w;
 
    if (amrex::almostEqual(time,ftime[0])) {
        fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
    } else if (amrex::almostEqual(time,ftime[1])) {
        fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
    } else {
        fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
    }
 
    const int  ncomp = mfs_vel[Vars::cons]->nComp();
 
    FillPatchSingleLevel(*mfs_vel[Vars::cons], ngvect_cons, time, fmf, IntVect(0,0,0), ftime,
                         0, 0, ncomp, geom[lev]);
 
    if (!cons_only) {
        if (amrex::almostEqual(time,ftime[0])) {
            fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
            fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
            fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
        } else if (amrex::almostEqual(time,ftime[1])) {
            fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
            fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
            fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
        } else {
            fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
            fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
            fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
        }
        FillPatchSingleLevel(*mfs_vel[Vars::xvel], ngvect_vels, time, fmf_u,
                             IntVect(0,0,0), ftime,  0, 0, 1, geom[lev]);
 
        FillPatchSingleLevel(*mfs_vel[Vars::yvel], ngvect_vels, time, fmf_v,
                             IntVect(0,0,0), ftime,  0, 0, 1, geom[lev]);
 
        FillPatchSingleLevel(*mfs_vel[Vars::zvel], ngvect_vels, time, fmf_w,
                             IntVect(0,0,0), ftime,  0, 0, 1, geom[lev]);
    } // !cons_only
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    int icomp_cons = 0;
    int ncomp_cons = mfs_vel[Vars::cons]->nComp();
 
    bool do_fb = true;
 
#ifdef ERF_USE_NETCDF
    if(solverChoice.use_real_bcs && (lev==0)) {
        fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
        do_fb = false;
    }
#endif
 
    if (m_r2d && !solverChoice.use_real_bcs) { fill_from_bndryregs(mfs_vel,time); }
 
    // We call this even if use_real_bcs is true because these will fill the vertical bcs
    // Note that we call FillBoundary inside the physbcs call
    (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
    if (!cons_only) {
        (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::xvel_bc, do_fb);
        (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::yvel_bc, do_fb);
        (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::zvel_bc, do_fb);
    }
}

Here is the call graph for this function:

FillPatchFineLevel()

void ERF::FillPatchFineLevel ( int  lev, double  time, const amrex::Vector< amrex::MultiFab * > &  mfs_vel, const amrex::Vector< amrex::MultiFab * > &  mfs_mom, const amrex::MultiFab &  old_base_state, const amrex::MultiFab &  new_base_state, bool  fillset = true, bool  cons_only = false  )

private

Fill valid and ghost data with the "state data" at the given time NOTE: THIS OPERATES ON VELOCITY (MOMENTA ARE JUST TEMPORARIES)

Parameters

[in]	lev	level of refinement at which to fill the data
[in]	time_d	time at which the data should be filled
[in,out]	mfs_vel	Vector of MultiFabs to be filled containing, in order: cons, xvel, yvel, and zvel
[in,out]	mfs_mom	Vector of MultiFabs to be filled containing, in order: cons, xmom, ymom, and zmom
[in]	old_base_state	base-state data at the old time
[in]	new_base_state	base-state data at the new time
[in]	fillset	whether to fill the coarse-fine set region before standard fillpatch
[in]	cons_only	whether to fill only conserved variables

{
    BL_PROFILE_VAR("ERF::FillPatchFineLevel()",ERF_FillPatchFineLevel);
 
    AMREX_ALWAYS_ASSERT(lev > 0);
 
    Real time = static_cast<Real>(time_d);
 
    Interpolater* mapper = nullptr;
 
    PhysBCFunctNoOp null_bc;
 
    //
    // ***************************************************************************
    // The first thing we do is interpolate the momenta on the "valid" faces of
    // the fine grids (where the interface is coarse/fine not fine/fine) -- this
    // will not be over-written below because the FillPatch operators see these as
    // valid faces.
    //
    // Note that we interpolate momentum not velocity, but all the other boundary
    // conditions are imposed on velocity, so we convert to momentum here then
    // convert back.
    // ***************************************************************************
    if (fillset) {
        if (cf_set_width > 0) {
            FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
        }
        if (cf_set_width >= 0 && !cons_only) {
 
            const MultiFab* c_vfrac = nullptr;
            if (solverChoice.terrain_type == TerrainType::EB) {
                c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
            }
 
            VelocityToMomentum(*mfs_vel[Vars::xvel], IntVect{0},
                               *mfs_vel[Vars::yvel], IntVect{0},
                               *mfs_vel[Vars::zvel], IntVect{0},
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom],
                               *mfs_mom[IntVars::ymom],
                               *mfs_mom[IntVars::zmom],
                               Geom(lev).Domain(),
                               domain_bcs_type, c_vfrac);
 
            FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
            FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
            FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
 
            MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom],
                               *mfs_mom[IntVars::ymom],
                               *mfs_mom[IntVars::zmom],
                               Geom(lev).Domain(),
                               domain_bcs_type, c_vfrac);
        }
    }
 
    IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
    IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
 
    Vector<Real> ftime    = {static_cast<Real>(t_old[lev  ]), static_cast<Real>(t_new[lev  ])};
    Vector<Real> ctime    = {static_cast<Real>(t_old[lev-1]), static_cast<Real>(t_new[lev-1])};
 
    amrex::Real small_dt = Real(1.e-8) * (ftime[1] - ftime[0]);
 
    Vector<MultiFab*> fmf;
    if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
        fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
    } else if (amrex::almostEqual(time,ftime[1])) {
        fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
    } else {
        fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
    }
    Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
 
    // We must fill a temporary then copy it back so we don't double add/subtract
    MultiFab mf_c(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
                  mfs_vel[Vars::cons]->nComp()   ,mfs_vel[Vars::cons]->nGrowVect());
 
    mapper = &cell_cons_interp;
 
    if (interpolation_type == StateInterpType::Perturbational)
    {
        // Divide (rho theta) by rho to get theta (before we subtract rho0 from rho!)
        if (!amrex::almostEqual(time,ctime[1])) {
            MultiFab::Divide(vars_old[lev-1][Vars::cons],vars_old[lev-1][Vars::cons],
                             Rho_comp,RhoTheta_comp,1,ngvect_cons);
            MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
                               BaseState::r0_comp,Rho_comp,1,ngvect_cons);
            MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
                               BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
        }
        if (!amrex::almostEqual(time,ctime[0])) {
            MultiFab::Divide(vars_new[lev-1][Vars::cons],vars_new[lev-1][Vars::cons],
                             Rho_comp,RhoTheta_comp,1,ngvect_cons);
            MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
                               BaseState::r0_comp,Rho_comp,1,ngvect_cons);
            MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
                               BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
        }
 
        if (!amrex::almostEqual(time,ftime[1])) {
            MultiFab::Divide(vars_old[lev  ][Vars::cons],vars_old[lev  ][Vars::cons],
                             Rho_comp,RhoTheta_comp,1,IntVect{0});
            MultiFab::Subtract(vars_old[lev  ][Vars::cons],old_base_state,
                               BaseState::r0_comp,Rho_comp,1,IntVect{0});
            MultiFab::Subtract(vars_old[lev  ][Vars::cons],old_base_state,
                               BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        }
        if (!amrex::almostEqual(time,ftime[0])) {
            MultiFab::Divide(vars_new[lev  ][Vars::cons],vars_new[lev  ][Vars::cons],
                             Rho_comp,RhoTheta_comp,1,IntVect{0});
            MultiFab::Subtract(vars_new[lev  ][Vars::cons],old_base_state,
                               BaseState::r0_comp,Rho_comp,1,IntVect{0});
            MultiFab::Subtract(vars_new[lev  ][Vars::cons],old_base_state,
                               BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        }
    }
 
    // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
    FillPatchTwoLevels(mf_c, ngvect_cons, IntVect(0,0,0),
                       time, cmf, ctime, fmf, ftime,
                       0, 0, mf_c.nComp(), geom[lev-1], geom[lev],
                       refRatio(lev-1), mapper, domain_bcs_type,
                       BCVars::cons_bc);
 
    if (interpolation_type == StateInterpType::Perturbational)
    {
        // Restore the coarse values to what they were
        if (!amrex::almostEqual(time,ctime[1])) {
            MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                          BaseState::r0_comp,Rho_comp,1,ngvect_cons);
            MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                          BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
            MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                               Rho_comp,RhoTheta_comp,1,ngvect_cons);
        }
        if (!amrex::almostEqual(time,ctime[0])) {
            MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                          BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
            MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                          BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
            MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                               Rho_comp,RhoTheta_comp,1,ngvect_cons);
        }
 
        if (!amrex::almostEqual(time,ftime[1])) {
            MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev  ],BaseState::r0_comp,Rho_comp,1,IntVect{0});
            MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev  ],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
            MultiFab::Multiply(vars_old[lev][Vars::cons], vars_old[lev][Vars::cons],
                               Rho_comp,RhoTheta_comp,1,IntVect{0});
        }
        if (!amrex::almostEqual(time,ftime[0])) {
            MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
            MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
            MultiFab::Multiply(vars_new[lev][Vars::cons], vars_new[lev][Vars::cons],
                               Rho_comp,RhoTheta_comp,1,IntVect{0});
        }
 
        // Set values in the cells outside the domain boundary so that we can do the Add
        //     without worrying about uninitialized values outside the domain -- these
        //     will be filled in the physbcs call
        mf_c.setDomainBndry(bogus_large_value,0,2,geom[lev]); // Do both rho and (rho theta) together
 
        // Add rho_0 back to rho and theta_0 back to theta
        MultiFab::Add(mf_c, new_base_state,BaseState::r0_comp,Rho_comp,1,ngvect_cons);
        MultiFab::Add(mf_c, new_base_state,BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
 
        // Multiply (theta) by rho to get (rho theta)
        MultiFab::Multiply(mf_c,mf_c,Rho_comp,RhoTheta_comp,1,ngvect_cons);
    }
 
    MultiFab::Copy(*mfs_vel[Vars::cons],mf_c,0,0,mf_c.nComp(),mf_c.nGrowVect());
 
    // ***************************************************************************************
 
    if (!cons_only)
    {
            mapper = &face_cons_linear_interp;
 
            MultiFab& mf_u = *mfs_vel[Vars::xvel];
            MultiFab& mf_v = *mfs_vel[Vars::yvel];
            MultiFab& mf_w = *mfs_vel[Vars::zvel];
 
            Vector<MultiFab*> fmf_u; Vector<MultiFab*> fmf_v; Vector<MultiFab*> fmf_w;
            Vector<MultiFab*> cmf_u; Vector<MultiFab*> cmf_v; Vector<MultiFab*> cmf_w;
 
            // **********************************************************************
 
            if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
                fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
                fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
                fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
            } else if ( amrex::almostEqual(time,ftime[1]) ) {
                fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
                fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
                fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
            } else {
                fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
                fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
                fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
            }
            cmf_u = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
            cmf_v = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
            cmf_w = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mf_u, ngvect_vels, IntVect(0,0,0),
                               time, cmf_u, ctime, fmf_u, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::xvel_bc);
 
            FillPatchTwoLevels(mf_v, ngvect_vels, IntVect(0,0,0),
                               time, cmf_v, ctime, fmf_v, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::yvel_bc);
 
            // We put these here because these may be used in constructing omega outside the
            // domain when fillpatching w
            bool do_fb = true;
            (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                              ngvect_vels,time,BCVars::xvel_bc, do_fb);
            (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                              ngvect_vels,time,BCVars::yvel_bc, do_fb);
 
            // **********************************************************************
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mf_w, ngvect_vels, IntVect(0,0,0),
                               time, cmf_w, ctime, fmf_w, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::zvel_bc);
    } // !cons_only
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    int icomp_cons = 0;
    int ncomp_cons = mfs_vel[Vars::cons]->nComp();
 
    bool do_fb = true;
 
    if (m_r2d && !solverChoice.use_real_bcs) fill_from_bndryregs(mfs_vel,time);
 
    // We call this even if use_real_bcs is true because these will fill the vertical bcs
    // Note that we call FillBoundary inside the physbcs call
    (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                         icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
    if (!cons_only) {
        // Note that we need to fill u and v in the case of terrain because we will use
        //      these in the call of WFromOmega in lateral ghost cells of the fine grid
        // (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
        //                   ngvect_vels,time,BCVars::xvel_bc, do_fb);
        // (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
        //                   ngvect_vels,time,BCVars::yvel_bc, do_fb);
        (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::zvel_bc, do_fb);
    }
}

Here is the call graph for this function:

FillPlot2DVars()

void ERF::FillPlot2DVars	(	int	lev,
		const amrex::Vector< std::string > &	plot_var_names,
		const amrex::Vector< std::string > &	varnames,
		const amrex::Vector< plotfile2d::Plotfile2DOutputDescriptor > &	output_descriptors,
		amrex::MultiFab &	mf_dst,
		int	ncomp_mf
	)

{
    // Make sure getPgivenRTh and getTgivenRandRTh don't fail
    if (check_for_nans) {
        check_for_negative_theta(vars_new[lev][Vars::cons]);
    }
 
    int mf_comp = 0;
 
    // Set all components to zero in case they aren't defined below
    mf_dst.setVal(0.0);
 
    // Expose domain khi and klo at each level
    int klo = geom[lev].Domain().smallEnd(2);
    int khi = geom[lev].Domain().bigEnd(2);
 
    const MultiFab* pblh_source = nullptr;
    const MultiFab* sens_flux_source = SFS_hfx3_lev[lev].get();
    const MultiFab* laten_flux_source = SFS_q1fx3_lev[lev].get();
    const MultiFab* shoc_ustar_source = nullptr;
    const MultiFab* shoc_olen_source = nullptr;
    const MultiFab* shoc_wthv_source = nullptr;
    const ShocDriver* native_shoc = native_shoc_driver[lev].get();
    SurfaceLayer* surf_layer = m_SurfaceLayer[Orientation(Direction::z,Orientation::low)].get();
    const bool native_shoc_owns_scalar_fluxes =
        native_shoc && native_shoc->owns_scalar_surface_fluxes();
    const bool native_shoc_has_consumed_flux_diagnostics =
        native_shoc && native_shoc->has_consumed_surface_flux_diagnostics();
    if (native_shoc && native_shoc->has_native_diagnostics()) {
        pblh_source = &native_shoc->pblh_diagnostics();
    }
    // Native SHOC state_update clears the host SFS arrays after consuming
    // them. Use SHOC's preserved snapshots only for flux components whose
    // corresponding host SFS field existed; otherwise keep the source null
    // so the 2D writer emits the documented -999 missing value.
    if (plotfile2d::use_native_shoc_consumed_flux_source(
            native_shoc_owns_scalar_fluxes,
            native_shoc_has_consumed_flux_diagnostics,
            SFS_hfx3_lev[lev] != nullptr)) {
        sens_flux_source = &native_shoc->consumed_sens_flux_diagnostics();
    }
    if (plotfile2d::use_native_shoc_consumed_flux_source(
            native_shoc_owns_scalar_fluxes,
            native_shoc_has_consumed_flux_diagnostics,
            SFS_q1fx3_lev[lev] != nullptr)) {
        laten_flux_source = &native_shoc->consumed_laten_flux_diagnostics();
    }
    if (native_shoc && native_shoc->has_native_diagnostics()) {
        shoc_ustar_source = &native_shoc->shoc_ustar_diagnostics();
        shoc_olen_source = &native_shoc->shoc_olen_diagnostics();
        shoc_wthv_source = &native_shoc->wthv_sec_diagnostics();
    }
    // pblh should follow the active PBL diagnostic provider. Native SHOC
    // diagnoses its own PBL height in state_update mode; SurfaceLayer
    // remains the fallback for non-SHOC configurations.
    //
    // NOTE: the SurfaceLayer pblh holds only the bogus value it was initialized
    //       with unless erf.most.pblh_calc was set (the default is "none"), so we
    //       leave the source null in that case and emit the missing value.
    if (!pblh_source && surf_layer && surf_layer->computes_pblh()) {
        pblh_source = surf_layer->get_pblh(lev);
    }
 
    if (containerHasElement(plot_var_names, "z_surf")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& derdat = mf_dst.array(mfi);
            const Array4<const Real>& z_phys_arr = z_phys_nd[lev]->const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
               derdat(i, j, k, mf_comp) = Compute_Z_AtWFace(i, j, 0, z_phys_arr);
            });
        }
        mf_comp++;
    }
 
    if (containerHasElement(plot_var_names, "landmask")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& derdat = mf_dst.array(mfi);
            const Array4<const int>& lmask_arr = lmask_lev[lev][0]->const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
               derdat(i, j, k, mf_comp) = static_cast<Real>(lmask_arr(i, j, 0));
            });
        }
        mf_comp++;
    }
 
    if (containerHasElement(plot_var_names, "mapfac")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& derdat = mf_dst.array(mfi);
            const Array4<Real>& mf_m   = mapfac[lev][MapFacType::m_x]->array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
               derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
            });
        }
        mf_comp++;
    }
 
    if (containerHasElement(plot_var_names, "lat_m")) {
        if (lat_m[lev]) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf_dst.array(mfi);
                const Array4<Real>& data   = lat_m[lev]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                   derdat(i, j, k, mf_comp) = data(i,j,0);
                });
            }
        }
        mf_comp++;
    } // lat_m
 
    if (containerHasElement(plot_var_names, "lon_m")) {
        if (lon_m[lev]) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf_dst.array(mfi);
                const Array4<Real>& data   = lon_m[lev]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                   derdat(i, j, k, mf_comp) = data(i,j,0);
                });
            }
        } else {
            mf_dst.setVal(0.0,mf_comp,1,0);
        }
 
        mf_comp++;
 
    } // lon_m
 
    ///////////////////////////////////////////////////////////////////////
    // Surface and single-level diagnostics use the fill helpers below.
    // Column reductions and future interpolated surfaces keep explicit
    // assembly paths until their contracts are isolated in dedicated
    // helpers.
    if (containerHasElement(plot_var_names, "u_star")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, surf_layer ? surf_layer->get_u_star(lev) : nullptr,
            0, -999);
        mf_comp++;
    } // u_star
 
    if (containerHasElement(plot_var_names, "w_star")) {
        // NOTE: w_star holds only the bogus value it was initialized with unless
        //       erf.most.include_wstar is on (it is off by default), so we must
        //       pass a null source in that case and emit the missing value
        const MultiFab* w_star_source = (surf_layer && surf_layer->computes_w_star())
                                      ? surf_layer->get_w_star(lev) : nullptr;
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, w_star_source, 0, -999);
        mf_comp++;
    } // w_star
 
    if (containerHasElement(plot_var_names, "t_star")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, surf_layer ? surf_layer->get_t_star(lev) : nullptr,
            0, -999);
        mf_comp++;
    } // t_star
 
    if (containerHasElement(plot_var_names, "q_star")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, surf_layer ? surf_layer->get_q_star(lev) : nullptr,
            0, -999);
        mf_comp++;
    } // q_star
 
    if (containerHasElement(plot_var_names, "Olen")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, surf_layer ? surf_layer->get_olen(lev) : nullptr,
            0, -999);
        mf_comp++;
    } // Olen
 
    if (containerHasElement(plot_var_names, "pblh")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, pblh_source, 0, -999);
        mf_comp++;
    } // pblh
 
    if (containerHasElement(plot_var_names, "t_surf")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, surf_layer ? surf_layer->get_t_surf(lev) : nullptr,
            0, -999);
        mf_comp++;
    } // t_surf
 
    if (containerHasElement(plot_var_names, "q_surf")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, surf_layer ? surf_layer->get_q_surf(lev) : nullptr,
            0, -999);
        mf_comp++;
    } // q_surf
 
    if (containerHasElement(plot_var_names, "z0")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, surf_layer ? surf_layer->get_z0(lev) : nullptr,
            0, -999);
        mf_comp++;
    } // z0
 
    if (containerHasElement(plot_var_names, "OLR")) {
        // Outgoing longwave: the upward LW at the top-of-atmosphere
        // interface, which rad_fluxes keeps in the z-ghost cell above the
        // top layer (index khi holds the lower interface of the top layer).
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, rad_fluxes[lev].get(), khi + 1, -999, 2);
        mf_comp++;
    } // OLR
 
    if (containerHasElement(plot_var_names, "sens_flux")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, sens_flux_source, klo, -999);
        mf_comp++;
    } // sens_flux
 
    // Keep the legacy output name "laten_flux"; it maps to the vertical
    // water-vapor surface flux field.
    if (containerHasElement(plot_var_names, "laten_flux")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, laten_flux_source, klo, -999);
        mf_comp++;
    } // laten_flux
 
    if (containerHasElement(plot_var_names, "surf_pres")) {
        bool moist = (solverChoice.moisture_type != MoistureType::None);
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const auto& derdat   = mf_dst.array(mfi);
            const auto& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                auto rt = cons_arr(i,j,klo,RhoTheta_comp);
                auto qv = (moist) ? cons_arr(i,j,klo,RhoQ1_comp)/cons_arr(i,j,klo,Rho_comp)
                                  : zero;
                derdat(i, j, k, mf_comp) = getPgivenRTh(rt, qv);
            });
        }
        mf_comp++;
    } // surf_pres
 
    if (containerHasElement(plot_var_names, "sea_level_pressure")) {
        sea_level_pressure_diagnostics::fill_sea_level_pressure(
            mf_dst, mf_comp, vars_new[lev][Vars::cons], *z_phys_nd[lev],
            solverChoice.moisture_indices.qv, klo);
        mf_comp++;
    } // sea_level_pressure
 
    const auto precip_sources =
        micro ? micro->Get_Surface_Precip_Accumulation_Ptrs(lev)
              : SurfacePrecipAccumulationSources{};
    const auto selected_precipitation =
        plotfile2d::selected_precipitation_accumulation_components(varnames,
                                                                   precip_sources);
    if (selected_precipitation.n > 0) {
        plotfile2d::fill_precipitation_accumulations(mf_dst,
                                                     precip_sources,
                                                     selected_precipitation,
                                                     klo);
    }
 
    if (containerHasElement(plot_var_names, "precip_total_accum"))   { mf_comp++; }
    if (containerHasElement(plot_var_names, "precip_rain_accum"))    { mf_comp++; }
    if (containerHasElement(plot_var_names, "precip_snow_accum"))    { mf_comp++; }
    if (containerHasElement(plot_var_names, "precip_graupel_accum")) { mf_comp++; }
    if (containerHasElement(plot_var_names, "precip_hail_accum"))    { mf_comp++; }
    if (containerHasElement(plot_var_names, "precip_frozen_accum"))  { mf_comp++; }
 
    if (containerHasElement(plot_var_names, "integrated_qv")) {
        // integrated_qv remains the legacy column-reduction example.
        // Scheme-aware condensed water paths use a dedicated helper below
        // rather than the single-k copy helpers.
        MultiFab mf_qv_int(mf_dst,make_alias,mf_comp,1);
        if (solverChoice.moisture_type != MoistureType::None) {
            volWgtColumnSum(lev, vars_new[lev][Vars::cons], RhoQ1_comp, mf_qv_int, *detJ_cc[lev]);
        } else {
            mf_qv_int.setVal(0.);
        }
        mf_comp++;
    }
 
    const auto selected_water_paths =
        plotfile2d::selected_condensed_water_path_components(varnames, solverChoice);
    if (selected_water_paths.n > 0) {
        // Condensed water paths are scheme-aware column reductions. Their
        // availability comes from solverChoice.moisture_indices, and the
        // reduction uses the same metric convention as integrated_qv.
        plotfile2d::fill_condensed_water_paths(mf_dst,
                                               vars_new[lev][Vars::cons],
                                               selected_water_paths,
                                               geom[lev],
                                               *detJ_cc[lev]);
    }
 
    if (containerHasElement(plot_var_names, "integrated_qc")) { mf_comp++; }
    if (containerHasElement(plot_var_names, "integrated_qi")) { mf_comp++; }
    if (containerHasElement(plot_var_names, "integrated_qr")) { mf_comp++; }
    if (containerHasElement(plot_var_names, "integrated_qs")) { mf_comp++; }
    if (containerHasElement(plot_var_names, "integrated_qg")) { mf_comp++; }
 
    if (containerHasElement(plot_var_names, "surface_diagnostic_source")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp,
            surf_layer ? surf_layer->get_surface_diagnostic_source(lev) : nullptr,
            0, 0);
        mf_comp++;
    } // surface_diagnostic_source
 
    if (containerHasElement(plot_var_names, "sensible_heat_flux")) {
        plotfile2d::fill_sensible_heat_flux_from_klevel_or_missing(
            mf_dst, mf_comp, sens_flux_source, klo, -999);
        mf_comp++;
    } // sensible_heat_flux
 
    if (containerHasElement(plot_var_names, "latent_heat_flux")) {
        plotfile2d::fill_latent_heat_flux_from_klevel_or_missing(
            mf_dst, mf_comp, laten_flux_source, klo, -999);
        mf_comp++;
    } // latent_heat_flux
 
    if (containerHasElement(plot_var_names, "shoc_u_star")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, shoc_ustar_source, klo, -999);
        mf_comp++;
    } // shoc_u_star
 
    if (containerHasElement(plot_var_names, "shoc_Olen")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, shoc_olen_source, klo, -999);
        mf_comp++;
    } // shoc_Olen
 
    if (containerHasElement(plot_var_names, "shoc_wthv_sfc")) {
        plotfile2d::fill_component_from_klevel_or_value(
            mf_dst, mf_comp, shoc_wthv_source, klo, -999);
        mf_comp++;
    } // shoc_wthv_sfc
 
    // Land-surface provider fields use the generic LandSurface name/index
    // interface. The catalog supplies metadata; this block only assembles
    // stored values and translates provider sentinels.
    int temperature_2m_comp = -1;
    int mixing_ratio_2m_comp = -1;
    int near_surface_source_comp = -1;
    for (const auto& name : plot_var_names) {
        const auto* descriptor = plotfile2d::find_diagnostic(name);
        if (!descriptor) {
            descriptor = plotfile2d::find_dynamic_soil_diagnostic(name);
        }
        if (!is_land_surface_diagnostic(descriptor)) {
            continue;
        }
 
        if (is_unified_near_surface_diagnostic(descriptor->id)) {
            if (descriptor->id == plotfile2d::DiagnosticID::Temperature2m) {
                temperature_2m_comp = mf_comp;
            } else if (descriptor->id == plotfile2d::DiagnosticID::WaterVaporMixingRatio2m) {
                mixing_ratio_2m_comp = mf_comp;
            } else {
                near_surface_source_comp = mf_comp;
            }
            ++mf_comp;
            continue;
        }
 
        plotfile2d::fill_land_surface_component_from_klevel_or_missing(
            mf_dst, mf_comp, lsm.Get_Data_Ptr(lev, name), 0, -999);
        ++mf_comp;
    }
 
    if (temperature_2m_comp >= 0 || mixing_ratio_2m_comp >= 0 ||
        near_surface_source_comp >= 0) {
        near_surface_diagnostics::Sources near_surface_sources;
        near_surface_sources.native_temperature_vegetated =
            lsm.Get_Data_Ptr(lev, "noahmp_temperature_2m_vegetated");
        near_surface_sources.native_temperature_bare =
            lsm.Get_Data_Ptr(lev, "noahmp_temperature_2m_bare");
        near_surface_sources.native_mixing_ratio_vegetated =
            lsm.Get_Data_Ptr(lev, "noahmp_water_vapor_mixing_ratio_2m_vegetated");
        near_surface_sources.native_mixing_ratio_bare =
            lsm.Get_Data_Ptr(lev, "noahmp_water_vapor_mixing_ratio_2m_bare");
        near_surface_sources.native_vegetation_fraction =
            lsm.Get_Data_Ptr(lev, "noahmp_vegetation_fraction");
        near_surface_sources.theta_surface =
            surf_layer ? surf_layer->get_t_surf(lev) : nullptr;
        near_surface_sources.theta_star =
            surf_layer ? surf_layer->get_t_star(lev) : nullptr;
        near_surface_sources.mixing_ratio_surface =
            surf_layer ? surf_layer->get_q_surf(lev) : nullptr;
        near_surface_sources.mixing_ratio_star =
            surf_layer ? surf_layer->get_q_star(lev) : nullptr;
        near_surface_sources.roughness_height =
            surf_layer ? surf_layer->get_z0(lev) : nullptr;
        near_surface_sources.obukhov_length =
            surf_layer ? surf_layer->get_olen(lev) : nullptr;
        near_surface_sources.source_mask =
            surf_layer ? surf_layer->get_surface_diagnostic_source(lev) : nullptr;
        near_surface_sources.land_mask =
            (lmask_lev[lev].empty()) ? nullptr : lmask_lev[lev][0].get();
        near_surface_sources.cons = &vars_new[lev][Vars::cons];
        near_surface_sources.z_phys_nd = z_phys_nd[lev].get();
        near_surface_sources.dz = geom[lev].CellSize(2);
        near_surface_sources.klo = klo;
        near_surface_sources.moist = solverChoice.moisture_type != MoistureType::None;
        near_surface_sources.has_lsm =
            near_surface_sources.native_temperature_vegetated != nullptr &&
            near_surface_sources.native_temperature_bare != nullptr &&
            near_surface_sources.native_mixing_ratio_vegetated != nullptr &&
            near_surface_sources.native_mixing_ratio_bare != nullptr &&
            near_surface_sources.native_vegetation_fraction != nullptr;
        near_surface_diagnostics::fill(mf_dst, temperature_2m_comp,
                                       mixing_ratio_2m_comp,
                                       near_surface_source_comp,
                                       near_surface_sources);
    }
 
    const int static_output_count = static_cast<int>(plot_var_names.size());
    // If this level is anelastic then the base state pressure -- not the compressible
    //    EOS -- defines the pressure, both for the sampled "pressure" field and for
    //    the pressure vertical coordinate.  This matches the 3-D plotfile path.
    MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
    const MultiFab* p_hse_ptr = (solverChoice.anelastic[lev] == 1) ? &p_hse : nullptr;
    for (int out_idx = static_output_count; out_idx < static_cast<int>(output_descriptors.size()); ++out_idx) {
        const auto& descriptor = output_descriptors[out_idx];
        // Sampled-level descriptors are dynamic. The interpolator owns
        // their vertical sampling so the writer remains an output
        // assembly layer.
        plotfile2d::SampledWindSources wind_sources;
        wind_sources.xvel = &vars_new[lev][Vars::xvel];
        wind_sources.yvel = &vars_new[lev][Vars::yvel];
        wind_sources.zvel = &vars_new[lev][Vars::zvel];
        plotfile2d::fill_sampled_level_component(
            mf_dst, mf_comp, descriptor,
            vars_new[lev][Vars::cons],
            z_phys_cc[lev].get(),
            *z_phys_nd[lev],
            z_phys_cc[lev] != nullptr,
            solverChoice.moisture_indices,
            klo, khi,
            wind_sources,
            p_hse_ptr);
        mf_comp++;
    }
 
    if (mf_comp != ncomp_mf) {
        Abort(plotfile2d::format_2d_component_count_error(lev, mf_comp, ncomp_mf));
    }
}

Here is the call graph for this function:

FillPlot3DVars()

void ERF::FillPlot3DVars	(	int	lev,
		const amrex::Vector< std::string > &	plot_var_names,
		Plot3DScratch &	scratch,
		amrex::MultiFab &	mf_dst,
		int	ncomp_mf,
		amrex::Real	tnew
	)

{
    auto& mf_cc_vel = scratch.mf_cc_vel;
    auto& mf_cc_tau = scratch.mf_cc_tau;
    auto& mf_cc_fx  = scratch.mf_cc_fx;
 
    // Make sure getPgivenRTh and getTgivenRandRTh don't fail
    if (check_for_nans) {
        check_for_negative_theta(vars_new[lev][Vars::cons]);
    }
 
    int mf_comp = 0;
    const int ncomp_cons_lev = vars_new[lev][Vars::cons].nComp();
 
    // The moisture map is the authority on which moisture variables exist;
    // this only re-checks that the components it names are inside the state
    // that was actually allocated, which would be a registration bug.
    const MoistureComponentIndices& mi = solverChoice.moisture_indices;
    auto assert_comps_in_state =
        [&](const char* /*field*/, const MoistureComponentIndices::CompList& list)
    {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            !list.empty(), "aggregate 3D plot variable requires at least one moist species");
        for (int n = 0; n < list.size; ++n) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
                list.comp[n] >= 0 && list.comp[n] < ncomp_cons_lev,
                "aggregate 3D plot variable names a moist component outside the state");
        }
    };
 
    // Sum a moisture component list into the next plot component as a mixing
    // ratio: the state holds rho-weighted species, so the sum is divided by
    // density on the way out.
    auto copy_moist_sum = [&](const MoistureComponentIndices::CompList& list)
    {
        MultiFab::Copy(mf_dst, vars_new[lev][Vars::cons], list.comp[0], mf_comp, 1, 0);
        for (int n = 1; n < list.size; ++n) {
            MultiFab::Add(mf_dst, vars_new[lev][Vars::cons], list.comp[n], mf_comp, 1, 0);
        }
        MultiFab::Divide(mf_dst, vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
        mf_comp += 1;
    };
 
    // Copy one moist species out of the state as a mixing ratio.
    auto copy_moist_species = [&](const std::string& name)
    {
        const int comp = mi.comp_for_var(name);
        if (containerHasElement(plot_var_names, name) && (comp >= 0)) {
            AMREX_ALWAYS_ASSERT(comp < ncomp_cons_lev);
            MultiFab::Copy(  mf_dst, vars_new[lev][Vars::cons], comp    , mf_comp, 1, 0);
            MultiFab::Divide(mf_dst, vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
            mf_comp += 1;
        }
    };
 
    // Copy one moist diagnostic out of the scheme's qmoist arrays.
    auto copy_moist_diagnostic = [&](const std::string& name)
    {
        const int idx = mi.qmoist_index_for_var(name);
        if (containerHasElement(plot_var_names, name) && (idx >= 0)) {
            AMREX_ALWAYS_ASSERT(idx < static_cast<int>(qmoist[lev].size()) &&
                                qmoist[lev][idx] != nullptr);
            MultiFab::Copy(mf_dst, *(qmoist[lev][idx]), 0, mf_comp, 1, 0);
            mf_comp += 1;
        }
    };
 
    BoxArray ba(vars_new[lev][Vars::cons].boxArray());
    DistributionMapping dm = vars_new[lev][Vars::cons].DistributionMap();
 
    // First, copy any of the conserved state variables into the output plotfile
    for (int i = 0; i < cons_names.size(); ++i) {
        if (containerHasElement(plot_var_names, cons_names[i])) {
            const int cons_comp = erf_plotfile::plot3d_conserved_component_index(cons_names[i]);
            AMREX_ALWAYS_ASSERT(cons_comp >= 0 && cons_comp < vars_new[lev][Vars::cons].nComp());
            MultiFab::Copy(mf_dst,vars_new[lev][Vars::cons],cons_comp,mf_comp,1,0);
            mf_comp++;
        }
    }
 
    // Next, check for velocities
    if (containerHasElement(plot_var_names, "x_velocity")) {
        MultiFab::Copy(mf_dst, mf_cc_vel[lev], 0, mf_comp, 1, 0);
        mf_comp += 1;
    }
    if (containerHasElement(plot_var_names, "y_velocity")) {
        MultiFab::Copy(mf_dst, mf_cc_vel[lev], 1, mf_comp, 1, 0);
        mf_comp += 1;
    }
    if (containerHasElement(plot_var_names, "z_velocity")) {
        MultiFab::Copy(mf_dst, mf_cc_vel[lev], 2, mf_comp, 1, 0);
        mf_comp += 1;
    }
 
    // Create multifabs for HSE and pressure fields used to derive other quantities
    MultiFab  r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
    MultiFab  p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
    MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
    MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
    MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
 
    MultiFab pressure;
 
    const bool needs_pressure = erf_plotfile::plot3d_needs_pressure(plot_var_names);
    if (solverChoice.anelastic[lev] == 0) {
        if (needs_pressure) {
            int ng = (containerHasElement(plot_var_names, "dpdx") || containerHasElement(plot_var_names, "dpdy") ||
                      containerHasElement(plot_var_names, "dpdz")) ? 1 : 0;
 
            // Allocate space for pressure
            pressure.define(ba,dm,1,ng);
 
            if (ng > 0) {
                // Default to p_hse as a way of filling ghost cells at domain boundaries
                MultiFab::Copy(pressure,p_hse,0,0,1,1);
            }
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& gbx = mfi.growntilebox(IntVect(ng,ng,0));
 
                const Array4<Real      >& p_arr = pressure.array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                const int qv_comp = mi.qv;
 
                ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    Real qv_for_p = (qv_comp >= 0) ? S_arr(i,j,k,qv_comp)/S_arr(i,j,k,Rho_comp) : 0;
                    const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                    p_arr(i, j, k) = getPgivenRTh(rhotheta,qv_for_p);
                });
           } // mfi
           pressure.FillBoundary(geom[lev].periodicity());
        } // compute compressible pressure
    } // not anelastic
    else {
        if (needs_pressure) {
            // Copy p_hse into pressure if using anelastic
            pressure.define(ba,dm,1,0);
            MultiFab::Copy(pressure,p_hse,0,0,1,0);
        }
    }
 
    // ***************************************************************************************
    // Finally, check for any derived quantities and compute them, inserting
    // them into our output multifab
    // ***************************************************************************************
    auto calculate_derived = [&](const std::string& der_name,
                                 MultiFab& src_mf,
                                 decltype(derived::erf_dernull)& der_function)
    {
        if (containerHasElement(plot_var_names, der_name)) {
            MultiFab dmf(mf_dst, make_alias, mf_comp, 1);
            //
            // NOTE: we must not tile in z here because some of the derived quantities
            //       ("precipitable", "mucape", "helicity", "max_reflectivity") are
            //       whole-column operations and require the full column in each box
            //
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(dmf, TileNoZ()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                auto& dfab = dmf[mfi];
                auto& sfab = src_mf[mfi];
                auto& zfab = (*z_phys_cc[lev])[mfi];
                der_function(bx, dfab, 0, 1, sfab, zfab, Geom(lev), tnew, nullptr, lev);
            }
 
            mf_comp++;
        }
    }; // end calculate_derived
    // ***************************************************************************************
 
    // *****************************************************************************************
    // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
    //       defined in ERF.H
    // *****************************************************************************************
 
    if (solverChoice.anelastic[lev]) {
        if (containerHasElement(plot_var_names, "temp")) {
            MultiFab dmf(mf_dst, make_alias, mf_comp, 1);
            const Real rdOcp = solverChoice.rdOcp;
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const auto temp = dmf.array(mfi);
                const auto state = vars_new[lev][Vars::cons].const_array(mfi);
                const auto p0 = p_hse.const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                    const Real theta = state(i,j,k,RhoTheta_comp) /
                                       state(i,j,k,Rho_comp);
                    temp(i,j,k,0) = getTgivenPandTh(p0(i,j,k), theta, rdOcp);
                });
            }
            ++mf_comp;
        }
    } else if (mi.has_moisture()) {
        calculate_derived("temp",        vars_new[lev][Vars::cons], derived::erf_dermoisttemp);
    } else {
        calculate_derived("temp",        vars_new[lev][Vars::cons], derived::erf_dertemp);
    }
    calculate_derived("theta",       vars_new[lev][Vars::cons], derived::erf_dertheta);
    calculate_derived("KE",          vars_new[lev][Vars::cons], derived::erf_derKE);
    calculate_derived("scalar",      vars_new[lev][Vars::cons], derived::erf_derscalar);
    calculate_derived("soundspeed",  vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
 
    if (containerHasElement(plot_var_names, "reflectivity"))
    {
        // NOTE: this tests the data layout the reflectivity kernel assumes rather
        //       than enumerating moisture models (see ERF_Tagging.cpp)
        if (solverChoice.moisture_indices.has_reflectivity_species()) {
            calculate_derived("reflectivity",      vars_new[lev][Vars::cons], derived::erf_derreflectivity);
        } else {
            mf_dst.setVal(zero, mf_comp, 1, 0);
            mf_comp++;
        }
    }
 
    if (containerHasElement(plot_var_names, "max_reflectivity"))
    {
        // NOTE: this tests the data layout the reflectivity kernel assumes rather
        //       than enumerating moisture models (see ERF_Tagging.cpp)
        if (solverChoice.moisture_indices.has_reflectivity_species()) {
            calculate_derived("max_reflectivity",  vars_new[lev][Vars::cons], derived::erf_dermaxreflectivity);
        } else {
            mf_dst.setVal(zero, mf_comp, 1, 0);
            mf_comp++;
        }
    }
 
    // Precipitable water is a column integral of vapor, so it needs a scheme
    // that carries vapor -- which is what has_moisture() tests.
    if (mi.has_moisture()) {
        calculate_derived("precipitable"   ,  vars_new[lev][Vars::cons], derived::erf_derprecipitable);
    }
    calculate_derived("mucape"         ,  vars_new[lev][Vars::cons], derived::erf_dermucape);
 
    calculate_derived("vorticity_x",    mf_cc_vel[lev], derived::erf_dervortx);
    calculate_derived("vorticity_y",    mf_cc_vel[lev], derived::erf_dervorty);
    calculate_derived("vorticity_z",    mf_cc_vel[lev], derived::erf_dervortz);
    calculate_derived("helicity",       mf_cc_vel[lev], derived::erf_derhelicity);
    calculate_derived("local_helicity", mf_cc_vel[lev], derived::erf_derlocalhelicity);
    calculate_derived("vort_stretching", mf_cc_vel[lev], derived::erf_dervortstretching);
    calculate_derived("magvel",         mf_cc_vel[lev], derived::erf_dermagvel);
 
    if (containerHasElement(plot_var_names, "divU"))
    {
        MultiFab dmf(mf_dst, make_alias, mf_comp, 1);
        Array<MultiFab const*, AMREX_SPACEDIM> u;
        u[0] = &(vars_new[lev][Vars::xvel]);
        u[1] = &(vars_new[lev][Vars::yvel]);
        u[2] = &(vars_new[lev][Vars::zvel]);
        compute_divergence(lev, dmf, u, *mapfac[lev][MapFacType::m_x],
                           *mapfac[lev][MapFacType::m_y], *mapfac[lev][MapFacType::v_x],
                           *mapfac[lev][MapFacType::u_y], *ax[lev], *ay[lev],
                           *detJ_cc[lev], geom[lev]);
        mf_comp += 1;
    }
 
    if (containerHasElement(plot_var_names, "pres_hse"))
    {
        MultiFab::Copy(mf_dst,p_hse,0,mf_comp,1,0);
        mf_comp += 1;
    }
    if (containerHasElement(plot_var_names, "dens_hse"))
    {
        MultiFab::Copy(mf_dst,r_hse,0,mf_comp,1,0);
        mf_comp += 1;
    }
    if (containerHasElement(plot_var_names, "theta_hse"))
    {
        MultiFab::Copy(mf_dst,th_hse,0,mf_comp,1,0);
        mf_comp += 1;
    }
    if (containerHasElement(plot_var_names, "pi_hse"))
    {
        MultiFab::Copy(mf_dst,pi_hse,0,mf_comp,1,0);
        mf_comp += 1;
    }
    if (containerHasElement(plot_var_names, "qv_hse"))
    {
        MultiFab::Copy(mf_dst,qv_hse,0,mf_comp,1,0);
        mf_comp += 1;
    }
 
    if (containerHasElement(plot_var_names, "pressure"))
    {
        if (solverChoice.anelastic[lev] == 1) {
            MultiFab::Copy(mf_dst, p_hse, 0, mf_comp, 1, 0);
        } else {
            MultiFab::Copy(mf_dst, pressure, 0, mf_comp, 1, 0);
        }
 
        mf_comp += 1;
    }
 
    if (containerHasElement(plot_var_names, "pert_pres"))
    {
        if (solverChoice.anelastic[lev] == 1) {
            MultiFab::Copy(mf_dst, pp_inc[lev], 0, mf_comp, 1, 0);
        } else {
            MultiFab::Copy(mf_dst, pressure, 0, mf_comp, 1, 0);
            MultiFab::Subtract(mf_dst,p_hse,0,mf_comp,1,IntVect{0});
        }
        mf_comp += 1;
    }
 
    if (containerHasElement(plot_var_names, "pert_dens"))
    {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& derdat  = mf_dst.array(mfi);
            const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
            const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
            });
        }
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "buoyancy"))
    {
        MultiFab     qt(mf_dst.boxArray(), mf_dst.DistributionMap(), 1, 1);
        MultiFab      b(mf_dst.boxArray(), mf_dst.DistributionMap(), 1, 0);
        MultiFab S_prim(mf_dst.boxArray(), mf_dst.DistributionMap(),
                         vars_new[lev][Vars::cons].nComp()-1, 1);
 
        qt.setVal(0.);
        int n_qstate_into_total = micro->Get_Qstate_Moist_Size() - micro->Get_Qstate_Moist_NumConc_Size();
        if (solverChoice.moisture_type != MoistureType::None) {
            make_qt(vars_new[lev][Vars::cons], qt, n_qstate_into_total);
        }
        //
        // NOTE: we must fill one ghost cell of S_prim here because make_buoyancy
        //       reads cell_prim(i,j,k-1) at the lower z face of every box -- with
        //       ng = 0 those ghost cells hold uninitialized data at the bottom of
        //       any box that doesn't touch the bottom of the domain.  (The ghost
        //       cells of vars_new[cons] are valid since we fillpatched above.)
        //
        cons_to_prim(vars_new[lev][Vars::cons], S_prim, 1);
 
        b.setVal(0.); // Need to initialize to zero because buoyancy not defined on faces at top and bottom of domain
        make_buoyancy(lev, vars_new[lev], S_prim, qt, b, geom[lev], solverChoice, base_state[lev], n_qstate_into_total,
                      get_eb(lev), solverChoice.anelastic[lev]);
        MultiFab::Copy(mf_dst, b, 0, mf_comp, 1, 0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "eq_pot_temp"))
    {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& derdat  = mf_dst.array(mfi);
            const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
            const Array4<Real const>& p_arr = pressure.const_array(mfi);
            const int qv_comp = mi.qv;
            const int qc_comp = mi.qc;
            const bool anelastic = solverChoice.anelastic[lev];
            const Real rdOcp = solverChoice.rdOcp;
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                Real qv = (qv_comp >= 0) ? S_arr(i,j,k,qv_comp)/S_arr(i,j,k,Rho_comp) : zero;
                Real qc = (qc_comp >= 0) ? S_arr(i,j,k,qc_comp)/S_arr(i,j,k,Rho_comp) : zero;
                Real T = anelastic ?
                    getTgivenPandTh(p_arr(i,j,k), S_arr(i,j,k,RhoTheta_comp) /
                                    S_arr(i,j,k,Rho_comp), rdOcp) :
                    getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                Real fac = Cp_d + Cp_l*(qv + qc);
                Real pv = erf_esatw(T)*Real(100.0);
 
                derdat(i, j, k, mf_comp) = T*std::pow((p_arr(i,j,k) - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
            });
        }
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "VPD"))
    {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& derdat  = mf_dst.array(mfi);
            const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
            const Array4<Real const>& p_arr = pressure.const_array(mfi);
            const int qv_comp = mi.qv;
            const bool anelastic = solverChoice.anelastic[lev];
            const Real rdOcp = solverChoice.rdOcp;
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                const Real qv       = (qv_comp >= 0) ? S_arr(i,j,k,qv_comp)/S_arr(i,j,k,Rho_comp) : zero;
 
                const Real T        = anelastic ?
                    getTgivenPandTh(p_arr(i,j,k), S_arr(i,j,k,RhoTheta_comp) /
                                    S_arr(i,j,k,Rho_comp), rdOcp) :
                    getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                const Real e_sat = Real(100.0) * erf_esatw_cc(T);
 
                const Real P     = p_arr(i,j,k);
                const Real e_act = P * qv / (RdoRv + qv);
 
                derdat(i,j,k,mf_comp) = std::max(amrex::Real(0), e_sat - e_act) * Real(0.001);
            });
        }
        mf_comp ++;
    }
 
#ifdef ERF_USE_WINDFARM
    if ( containerHasElement(plot_var_names, "num_turb") and
         (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
          solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
    {
        MultiFab::Copy(mf_dst,Nturb[lev],0,mf_comp,1,0);
        mf_comp ++;
    }
 
    if ( containerHasElement(plot_var_names, "SMark0") and
         (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
          solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
    {
        MultiFab::Copy(mf_dst,SMark[lev],0,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "SMark1") and
       (solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD))
    {
        MultiFab::Copy(mf_dst,SMark[lev],1,mf_comp,1,0);
        mf_comp ++;
    }
#endif
 
    // **********************************************************************************************
    // Allocate space if we are computing any pressure gradients
    // **********************************************************************************************
 
    Vector<MultiFab> gradp_temp;  gradp_temp.resize(AMREX_SPACEDIM);
    if (containerHasElement(plot_var_names, "dpdx")       ||
        containerHasElement(plot_var_names, "dpdy")       ||
        containerHasElement(plot_var_names, "dpdz")       ||
        containerHasElement(plot_var_names, "pres_hse_x") ||
        containerHasElement(plot_var_names, "pres_hse_y"))
    {
        gradp_temp[GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp_temp[GpVars::gpx].setVal(0.);
        gradp_temp[GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp_temp[GpVars::gpy].setVal(0.);
        gradp_temp[GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp_temp[GpVars::gpz].setVal(0.);
    }
 
    // **********************************************************************************************
    // These are based on computing gradient of full pressure
    // **********************************************************************************************
 
    if (solverChoice.anelastic[lev] == 0) {
        if ( (containerHasElement(plot_var_names, "dpdx")) ||
             (containerHasElement(plot_var_names, "dpdy")) ||
             (containerHasElement(plot_var_names, "dpdz")) ) {
            compute_gradp(pressure, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
                          get_eb(lev), gradp_temp, solverChoice);
        }
    }
 
    if (containerHasElement(plot_var_names, "dpdx"))
    {
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real      >&   derdat  = mf_dst.array(mfi);
            const Array4<Real const>&   gpx_arr = (solverChoice.anelastic[lev] == 1) ?
                  gradp[lev][GpVars::gpx].array(mfi) : gradp_temp[GpVars::gpx].array(mfi);
            const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                derdat(i ,j ,k, mf_comp) = myhalf * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
            });
        }
        mf_comp ++;
    } // dpdx
    if (containerHasElement(plot_var_names, "dpdy"))
    {
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real      >&   derdat  = mf_dst.array(mfi);
            const Array4<Real const>&   gpy_arr = (solverChoice.anelastic[lev] == 1) ?
                  gradp[lev][GpVars::gpy].array(mfi) : gradp_temp[GpVars::gpy].array(mfi);
            const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                derdat(i ,j ,k, mf_comp) = myhalf * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
            });
        }
        mf_comp ++;
    } // dpdy
    if (containerHasElement(plot_var_names, "dpdz"))
    {
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real      >&  derdat  = mf_dst.array(mfi);
            const Array4<Real const>&  gpz_arr = (solverChoice.anelastic[lev] == 1) ?
                  gradp[lev][GpVars::gpz].array(mfi) : gradp_temp[GpVars::gpz].array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                derdat(i ,j ,k, mf_comp) = myhalf * (gpz_arr(i,j,k+1) + gpz_arr(i,j,k));
            });
        }
        mf_comp ++;
    } // dpdz
 
    // **********************************************************************************************
    // These are based on computing gradient of basestate pressure
    // **********************************************************************************************
 
    if ( (containerHasElement(plot_var_names, "pres_hse_x")) ||
         (containerHasElement(plot_var_names, "pres_hse_y")) ) {
        compute_gradp(p_hse, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
                      get_eb(lev), gradp_temp, solverChoice);
    }
 
    if (containerHasElement(plot_var_names, "pres_hse_x"))
    {
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real      >&  derdat  = mf_dst.array(mfi);
            const Array4<Real const>&  gpx_arr = gradp_temp[0].array(mfi);
            const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                derdat(i ,j ,k, mf_comp) = myhalf * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
            });
        }
        mf_comp += 1;
    } // pres_hse_x
 
    if (containerHasElement(plot_var_names, "pres_hse_y"))
    {
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real      >&  derdat  = mf_dst.array(mfi);
            const Array4<Real const>&  gpy_arr = gradp_temp[1].array(mfi);
            const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                derdat(i ,j ,k, mf_comp) = myhalf * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
            });
        }
        mf_comp += 1;
    } // pres_hse_y
 
    // **********************************************************************************************
    // Metric terms
    // **********************************************************************************************
 
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        if (containerHasElement(plot_var_names, "z_phys"))
        {
            MultiFab::Copy(mf_dst,*z_phys_cc[lev],0,mf_comp,1,0);
            mf_comp ++;
        }
 
        if (containerHasElement(plot_var_names, "detJ"))
        {
            MultiFab::Copy(mf_dst,*detJ_cc[lev],0,mf_comp,1,0);
            mf_comp ++;
        }
 
        //
        // Cell-centered averages of the terrain metric terms h_xi, h_eta and h_zeta
        //
        for (int imet(0); imet < 3; ++imet)
        {
            const std::string met_name = (imet == 0) ? "h_xi" : ((imet == 1) ? "h_eta" : "h_zeta");
 
            if (containerHasElement(plot_var_names, met_name))
            {
                const GpuArray<Real, AMREX_SPACEDIM> dxInv = geom[lev].InvCellSizeArray();
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real      >& derdat = mf_dst.array(mfi);
                    const Array4<Real const>& z_nd   = z_phys_nd[lev]->const_array(mfi);
                    if (imet == 0) {
                        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                            derdat(i,j,k,mf_comp) = Compute_h_xi_AtCellCenter  (i,j,k,dxInv,z_nd);
                        });
                    } else if (imet == 1) {
                        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                            derdat(i,j,k,mf_comp) = Compute_h_eta_AtCellCenter (i,j,k,dxInv,z_nd);
                        });
                    } else {
                        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                            derdat(i,j,k,mf_comp) = Compute_h_zeta_AtCellCenter(i,j,k,dxInv,z_nd);
                        });
                    }
                }
                mf_comp ++;
            }
        } // h_xi, h_eta, h_zeta
    } // use_terrain
 
    if (containerHasElement(plot_var_names, "mapfac")) {
        amrex::Print() << "You are plotting a 3D version of mapfac; we suggest using the 2D plotfile instead" << std::endl;
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& derdat = mf_dst.array(mfi);
            const Array4<Real>& mf_m   = mapfac[lev][MapFacType::m_x]->array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
               derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
            });
        }
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "lat_m")) {
        amrex::Print() << "You are plotting a 3D version of lat_m; we suggest using the 2D plotfile instead" << std::endl;
        if (lat_m[lev]) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf_dst.array(mfi);
                const Array4<Real>& data   = lat_m[lev]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                   derdat(i, j, k, mf_comp) = data(i,j,0);
                });
            }
        } else {
            mf_dst.setVal(0.0,mf_comp,1,0);
        }
        mf_comp++;
    } // lat_m
 
    if (containerHasElement(plot_var_names, "lon_m")) {
        amrex::Print() << "You are plotting a 3D version of lon_m; we suggest using the 2D plotfile instead" << std::endl;
        if (lon_m[lev]) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf_dst.array(mfi);
                const Array4<Real>& data   = lon_m[lev]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                   derdat(i, j, k, mf_comp) = data(i,j,0);
                });
            }
        } else {
            mf_dst.setVal(0.0,mf_comp,1,0);
        }
        mf_comp++;
    } // lon_m
 
    if (solverChoice.time_avg_vel) {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            vel_t_avg[lev] != nullptr,
            "time-average plot variables require vel_t_avg storage on every AMR level");
        const Real norm = static_cast<Real>(t_avg_cnt[lev]);
        auto copy_time_average = [&](const int source_comp) {
            if (norm > Real(0.0)) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf_dst.array(mfi);
                    const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        derdat(i ,j ,k, mf_comp) = data(i,j,k,source_comp) / norm;
                    });
                }
            } else {
                // A plot requested before the first accumulation is defined as zero.
                mf_dst.setVal(Real(0.0), mf_comp, 1, 0);
            }
            ++mf_comp;
        };
 
        if (containerHasElement(plot_var_names, "u_t_avg")) copy_time_average(0);
        if (containerHasElement(plot_var_names, "v_t_avg")) copy_time_average(1);
        if (containerHasElement(plot_var_names, "w_t_avg")) copy_time_average(2);
        if (containerHasElement(plot_var_names, "umag_t_avg")) copy_time_average(3);
    }
 
    if (solverChoice.compute_mean_vars) {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            interval_means[lev] != nullptr,
            "interval-mean plot variables require storage on every AMR level");
 
        const Real norm = static_cast<Real>(t_mean_cnt[lev]);
        const Real inv_norm = norm > Real(0.0) ? Real(1.0) / norm : Real(0.0);
 
        auto copy_interval_mean = [&](const int source_comp) {
            const int dest_comp = mf_comp++;
            if (norm <= Real(0.0)) {
                mf_dst.setVal(Real(0.0), dest_comp, 1, 0);
                return;
            }
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(mf_dst, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& out = mf_dst.array(mfi);
                const Array4<const Real>& moments = interval_means[lev]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                    out(i,j,k,dest_comp) = moments(i,j,k,source_comp) * inv_norm;
                });
            }
        };
 
        static constexpr const char* mean_names[] = {
            "u_mean", "v_mean", "w_mean", "theta_mean",
            "uu_mean", "vv_mean", "ww_mean", "uw_mean", "vw_mean", "wtheta_mean"
        };
        for (int source_comp = 0; source_comp < 10; ++source_comp) {
            if (containerHasElement(plot_var_names, mean_names[source_comp])) {
                copy_interval_mean(source_comp);
            }
        }
 
        auto copy_fluctuation = [&](const int product_comp,
                                    const int first_comp,
                                    const int second_comp) {
            const int dest_comp = mf_comp++;
            if (norm <= Real(0.0)) {
                mf_dst.setVal(Real(0.0), dest_comp, 1, 0);
                return;
            }
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(mf_dst, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& out = mf_dst.array(mfi);
                const Array4<const Real>& moments = interval_means[lev]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                    const Real product_mean = moments(i,j,k,product_comp) * inv_norm;
                    const Real first_mean = moments(i,j,k,first_comp) * inv_norm;
                    const Real second_mean = moments(i,j,k,second_comp) * inv_norm;
                    out(i,j,k,dest_comp) = product_mean - first_mean * second_mean;
                });
            }
        };
 
        if (containerHasElement(plot_var_names, "uu_fluct")) copy_fluctuation(4, 0, 0);
        if (containerHasElement(plot_var_names, "vv_fluct")) copy_fluctuation(5, 1, 1);
        if (containerHasElement(plot_var_names, "ww_fluct")) copy_fluctuation(6, 2, 2);
        if (containerHasElement(plot_var_names, "uw_fluct")) copy_fluctuation(7, 0, 2);
        if (containerHasElement(plot_var_names, "vw_fluct")) copy_fluctuation(8, 1, 2);
        if (containerHasElement(plot_var_names, "wtheta_fluct")) copy_fluctuation(9, 2, 3);
 
        if (containerHasElement(plot_var_names, "tke_resolved")) {
            const int dest_comp = mf_comp++;
            if (norm <= Real(0.0)) {
                mf_dst.setVal(Real(0.0), dest_comp, 1, 0);
            } else {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for (MFIter mfi(mf_dst, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& out = mf_dst.array(mfi);
                    const Array4<const Real>& moments = interval_means[lev]->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                        const Real u_mean = moments(i,j,k,0) * inv_norm;
                        const Real v_mean = moments(i,j,k,1) * inv_norm;
                        const Real w_mean = moments(i,j,k,2) * inv_norm;
                        const Real uu_fluct = moments(i,j,k,4) * inv_norm - u_mean * u_mean;
                        const Real vv_fluct = moments(i,j,k,5) * inv_norm - v_mean * v_mean;
                        const Real ww_fluct = moments(i,j,k,6) * inv_norm - w_mean * w_mean;
                        out(i,j,k,dest_comp) = Real(0.5) * (uu_fluct + vv_fluct + ww_fluct);
                    });
                }
            }
        }
    }
 
    const MultiFab* eta_src = nullptr;
    const bool have_native_shoc_diagnostics =
        solverChoice.turbChoice[lev].uses_native_shoc() &&
        native_shoc_driver[lev] &&
        native_shoc_driver[lev]->has_native_diagnostics();
    if (solverChoice.turbChoice[lev].use_kturb) {
        if (have_native_shoc_diagnostics) {
            eta_src = &native_shoc_driver[lev]->native_diagnostics();
        } else
        {
            eta_src = eddyDiffs_lev[lev].get();
        }
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            eta_src != nullptr,
            "eddy-diffusivity plot variables require an eddy diagnostic source");
    }
 
    if (containerHasElement(plot_var_names, "nut")) {
        MultiFab dmf(mf_dst, make_alias, mf_comp, 1);
        MultiFab cmf(vars_new[lev][Vars::cons], make_alias, 0, 1); // to provide rho only
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            auto       prim = dmf[mfi].array();
            auto const cons = cmf[mfi].const_array();
            auto const diff = (eta_src) ? eta_src->const_array(mfi) :
                                          Array4<const Real>{};
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                const Real rho = cons(i, j, k, Rho_comp);
                const Real Kmv = diff(i, j, k, EddyDiff::Mom_v);
                prim(i,j,k) = Kmv / rho;
            });
        }
 
        mf_comp++;
    }
 
    const MultiFab* shoc_or_host_eddy = have_native_shoc_diagnostics
        ? &native_shoc_driver[lev]->native_diagnostics()
        : eddyDiffs_lev[lev].get();
 
    if (containerHasElement(plot_var_names, "Kmv") ||
        containerHasElement(plot_var_names, "Kmh") ||
        containerHasElement(plot_var_names, "Khv") ||
        containerHasElement(plot_var_names, "Khh") ||
        containerHasElement(plot_var_names, "Lturb")) {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            shoc_or_host_eddy != nullptr,
            "eddy diagnostic plot variables require an eddy diagnostic source");
        if (containerHasElement(plot_var_names, "Kmh") ||
            containerHasElement(plot_var_names, "Khh")) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
                eddyDiffs_lev[lev] != nullptr,
                "horizontal eddy diagnostic plot variables require eddyDiffs storage");
        }
        if (!have_native_shoc_diagnostics) {
            AMREX_ALWAYS_ASSERT(eddyDiffs_lev[lev] != nullptr);
        }
    }
 
    if (containerHasElement(plot_var_names, "Kmv")) {
        MultiFab::Copy(mf_dst,*shoc_or_host_eddy,EddyDiff::Mom_v,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "Kmh")) {
        MultiFab::Copy(mf_dst,*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "Khv")) {
        MultiFab::Copy(mf_dst,*shoc_or_host_eddy,EddyDiff::Theta_v,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "Khh")) {
        MultiFab::Copy(mf_dst,*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "Lturb")) {
        MultiFab::Copy(mf_dst,*shoc_or_host_eddy,EddyDiff::Turb_lengthscale,mf_comp,1,0);
        mf_comp ++;
    }
    // k-eqn RANS diagnostics (zero unless the closure is running)
    if (containerHasElement(plot_var_names, "Rt")) {
        AMREX_ALWAYS_ASSERT(eddyDiffs_lev[lev] != nullptr);
        MultiFab::Copy(mf_dst,*eddyDiffs_lev[lev],EddyDiff::RANS_Rt,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "cmu")) {
        AMREX_ALWAYS_ASSERT(eddyDiffs_lev[lev] != nullptr);
        MultiFab::Copy(mf_dst,*eddyDiffs_lev[lev],EddyDiff::RANS_cmu,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "cmu_prime")) {
        AMREX_ALWAYS_ASSERT(eddyDiffs_lev[lev] != nullptr);
        MultiFab::Copy(mf_dst,*eddyDiffs_lev[lev],EddyDiff::RANS_cmu_prime,mf_comp,1,0);
        mf_comp ++;
    }
    auto copy_native_shoc_diagnostic = [&](const MultiFab* src) {
        if (src != nullptr) {
            MultiFab::Copy(mf_dst, *src, 0, mf_comp, 1, 0);
        } else {
            mf_dst.setVal(-999, mf_comp, 1, 0);
        }
        mf_comp ++;
    };
    // Native SHOC pblh is diagnosed in meters AGL and is copied through
    // unchanged into the plotfile diagnostic field.
    if (containerHasElement(plot_var_names, "pblh")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->pblh_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "shoc_cldfrac")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->shoc_cldfrac_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "shoc_ql")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->shoc_ql_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "shoc_ql2")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->shoc_ql2_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "shoc_cond")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->shoc_cond_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "wqls_sec")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->wqls_sec_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "wthv_sec")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->wthv_sec_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "w_sec")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->w_sec_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "thl_sec")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->thl_sec_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "qw_sec")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->qw_sec_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "qwthl_sec")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->qwthl_sec_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "wthl_sec")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->wthl_sec_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "wqw_sec")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->wqw_sec_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "w3")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->w3_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "brunt")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->brunt_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "isotropy")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->isotropy_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "shear_prod")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->shear_prod_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "buoy_prod")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->buoy_prod_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "diss_tke")) {
        copy_native_shoc_diagnostic(have_native_shoc_diagnostics
                                    ? &native_shoc_driver[lev]->diss_tke_diagnostics()
                                    : nullptr);
    }
    if (containerHasElement(plot_var_names, "walldist")) {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            walldist[lev] != nullptr,
            "walldist plot variable requires wall-distance storage");
        MultiFab::Copy(mf_dst,*walldist[lev],0,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "diss")) {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            SFS_diss_lev[lev] != nullptr,
            "diss plot variable requires dissipation storage");
        MultiFab::Copy(mf_dst,*SFS_diss_lev[lev],0,mf_comp,1,0);
        mf_comp ++;
    }
 
    // ***************************************************************************************
    // Moisture variables.
    //
    // Which of these exist -- and where each one's data lives -- comes entirely from
    // solverChoice.moisture_indices (see the MoistureComponentIndices class comment).
    // Nothing below tests moisture_type, and nothing below is gated on the allocated
    // state width, so a component that a scheme allocates but never integrates is
    // never published as data.
    //
    // NOTE: the order here **MUST MATCH** the moisture entries of "derived_names" in
    //       ERF.H, which is what supplies the plotfile header names.  When the two
    //       disagree, every name is paired with another field's data.
    // ***************************************************************************************
    if (mi.has_moisture()) {
 
        // Moist density: dry density plus vapor and the suspended condensate
        if (containerHasElement(plot_var_names, "moist_density"))
        {
            const auto list = mi.nonprecipitating_comps();
            assert_comps_in_state("moist_density", list);
            MultiFab::Copy(mf_dst, vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
            for (int n = 0; n < list.size; ++n) {
                MultiFab::Add(mf_dst, vars_new[lev][Vars::cons], list.comp[n], mf_comp, 1, 0);
            }
            mf_comp += 1;
        }
 
        // Mass mixing ratios of the individual species
        //--------------------------------------------------------------------------
        copy_moist_species("qv");
        copy_moist_species("qc");
        copy_moist_species("qi");
        copy_moist_species("qrain");
        copy_moist_species("qsnow");
        copy_moist_species("qgraup");
 
        // Total water: every mass species the scheme carries
        //--------------------------------------------------------------------------
        if (containerHasElement(plot_var_names, "qt"))
        {
            const auto list = mi.total_water_comps();
            assert_comps_in_state("qt", list);
            copy_moist_sum(list);
        }
 
        // Non-precipitating water: vapor plus the suspended condensate
        //--------------------------------------------------------------------------
        if (containerHasElement(plot_var_names, "qn"))
        {
            const auto list = mi.nonprecipitating_comps();
            assert_comps_in_state("qn", list);
            copy_moist_sum(list);
        }
 
        // Precipitating water: the falling species
        //--------------------------------------------------------------------------
        if (containerHasElement(plot_var_names, "qp"))
        {
            const auto list = mi.precipitating_comps();
            assert_comps_in_state("qp", list);
            copy_moist_sum(list);
        }
 
        if (containerHasElement(plot_var_names, "qsat"))
        {
            const bool anelastic = solverChoice.anelastic[lev];
            const Real rdOcp = solverChoice.rdOcp;
            const int qv_comp = mi.qv;
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf_dst.array(mfi);
                const Array4<Real const>& p_arr = pressure.array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    Real qv = S_arr(i,j,k,qv_comp) / S_arr(i,j,k,Rho_comp);
                    Real T  = anelastic ?
                        getTgivenPandTh(p_arr(i,j,k), S_arr(i,j,k,RhoTheta_comp) /
                                        S_arr(i,j,k,Rho_comp), rdOcp) :
                        getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                    Real p  = p_arr(i,j,k) * Real(0.01);
                    erf_qsatw(T, p, derdat(i,j,k,mf_comp));
                });
            }
            mf_comp ++;
        }
 
        // Number concentrations
        //--------------------------------------------------------------------------
        // Note that nn, the CCN / total aerosol reservoir, may share a conserved
        // component with the cloud ice number of another scheme; the index map keeps
        // the two apart, so nothing here needs to know which scheme is running.
        copy_moist_species("nc");
        copy_moist_species("ni");
        copy_moist_species("nr");
        copy_moist_species("ns");
        copy_moist_species("ng");
        copy_moist_species("nn");
 
        // Surface accumulations, from the scheme's qmoist arrays
        //--------------------------------------------------------------------------
        copy_moist_diagnostic("rain_accum");
        copy_moist_diagnostic("snow_accum");
        copy_moist_diagnostic("graup_accum");
 
        // Relative humidity.  A scheme either publishes it in a qmoist array or, as
        // SatAdj does, publishes no qmoist arrays at all and leaves it to be
        // recovered here from the conserved state.
        if (mi.rel_hum == MoistureComponentIndices::computed_from_state)
        {
            if (containerHasElement(plot_var_names, "rel_humidity"))
            {
                const Real rdOcp = solverChoice.rdOcp;
                const bool anelastic = solverChoice.anelastic[lev];
                const int qv_comp = mi.qv;
                for (MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    const Box& bx = mfi.tilebox();
                    const auto derdat = mf_dst.array(mfi);
                    const auto state = vars_new[lev][Vars::cons].const_array(mfi);
                    const auto p0 = p_hse.const_array(mfi);
                    const auto pfield = pressure.const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                        const Real rho = state(i,j,k,Rho_comp);
                        const Real qv = state(i,j,k,qv_comp) / rho;
                        const Real theta = state(i,j,k,RhoTheta_comp) / rho;
                        const Real p = anelastic ? p0(i,j,k) : pfield(i,j,k);
                        const Real T = anelastic ?
                            getTgivenPandTh(p0(i,j,k), theta, rdOcp) :
                            getTgivenRandRTh(rho, state(i,j,k,RhoTheta_comp), qv);
                        const Real vapor_pressure = p * qv / (RdoRv + qv);
                        derdat(i,j,k,mf_comp) = vapor_pressure /
                            (Real(100.0) * erf_esatw(T));
                    });
                }
                ++mf_comp;
            }
        } else {
            copy_moist_diagnostic("rel_humidity");
        }
 
        copy_moist_diagnostic("condensation_rate");
 
    } // if has_moisture
 
    if (containerHasElement(plot_var_names, "terrain_IB_mask"))
    {
        MultiFab* terrain_blank = terrain_blanking[lev].get();
        MultiFab::Copy(mf_dst,*terrain_blank,0,mf_comp,1,0);
        mf_comp ++;
    }
 
    // Immersed-boundary surface energy balance: faces per cell and their mean skin temperature
    if (containerHasElement(plot_var_names, "ibseb_nfaces") ||
        containerHasElement(plot_var_names, "ibseb_tskin"))
    {
        MultiFab nfaces(grids[lev], dmap[lev], 1, 0);
        MultiFab tskin (grids[lev], dmap[lev], 1, 0);
        if (ibseb_params.enable && lev < static_cast<int>(m_ibseb.size()) && m_ibseb[lev]) {
            m_ibseb[lev]->scatter_diagnostics(nfaces, tskin);
        } else {
            nfaces.setVal(0.0);
            tskin.setVal(0.0);
        }
        if (containerHasElement(plot_var_names, "ibseb_nfaces")) {
            MultiFab::Copy(mf_dst, nfaces, 0, mf_comp, 1, 0);
            mf_comp++;
        }
        if (containerHasElement(plot_var_names, "ibseb_tskin")) {
            MultiFab::Copy(mf_dst, tskin, 0, mf_comp, 1, 0);
            mf_comp++;
        }
    }
    // Radiation on the faces, per-cell means: absorbed shortwave, shadow
    // flag, net longwave, sky view fraction
    for (const char* nm : {"ibseb_sw_abs", "ibseb_shadow", "ibseb_lw_net", "ibseb_f_sky", "ibseb_H", "ibseb_G"}) {
        if (!containerHasElement(plot_var_names, nm)) { continue; }
        MultiFab tmp(grids[lev], dmap[lev], 1, 0);
        if (ibseb_params.enable && lev < static_cast<int>(m_ibseb.size()) && m_ibseb[lev]) {
            const std::string s(nm);
            const auto& v = (s == "ibseb_sw_abs") ? m_ibseb[lev]->d_SW_abs
                          : (s == "ibseb_shadow") ? m_ibseb[lev]->d_shadow
                          : (s == "ibseb_lw_net") ? m_ibseb[lev]->d_LW_net
                          : (s == "ibseb_H")      ? m_ibseb[lev]->d_H
                          : (s == "ibseb_G")      ? m_ibseb[lev]->d_G
                          :                         m_ibseb[lev]->d_f_sky;
            m_ibseb[lev]->scatter_field(v, tmp);
        } else {
            tmp.setVal(0.0);
        }
        MultiFab::Copy(mf_dst, tmp, 0, mf_comp, 1, 0);
        mf_comp++;
    }
 
    if (containerHasElement(plot_var_names, "volfrac")) {
        if ( solverChoice.terrain_type == TerrainType::EB ||
             solverChoice.terrain_type == TerrainType::ImmersedForcing)
        {
            MultiFab::Copy(mf_dst, EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
        } else {
            mf_dst.setVal(1.0, mf_comp, 1, 0);
        }
        mf_comp += 1;
    }
 
#ifdef ERF_COMPUTE_ERROR
    // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
    if (containerHasElement(plot_var_names, "xvel_err") ||
        containerHasElement(plot_var_names, "yvel_err") ||
        containerHasElement(plot_var_names, "zvel_err"))
    {
        //
        // Moving terrain ANALYTICAL
        //
        Real H           = geom[lev].ProbHi()[2];
        Real Ampl        = Real(0.16);
        Real wavelength  = Real(100.);
        Real kp          = two * PI / wavelength;
        Real g           = CONST_GRAV;
        Real omega       = std::sqrt(g * kp);
        Real omega_t     = omega * t_new[lev];
 
        const auto dx = geom[lev].CellSizeArray();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for (MFIter mfi(mf_dst, TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.validbox();
            Box xbx(bx); xbx.surroundingNodes(0);
            const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
            const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
 
            const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
 
            ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real x = i * dx[0];
                Real z = fourth * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
 
                Real z_base = Ampl * std::sin(kp * x - omega_t);
                z -= z_base;
 
                Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
 
                xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
            });
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real x   = (i + myhalf) * dx[0];
                Real z   = fourth * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
 
                Real z_base = Ampl * std::sin(kp * x - omega_t);
                z -= z_base;
 
                Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
 
                zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
            });
        }
 
        MultiFab temp_mf(mf_dst.boxArray(), mf_dst.DistributionMap(), AMREX_SPACEDIM, 0);
        average_face_to_cellcenter(temp_mf,0,
            Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
 
        if (containerHasElement(plot_var_names, "xvel_err")) {
            MultiFab::Copy(mf_dst,temp_mf,0,mf_comp,1,0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "yvel_err")) {
            MultiFab::Copy(mf_dst,temp_mf,1,mf_comp,1,0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "zvel_err")) {
            MultiFab::Copy(mf_dst,temp_mf,2,mf_comp,1,0);
            mf_comp += 1;
        }
 
        // Now restore the velocities to what they were
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for (MFIter mfi(mf_dst, TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.validbox();
            Box xbx(bx); xbx.surroundingNodes(0);
 
            const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
            const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
 
            const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
 
            ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real x = i * dx[0];
                Real z = fourth * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
                Real z_base = Ampl * std::sin(kp * x - omega_t);
 
                z -= z_base;
 
                Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
            });
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real x   = (i + myhalf) * dx[0];
                Real z   = fourth * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
                Real z_base = Ampl * std::sin(kp * x - omega_t);
 
                z -= z_base;
                Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
 
                zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
            });
        }
    } // end xvel_err, yvel_err, zvel_err
 
    if (containerHasElement(plot_var_names, "pp_err"))
    {
        // Moving terrain ANALYTICAL
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(mf_dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& derdat = mf_dst.array(mfi);
            const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
            const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
 
            const auto dx = geom[lev].CellSizeArray();
            const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
            const Array4<Real const>&  p_arr = pressure.const_array(mfi);
            const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
 
            Real H           = geom[lev].ProbHi()[2];
            Real Ampl        = Real(0.16);
            Real wavelength  = Real(100.);
            Real kp          = two * PI / wavelength;
            Real g           = CONST_GRAV;
            Real omega       = std::sqrt(g * kp);
            Real omega_t     = omega * t_new[lev];
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                derdat(i, j, k, mf_comp) = p_arr(i,j,k) - p0_arr(i,j,k);
 
                Real rho_hse     = r0_arr(i,j,k);
 
                Real x   = (i + myhalf) * dx[0];
                Real z   = Real(0.125) * ( z_nd(i,j,k  ) + z_nd(i+1,j,k  ) + z_nd(i,j+1,k  ) + z_nd(i+1,j+1,k  )
                                    +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
                Real z_base = Ampl * std::sin(kp * x - omega_t);
 
                z -= z_base;
                Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
                                          std::sin(kp * x - omega_t) * r0_arr(i,j,k);
 
                derdat(i,j,k,mf_comp) -= pprime_exact;
            });
        }
        mf_comp += 1;
    }
#endif
 
    if (solverChoice.rad_type != RadiationType::None) {
        if (containerHasElement(plot_var_names, "qsrc_sw") ||
            containerHasElement(plot_var_names, "qsrc_lw")) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
                qheating_rates[lev] != nullptr,
                "radiation source plot variables require qheating_rates storage");
        }
        if (containerHasElement(plot_var_names, "qsrc_sw")) {
            MultiFab::Copy(mf_dst, *(qheating_rates[lev]), 0, mf_comp, 1, 0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "qsrc_lw")) {
            MultiFab::Copy(mf_dst, *(qheating_rates[lev]), 1, mf_comp, 1, 0);
            mf_comp += 1;
        }
    }
 
    const bool plot_tau =
        containerHasElement(plot_var_names, "Tau11") ||
        containerHasElement(plot_var_names, "Tau12") ||
        containerHasElement(plot_var_names, "Tau13") ||
        containerHasElement(plot_var_names, "Tau21") ||
        containerHasElement(plot_var_names, "Tau22") ||
        containerHasElement(plot_var_names, "Tau23") ||
        containerHasElement(plot_var_names, "Tau31") ||
        containerHasElement(plot_var_names, "Tau32") ||
        containerHasElement(plot_var_names, "Tau33");
    if (plot_tau) {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            Tau[lev][TauType::tau11] != nullptr,
            "Tau plot variables require diffusion to be enabled");
    }
 
    if (containerHasElement(plot_var_names, "Tau11")) {
        MultiFab::Copy(mf_dst,*Tau[lev][TauType::tau11],0,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "Tau12")) {
        MultiFab::Copy(mf_dst,mf_cc_tau[lev],TauType::tau12,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "Tau13")) {
        MultiFab::Copy(mf_dst,mf_cc_tau[lev],TauType::tau13,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "Tau21")) {
        MultiFab::Copy(mf_dst,mf_cc_tau[lev],TauType::tau21,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "Tau22")) {
        MultiFab::Copy(mf_dst,*Tau[lev][TauType::tau22],0,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "Tau23")) {
        MultiFab::Copy(mf_dst,mf_cc_tau[lev],TauType::tau23,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "Tau31")) {
        MultiFab::Copy(mf_dst,mf_cc_tau[lev],TauType::tau31,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "Tau32")) {
        MultiFab::Copy(mf_dst,mf_cc_tau[lev],TauType::tau32,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "Tau33")) {
        MultiFab::Copy(mf_dst,*Tau[lev][TauType::tau33],0,mf_comp,1,0);
        mf_comp ++;
    }
 
    if (containerHasElement(plot_var_names, "hfx1")) {
        MultiFab::Copy(mf_dst,mf_cc_fx[lev],0,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "hfx2")) {
        MultiFab::Copy(mf_dst,mf_cc_fx[lev],1,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "hfx3")) {
        MultiFab::Copy(mf_dst,mf_cc_fx[lev],2,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "q1fx1")) {
        MultiFab::Copy(mf_dst,mf_cc_fx[lev],3,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "q1fx2")) {
        MultiFab::Copy(mf_dst,mf_cc_fx[lev],4,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "q1fx3")) {
        MultiFab::Copy(mf_dst,mf_cc_fx[lev],5,mf_comp,1,0);
        mf_comp ++;
    }
    if (containerHasElement(plot_var_names, "q2fx3")) {
        MultiFab::Copy(mf_dst,mf_cc_fx[lev],6,mf_comp,1,0);
        mf_comp ++;
    }
 
    // *****************************************************************************************
    // End of derived variables corresponding to "derived_names" in ERF.H
    //
    // Particles and microphysics can provide additional outputs, which are handled below.
    // *****************************************************************************************
 
#ifdef ERF_USE_PARTICLES
    Vector<std::string> configured_particle_names;
    for (const auto& name : particleData.getNamesUnalloc()) {
        configured_particle_names.push_back(name);
    }
    for (const auto& name : particleData.getNames()) {
        if (!containerHasElement(configured_particle_names, name)) {
            configured_particle_names.push_back(name);
        }
    }
 
    const Vector<std::string> requested_particle_counts =
        erf_plotfile::plot3d_selected_particle_count_names(plot_var_names,
                                                            configured_particle_names);
    for (const auto& name : requested_particle_counts) {
        MultiFab temp_dat(mf_dst.boxArray(), mf_dst.DistributionMap(), 1, 0);
        temp_dat.setVal(0);
        if (particleData.HasSpecies(name)) {
            particleData[name]->Increment(temp_dat, lev);
        }
        MultiFab::Copy(mf_dst, temp_dat, 0, mf_comp, 1, 0);
        mf_comp += 1;
    }
 
    Vector<std::string> particle_mesh_plot_names(0);
    particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
 
    for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
        std::string plot_var_name(particle_mesh_plot_names[i]);
        if (containerHasElement(plot_var_names, plot_var_name) ) {
            MultiFab temp_dat(mf_dst.boxArray(), mf_dst.DistributionMap(), 1, 1);
            temp_dat.setVal(0);
            particleData.GetMeshPlotVar(plot_var_name, temp_dat, *z_phys_nd[lev], lev);
            MultiFab::Copy(mf_dst, temp_dat, 0, mf_comp, 1, 0);
            mf_comp += 1;
        }
    }
#endif
 
    {
        Vector<std::string> microphysics_plot_names;
        micro->GetPlotVarNames(microphysics_plot_names);
        for (auto& plot_name : microphysics_plot_names) {
            if (containerHasElement(plot_var_names, plot_name)) {
                MultiFab temp_dat(mf_dst.boxArray(), mf_dst.DistributionMap(), 1, 1);
                temp_dat.setVal(0);
                micro->GetPlotVar(plot_name, temp_dat, lev);
                MultiFab::Copy(mf_dst, temp_dat, 0, mf_comp, 1, 0);
                mf_comp += 1;
            }
        }
    }
 
    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
        mf_comp == ncomp_mf,
        "3D plotfile component assembly does not match selected variable names");
}

Here is the call graph for this function:

FillSurfaceStateMultiFabs()

void ERF::FillSurfaceStateMultiFabs	(	const int	lev,
		const std::string &	filename,
		amrex::Vector< amrex::MultiFab > &	surface_state
	)

Read surface state data from a binary file and interpolate onto MultiFabs.

Parameters

[in]	lev	Current level.
[in]	filename	Path to the binary data file.
[out]	surface_state	MultiFabs to be filled with interpolated surface data.

{
       // Open the binary file in input mode
    std::ifstream infile(filename, std::ios::binary);
    if (!infile) {
        std::cerr << "Error: Could not open file " << filename << std::endl;
    }
    Vector<Real> xvec_h, yvec_h, zvec_h;
    Vector<Real> sst_h, q_star_h, t_star_h, u_star_h, ls_mask_h;
 
    int nx, ny, nz, ndata;
    float value;
 
    // Read the four integers
    infile.read(reinterpret_cast<char*>(&nx), sizeof(int));
    infile.read(reinterpret_cast<char*>(&ny), sizeof(int));
    infile.read(reinterpret_cast<char*>(&nz), sizeof(int));
    infile.read(reinterpret_cast<char*>(&ndata), sizeof(int));
 
    amrex::Gpu::DeviceVector<Real> xvec_d(nx*ny*nz), yvec_d(nx*ny*nz), zvec_d(nz);
    for(int i=0; i<nx; i++) {
        infile.read(reinterpret_cast<char*>(&value), sizeof(float));
        xvec_h.emplace_back(value);
    }
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, xvec_h.begin(), xvec_h.end(), xvec_d.begin());
 
    for(int j=0; j<ny; j++) {
        infile.read(reinterpret_cast<char*>(&value), sizeof(float));
        yvec_h.emplace_back(value);
    }
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, yvec_h.begin(), yvec_h.end(), yvec_d.begin());
 
    for(int k=0; k<nz; k++) {
        infile.read(reinterpret_cast<char*>(&value), sizeof(float));
        zvec_h.emplace_back(value);
    }
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, zvec_h.begin(), zvec_h.end(), zvec_d.begin());
 
         // Vector to store the data
 
    Vector<Real>* data_h = nullptr; // Declare pointer outside the loop
 
    Real* xvec_d_ptr = xvec_d.data();
    Real* yvec_d_ptr = yvec_d.data();
 
    Real dxvec = (xvec_h[nx-1]-xvec_h[0])/(nx-1);
    Real dyvec = (yvec_h[ny-1]-yvec_h[0])/(ny-1);
 
    // Read the file
    for(int idx=0; idx<ndata; idx++){
        if(idx == 0){
            data_h = &sst_h;
        } else if (idx==1) {
            data_h = &q_star_h;
        } else if (idx==2) {
            data_h = &t_star_h;
        } else if (idx==3) {
            data_h = &u_star_h;
        } else if(idx==4) {
            data_h = &ls_mask_h;
        }
        for(int k=0; k<nz; k++) {
            for(int j=0; j<ny; j++) {
                for(int i=0; i<nx; i++) {
                    infile.read(reinterpret_cast<char*>(&value), sizeof(float));
                    //if(idx == 3) {
                        //printf("theta is %0.15g, %0.15g, %0.15g %0.15g\n", xvec_h[i], yvec_h[j], zvec_h[k], value);
                    //}
                    data_h->emplace_back(value);
                }
            }
        }
    }
 
    infile.close();
 
    amrex::Gpu::DeviceVector<Real> ls_mask_d(nx*ny*nz), sst_d(nx*ny*nz);
 
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, ls_mask_h.begin(), ls_mask_h.end(), ls_mask_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, sst_h.begin(), sst_h.end(), sst_d.begin());
 
    Real* ls_mask_d_ptr = ls_mask_d.data();
    Real* sst_d_ptr   = sst_d.data();
 
    const auto prob_lo  = geom[lev].ProbLo();
    const auto dx       = geom[lev].CellSize();
 
    for (amrex::MFIter mfi(surface_state[lev]); mfi.isValid(); ++mfi) {
        const Box gbx = mfi.growntilebox();
        const Array4<Real>& surf_arr = surface_state[lev].array(mfi);
 
        ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
 
            if(k == 0) {
                const Real x        = prob_lo[0] + (i + myhalf) * dx[0];
                const Real y        = prob_lo[1] + (j + myhalf) * dx[1];
 
                // First interpolate where the weather data is available from
                Real tmp_ls_mask, tmp_sst;
 
                bilinear_interpolation_2d(xvec_d_ptr, yvec_d_ptr,
                                          dxvec, dyvec,
                                          nx, ny,
                                          x, y,
                                          ls_mask_d_ptr, tmp_ls_mask);
 
                bilinear_interpolation_2d(xvec_d_ptr, yvec_d_ptr,
                                          dxvec, dyvec,
                                          nx, ny,
                                          x, y,
                                          sst_d_ptr, tmp_sst);
 
                surf_arr(i, j, 0, 0) = std::min(tmp_ls_mask, amrex::Real(1.0));
                surf_arr(i, j, 0, 1) = tmp_sst;
            }
        });
    }
 
}

Here is the call graph for this function:

FindInitialEye()

bool ERF::FindInitialEye	(	int	lev,
		const amrex::MultiFab &	cc_vel,
		const amrex::Real	velmag_threshold,
		amrex::Real &	eye_x,
		amrex::Real &	eye_y
	)

Locate the initial position of the hurricane eye.

Parameters

	levc	Current level.
	mf_cc_vel	MultiFab containing cell-centered velocity.
	velmag_threshold	Velocity magnitude threshold for eye identification.
[out]	eye_x	Found x-coordinate of the eye.
[out]	eye_y	Found y-coordinate of the eye.

Returns

True if the eye was found, false otherwise.

{
    const auto dx = geom[levc].CellSizeArray();
    const auto prob_lo = geom[levc].ProbLoArray();
 
    Gpu::DeviceVector<Real> d_coords(2, zero);
    Gpu::DeviceVector<int>  d_found(1,0);
 
    Real* d_coords_ptr = d_coords.data();
    int*   d_found_ptr = d_found.data();
 
    for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi)
    {
        const Box& box = mfi.validbox();
        const Array4<const Real>& vel_arr = mf_cc_vel.const_array(mfi);
 
        ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k)
        {
            Real magnitude = std::sqrt(vel_arr(i,j,k,0) * vel_arr(i,j,k,0) +
                                       vel_arr(i,j,k,1) * vel_arr(i,j,k,1) +
                                       vel_arr(i,j,k,2) * vel_arr(i,j,k,2));
 
            magnitude *= Real(3.6);
 
            Real z = prob_lo[2] + (k + myhalf) * dx[2];
 
            // Check if magnitude exceeds threshold
            if (z < Real(2000.) && magnitude > velmag_threshold) {
                // Use atomic operations to set found flag and store coordinates
                Gpu::Atomic::Add(&d_found_ptr[0], 1); // Mark as found
 
                Real x = prob_lo[0] + (i + myhalf) * dx[0];
                Real y = prob_lo[1] + (j + myhalf) * dx[1];
 
                // Store coordinates
                Gpu::Atomic::Add(&d_coords_ptr[0],x); // Store x index
                Gpu::Atomic::Add(&d_coords_ptr[1],y); // Store x index
            }
        });
    }
 
    // Synchronize to ensure all threads complete their execution
    amrex::Gpu::streamSynchronize(); // Wait for all GPU threads to finish
 
    Vector<int> h_found(1,0);
    Gpu::copy(Gpu::deviceToHost, d_found.begin(), d_found.end(), h_found.begin());
    ParallelAllReduce::Sum(h_found.data(), h_found.size(), ParallelContext::CommunicatorAll());
 
    // Broadcast coordinates if found
    if (h_found[0] > 0) {
        Vector<Real> h_coords(2,-bogus_large_value);
        Gpu::copy(Gpu::deviceToHost, d_coords.begin(), d_coords.end(), h_coords.begin());
 
        ParallelAllReduce::Sum(h_coords.data(), h_coords.size(), ParallelContext::CommunicatorAll());
 
        eye_x = h_coords[0]/h_found[0];
        eye_y = h_coords[1]/h_found[0];
 
    } else {
        // Random large negative numbers so we don't trigger refinement in this case
        eye_x = -bogus_large_value;
        eye_y = -bogus_large_value;
    }
 
    return (h_found[0] > 0);
}

Here is the call graph for this function:

flush_stations()

void ERF::flush_stations

(

)

const

{
    if (station_sampler) { station_sampler->flush(); }
}

get_eb()

eb_ const& ERF::get_eb ( int  lev ) const

inlineprivatenoexcept

                                                             {
        AMREX_ASSERT(lev >= 0 && lev < eb.size() && eb[lev] != nullptr);
        return *eb[lev];
    }

getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int  lev )

inlineprivate

    {
        return advflux_reg[lev];
    }

getCPUTime()

static double ERF::getCPUTime ( )

inlinestaticprivate

    {
      int numCores = amrex::ParallelDescriptor::NProcs();
#ifdef _OPENMP
      numCores = numCores * omp_get_max_threads();
#endif
 
      double T =
        numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
        previousCPUTimeUsed;
 
      return T;
    }

GetEnsembleCheckpointName()

void ERF::GetEnsembleCheckpointName	(	int	da_iter,
		int	ens_no
	)

GetFillableAtmosToOceanLanes()

void ERF::GetFillableAtmosToOceanLanes	(	bool	state_contract,
		amrex::Vector< int > &	can_fill
	)		const

{
    // Each entry restates PackAtmosphericStates' own gate for that lane, which
    // is why this lives beside it: changing a pack gate - re-enabling an #if 0
    // block included - means changing the matching line here. Report false and
    // the driver withholds the lane; report true wrongly and the ocean reads the
    // driver's prefill as an atmospheric value.
    //
    // A configuration predicate, like HasRadiation()/HasCloudWater(): the lane
    // will be written, not that any physics step has run.
    const int lev = 0;
    const bool has_moisture  = (solverChoice.moisture_type != MoistureType::None);
    const bool has_radiation = HasRadiation();
    const bool has_sfs_heat  = (!SFS_hfx3_lev.empty()  && SFS_hfx3_lev[lev]  != nullptr);
    const bool has_sfs_moist = (!SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev] != nullptr);
 
    if (state_contract) {
        can_fill.resize(nStateLanes, 0);
        can_fill[iUwind] = 1;                       // always packed from Vars::xvel
        can_fill[iVwind] = 1;                       // always packed from Vars::yvel
        can_fill[iPatm]  = 1;                       // always packed via getPgivenRTh
        can_fill[iTair]  = 1;                       // always packed via getTgivenRandRTh
        // #if 0 below, pending Qair (RH vs specific humidity) and rain-rate
        // semantics.
        can_fill[iRH]    = 0;
        can_fill[iRain]  = 0;
        can_fill[iCloud] = (has_moisture &&
                            (solverChoice.moisture_indices.qc != -1 ||
                             solverChoice.moisture_indices.qi != -1)) ? 1 : 0;
        can_fill[iSWrad] = has_radiation ? 1 : 0;
        can_fill[iLWrad] = has_radiation ? 1 : 0;
    } else {
        can_fill.resize(nFluxLanes, 0);
        // tau_x/tau_y assert rather than skip, so gate on what they assert on.
        can_fill[iTauX]      = (Tau[lev][TauType::tau13] != nullptr) ? 1 : 0;
        can_fill[iTauY]      = (Tau[lev][TauType::tau23] != nullptr) ? 1 : 0;
        can_fill[iSHflux]    = has_sfs_heat  ? 1 : 0;
        can_fill[iLHflux]    = has_sfs_moist ? 1 : 0;
        can_fill[iFluxSWrad] = has_radiation ? 1 : 0;
        can_fill[iFluxLWrad] = has_radiation ? 1 : 0;
        can_fill[iFluxRain]  = 0;                   // pack block is #if 0
        can_fill[iFluxEvap]  = has_sfs_moist ? 1 : 0;
    }
}

Here is the call graph for this function:

GetLandMask()

void ERF::GetLandMask

(

const amrex::iMultiFab *&

lmask

)

const

{
    if (lmask_lev.empty() || lmask_lev[0].empty() || lmask_lev[0][0] == nullptr) {
        lmask = nullptr;
        return;
    }
    lmask = lmask_lev[0][0].get();
}

GetOceanToAtmosCellCenterLonLat()

void ERF::GetOceanToAtmosCellCenterLonLat	(	const amrex::MultiFab *&	lon_cc,
		const amrex::MultiFab *&	lat_cc
	)		const

{
    if (lon_m.empty() || lat_m.empty() ||
        lon_m[0] == nullptr || lat_m[0] == nullptr) {
        lon_cc = nullptr;
        lat_cc = nullptr;
        return;
    }
    lon_cc = lon_m[0].get();
    lat_cc = lat_m[0].get();
}

GetOceanToAtmosSurfaceLayout()

void ERF::GetOceanToAtmosSurfaceLayout	(	amrex::BoxArray &	ba,
		amrex::DistributionMapping &	dm
	)

{
    // The level-0 flattened cell layout.
    // We take ba2d[0] directly and assert the shape the driver depends on.
    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
        !ba2d.empty() && ba2d[0].size() == grids[0].size(),
        "ERF::GetOceanToAtmosSurfaceLayout requires the level-0 2D layout after InitData.");
    ba = ba2d[0];
    dm = dmap[0];
}

Referenced by ApplyOceanSurfaceState().

Here is the call graph for this function:

Here is the caller graph for this function:

GotoNextLine()

void ERF::GotoNextLine ( std::istream &  is )

staticprivate

Utility to skip to next line in Header file input stream.

{
    constexpr std::streamsize bl_ignore_max { 100000 };
    is.ignore(bl_ignore_max, '\n');
}

grids_are_split_in_z()

bool ERF::grids_are_split_in_z ( const amrex::BoxArray &  ba )

staticprivate

Is any (i,j) column of this BoxArray covered by more than one box?

Projecting every box onto a common z index turns "two boxes stacked in z" into "two overlapping 2D boxes", so a non-disjoint projection is exactly the test for a grid that has been decomposed in the vertical.

Parameters

[in]

ba

BoxArray to test

{
    BoxList bl(ba.ixType());
    for (int i(0); i < ba.size(); ++i) {
        Box b(ba[i]); b.setRange(2,0);
        bl.push_back(b);
    }
    return !(BoxArray(std::move(bl)).isDisjoint());
}

has_own_init_file()

bool ERF::has_own_init_file

(

int

lev

)

const

{
    if (lev < 0 || lev >= static_cast<int>(nc_init_file.size())) { return false; }
    return !nc_init_file[lev].empty();
}

has_surface_layer_inputs()

bool ERF::has_surface_layer_inputs ( const std::string &  prefix ) const

private

Whether the user wrote any surface-layer input under the given prefix.

This asks the snapshot taken in ReadParameters, not the live ParmParse table. The live table cannot answer the question: ERF_InputSoundingData.H queries erf.most.surf_temp and erf.most.surf_moist with queryAdd while reading the sounding, which runs during InitData_pre and leaves both keys in the table holding their negative sentinels. Testing the live table would therefore report unqualified MOST inputs for every input_sounding run, whether or not the user wrote any.

Parameters

[in]

prefix

the prefix to test, e.g. "erf" or "erf.zlo"

{
    const std::string most_prefix = prefix + ".most.";
    const std::string surface_layer_prefix = prefix + ".surface_layer.";
 
    for (const auto& key : user_specified_inputs) {
        if (key.compare(0, most_prefix.size(), most_prefix) == 0 ||
            key.compare(0, surface_layer_prefix.size(), surface_layer_prefix) == 0) {
            return true;
        }
    }
    return false;
}

HasCloudWater()

bool ERF::HasCloudWater ( ) const

inline

                                              {
        return (solverChoice.moisture_type != MoistureType::None) &&
               (solverChoice.moisture_indices.qc != -1 ||
                solverChoice.moisture_indices.qi != -1);
    }

HasRadiation()

bool ERF::HasRadiation ( ) const

inline

                                             {
        return (!rad_fluxes.empty() && rad_fluxes[0] != nullptr);
    }

Referenced by GetFillableAtmosToOceanLanes(), and PackAtmosphericStates().

Here is the caller graph for this function:

HurricaneEyeTracker()

void ERF::HurricaneEyeTracker

(

const SolverChoice &

sc

)

Wrapper to track the hurricane eye position over time.

Parameters

[in]

sc

Solver choices

{
    static bool is_start = true;
    int levc=finest_level;
 
    const MoistureType moisture_type   = sc.moisture_type;
    const Real hurricane_eye_latitude  = sc.hurricane_eye_latitude;
    const Real hurricane_eye_longitude = sc.hurricane_eye_longitude;
 
    if(is_start and restart_chkfile.empty()){
        HurricaneEyeTrackerInitial(sc, geom[levc],
                                   vars_new[levc],
                                   hurricane_eye_latitude,
                                   hurricane_eye_longitude);
        is_start = false;
    } else {
         if(!restart_chkfile.empty()) {
            ReadStormTrackerRestart();
        }
        HurricaneEyeTrackerNotInitial(sc, geom[levc], vars_new[levc],
                                      moisture_type);
    }
    HurricaneTrackerCircle();
}

HurricaneEyeTrackerInitial()

void ERF::HurricaneEyeTrackerInitial	(	const SolverChoice &	sc,
		const amrex::Geometry &	geom,
		const amrex::Vector< amrex::MultiFab > &	S_data,
		const amrex::Real &	hurricane_eye_latitude,
		const amrex::Real &	hurricane_eye_longitude
	)

Initialize the hurricane eye tracker using a given latitude and longitude.

Parameters

[in]	sc	Solver choices
[in]	lev_geom	Geometry of the current level
[in]	S_data	Conservative state data
[in]	hurricane_eye_latitude	Target latitude for the eye
[in]	hurricane_eye_longitude	Target longitude for the eye

{
    int levc = finest_level;
    Gpu::DeviceScalar<Real> d_val_min(1e10);
    Gpu::DeviceScalar<int> d_i_min(-1), d_j_min(-1);
 
    Real* d_val_min_ptr = d_val_min.dataPtr();
    int* d_i_min_ptr = d_i_min.dataPtr();
    int* d_j_min_ptr = d_j_min.dataPtr();
 
    // NOTE: the arg-min is done in two passes. A single pass that takes an
    //       atomic min of the distance and then plainly stores i and j is
    //       racy: a thread with a worse distance can still win the store and
    //       leave an eye location that does not belong to the recorded
    //       minimum. The first pass reduces the distance only; the second
    //       records the location of the cells that attain it, through one
    //       atomic on a packed index so value and location stay consistent.
    const Dim3 dlo = lbound(lev_geom.Domain());
    const int nx = lev_geom.Domain().length(0);
 
    Gpu::DeviceScalar<Long> d_idx_min(std::numeric_limits<Long>::max());
    Long* d_idx_min_ptr = d_idx_min.dataPtr();
 
    if(sc.init_type == InitType::WRFInput){
        for (MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
            const Box& box = mfi.validbox();
            FArrayBox& fab_lat = (*(lat_m[levc]))[mfi];
            FArrayBox& fab_lon = (*(lon_m[levc]))[mfi];
            const Array4<Real>& lat_arr = fab_lat.array();
            const Array4<Real>& lon_arr = fab_lon.array();
 
            ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                if (k==0) {
 
                    Real dlat = lat_arr(i,j,0) - hurricane_eye_latitude;
                    Real dlon = lon_arr(i,j,0) - hurricane_eye_longitude;
                    Real dist = std::sqrt(dlat*dlat + dlon*dlon);
                    Gpu::Atomic::Min(&d_val_min_ptr[0], dist);
                }
            });
        }
 
        // The minimum over every box on this rank must be known before the
        // locating pass below can test against it.
        Gpu::synchronize();
 
        for (MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
            const Box& box = mfi.validbox();
            FArrayBox& fab_lat = (*(lat_m[levc]))[mfi];
            FArrayBox& fab_lon = (*(lon_m[levc]))[mfi];
            const Array4<Real>& lat_arr = fab_lat.array();
            const Array4<Real>& lon_arr = fab_lon.array();
 
            ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                if (k==0) {
 
                    Real dlat = lat_arr(i,j,0) - hurricane_eye_latitude;
                    Real dlon = lon_arr(i,j,0) - hurricane_eye_longitude;
                    Real dist = std::sqrt(dlat*dlat + dlon*dlon);
                    if (dist == d_val_min_ptr[0]) {
                        Gpu::Atomic::Min(d_idx_min_ptr, pack_ij(i,j,nx,dlo));
                    }
                }
            });
        }
    }
 
    if(sc.init_type == InitType::HindCast){
        for (amrex::MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
            const amrex::Box& box = mfi.validbox();
            const auto& mf_latlon = forecast_state_interp[levc][4];
            const auto latlon_arr = mf_latlon.array(mfi);
 
            amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                if (k==0) {
 
                    amrex::Real dlat = latlon_arr(i,j,k,0) - hurricane_eye_latitude;
                    amrex::Real dlon = latlon_arr(i,j,k,1) - hurricane_eye_longitude;
                    amrex::Real dist = std::sqrt(dlat*dlat + dlon*dlon);
                    amrex::Gpu::Atomic::Min(&d_val_min_ptr[0], dist);
                }
            });
        }
 
        // The minimum over every box on this rank must be known before the
        // locating pass below can test against it.
        Gpu::synchronize();
 
        for (amrex::MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
            const amrex::Box& box = mfi.validbox();
            const auto& mf_latlon = forecast_state_interp[levc][4];
            const auto latlon_arr = mf_latlon.array(mfi);
 
            amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                if (k==0) {
 
                    amrex::Real dlat = latlon_arr(i,j,k,0) - hurricane_eye_latitude;
                    amrex::Real dlon = latlon_arr(i,j,k,1) - hurricane_eye_longitude;
                    amrex::Real dist = std::sqrt(dlat*dlat + dlon*dlon);
                    if (dist == d_val_min_ptr[0]) {
                        amrex::Gpu::Atomic::Min(d_idx_min_ptr, pack_ij(i,j,nx,dlo));
                    }
                }
            });
        }
    }
 
    Gpu::synchronize();
 
    // Unpack the located index back into the (i,j) device scalars that
    // ComputeGlobalMinLocation reads.
    {
        int h_i_min, h_j_min;
        unpack_ij(d_idx_min.dataValue(), nx, dlo, h_i_min, h_j_min);
        Gpu::copy(Gpu::hostToDevice, &h_i_min, &h_i_min + 1, d_i_min_ptr);
        Gpu::copy(Gpu::hostToDevice, &h_j_min, &h_j_min + 1, d_j_min_ptr);
    }
 
    Real global_val_min;
    int global_i_min, global_j_min;
 
    ComputeGlobalMinLocation(sc, lev_geom, S_data,
                             d_val_min_ptr, d_i_min_ptr, d_j_min_ptr,
                             global_val_min, global_i_min, global_j_min);
}

Here is the call graph for this function:

HurricaneEyeTrackerNotInitial()

void ERF::HurricaneEyeTrackerNotInitial	(	const SolverChoice &	sc,
		const amrex::Geometry &	geom,
		const amrex::Vector< amrex::MultiFab > &	S_data,
		MoistureType	moisture_type
	)

Track the hurricane eye by searching for minimum pressure near the previous position.

Parameters

[in]	sc	Solver choices
[in]	lev_geom	Geometry of the current level
[in]	S_data	Conservative state data
[in]	moisture_type	Moisture model type

{
 
    if (hurricane_eye_track_xy.empty()) {
        Print() << "Error: hurricane_eye_track_xy is empty!\n";
        Abort("Attempted to access hurricane_eye_track_xy[0]");
    }
 
    Real tmp_x_eye = hurricane_eye_track_xy.back()[0];
    Real tmp_y_eye = hurricane_eye_track_xy.back()[1];
 
    if(ParallelDescriptor::IOProcessor()){
        std::cout << "The value of x y are " << tmp_x_eye << " " << tmp_y_eye << std::endl;
    }
 
    Gpu::DeviceScalar<Real> d_val_min(1e10);
    Gpu::DeviceScalar<int> d_i_min(-1), d_j_min(-1);
 
    Real* d_val_min_ptr = d_val_min.dataPtr();
    int* d_i_min_ptr = d_i_min.dataPtr();
    int* d_j_min_ptr = d_j_min.dataPtr();
 
    bool use_moisture = (moisture_type != MoistureType::None);
    const int ncomp = S_data[IntVars::cons].nComp();
 
    const auto dx = lev_geom.CellSizeArray();
    const auto prob_lo = lev_geom.ProbLoArray();
 
    // NOTE: see HurricaneEyeTrackerInitial -- the arg-min is split into a
    //       reducing pass and a locating pass so that the recorded (i,j)
    //       always belongs to the cell holding the recorded minimum.
    const Dim3 dlo = lbound(lev_geom.Domain());
    const int nx = lev_geom.Domain().length(0);
 
    Gpu::DeviceScalar<Long> d_idx_min(std::numeric_limits<Long>::max());
    Long* d_idx_min_ptr = d_idx_min.dataPtr();
 
    for (MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
        const Box& box = mfi.validbox();
        const Array4<Real const>& S_arr = S_data[IntVars::cons].const_array(mfi);
 
        ParallelFor(box,[=] AMREX_GPU_DEVICE(int i, int j, int k) {
            if(k==0) {
                Real x =  prob_lo[0] + (i+myhalf)*dx[0];
                Real y =  prob_lo[1] + (j+myhalf)*dx[1];
                Real dist = std::sqrt((x-tmp_x_eye)*(x-tmp_x_eye) + (y-tmp_y_eye)*(y-tmp_y_eye));
                if(dist < 200e3) {
                    Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
                    const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                    Real pressure = getPgivenRTh(rhotheta,qv_for_p);
                    Gpu::Atomic::Min(&d_val_min_ptr[0], pressure);
                }
            }
        });
    }
 
    // The minimum over every box on this rank must be known before the
    // locating pass below can test against it.
    Gpu::synchronize();
 
    for (MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
        const Box& box = mfi.validbox();
        const Array4<Real const>& S_arr = S_data[IntVars::cons].const_array(mfi);
 
        ParallelFor(box,[=] AMREX_GPU_DEVICE(int i, int j, int k) {
            if(k==0) {
                Real x =  prob_lo[0] + (i+myhalf)*dx[0];
                Real y =  prob_lo[1] + (j+myhalf)*dx[1];
                Real dist = std::sqrt((x-tmp_x_eye)*(x-tmp_x_eye) + (y-tmp_y_eye)*(y-tmp_y_eye));
                if(dist < 200e3) {
                    Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
                    const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                    Real pressure = getPgivenRTh(rhotheta,qv_for_p);
                    if (pressure == d_val_min_ptr[0]) {
                        Gpu::Atomic::Min(d_idx_min_ptr, pack_ij(i,j,nx,dlo));
                    }
                }
            }
        });
    }
 
    Gpu::synchronize();
 
    // Unpack the located index back into the (i,j) device scalars that
    // ComputeGlobalMinLocation reads.
    {
        int h_i_min, h_j_min;
        unpack_ij(d_idx_min.dataValue(), nx, dlo, h_i_min, h_j_min);
        Gpu::copy(Gpu::hostToDevice, &h_i_min, &h_i_min + 1, d_i_min_ptr);
        Gpu::copy(Gpu::hostToDevice, &h_j_min, &h_j_min + 1, d_j_min_ptr);
    }
 
    Real global_val_min;
    int global_i_min, global_j_min;
 
    ComputeGlobalMinLocation(sc, lev_geom, S_data,
                             d_val_min_ptr, d_i_min_ptr, d_j_min_ptr,
                             global_val_min, global_i_min, global_j_min);
}

Here is the call graph for this function:

HurricaneMaxVelTracker()

void ERF::HurricaneMaxVelTracker	(	const amrex::Geometry &	geom,
		const amrex::MultiFab &	mf_cc_vel,
		const double &	time
	)

Compute and track the maximum wind velocity near the hurricane eye.

Parameters

[in]	lev_geom	Geometry of the current level
[in]	mf_cc_vel	MultiFab containing cell-centered velocity
[in]	time	Current simulation time

{
    const int ncomp = AMREX_SPACEDIM;
 
    Real* d_val_max_ptr;
    Gpu::DeviceVector<Real> d_val_max(1, -bogus_large_value);
    d_val_max_ptr = d_val_max.data();
 
    const auto [x_last, y_last] = hurricane_eye_track_xy.back();
    const auto dx = lev_geom.CellSizeArray();
    const auto prob_lo = lev_geom.ProbLoArray();
 
    Real x_eye = x_last;
    Real y_eye = y_last;
 
    for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi) {
        const Box& box = mfi.validbox();
        const auto& vel_arr = mf_cc_vel.const_array(mfi);
 
        ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
            Real x = prob_lo[0] + (i+myhalf)*dx[0];
            Real y = prob_lo[1] + (j+myhalf)*dx[1];
            Real dist = std::sqrt((x-x_eye)*(x-x_eye) +
                                         (y-y_eye)*(y-y_eye));
            if(k==1 && dist < 200e3) {
                Real velmag = zero;
                for (int comp = 0; comp < ncomp; ++comp) {
                    Real vel = vel_arr(i, j, k, comp);
                    velmag += vel * vel;
                }
                velmag = std::sqrt(velmag)*Real(3.6); // km/hr
                Gpu::Atomic::Max(&d_val_max_ptr[0], velmag);
            }
        });
    }
 
    Gpu::synchronize();
 
    Real h_val_max_local = -bogus_large_value;
    Gpu::copy(Gpu::deviceToHost, d_val_max.begin(), d_val_max.end(), &h_val_max_local);
 
    Real h_val_max_global = -bogus_large_value;
    #ifdef AMREX_USE_MPI
        h_val_max_global = h_val_max_local;
        amrex::ParallelDescriptor::ReduceRealMax(h_val_max_global);
    #else
        h_val_max_global = h_val_max_local;
    #endif
 
    double time_in_hrs = time / 3600.0;
    hurricane_maxvel_vs_time.push_back({static_cast<Real>(time_in_hrs), h_val_max_global});
}

Here is the call graph for this function:

HurricaneMinPressureTracker()

void ERF::HurricaneMinPressureTracker	(	MoistureType	moisture_type,
		const amrex::Geometry &	geom,
		const amrex::MultiFab &	mf_cons_var,
		const double &	time
	)

Compute and track the minimum pressure near the hurricane eye.

Parameters

[in]	moisture_type	Moisture model type
[in]	lev_geom	Geometry of the current level
[in]	mf_cons_var	MultiFab containing conservative variables
[in]	time	Current simulation time

{
 
 
    Real* d_val_min_ptr;
    Gpu::DeviceVector<Real> d_val_min(1, bogus_large_value);
    d_val_min_ptr = d_val_min.data();
 
    const Real x_last = hurricane_eye_track_xy.back()[0];
    const Real y_last = hurricane_eye_track_xy.back()[1];
    const auto dx = lev_geom.CellSizeArray();
    const auto prob_lo = lev_geom.ProbLoArray();
 
    const int ncomp = mf_cons_var.nComp();
    bool use_moisture = (moisture_type != MoistureType::None);
 
    for (MFIter mfi(mf_cons_var); mfi.isValid(); ++mfi) {
        const Box& box = mfi.validbox();
        const auto& S_arr = mf_cons_var.const_array(mfi);
 
        ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
            Real x = prob_lo[0] + (i+myhalf)*dx[0];
            Real y = prob_lo[1] + (j+myhalf)*dx[1];
            Real dist2 = (x-x_last)*(x-x_last) +
                                (y-y_last)*(y-y_last);
            if(k==1 && dist2 < 200e3*200e3) {
                const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                const Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
                const Real pressure = getPgivenRTh(rhotheta,qv_for_p);
                Gpu::Atomic::Min(&d_val_min_ptr[0], pressure);
            }
        });
    }
 
    Gpu::synchronize();
 
    Real h_val_min_local = bogus_large_value;
    Gpu::copy(Gpu::deviceToHost, d_val_min.begin(), d_val_min.end(), &h_val_min_local);
 
    // NOTE: use the amrex wrapper rather than a hard-coded MPI_DOUBLE, which is
    //       a type mismatch when amrex::Real is float (ERF_PRECISION=SINGLE).
    Real h_val_min_global = h_val_min_local;
    ParallelDescriptor::ReduceRealMin(h_val_min_global);
 
    double time_in_hrs = time / 3600.0;
    hurricane_minpressure_vs_time.push_back({static_cast<Real>(time_in_hrs), h_val_min_global});
}

Here is the call graph for this function:

HurricaneTracker()

void ERF::HurricaneTracker	(	int	lev,
		double	time,
		const amrex::MultiFab &	cc_vel,
		const amrex::Real	velmag_threshold,
		amrex::TagBoxArray *	tags = nullptr
	)

Track the hurricane eye and tag cells for refinement.

Parameters

	levc	Current level.
	time	Current simulation time.
	mf_cc_vel	MultiFab containing cell-centered velocity.
	velmag_threshold	Velocity magnitude threshold for eye identification.
[out]	tags	TagBoxArray to be filled for refinement.

{
    bool is_found;
 
    Real eye_x, eye_y;
 
    if (time==zero || hurricane_eye_track_xy.empty()) {
        is_found = FindInitialEye(levc, mf_cc_vel, velmag_threshold, eye_x, eye_y);
    } else {
        is_found = true;
        const auto& last = hurricane_eye_track_xy.back();
        eye_x = last[0];
        eye_y = last[1];
    }
 
    if (is_found) {
        const int exponent = max_level-1-levc;
        Real rad_tag = std::ldexp(Real(4.e5), exponent);
        tag_on_distance_from_eye(geom[levc], tags, eye_x, eye_y, rad_tag);
    }
}

Here is the call graph for this function:

HurricaneTrackerCircle()

void ERF::HurricaneTrackerCircle

(

)

Generate a circular set of points around the last known eye position.

{
    // Check that there is at least one eye position
    if (hurricane_eye_track_xy.empty()) return;
 
    // Get the last known (x, y) position of the eye
    const auto [x_last, y_last] = hurricane_eye_track_xy.back();
 
    // Define circle properties
    const int n_points = 100;        // number of points on the circle
    const Real radius = 200e3; // radius in meters (example: 50 km)
 
    // Clear previous points and reserve space
    hurricane_tracker_circle.clear();
    hurricane_tracker_circle.reserve(n_points);
 
    // Fill the circle points
    for (int i = 0; i < n_points; ++i) {
        Real theta = two * static_cast<Real>(M_PI) * static_cast<Real>(i) / static_cast<Real>(n_points);
        Real x = x_last + radius * std::cos(theta);
        Real y = y_last + radius * std::sin(theta);
        hurricane_tracker_circle.push_back({x, y});
    }
}

ibseb_advance()

void ERF::ibseb_advance	(	int	lev,
		amrex::Real	time,
		amrex::Real	dt_lev,
		const amrex::MultiFab &	cons,
		const amrex::MultiFab &	xvel,
		const amrex::MultiFab &	yvel,
		const amrex::MultiFab &	zvel
	)

Per-step update of one level, called at the start of ERF::Advance() with the state at the start of the step: shortwave, longwave and the wall function on the faces, then either the prognostic balance (which finds the skin temperature at the end of the step and advances the slab with it) or, with erf.ibseb.prognostic = false, the slab alone under the fixed skin. The sensible flux left in the set is what add_heat_flux_to_source() deposits at every slow stage of the step, so the air receives exactly the H of the closed balance. The atmosphere is seen at the start of the step and the skin is implicit within it, the usual coupling of a land-surface model.

Parameters

[in]	lev	AMR level to update; a no-op if it has no face set.
[in]	time	Time at the start of the step [s], at which the sun position of the shortwave is taken.
[in]	dt_lev	Length of the level's step [s], over which the slab is advanced and within which the skin is implicit.
[in]	cons	Conserved state at the start of the step: the air temperature of the longwave and the wall function, and the profile of the bulk Richardson depth.
[in]	xvel	Face-centred x velocity at the start of the step.
[in]	yvel	Face-centred y velocity at the start of the step.
[in]	zvel	Face-centred z velocity at the start of the step; the three together drive the wall function of the sensible flux.

{
    if (!ibseb_params.enable || lev >= static_cast<int>(m_ibseb.size()) || !m_ibseb[lev]) { return; }
    const double t_wall0 = ParallelDescriptor::second();
    m_ibseb[lev]->compute_shortwave(time);
    m_ibseb[lev]->compute_longwave(cons);
    // The ground surface layer's fields and the mixed-layer depth
    // for the wall function beyond neutral (all null / zero unless asked).
    const MultiFab* olen2d = nullptr;
    const MultiFab* pblh2d = nullptr;
    Real z_i_bulk = 0.0;
    // The surface layer now exists per domain face.  What the wall function wants
    // here is the ground beneath the buildings, so take zlo; a surface layer on a
    // lateral or upper wall says nothing about the stability of this column.
    const auto& ground_sl = m_SurfaceLayer[Orientation(Direction::z, Orientation::low)];
    if (ground_sl && ibseb_params.stability_correction) { olen2d = ground_sl->get_olen(lev); }
    if (ibseb_params.convective_velocity == "deardorff") {
        if (ground_sl && ground_sl->computes_pblh() && ibseb_params.z_i_mode == "pblh") {
            pblh2d = ground_sl->get_pblh(lev);
        }
        z_i_bulk = (ibseb_params.z_i_mode == "fixed") ? ibseb_params.z_i
                                                     : ibseb_bulk_richardson_height(lev, cons, xvel, yvel);
        if (ibseb_params.debug) {
            Print() << "[IBSEB DEBUG] lev=" << lev << " mixed-layer depth for w*: " << z_i_bulk << " m ("
                    << ibseb_params.z_i_mode << (pblh2d ? ", pblh per column" : "") << ")\n";
        }
    }
    m_ibseb[lev]->compute_sensible(cons, xvel, yvel, zvel, solverChoice.c_p, olen2d, pblh2d, z_i_bulk);
    if (ibseb_params.prognostic) {
        m_ibseb[lev]->solve_balance(dt_lev);
    } else {
        m_ibseb[lev]->compute_ground(dt_lev);
    }
    m_ibseb[lev]->add_cost(ParallelDescriptor::second() - t_wall0);
}

ibseb_bulk_richardson_height()

Real ERF::ibseb_bulk_richardson_height	(	int	lev,
		const amrex::MultiFab &	cons,
		const amrex::MultiFab &	xvel,
		const amrex::MultiFab &	yvel
	)

Mixed-layer depth of a level by the bulk Richardson method on the horizontal-mean profile (Troen and Mahrt; Vogelezang and Holtslag).

With the first level as the reference, Ri_b(z) = g (z - z_1) (theta(z)

• theta_1) / (theta_1 (|U(z) - U_1|^2 + 100 u*^2)) with u* = 0.1 m/s, and the depth is the first cell centre where it exceeds erf.ibseb.ri_crit, or the domain depth when it never does (a neutral profile); both are heights above the domain bottom. The profile is the plane average of the conserved state and the face velocities, uniform vertical spacing assumed as elsewhere in the balance; called once per step and level when the convective velocity scale is on and z_i is not fixed, also as the fallback of the pblh mode.

Parameters

[in]	lev	AMR level whose horizontal-mean profile is taken.
[in]	cons	Conserved state of the level; the Rho_comp and RhoTheta_comp averages give the mean potential temperature.
[in]	xvel	Face-centred x velocity, for the mean wind profile.
[in]	yvel	Face-centred y velocity, for the mean wind profile.

Returns

Mixed-layer depth above the domain bottom [m]; the depth of the domain when Ri_b never exceeds erf.ibseb.ri_crit.

{
    MultiFab c2(cons, make_alias, Rho_comp, 2);   // rho and rho theta are the first two components
    PlaneAverage r_ave(&c2, geom[lev], 2);
    r_ave.compute_averages(ZDir(), r_ave.field());
    PlaneAverage u_ave(&xvel, geom[lev], 2);
    u_ave.compute_averages(ZDir(), u_ave.field());
    PlaneAverage v_ave(&yvel, geom[lev], 2);
    v_ave.compute_averages(ZDir(), v_ave.field());
    const int nz = r_ave.ncell_line();
    Gpu::HostVector<Real> rho(nz), rth(nz), uu(u_ave.ncell_line()), vv(v_ave.ncell_line());
    r_ave.line_average(0, rho);
    r_ave.line_average(1, rth);
    u_ave.line_average(0, uu);
    v_ave.line_average(0, vv);
    const Real dz = geom[lev].CellSize(2);
    const Real z_top = geom[lev].ProbHi(2) - geom[lev].ProbLo(2);   // depth of the domain
    const Real th1 = rth[0] / rho[0];
    const Real ustar_floor2 = 100.0 * 0.1 * 0.1;
    Real z_i = z_top;
    for (int k = 1; k < nz; ++k) {
        const Real th = rth[k] / rho[k];
        // |U(z) - U_1|^2 of the wind vector, so a veering wind of constant speed
        // still counts as shear.
        const Real dU2 = (uu[k] - uu[0]) * (uu[k] - uu[0]) + (vv[k] - vv[0]) * (vv[k] - vv[0]);
        const Real rib = CONST_GRAV * (k * dz) * (th - th1) / (th1 * (dU2 + ustar_floor2));
        if (rib > ibseb_params.ri_crit) { z_i = (k + 0.5) * dz; break; }
    }
    return z_i;
}

Here is the call graph for this function:

ibseb_report()

void ERF::ibseb_report	(	int	nstep,
		amrex::Real	time
	)

Periodic report from ERF::post_timestep(), with nstep the number of completed steps (the plotfiles' numbering; the initial state is the step-0 report of init_ibseb()): after every erf.ibseb.csv_int-th step, print the summary of each level and append its CSV rows. A non-positive interval disables both.

Parameters

[in]	nstep	Number of completed steps, tested against erf.ibseb.csv_int and written to the CSV rows.
[in]	time	Simulation time at the end of the step [s], written to the summary and the CSV rows.

{
    if (!ibseb_params.enable) { return; }
    // With debug on the summary is printed every step, as the fire module
    // does; the CSV rows keep their interval.
    const bool csv_now = (ibseb_params.csv_int > 0) && (nstep % ibseb_params.csv_int == 0);
    if (!csv_now && !ibseb_params.debug) { return; }
    for (int lev = 0; lev <= finest_level && lev < static_cast<int>(m_ibseb.size()); ++lev) {
        if (m_ibseb[lev]) { m_ibseb[lev]->report(time, nstep, csv_now); }
    }
}

ibseb_write_checkpoint()

void ERF::ibseb_write_checkpoint	(	const std::string &	checkpointname,
		int	lev
	)		const

Write the face state of one level into the checkpoint as IBSEBState, a field on the blocks around the buildings (IBFaceSet::state_boxarray()) clipped to the cells that own faces, so it scales with the shell of the buildings rather than with the level. Called inside the level loop of ERF::WriteCheckpointFile(); a no-op unless the balance is on and the level has faces.

Parameters

[in]	checkpointname	Path of the checkpoint directory being written.
[in]	lev	AMR level whose face state is written, as the IBSEBState field of its Level_ group.

{
    if (!ibseb_params.enable || lev >= static_cast<int>(m_ibseb.size()) || !m_ibseb[lev]) { return; }
    // A level without faces has no field to write (and nothing to restore).
    if (!m_ibseb[lev]->has_state()) { return; }
    MultiFab state = m_ibseb[lev]->make_state();
    m_ibseb[lev]->save_state(state);
    VisMF::Write(state, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "IBSEBState"));
}

ImposeBCsOnPhi()

void ERF::ImposeBCsOnPhi	(	int	lev,
		amrex::MultiFab &	phi,
		const amrex::Box &	subdomain
	)

Impose bc's on the pressure that comes out of the solve

Parameters

lev	Level index for the pressure field
phi	Pressure increment with ghost cells to fill
subdomain	Subdomain over which the solve was performed

{
    BL_PROFILE("ERF::ImposeBCsOnPhi()");
 
    auto const sub_lo = lbound(subdomain);
    auto const sub_hi = ubound(subdomain);
 
    auto const dom_lo = lbound(geom[lev].Domain());
    auto const dom_hi = ubound(geom[lev].Domain());
 
    phi.setBndry(Real(1.e25));
    phi.FillBoundary(geom[lev].periodicity());
 
    // ****************************************************************************
    // Impose bc's on pprime
    // ****************************************************************************
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        Array4<Real> const& pp_arr  = phi.array(mfi);
        Box const& bx    = mfi.tilebox();
        auto const bx_lo = lbound(bx);
        auto const bx_hi = ubound(bx);
 
        auto bc_type_xlo = domain_bc_type[Orientation(0,Orientation::low)];
        auto bc_type_xhi = domain_bc_type[Orientation(0,Orientation::high)];
        auto bc_type_ylo = domain_bc_type[Orientation(1,Orientation::low)];
        auto bc_type_yhi = domain_bc_type[Orientation(1,Orientation::high)];
        auto bc_type_zhi = domain_bc_type[Orientation(2,Orientation::high)];
 
        if ( (bx_lo.x == dom_lo.x) && (bc_type_xlo == "Outflow" || bc_type_xlo == "Open") && !solverChoice.use_real_bcs) {
            ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i-1,j,k) = -pp_arr(i,j,k);
            });
        } else if (bx_lo.x == sub_lo.x) {
            ParallelFor(makeSlab(bx,0,sub_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i-1,j,k) = pp_arr(i,j,k);
            });
        }
 
        if ( (bx_hi.x == dom_hi.x) && (bc_type_xhi == "Outflow" || bc_type_xhi == "Open") && !solverChoice.use_real_bcs) {
            ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i+1,j,k) = -pp_arr(i,j,k);
            });
        } else if (bx_hi.x == sub_hi.x) {
            ParallelFor(makeSlab(bx,0,sub_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i+1,j,k) = pp_arr(i,j,k);
            });
        }
 
        if ( (bx_lo.y == dom_lo.y) && (bc_type_ylo == "Outflow" || bc_type_ylo == "Open") && !solverChoice.use_real_bcs) {
            ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j-1,k) = -pp_arr(i,j,k);
            });
        } else if (bx_lo.y == sub_lo.y) {
            ParallelFor(makeSlab(bx,1,sub_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j-1,k) = pp_arr(i,j,k);
            });
        }
 
        if ( (bx_hi.y == dom_hi.y) && (bc_type_yhi == "Outflow" || bc_type_yhi == "Open") && !solverChoice.use_real_bcs) {
            ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j+1,k) = -pp_arr(i,j,k);
            });
        } else if (bx_hi.y == sub_hi.y) {
            ParallelFor(makeSlab(bx,1,sub_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j+1,k) = pp_arr(i,j,k);
            });
        }
 
        // At low z we are always Neumann whether the box touches the bottom boundary or not
        Box zbx(bx); zbx.grow(0,1); zbx.grow(1,1); // Grow in x-dir and y-dir because we have filled that above
        if (bx_lo.z == sub_lo.z) {
            ParallelFor(makeSlab(zbx,2,sub_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j,k-1) = pp_arr(i,j,k);
            });
        }
 
        if ( (bx_hi.z == dom_hi.z) && (bc_type_zhi == "Outflow" || bc_type_zhi == "Open") ) {
            ParallelFor(makeSlab(bx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j,k+1) = -pp_arr(i,j,k);
            });
        } else if (bx_hi.z == sub_hi.z) {
            ParallelFor(makeSlab(bx,2,sub_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j,k+1) = pp_arr(i,j,k);
            });
        }
    } // mfi
 
    // Now overwrite with periodic fill outside domain and fine-fine fill inside
    phi.FillBoundary(geom[lev].periodicity());
}

Here is the call graph for this function:

init1DArrays()

void ERF::init1DArrays ( )

private

init_bcs()

void ERF::init_bcs ( )

private

Initialize ERF physical and component boundary condition data.

{
    bool read_prim_theta  = true;
    bool use_surfacelayer = false;
 
    init_phys_bcs(read_prim_theta);
 
    // Deardorff LES, RANS, and PBL models consume lower-z SurfaceLayer fields
    // explicitly. They cannot safely coexist with a lateral or upper
    // SurfaceLayer wall. Smagorinsky LES uses the generic all-face diffusion
    // path.
    bool has_non_zlo_surface_layer = false;
    for (OrientationIter oit; oit; ++oit) {
        const Orientation ori = oit();
        const bool is_zlo = (ori.coordDir() == static_cast<int>(Direction::z) &&
                             ori.faceDir() == Orientation::low);
        if (!is_zlo && phys_bc_type[ori] == ERF_BC::surface_layer) {
            has_non_zlo_surface_layer = true;
            break;
        }
    }
 
    if (has_non_zlo_surface_layer) {
        for (int lev = 0; lev <= max_level; ++lev) {
            const auto& turb_choice = solverChoice.turbChoice[lev];
            if (turb_choice.les_type  == LESType::Deardorff ||
                turb_choice.rans_type != RANSType::None ||
                turb_choice.pbl_type  != PBLType::None) {
                Abort("Deardorff LES, RANS, and PBL models (including SHOC) support "
                      "SurfaceLayer only at zlo. Remove non-zlo surface_layer boundaries "
                      "or select Smagorinsky LES.");
            }
        }
    }
 
    bool keqn_dir = (solverChoice.turbChoice[max_level].rans_type == RANSType::kEqn &&
                     solverChoice.turbChoice[max_level].dirichlet_k == true);
    if (keqn_dir) {
        // The wall value of k (AL01 Eq. 16) is written into the first cell by
        // SurfaceLayer::update_fluxes and held there through every RK stage
        // (erf_slow_rhs_post, ImplicitDiffForStateLU_*). The logical BC for
        // RhoKE at the wall stays foextrap, so the ghost cell carries the same
        // value and the surface-layer branch of the diffusion sets a zero flux
        // through the wall face. Assume for now that all levels are RANS.
        for (int lev = 0; lev < max_level; ++lev) {
            if (solverChoice.turbChoice[lev].rans_type != RANSType::kEqn) {
                Error("If using one-eqn RANS, all levels must be RANS for now");
            }
        }
        // The AL01 Eq. 16 wall value is computed only by SurfaceLayer::update_fluxes,
        // so without a surface layer at zlo nothing ever writes it. The pin in
        // erf_slow_rhs_post is gated on SurfLayer != nullptr and would silently do
        // nothing, while ImplicitDiffForStateLU_* would still collapse the klo row
        // and freeze the first cell at whatever it was initialized to. Rather than
        // let the two paths disagree, require the surface layer.
        if (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::surface_layer) {
            Error("erf.dirichlet_k = true requires zlo.type = surface_layer: the wall value "
                  "of k (Axell & Liungman Eq. 16) is computed by the surface layer model");
        }
        Print() << "Using dirichlet wall value for the k equation (held in the first cell)" << std::endl;
    }
 
    // k-eqn RANS under a surface layer without the Dirichlet wall value: the
    // first cell cannot resolve the near-wall shear production, so k there
    // settles at about half the AL01 equilibrium u*^2/Cmu0^2 (the mean wind
    // still follows the log law because MOST supplies the stress). Warn.
    for (int lev = 0; lev <= max_level; ++lev) {
        if (solverChoice.turbChoice[lev].rans_type == RANSType::kEqn &&
            !solverChoice.turbChoice[lev].dirichlet_k &&
            phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
            Warning("erf.rans_type = kEqn with zlo.type = surface_layer but erf.dirichlet_k = false: "
                    "near-wall TKE will be about half the Axell & Liungman equilibrium value; "
                    "set erf.dirichlet_k = true");
        }
    }
 
    // *****************************************************************************
    //
    // Here we translate the physical boundary conditions -- one type per face --
    //     into logical boundary conditions for each velocity component
    //
    // *****************************************************************************
    {
        domain_bcs_type.resize(AMREX_SPACEDIM+NBCVAR_max);
        domain_bcs_type_d.resize(AMREX_SPACEDIM+NBCVAR_max);
 
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            int dir = ori.coordDir();
            Orientation::Side side = ori.faceDir();
            auto const bct = phys_bc_type[ori];
            if ( bct == ERF_BC::symmetry )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::reflect_even);
                    }
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::reflect_odd);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::reflect_even);
                    }
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::reflect_odd);
                }
            }
            else if (bct == ERF_BC::outflow or bct == ERF_BC::ho_outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                    if (!solverChoice.anelastic[0]) {
                        domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::neumann_int);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                    if (!solverChoice.anelastic[0]) {
                        domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::neumann_int);
                    }
                }
            }
            else if (bct == ERF_BC::open)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::open);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::open);
                }
            }
            else if (bct == ERF_BC::inflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                        if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                            domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                        }
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                        if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                            domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                        }
                    }
                }
            }
            else if (bct == ERF_BC::inflow_outflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_upwind);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_upwind);
                    }
                }
            }
            else if (bct == ERF_BC::no_slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                    }
                }
            }
            else if (bct == ERF_BC::slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::ext_dir);
 
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::ext_dir);
                }
            }
            else if (bct == ERF_BC::periodic)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::int_dir);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::int_dir);
                    }
                }
            }
            else if ( bct == ERF_BC::surface_layer )
            {
                use_surfacelayer = true;
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::hoextrap);
                    }
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::ext_dir);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::hoextrap);
                    }
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::ext_dir);
                }
            }
        }
    }
 
    // *****************************************************************************
    //
    // Here we translate the physical boundary conditions -- one type per face --
    //     into logical boundary conditions for each cell-centered variable
    //     (including the base state variables)
    // NOTE: all "scalars" share the same type of boundary condition
    //
    // *****************************************************************************
    {
        auto wall_scalar_type = [this] (int bc_comp, Orientation ori) {
            const auto intent = m_wall_scalar_bc_intent[bc_comp][ori];
            if (intent == erf_wall_scalar_bc::WallScalarBCIntent::DirichletConserved) {
                return ERFBCType::ext_dir;
            } else if (intent == erf_wall_scalar_bc::WallScalarBCIntent::DirichletPrimitive) {
                return ERFBCType::ext_dir_prim;
            } else if (intent == erf_wall_scalar_bc::WallScalarBCIntent::Neumann) {
                return ERFBCType::neumann;
            }
            return ERFBCType::foextrap;
        };
 
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            int dir = ori.coordDir();
            Orientation::Side side = ori.faceDir();
            auto const bct = phys_bc_type[ori];
            if ( bct == ERF_BC::symmetry )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::reflect_even);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::reflect_even);
                    }
                }
            }
            else if ( bct == ERF_BC::outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                }
            }
            else if ( bct == ERF_BC::ho_outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::hoextrap);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::hoextrap);
                    }
                }
            }
            else if ( bct == ERF_BC::open )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::open);
                } else {
                    for (int i = 0; i < NBCVAR_max; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::open);
                }
            }
            else if ( bct == ERF_BC::no_slip_wall )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, wall_scalar_type(BCVars::cons_bc+i, ori));
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, wall_scalar_type(BCVars::cons_bc+i, ori));
                    }
                }
            }
            else if (bct == ERF_BC::slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, wall_scalar_type(BCVars::cons_bc+i, ori));
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, wall_scalar_type(BCVars::cons_bc+i, ori));
                    }
                }
            }
            else if (bct == ERF_BC::inflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                        if ((BCVars::cons_bc+i == RhoTheta_comp) &&
                            (th_bc_data[0].data() != nullptr))
                        {
                            if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
                        }
                        else if (input_bndry_planes && dir < 2 && (
                           ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )) )
                        {
                            domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                        }
                        else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        }
                    }
                    // Non-reflecting: extrapolate RhoTheta from interior so
                    // pressure is not prescribed, allowing acoustic waves out
                    if (m_bc_nonreflecting[ori]) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::foextrap);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                        if ((BCVars::cons_bc+i == RhoTheta_comp) &&
                            (th_bc_data[0].data() != nullptr))
                        {
                            if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
                        }
                        else if (input_bndry_planes && dir < 2 && (
                           ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )
                           ) )
                        {
                            domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                        }
                        else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        }
                    }
                    // Non-reflecting: extrapolate RhoTheta from interior
                    if (m_bc_nonreflecting[ori]) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::foextrap);
                    }
                }
            }
            else if (bct == ERF_BC::inflow_outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_upwind);
                        if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        }
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_upwind);
                        if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        }
                    }
                }
            }
            else if (bct == ERF_BC::periodic)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::int_dir);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                       domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::int_dir);
                    }
                }
            }
            else if ( bct == ERF_BC::surface_layer )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                }
                // NOTE: with erf.dirichlet_k the RhoKE wall value lives in the
                //       first cell (see above); foextrap is the right logical BC.
            }
        }
    }
 
    // Sanity check that implicit diffusion is consistent with the BC types.
    // Turn off implicit diffusion for a component if its BCs don't match
    // those allowed by the tridiagonal solver.
    const BCRec* bc_ptr = domain_bcs_type.data();
    Vector<int> cc_comp_map = {BCVars::RhoTheta_bc_comp, BCVars::RhoKE_bc_comp, BCVars::RhoQ1_bc_comp};
    Vector<std::string> cc_comp_name = {"Theta", "KE", "Qv"};
    for (int icomp(0); icomp < cc_comp_map.size(); ++icomp) {
        int bc_comp = cc_comp_map[icomp];
        bool foextrap_on_zlo = (bc_ptr[bc_comp].lo(2) == ERFBCType::foextrap);
        bool foextrap_on_zhi = (bc_ptr[bc_comp].hi(2) == ERFBCType::foextrap);
        bool neumann_on_zlo  = (bc_ptr[bc_comp].lo(2) == ERFBCType::neumann);
        bool neumann_on_zhi  = (bc_ptr[bc_comp].hi(2) == ERFBCType::neumann);
        if ( (!foextrap_on_zlo && !neumann_on_zlo && !use_surfacelayer) ||
             (!foextrap_on_zhi && !neumann_on_zhi) ) {
            Print() << "WARNING: Selected BCs for " << cc_comp_name[icomp] << " are not supported with implicit diffusion. Turning off implicit diffusion for this component." << "\n";
            if (bc_comp == BCVars::RhoTheta_bc_comp) { solverChoice.implicit_thermal_diffusion  = false; }
            if (bc_comp == BCVars::RhoKE_bc_comp)    { solverChoice.implicit_ke_diffusion       = false; }
            if (bc_comp == BCVars::RhoQ1_bc_comp)    { solverChoice.implicit_moisture_diffusion = false; }
        }
    }
    Vector<int> vel_comp_map = {BCVars::xvel_bc, BCVars::yvel_bc};
    Vector<std::string> vel_comp_name = {"xvel", "yvel"};
    for (int icomp(0); icomp < vel_comp_map.size(); ++icomp) {
        int bc_comp = vel_comp_map[icomp];
        bool ext_dir_on_zlo  = (bc_ptr[bc_comp].lo(2) == ERFBCType::ext_dir ||
                                bc_ptr[bc_comp].lo(2) == ERFBCType::ext_dir_prim);
        bool ext_dir_on_zhi  = (bc_ptr[bc_comp].hi(2) == ERFBCType::ext_dir ||
                                bc_ptr[bc_comp].hi(2) == ERFBCType::ext_dir_prim);
        bool foextrap_on_zlo = (bc_ptr[bc_comp].lo(2) == ERFBCType::foextrap);
        bool foextrap_on_zhi = (bc_ptr[bc_comp].hi(2) == ERFBCType::foextrap);
        if ( (!foextrap_on_zlo && !ext_dir_on_zlo && !use_surfacelayer) ||
             (!foextrap_on_zhi && !ext_dir_on_zhi) ) {
            Print() << "WARNING: Selected BCs for " << vel_comp_name[icomp] << " are not supported with implicit diffusion. Turning off implicit diffusion for this component." << "\n";
            solverChoice.implicit_momentum_diffusion  = false;
        }
    }
 
    // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
    Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
}

init_custom()

void ERF::init_custom

(

int

lev

)

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
 
    MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
                       lev_new[Vars::cons].nComp()   , lev_new[Vars::cons].nGrow());
    MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
    MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
    MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
 
    // Default all perturbations to zero
    cons_pert.setVal(0.);
    xvel_pert.setVal(0.);
    yvel_pert.setVal(0.);
    zvel_pert.setVal(0.);
 
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
    {
        const Box &bx  = mfi.tilebox();
        const Box &xbx = mfi.tilebox(IntVect(1,0,0));
        const Box &ybx = mfi.tilebox(IntVect(0,1,0));
        const Box &zbx = mfi.tilebox(IntVect(0,0,1));
 
        const auto &cons_pert_arr = cons_pert.array(mfi);
        const auto &xvel_pert_arr = xvel_pert.array(mfi);
        const auto &yvel_pert_arr = yvel_pert.array(mfi);
        const auto &zvel_pert_arr = zvel_pert.array(mfi);
 
        Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
        Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
        Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
 
        // Here we arbitrarily choose the x-oriented map factor -- this should be generalized
        Array4<Real const> mf_m     = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        Array4<Real const> mf_u     = mapfac[lev][MapFacType::u_x]->const_array(mfi);
        Array4<Real const> mf_v     = mapfac[lev][MapFacType::v_y]->const_array(mfi);
 
        Array4<Real> r_hse_arr = r_hse.array(mfi);
        Array4<Real> p_hse_arr = p_hse.array(mfi);
 
        prob->init_custom_pert(bx, cons_arr, cons_pert_arr,
                               r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
                               geom[lev].data(), mf_m, solverChoice, lev);
        prob->init_custom_pert_vels(xbx, ybx, zbx,
                                    xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
                                    z_nd_arr, geom[lev].data(), mf_u, mf_v,
                                    solverChoice, lev);
 
        // Zero out perturbations in covered cells in EB
        if (solverChoice.terrain_type == TerrainType::EB) {
 
            Array4<const EBCellFlag> c_cellflg = (get_eb(lev).get_const_factory())->getMultiEBCellFlagFab()[mfi].const_array();
            Array4<const EBCellFlag> u_cellflg = (get_eb(lev).get_u_const_factory())->getMultiEBCellFlagFab()[mfi].const_array();
            Array4<const EBCellFlag> v_cellflg = (get_eb(lev).get_v_const_factory())->getMultiEBCellFlagFab()[mfi].const_array();
            Array4<const EBCellFlag> w_cellflg = (get_eb(lev).get_w_const_factory())->getMultiEBCellFlagFab()[mfi].const_array();
 
            ParallelFor(bx,
            [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                if (c_cellflg(i,j,k).isCovered()) {
                    cons_pert_arr(i,j,k,RhoTheta_comp) = 0.0;
                }
            });
 
            ParallelFor(xbx, ybx, zbx,
            [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                if (u_cellflg(i,j,k).isCovered()) {
                    xvel_pert_arr(i,j,k) = 0.0;
                }
            },
            [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                if (v_cellflg(i,j,k).isCovered()) {
                    yvel_pert_arr(i,j,k) = 0.0;
                }
            },
            [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                if (w_cellflg(i,j,k).isCovered()) {
                    zvel_pert_arr(i,j,k) = 0.0;
                }
            });
        }
 
    } //mfi
 
 
    // Add problem-specific perturbation to background flow if not doing anelastic with fixed-in-time density
    if (!solverChoice.fixed_density[lev]) {
        MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp,      Rho_comp,             1, cons_pert.nGrow());
    }
    MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp,        1, cons_pert.nGrow());
    MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,NSCALARS, cons_pert.nGrow());
 
    // RhoKE is relevant if using Deardorff with LES, k-equation for RANS, MYJ, SHOC, MYNN2.5 or MYNN-EDMF
    // Here we initialize TKE to the tke_min value and then add problem-specific perturbations
    if (solverChoice.turbChoice[lev].use_tke) {
        lev_new[Vars::cons].setVal(solverChoice.turbChoice[lev].tke_min, RhoKE_comp, 1);
        MultiFab::Multiply(lev_new[Vars::cons],lev_new[Vars::cons],Rho_comp,RhoKE_comp,1,lev_new[Vars::cons].nGrowVect());
        MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp,    RhoKE_comp,    1, cons_pert.nGrow());
    }
 
    if (solverChoice.moisture_type != MoistureType::None) {
        int qstate_size = micro->Get_Qstate_Size();
        for (int q_offset(0); q_offset<qstate_size; ++q_offset) {
            int q_idx = RhoQ1_comp+q_offset;
            MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
        }
    }
 
    // Should we initialize the velocities from a checkpoint file?
    static std::string init_vels_from_checkpoint;
    ParmParse pp("erf");
    if (pp.queryAdd("init_vels_from_checkpoint",init_vels_from_checkpoint)) {
        ReadVelsOnlyFromCheckpointFile(lev,init_vels_from_checkpoint);
    } else {
        MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0,             0,             1, xvel_pert.nGrowVect());
        MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0,             0,             1, yvel_pert.nGrowVect());
        MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0,             0,             1, zvel_pert.nGrowVect());
    }
 
    // If initializing for ensemble simluations, then
    // 1. Create cell-centered random perturbations
    // 2. Create cell-centered spatially correlated perturbations
    // 3. Read in the coarse background state from the coarse data file,
    //    interpolate the state data onto the current mesh, and
    //    add the perturbations to the background state and then populate the "pert variables
 
    if(solverChoice.is_init_for_ensemble) {
        Vector<std::string> varnames = {"density","theta", "x_velocity","y_velocity","z_velocity", "qv", "qc", "qrain"};
        MultiFab mf_cc_pert;
        create_random_perturbations(lev, mf_cc_pert);
        WriteSingleLevelPlotfile("plt_pert_correlated_gaussian", mf_cc_pert, varnames, geom[lev], zero, 0);
        apply_gaussian_smoothing_to_perturbations(lev, mf_cc_pert);
        //WriteSingleLevelPlotfile("plt_pert_correlated_gaussian", mf_cc_pert, varnames, geom[lev], zero, 0);
        create_background_state_for_ensemble(lev, mf_cc_pert, lev_new[Vars::cons], lev_new[Vars::xvel], lev_new[Vars::yvel], lev_new[Vars::zvel]);
    }
}

Here is the call graph for this function:

init_Dirichlet_bc_data()

void ERF::init_Dirichlet_bc_data ( const std::string  input_file )

private

Read vertical Dirichlet boundary data and interpolate it to ERF levels.

Parameters

input_file

Path to the Dirichlet input profile file

{
    // Read the dirichlet_input file
    Print() << "dirichlet_input file location : " << input_file << std::endl;
    std::ifstream input_reader(input_file);
    if (!input_reader.is_open()) {
        amrex::Abort("Error opening the dirichlet_input file.\n");
    }
 
    Print() << "Successfully opened the dirichlet_input file. Now reading... " << std::endl;
    std::string line;
 
    // Size of Ninp (number of z points in input file)
    Vector<Real> z_inp_tmp, u_inp_tmp, v_inp_tmp, w_inp_tmp, th_inp_tmp;
 
    // Top and bot for domain
    const int  klo  = geom[0].Domain().smallEnd()[2];
    const int  khi  = geom[0].Domain().bigEnd()[2];
    const Real zbot = zlevels_stag[0][klo];
    const Real ztop = zlevels_stag[0][khi+1];
 
    // Flag if theta input
    Real th_init = -Real(300.0);
    bool th_read{false};
 
    // Add surface
    z_inp_tmp.push_back(zbot); // height above sea level [m]
    u_inp_tmp.push_back(zero);
    v_inp_tmp.push_back(zero);
    w_inp_tmp.push_back(zero);
    th_inp_tmp.push_back(th_init);
 
    // Read the vertical profile at each given height
    Real z, u, v, w, th;
    while(std::getline(input_reader, line)) {
        std::istringstream iss_z(line);
 
        Vector<Real> rval_v;
        Real rval;
        while (iss_z >> rval) {
            rval_v.push_back(rval);
        }
        if ((rval_v.size() != 4) && (rval_v.size() != 5)) {
            Abort("Unknown inflow file format!");
        }
        z = rval_v[0];
        u = rval_v[1];
        v = rval_v[2];
        w = rval_v[3];
 
        // Format without theta
        if (rval_v.size() == 4) {
            if (z == zbot) {
                u_inp_tmp[0] = u;
                v_inp_tmp[0] = v;
                w_inp_tmp[0] = w;
            } else {
                AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
                z_inp_tmp.push_back(z);
                u_inp_tmp.push_back(u);
                v_inp_tmp.push_back(v);
                w_inp_tmp.push_back(w);
                if (z >= ztop) break;
            }
        } else if (rval_v.size() == 5) {
            th_read = true;
            th = rval_v[4];
            if (z == zbot) {
                u_inp_tmp[0]  = u;
                v_inp_tmp[0]  = v;
                w_inp_tmp[0]  = w;
                th_inp_tmp[0] = th;
            } else {
                AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
                z_inp_tmp.push_back(z);
                u_inp_tmp.push_back(u);
                v_inp_tmp.push_back(v);
                w_inp_tmp.push_back(w);
                th_inp_tmp.push_back(th);
                if (z >= ztop) break;
            }
        } else {
            Abort("Unknown inflow file format!");
        }
    }
 
    // Ensure we set a reasonable theta surface
    if (th_read) {
        if (th_inp_tmp[0] == th_init) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE((th_inp_tmp.size() > 2) && (z_inp_tmp.size() > 2),
                "Need at least 3 theta profile points to extrapolate surface theta");
            Real slope = (th_inp_tmp[2] - th_inp_tmp[1]) / (z_inp_tmp[2] - z_inp_tmp[1]);
            Real dz    = z_inp_tmp[0] - z_inp_tmp[1];
            th_inp_tmp[0] = slope * dz + th_inp_tmp[1];
        }
    }
 
    amrex::Print() << "Successfully read and interpolated the dirichlet_input file..." << std::endl;
    input_reader.close();
 
    for (int lev = 0; lev <= max_level; lev++) {
 
        const int Nz  = geom[lev].Domain().size()[2];
 
        // Size of Nz (domain grid)
        Vector<Real> zcc_inp(Nz  );
        Vector<Real> znd_inp(Nz+1);
        Vector<Real> u_inp(Nz  ); xvel_bc_data[lev].resize(Nz  ,zero);
        Vector<Real> v_inp(Nz  ); yvel_bc_data[lev].resize(Nz  ,zero);
        Vector<Real> w_inp(Nz+1); zvel_bc_data[lev].resize(Nz+1,zero);
        Vector<Real> th_inp;
        if (th_read) {
            th_inp.resize(Nz);
            th_bc_data[lev].resize(Nz, zero);
        }
 
        // At this point, we have an input from zbot up to
        // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights
        const int Ninp = z_inp_tmp.size();
        for (int k(0); k<Nz; ++k) {
            zcc_inp[k] = myhalf * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
            znd_inp[k] = zlevels_stag[lev][k];
            u_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), u_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
            v_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), v_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
            w_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), znd_inp[k], Ninp);
            if (th_read) {
                th_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), th_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
            }
        }
        znd_inp[Nz] = ztop;
        w_inp[Nz]   = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), ztop, Ninp);
 
        // Copy host data to the device
        Gpu::copy(Gpu::hostToDevice, u_inp.begin(), u_inp.end(), xvel_bc_data[lev].begin());
        Gpu::copy(Gpu::hostToDevice, v_inp.begin(), v_inp.end(), yvel_bc_data[lev].begin());
        Gpu::copy(Gpu::hostToDevice, w_inp.begin(), w_inp.end(), zvel_bc_data[lev].begin());
        if (th_read) {
           Gpu::copy(Gpu::hostToDevice, th_inp.begin(), th_inp.end(), th_bc_data[lev].begin());
        }
 
        // NOTE: These device vectors are passed to the PhysBC constructors when that
        //       class is instantiated in ERF_MakeNewArrays.cpp.
    } // lev
}

Here is the call graph for this function:

init_from_hse()

void ERF::init_from_hse

(

int

lev

)

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

• call prob->erf_init_dens_hse_dry(...)
  • call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
  • save r_hse
• call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse and th_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
 
    MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
    {
        const Box &gbx = mfi.growntilebox(1);
        const Array4<Real      >& cons_arr  = lev_new[Vars::cons].array(mfi);
        const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
        const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
 
        ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            cons_arr(i,j,k,Rho_comp)      = r_hse_arr(i,j,k);
            cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
        });
    } //mfi
}

Here is the call graph for this function:

init_from_input_sounding()

void ERF::init_from_input_sounding

(

int

lev

)

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters

lev	Integer specifying the current level

{
    // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
    if (lev == 0) {
        if (input_sounding_data.input_sounding_file.empty()) {
            Error("input_sounding file name must be provided via input");
        }
 
        input_sounding_data.resize_arrays();
 
        bool is_moist = (solverChoice.moisture_type != MoistureType::None);
 
        // this will interpolate the input profiles to the nominal height levels
        // (ranging from 0 to the domain top)
        for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
            input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n, is_moist);
        }
 
        //input_sounding_data.set_start_time(start_time);
 
        // this will calculate the hydrostatically balanced density and pressure
        // profiles following WRF ideal.exe
        if (solverChoice.sounding_type == SoundingType::Ideal) {
            input_sounding_data.calc_rho_p(0);
        } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
                   solverChoice.sounding_type == SoundingType::DryIsentropic) {
            input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
            input_sounding_data.calc_rho_p_isentropic(0);
        }
 
    } else {
        //
        // We need to do this interp from coarse level in order to set the values of
        // the base state inside the domain but outside of the fine region
        //
        base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
        //
        // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
        //       have been pre-filled - this includes ghost cells both inside and outside
        //       the domain
        //
        InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                              IntVect(0,0,0), // do not fill ghost cells outside the domain
                              base_state[lev-1], 0, 0, base_state[lev].nComp(),
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BaseBCVars::rho0_bc_comp);
 
         // We need to do this here because the interpolation above may leave corners unfilled
         //    when the corners need to be filled by, for example, reflection of the fine ghost
         //    cell outside the fine region but inide the domain.
         (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
    }
 
    auto& lev_new = vars_new[lev];
 
    const bool l_isentropic = (solverChoice.sounding_type == SoundingType::Isentropic ||
                               solverChoice.sounding_type == SoundingType::DryIsentropic);
 
    const bool constant_density_sounding = (solverChoice.sounding_type == SoundingType::ConstantDensity);
 
    MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
    MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
    MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
    MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
 
    const Real l_gravity = solverChoice.gravity;
    const Real l_rdOcp   = solverChoice.rdOcp;
    const bool l_moist   = (solverChoice.moisture_type != MoistureType::None);
 
    int ngz = r_hse.nGrow(2);
 
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box &bx = mfi.tilebox();
        const auto &cons_arr = lev_new[Vars::cons].array(mfi);
        Array4<Real>  r_hse_arr =  r_hse.array(mfi);
        Array4<Real>  p_hse_arr =  p_hse.array(mfi);
        Array4<Real> pi_hse_arr = pi_hse.array(mfi);
        Array4<Real> th_hse_arr = th_hse.array(mfi);
        Array4<Real> qv_hse_arr = qv_hse.array(mfi);
 
        Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
 
        if (constant_density_sounding) {
            // This assumes rho_0 = one
            // HSE will be calculated later with call to initHSE
            init_state_from_input_sounding(bx, cons_arr, geom[lev].data(), z_cc_arr,
                                           l_moist, input_sounding_data);
        }
        else
        {
            // HSE will be initialized here, interpolated from values previously
            // calculated by calc_rho_p or calc_rho_p_isentropic
            init_state_from_input_sounding_hse(bx, cons_arr,
                r_hse_arr, p_hse_arr, pi_hse_arr, th_hse_arr, qv_hse_arr,
                geom[lev].data(), z_cc_arr,
                l_gravity, l_rdOcp, l_moist, input_sounding_data,
                l_isentropic, ngz);
        }
    }
 
    if (!constant_density_sounding) {
        // Enforce HSE on the base state -- holding th_hse and qv_hse constant
        bool maintain_Th = true;
        rebalance_columns(r_hse, th_hse, qv_hse, qv_hse, z_phys_nd[lev].get(), geom[lev], maintain_Th);
 
        // Update rho in the state from base state
        MultiFab::Copy(lev_new[Vars::cons], r_hse, 0, Rho_comp, 1, 1);
 
        // Update (rho theta) in the state from base state
        MultiFab::Copy(lev_new[Vars::cons], th_hse, 0, RhoTheta_comp, 1, 1);
        MultiFab::Multiply(lev_new[Vars::cons], lev_new[Vars::cons], Rho_comp, RhoTheta_comp, 1, 1);
 
        if (l_moist) {
            // Update (rho qv) in the state from base state
            MultiFab::Copy(lev_new[Vars::cons], qv_hse, 0, RhoQ1_comp, 1, 1);
            MultiFab::Multiply(lev_new[Vars::cons], lev_new[Vars::cons], Rho_comp, RhoQ1_comp, 1, 1);
        }
    }
 
    for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box &bx = mfi.tilebox();
        const auto& xvel_arr = lev_new[Vars::xvel].array(mfi);
        const auto& yvel_arr = lev_new[Vars::yvel].array(mfi);
        const auto& zvel_arr = lev_new[Vars::zvel].array(mfi);
        const auto& z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
        init_velocities_from_input_sounding(bx, xvel_arr, yvel_arr, zvel_arr,
                                            geom[lev].data(), z_nd_arr, input_sounding_data);
    }
 
    // *****************************************************************************
    // Re-compute p_hse and pi_hse
    // *****************************************************************************
    for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        Box bx = mfi.tilebox();
 
        const Array4<const Real>&  r_hse_arr =  r_hse.const_array(mfi);
        const Array4<const Real>& th_hse_arr = th_hse.const_array(mfi);
        const Array4<const Real>& qv_hse_arr = qv_hse.const_array(mfi);
 
        const Array4<      Real>&  p_hse_arr =  p_hse.array(mfi);
        const Array4<      Real>& pi_hse_arr = pi_hse.array(mfi);
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            Real rhotheta = r_hse_arr(i,j,k) * th_hse_arr(i,j,k);
             p_hse_arr(i,j,k) = getPgivenRTh(rhotheta,qv_hse_arr(i,j,k));
            pi_hse_arr(i,j,k) = getExnergivenRTh(rhotheta, l_rdOcp, qv_hse_arr(i,j,k));
        });
    }
}

Here is the call graph for this function:

init_geo_wind_profile()

void ERF::init_geo_wind_profile ( const std::string  input_file, amrex::Vector< amrex::Real > &  u_geos, amrex::Gpu::DeviceVector< amrex::Real > &  u_geos_d, amrex::Vector< amrex::Real > &  v_geos, amrex::Gpu::DeviceVector< amrex::Real > &  v_geos_d, const amrex::Geometry &  lgeom, const amrex::Vector< amrex::Real > &  zlev_stag  )

private

Read and interpolate a geostrophic wind profile onto ERF vertical levels.

Parameters

input_file	Path to the geostrophic wind profile
u_geos	Host vector storing interpolated geostrophic x-wind
u_geos_d	Device vector storing interpolated geostrophic x-wind
v_geos	Host vector storing interpolated geostrophic y-wind
v_geos_d	Device vector storing interpolated geostrophic y-wind
lgeom	Geometry defining the vertical domain
zlev_stag	Optional stretched-grid staggered height levels

{
    const int klo = 0;
    const int khi = lgeom.Domain().bigEnd()[AMREX_SPACEDIM-1];
    const amrex::Real dz = lgeom.CellSize()[AMREX_SPACEDIM-1];
 
    const bool grid_stretch = (zlev_stag.size() > 0);
    const Real zbot = (grid_stretch) ? zlev_stag[klo]   : lgeom.ProbLo(AMREX_SPACEDIM-1);
    const Real ztop = (grid_stretch) ? zlev_stag[khi+1] : lgeom.ProbHi(AMREX_SPACEDIM-1);
 
    amrex::Print() << "Reading geostrophic wind profile from " << input_file << std::endl;
    std::ifstream profile_reader(input_file);
    if(!profile_reader.is_open()) {
        amrex::Error("Error opening the abl_geo_wind_table\n");
    }
 
    // First, read the input data into temp vectors
    std::string line;
    Vector<Real> z_inp, Ug_inp, Vg_inp;
    Real z, Ug, Vg;
    amrex::Print() << "z  Ug  Vg" << std::endl;
    while(std::getline(profile_reader, line)) {
        std::istringstream iss(line);
        iss >> z >> Ug >> Vg;
        amrex::Print() << z << " " << Ug << " " << Vg << std::endl;
        z_inp.push_back(z);
        Ug_inp.push_back(Ug);
        Vg_inp.push_back(Vg);
        if (z >= ztop) break;
    }
 
    const int Ninp = z_inp.size();
    AMREX_ALWAYS_ASSERT(z_inp[0] <= zbot);
    AMREX_ALWAYS_ASSERT(z_inp[Ninp-1] >= ztop);
 
    // Now, interpolate vectors to the cell centers
    for (int k = 0; k <= khi; k++) {
        z = (grid_stretch) ? myhalf * (zlev_stag[k] + zlev_stag[k+1])
                           : zbot + (k + myhalf) * dz;
        u_geos[k] = interpolate_1d(z_inp.dataPtr(), Ug_inp.dataPtr(), z, Ninp);
        v_geos[k] = interpolate_1d(z_inp.dataPtr(), Vg_inp.dataPtr(), z, Ninp);
    }
 
    // Copy from host version to device version
    Gpu::copy(Gpu::hostToDevice, u_geos.begin(), u_geos.end(), u_geos_d.begin());
    Gpu::copy(Gpu::hostToDevice, v_geos.begin(), v_geos.end(), v_geos_d.begin());
 
    profile_reader.close();
}

Here is the call graph for this function:

init_ibseb()

void ERF::init_ibseb

(

)

Build the face set of every level from the blanking and, on a restart, refill its state from the checkpoint.

Called from ERF::InitData_post() after restart(), which is the first point where both paths (fresh start and restart) have the blanking of every level built and ghost-filled. The face list is always rebuilt from the blanking rather than read back, so the checkpoint carries only the state (IBSEBState, see IBFaceSet::state_ncomp()) and a restart on a different number of ranks works. A checkpoint from a run without the balance has no such field; the initial state is kept and a note is printed.

{
    if (!ibseb_params.enable) { return; }
    if (solverChoice.buildings_type != BuildingsType::ImmersedForcing) {
        Abort("erf.ibseb.enable needs erf.buildings_type = ImmersedForcing");
    }
    // The face detection takes every solid column of the blanking for a
    // building, so terrain by immersed forcing would be put under the
    // balance as well; it is not supported.
    if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
        Abort("erf.ibseb.enable does not support erf.terrain_type = ImmersedForcing: "
              "the balance runs on building faces only");
    }
    // The face areas, the face heights, the wall-function distance and the
    // ray cast all take the level's constant cell sizes.
    if (solverChoice.mesh_type != MeshType::ConstantDz) {
        Abort("erf.ibseb.enable needs a uniform vertical grid (no erf.terrain_z_levels or stretched mesh): "
              "the face geometry and the ray cast assume constant dz");
    }
    // The face list is built once here from the blanking; a regrid would
    // leave it indexing the old boxes.
    if (regrid_int > 0) {
        Abort("erf.ibseb.enable does not support regridding (erf.regrid_int > 0): the face list is built once at initialisation");
    }
    // The immersed forcing's own surface-temperature conditions would fight
    // the face balance for the same cells.
    if (solverChoice.if_init_surf_temp > 0.0 ||
        solverChoice.if_surf_temp_flux != Real(1.e-8) ||
        solverChoice.if_Olen_in != Real(1.e-8)) {
        Abort("erf.ibseb.enable: remove erf.if_init_surf_temp, erf.if_surf_temp_flux and erf.if_Olen; "
              "the face balance sets the temperature condition at the buildings");
    }
    m_ibseb.resize(finest_level + 1);
    for (int lev = 0; lev <= finest_level; ++lev) {
        m_ibseb[lev] = std::make_unique<IBFaceSet>(ibseb_params, lev);
        const double t_init0 = ParallelDescriptor::second();
        m_ibseb[lev]->build(*terrain_blanking[lev], geom[lev]);
        std::unique_ptr<MultiFab> restored;
        if (!restart_chkfile.empty()) {
            const std::string name = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "IBSEBState");
            if (FileExists(name + "_H")) {
                // The field's width is n_slots x (2 + n_slab_layers) and its
                // boxes follow the buildings; a checkpoint written with
                // another layer count or another building set cannot be
                // unpacked.
                const VisMF header(name);
                const int ncomp_chk = header.nComp();
                if (ncomp_chk != m_ibseb[lev]->state_ncomp()) {
                    Abort("erf.ibseb: IBSEBState in " + restart_chkfile + " has " + std::to_string(ncomp_chk)
                          + " components; the deck sets erf.ibseb.n_slab_layers = " + std::to_string(ibseb_params.n_slab_layers)
                          + ", which with the " + std::to_string(m_ibseb[lev]->n_slots())
                          + " face slots per cell of this blanking needs " + std::to_string(m_ibseb[lev]->state_ncomp())
                          + ". The checkpoint was written with another erf.ibseb.n_slab_layers (restart with the"
                            " checkpoint's value) or for another building set.");
                }
                if (header.boxArray() != m_ibseb[lev]->state_boxarray()) {
                    Abort("erf.ibseb: IBSEBState in " + restart_chkfile + " was written for a different building layout ("
                          + std::to_string(header.boxArray().size()) + " boxes against the "
                          + std::to_string(m_ibseb[lev]->state_boxarray().size()) + " the blanking gives); restart from a checkpoint of the same buildings");
                }
                restored = std::make_unique<MultiFab>(m_ibseb[lev]->make_state());
                VisMF::Read(*restored, name);
                m_ibseb[lev]->load_state(*restored);
                Print() << "[IBSEB] Face state restored from " << restart_chkfile << "\n";
            } else {
                Print() << "[IBSEB] Checkpoint has no IBSEBState; keeping the initial face state.\n";
            }
        }
        m_ibseb[lev]->assign_materials();
        m_ibseb[lev]->compute_view_fractions();
        m_ibseb[lev]->set_init_cost(ParallelDescriptor::second() - t_init0);
        // Initial diagnostics for the first report. On a restart they
        // overwrite the sensible flux with a diagnostic value, so the
        // checkpointed flux (which the convective velocity scale of the next
        // step reads as the previous step's) is put back afterwards.
        m_ibseb[lev]->compute_shortwave(t_new[lev]);
        m_ibseb[lev]->compute_longwave(vars_new[lev][Vars::cons]);
        m_ibseb[lev]->compute_sensible(vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel],
                                       vars_new[lev][Vars::yvel], vars_new[lev][Vars::zvel], solverChoice.c_p);
        if (restored) { m_ibseb[lev]->load_state(*restored); }
        m_ibseb[lev]->report(t_new[lev], istep[lev], ibseb_params.csv_int > 0);
    }
}

init_immersed_forcing()

void ERF::init_immersed_forcing

(

int

lev

)

Set velocities in cells that are immersed to be 0 (or a very small number)

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
    MultiFab* terrain_blank = terrain_blanking[lev].get();
    MultiFab* terrain_blank_xface = terrain_blanking_xface[lev].get();
    MultiFab* terrain_blank_yface = terrain_blanking_yface[lev].get();
    MultiFab* terrain_blank_zface = terrain_blanking_zface[lev].get();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
    {
        const Box &xbx = mfi.tilebox(IntVect(1,0,0));
        const Box &ybx = mfi.tilebox(IntVect(0,1,0));
        const Box &zbx = mfi.tilebox(IntVect(0,0,1));
        const Real epsilon = Real(1e-2);
 
        const Array4<const Real>& t_blank_arr = terrain_blank->const_array(mfi);
        const Array4<const Real>& t_blank_xface_arr = (terrain_blank_xface) ? terrain_blank_xface->const_array(mfi) :
                                                                               Array4<const Real>{};
        const Array4<const Real>& t_blank_yface_arr = (terrain_blank_yface) ? terrain_blank_yface->const_array(mfi) :
                                                                               Array4<const Real>{};
        const Array4<const Real>& t_blank_zface_arr = (terrain_blank_zface) ? terrain_blank_zface->const_array(mfi) :
                                                                               Array4<const Real>{};
 
        const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
        const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
        const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
 
        // Set the x,y,z-velocities
        ParallelFor(xbx, ybx, zbx,
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
            const Real t_blank = (t_blank_xface_arr) ? t_blank_xface_arr(i, j, k) :
                                 myhalf * (t_blank_arr(i, j, k) + t_blank_arr(i-1, j, k));
            if (t_blank == one) { xvel_arr(i, j, k) = epsilon; }
        },
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
            const Real t_blank = (t_blank_yface_arr) ? t_blank_yface_arr(i, j, k) :
                                 myhalf * (t_blank_arr(i, j, k) + t_blank_arr(i, j-1, k));
            if (t_blank == one) { yvel_arr(i, j, k) = epsilon; }
        },
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
            const Real t_blank = (t_blank_zface_arr) ? t_blank_zface_arr(i, j, k) :
                                 myhalf * (t_blank_arr(i, j, k) + t_blank_arr(i, j, k-1));
            if (t_blank == one) { zvel_arr(i, j, k) = epsilon; }
        });
    } //mfi
}

Here is the call graph for this function:

init_only()

void ERF::init_only	(	int	lev,
		double	time
	)

{
    t_new[lev] = elapsed_time;
    t_old[lev] = elapsed_time - bogus_large_value;
 
    auto& lev_new = vars_new[lev];
    auto& lev_old = vars_old[lev];
 
    // Loop over grids at this level to initialize our grid data
    lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
    lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
    lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
    lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
 
    // Initialize background flow (optional)
    if (solverChoice.init_type == InitType::Input_Sounding) {
        // The physbc's need the terrain but are needed for initHSE
        // We have already made the terrain in the call to init_zphys
        //    in MakeNewLevelFromScratch
        make_physbcs(lev);
 
        // Now init the base state and the data itself
        init_from_input_sounding(lev);
 
        // The base state has been initialized by integrating vertically
        // through the sounding for ideal (like WRF) or isentropic approaches
        if (solverChoice.sounding_type == SoundingType::Ideal ||
            solverChoice.sounding_type == SoundingType::Isentropic ||
            solverChoice.sounding_type == SoundingType::DryIsentropic) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
                "Gravity should be on to be consistent with sounding initialization.");
        } else { // SoundingType::ConstantDensity
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!solverChoice.use_gravity || (solverChoice.anelastic[lev] == 1),
                "Constant density probably doesn't make sense for compressible flow with gravity");
            initHSE();
        }
 
#ifdef ERF_USE_NETCDF
    }
    else if (solverChoice.init_type == InitType::WRFInput && !nc_init_file[lev].empty())
    {
        // The base state is initialized from WRF wrfinput data, output by
        // ideal.exe or real.exe
 
        init_from_wrfinput(lev, *mf_PSFC[lev]);
 
        // The physbc's need the terrain but are needed for initHSE
        make_physbcs(lev);
        (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
    }
    else if (solverChoice.init_type == InitType::WRFInput && nc_init_file[lev].empty())
    {
        //
        // A refined level with no wrfinput file of its own: the level was created by
        // ERF's own refinement machinery rather than read from a wrfinput_d0*.  Level 0
        // must always have a file (checked in ParameterSanityChecks), so this is lev > 0.
        //
        // Note the caller has already built the terrain at this level for us -- see the
        // ordering in MakeNewLevelFromScratch -- because unlike the has-a-file case there
        // is no file here to build z_phys from, and the base state below needs it.
        //
        AMREX_ALWAYS_ASSERT(lev > 0);
        AMREX_ALWAYS_ASSERT(z_phys_cc[lev] != nullptr);
 
        make_physbcs(lev);
 
        // Fill the part of this level that lies inside the domain but outside the fine
        // grids, and the fine ghost cells, from the coarser level.
        interp_base_state_from_coarse(lev);
 
        // Now build the base state on the fine grids by exactly the construction level 0
        // used: the analytic reference profile from the level-0 parameters, evaluated at
        // this level's cell-centered heights, followed by the discrete hydrostatic
        // rebalance.  Running the same procedure on both levels is what makes their base
        // states agree; interpolating the coarse base state instead would leave this level
        // not discretely hydrostatic on its own mesh.
        rebuild_base_state_from_wrfinput(lev, base_state[lev]);
        (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
 
        // The state itself is interpolated from the coarser level.  This must come after the
        // base state, since it interpolates perturbational quantities relative to it.
        FillCoarsePatch(lev, elapsed_time);
 
        // The 2D/surface arrays likewise have no file to come from at this level.  That
        // includes PSFC and the land mask / land type / soil type, which Interp2DArrays
        // takes from the parent for a level with no init file of its own.
        Interp2DArrays(lev, ba2d[lev], dmap[lev]);
    }
    else if (solverChoice.init_type == InitType::NCFile)
    {
        // The state is initialized by reading from a Netcdf file
        init_from_ncfile(lev);
 
        // The physbc's need the terrain but are needed for initHSE
        make_physbcs(lev);
    }
    else if (solverChoice.init_type == InitType::Metgrid)
    {
        // The base state is initialized from data output by WPS metgrid;
        // we will rebalance after interpolation
        init_from_metgrid(lev);
#endif
    } else if ( (solverChoice.init_type == InitType::Uniform        ) ||
                (solverChoice.init_type == InitType::ConstantDensity) ||
                (solverChoice.init_type == InitType::Isentropic     ) ||
                (solverChoice.init_type == InitType::ConstantDensityLinearTheta     ) ||
                (solverChoice.init_type == InitType::HindCast       ) ||
                (solverChoice.init_type == InitType::MoistBaseState ) ) {
        // Initialize a uniform density/entropy background field and base state
        // based on the problem-specified reference density and temperature
 
        // The physbc's need the terrain but are needed for initHSE
        make_physbcs(lev);
 
        // We will initialize the state from the background state so must set that first
        // The choice between constant rho and constant theta will be made inside initHSE
        initHSE(lev);
 
        // Copy rho and rhotheta from rho_hse and p_hse
        init_from_hse(lev);
 
        if (lev == 0 && cloud_chamber_config.active &&
            cloud_chamber_config.physical_initialization) {
            MultiFab p0(base_state[lev], make_alias, BaseState::p0_comp, 1);
            const Real p0_min = p0.min(0);
            const Real p0_max = p0.max(0);
            if (ParallelDescriptor::IOProcessor()) {
                Print() << "Cloud Chamber base pressure [Pa]: min=" << p0_min
                        << " max=" << p0_max
                        << " lower-boundary=" << p0_max << "\n";
            }
        }
 
    } else {
        Abort("Unknown init_type!");
    }
 
    // Add problem-specific flow features
    //
    // Notes:
    // - This calls init_custom_pert that is defined for each problem
    // - This may modify the base state
    // - The fields set by init_custom_pert are **perturbations** to the
    //   background flow set based on init_type
    if (solverChoice.init_type != InitType::NCFile) {
        init_custom(lev);
    }
 
    // Ensure that the face-based data are the same on both sides of a periodic domain.
    // The data associated with the lower grid ID is considered the correct value.
    lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
    lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
    lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
 
    if (solverChoice.spongeChoice.sponge_type == SpongeType::Input_Sponge)
    {
        input_sponge(lev);
    }
 
    // Initialize turbulent perturbation
    if (solverChoice.use_perturbation(lev)) {
        turbPert_update(lev, zero);
        if (solverChoice.use_wvel_perturbation(lev)) {
            turbPert_amplitude_w(lev);
        } else {
            turbPert_amplitude(lev);
        }
    }
 
    // Set initial velocity field for immersed cells to be close to 0
    if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
        solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
        init_immersed_forcing(lev);
    }
}

Here is the call graph for this function:

init_phys_bcs()

void ERF::init_phys_bcs ( bool &  read_prim_theta )

private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, InflowOutflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

Parameters

[in,out]

read_prim_theta

Whether external Dirichlet theta values should be read as primitive theta.

{
    bool anelastic = false;
    for (int lev = 0; lev <= max_level; ++lev) {
        anelastic = anelastic || (solverChoice.anelastic[lev] != 0);
    }
 
    auto f = [this,&read_prim_theta,anelastic,one_d=one,zero_d=zero] (std::string const& bcid, Orientation ori)
    {
        // These are simply defaults for Dirichlet faces -- they should be over-written below
        m_bc_extdir_vals[BCVars::Rho_bc_comp][ori]       =  one_d;
        m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori]  = -one_d; // It is important to set this negative
                                                                 // because the sign is tested on below
        for (int n = BCVars::RhoKE_bc_comp; n < BCVars::xvel_bc; n++) {
            m_bc_extdir_vals[n][ori]                     = zero_d;
        }
 
        m_bc_extdir_vals[BCVars::xvel_bc][ori] = zero_d; // default
        m_bc_extdir_vals[BCVars::yvel_bc][ori] = zero_d;
        m_bc_extdir_vals[BCVars::zvel_bc][ori] = zero_d;
 
        // These are simply defaults for Neumann gradients -- they should be over-written below
        m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]       = zero_d;
        m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]  = zero_d;
 
        m_bc_neumann_vals[BCVars::RhoKE_bc_comp][ori]     = zero_d;
        m_bc_neumann_vals[BCVars::RhoScalar_bc_comp][ori] = zero_d;
        m_bc_neumann_vals[BCVars::RhoQ1_bc_comp][ori]     = zero_d;
        m_bc_neumann_vals[BCVars::RhoQ2_bc_comp][ori]     = zero_d;
        m_bc_neumann_vals[BCVars::RhoQ3_bc_comp][ori]     = zero_d;
        m_bc_neumann_vals[BCVars::RhoQ4_bc_comp][ori]     = zero_d;
        m_bc_neumann_vals[BCVars::RhoQ5_bc_comp][ori]     = zero_d;
        m_bc_neumann_vals[BCVars::RhoQ6_bc_comp][ori]     = zero_d;
 
        m_bc_neumann_vals[BCVars::xvel_bc][ori] = zero_d;
        m_bc_neumann_vals[BCVars::yvel_bc][ori] = zero_d;
        m_bc_neumann_vals[BCVars::zvel_bc][ori] = zero_d;
 
        for (int n = 0; n < NBCVAR_max; ++n) {
            m_wall_scalar_bc_intent[n][ori] = erf_wall_scalar_bc::WallScalarBCIntent::Unspecified;
        }
 
        std::string pp_text = pp_prefix + "." + bcid;
        ParmParse pp(pp_text);
 
        std::string bc_type_in;
        if (pp.queryAdd("type", bc_type_in) <= 0)
        {
            pp_text = bcid;
            pp = ParmParse(pp_text);
            pp.queryAdd("type", bc_type_in);
        }
 
        std::string bc_type = amrex::toLower(bc_type_in);
 
        auto apply_wall_scalar = [this,ori] (int comp,
                                             const erf_wall_scalar_bc::ParsedWallScalarBC& parsed)
        {
            using Intent = erf_wall_scalar_bc::WallScalarBCIntent;
            if (parsed.intent == Intent::DirichletConserved ||
                parsed.intent == Intent::DirichletPrimitive) {
                m_bc_extdir_vals[comp][ori] = parsed.stored_value;
            } else if (parsed.intent == Intent::Neumann) {
                m_bc_neumann_vals[comp][ori] = parsed.stored_value;
            }
            m_wall_scalar_bc_intent[comp][ori] = parsed.intent;
        };
 
        if (bc_type == "symmetry")
        {
            // Print() << bcid << " set to symmetry.\n";
              phys_bc_type[ori] = ERF_BC::symmetry;
            domain_bc_type[ori] = "Symmetry";
        }
        else if ( (ori.coordDir() != 2 && solverChoice.use_real_bcs) || (bc_type == "outflow") )
        {
            // Print() << bcid << " set to outflow.\n";
              phys_bc_type[ori] = ERF_BC::outflow;
            domain_bc_type[ori] = "Outflow";
        }
        else if (bc_type == "open")
        {
            // Print() << bcid << " set to open.\n";
            AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
              phys_bc_type[ori] = ERF_BC::open;
            domain_bc_type[ori] = "Open";
        }
        else if (bc_type == "ho_outflow")
        {
            phys_bc_type[ori] = ERF_BC::ho_outflow;
            domain_bc_type[ori] = "HO_Outflow";
        }
 
        else if (bc_type == "inflow" || bc_type == "inflow_outflow")
        {
            if (bc_type == "inflow") {
                // Print() << bcid << " set to inflow.\n";
                  phys_bc_type[ori] = ERF_BC::inflow;
                domain_bc_type[ori] = "Inflow";
            } else {
                // Print() << bcid << " set to inflow_outflow.\n";
                  phys_bc_type[ori] = ERF_BC::inflow_outflow;
                domain_bc_type[ori] = "InflowOutflow";
            }
 
            std::vector<Real> v;
            if (input_bndry_planes && m_r2d->ingested_velocity()) {
                m_bc_extdir_vals[BCVars::xvel_bc][ori] = zero_d;
                m_bc_extdir_vals[BCVars::yvel_bc][ori] = zero_d;
                m_bc_extdir_vals[BCVars::zvel_bc][ori] = zero_d;
            } else {
                // Test for input data file if at xlo face
                std::string dirichlet_file;
                auto file_exists = pp.queryAdd("dirichlet_file", dirichlet_file);
                if (file_exists) {
                    pp.queryAdd("read_prim_theta", read_prim_theta);
                    init_Dirichlet_bc_data(dirichlet_file);
                } else {
                    pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
                    m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                    m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                    m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
                }
            }
 
            Real rho_in = zero_d;
            if (input_bndry_planes && m_r2d->ingested_density()) {
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = zero_d;
            } else {
                // Negative sentinel rather than the queryAdd return value: a density is
                // always positive -- the "rho_in > 0" test just below already reads
                // presence off the value -- whereas the return value only reports whether
                // the key existed before this call.  init_bcs() runs twice on restart, so
                // the second pass would otherwise see every key as user-specified.
                rho_in = Real(-1.0);
                pp.queryAdd("density", rho_in);
                if (rho_in <= zero_d) {
                    rho_in = zero_d;
                    amrex::Print() << "Using interior values to set conserved vars" << std::endl;
                }
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
            }
 
            bool th_read  = (th_bc_data[0].data()!=nullptr);
            Real theta_in = zero_d;
            if (input_bndry_planes && m_r2d->ingested_theta()) {
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = zero_d;
            } else if (!th_read) {
                if (rho_in > 0) {
                    pp.get("theta", theta_in);
                }
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
            }
 
            // Non-reflecting inflow: prescribe velocity and density but
            // extrapolate RhoTheta (and hence pressure) from the interior.
            // This lets upstream-propagating acoustic waves exit the domain
            // instead of reflecting off the rigid Dirichlet boundary.
            bool nonreflecting = false;
            pp.queryAdd("nonreflecting", nonreflecting);
            m_bc_nonreflecting[ori] = nonreflecting;
 
            Real scalar_in = zero_d;
            if (input_bndry_planes && m_r2d->ingested_scalar()) {
                m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = zero_d;
            } else {
                if (pp.query("scalar", scalar_in))
                m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
            }
 
            if (solverChoice.moisture_type != MoistureType::None) {
                Real qv_in = Real(-1.0);
                if (input_bndry_planes && m_r2d->ingested_q1()) {
                    m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = zero_d;
                } else {
                    // Negative sentinel rather than the queryAdd return value: a mixing
                    // ratio is never negative, and the return value only reports whether
                    // the key existed before this call (init_bcs() runs twice on restart).
                    pp.queryAdd("qv", qv_in);
                    if (qv_in >= zero_d) {
                        m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
                    }
                }
                Real qc_in = Real(-1.0);
                if (input_bndry_planes && m_r2d->ingested_q2()) {
                    m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = zero_d;
                } else {
                    // Negative sentinel rather than the queryAdd return value: a mixing
                    // ratio is never negative, and the return value only reports whether
                    // the key existed before this call (init_bcs() runs twice on restart).
                    pp.queryAdd("qc", qc_in);
                    if (qc_in >= zero_d) {
                        m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
                    }
                }
            }
 
            Real KE_in = Real(-1.0);
            if (input_bndry_planes && m_r2d->ingested_KE()) {
                m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = zero_d;
            } else {
                // Negative sentinel rather than the queryAdd return value: a turbulent
                // kinetic energy is never negative, and the return value only reports
                // whether the key existed before this call (init_bcs() runs twice on
                // restart).
                pp.queryAdd("KE", KE_in);
                if (KE_in >= zero_d) {
                    m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
                }
            }
        }
        else if (bc_type == "noslipwall")
        {
            // Print() << bcid <<" set to no-slip wall.\n";
              phys_bc_type[ori] = ERF_BC::no_slip_wall;
            domain_bc_type[ori] = "NoSlipWall";
 
            std::vector<Real> v;
 
            // The values of m_bc_extdir_vals default to zero
            // But if we find "velocity" in the inputs file, use those values instead.
            if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
            {
                v[ori.coordDir()] = zero_d;
                m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
            }
 
            const auto parsed = erf_wall_scalar_bc::parse_wall_face_scalars(
                pp, bcid, erf_wall_scalar_bc::SolidWallKind::NoSlip, anelastic);
            if (!parsed.ok()) {
                amrex::Error(parsed.error);
            }
            apply_wall_scalar(BCVars::Rho_bc_comp, parsed.scalars.density);
            apply_wall_scalar(BCVars::RhoTheta_bc_comp, parsed.scalars.theta);
            apply_wall_scalar(BCVars::RhoQ1_bc_comp, parsed.scalars.qv);
        }
        else if (bc_type == "slipwall")
        {
            // Print() << bcid <<" set to slip wall.\n";
 
              phys_bc_type[ori] = ERF_BC::slip_wall;
            domain_bc_type[ori] = "SlipWall";
 
            const auto parsed = erf_wall_scalar_bc::parse_wall_face_scalars(
                pp, bcid, erf_wall_scalar_bc::SolidWallKind::Slip, anelastic);
            if (!parsed.ok()) {
                amrex::Error(parsed.error);
            }
            apply_wall_scalar(BCVars::Rho_bc_comp, parsed.scalars.density);
            apply_wall_scalar(BCVars::RhoTheta_bc_comp, parsed.scalars.theta);
        }
        else if (bc_type == "surface_layer")
        {
              phys_bc_type[ori] = ERF_BC::surface_layer;
            domain_bc_type[ori] = "surface_layer";
        }
        else
        {
            phys_bc_type[ori] = ERF_BC::undefined;
        }
 
        if (geom[0].isPeriodic(ori.coordDir())) {
            domain_bc_type[ori] = "Periodic";
            if (phys_bc_type[ori] == ERF_BC::undefined)
            {
                phys_bc_type[ori] = ERF_BC::periodic;
            } else {
                Abort("Wrong BC type for periodic boundary");
            }
        }
 
        if (phys_bc_type[ori] == ERF_BC::undefined)
        {
             if (solverChoice.use_real_bcs && ori.coordDir() < 2) {
                 Print() << "We are using real bc's so don't need to set lateral bc's" << std::endl;
             } else {
                 Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
                 Abort("This BC type is unknown");
             }
        }
    };
 
    f("xlo", Orientation(Direction::x,Orientation::low));
    f("xhi", Orientation(Direction::x,Orientation::high));
    f("ylo", Orientation(Direction::y,Orientation::low));
    f("yhi", Orientation(Direction::y,Orientation::high));
    f("zlo", Orientation(Direction::z,Orientation::low));
    f("zhi", Orientation(Direction::z,Orientation::high));
}

Here is the call graph for this function:

init_stations()

void ERF::init_stations

(

)

{
    if (!station_sampler) { return; }
 
    auto& stations = station_sampler->stations();
 
    // The set of 2D diagnostics this configuration can actually fill, selected
    // exactly as Write2DPlotFile selects them.
    const bool has_surface_layer =
        phys_bc_type[Orientation(Direction::z, Orientation::low)] == ERF_BC::surface_layer;
    const auto active_lsm_names = lsm.Get_DataNames();
    const auto available_2d = plotfile2d::available_diagnostic_names(solverChoice,
                                                                    has_surface_layer,
                                                                    active_lsm_names);
 
    Vector<std::string> requested_3d;
    Vector<std::string> requested_2d;
 
    for (auto& station : stations)
    {
        for (auto& var : station.vars)
        {
            // Is it a 3D plot variable that this configuration can produce?
            Vector<std::string> probe{var.name};
            canonicalizePlot3DVariables(probe);
            appendPlot3DVariables(Vector<std::string>{var.name}, probe);
            if (containerHasElement(probe, var.name)) {
                var.is_2d = false;
                if (!containerHasElement(requested_3d, var.name)) {
                    requested_3d.push_back(var.name);
                }
                continue;
            }
 
            // Is it a 2D diagnostic?
            const auto* descriptor = plotfile2d::find_diagnostic(var.name);
            if (descriptor == nullptr) {
                descriptor = plotfile2d::find_dynamic_soil_diagnostic(var.name);
            }
            if (descriptor != nullptr) {
                var.is_2d  = true;
                var.units  = descriptor->units;
                if (containerHasElement(available_2d, var.name)) {
                    if (!containerHasElement(requested_2d, var.name)) {
                        requested_2d.push_back(var.name);
                    }
                } else {
                    // Same convention as the 2D plotfile: a diagnostic that this
                    // run does not compute is written as its documented missing
                    // value rather than dropped.
                    var.is_missing = true;
                    var.missing_value =
                        (descriptor->missing_policy == plotfile2d::MissingPolicy::FillZeroWhenUnavailable)
                        ? Real(0.0) : Real(-999.0);
                }
                continue;
            }
 
            // Neither: say which of the two failures it is.
            if (containerHasElement(derived_names, var.name) ||
                containerHasElement(cons_names, var.name)    ||
                var.name == "x_velocity" || var.name == "y_velocity" || var.name == "z_velocity")
            {
                Abort("Station '" + station.name + "': '" + var.name +
                      "' is a 3D plotfile variable but is not available in this configuration");
            }
            Abort("Station '" + station.name + "': '" + var.name +
                  "' is neither a 3D plotfile variable nor a built-in 2D diagnostic, which "
                  "are the two catalogs a station can draw on.  Note that a 2D plotfile can "
                  "also carry sampled-level fields, named for the field and the level such "
                  "as 'theta_z100m', and those are not among them: ask for the 3D variable "
                  "itself and give the station a height instead.  See erf.plot_vars_1 and "
                  "erf.plot2d_vars_1 in the documentation for the names that can be requested");
        }
    }
 
    // The fill lists, in the order the fill routines produce their components.
    station_vars_3d = requested_3d;
    canonicalizePlot3DVariables(station_vars_3d);
    appendPlot3DVariables(requested_3d, station_vars_3d);
 
    station_vars_2d = plotfile2d::select_requested_plot_variables(requested_2d, available_2d).accepted;
 
    for (auto& station : stations) {
        for (auto& var : station.vars) {
            if (var.is_missing) { continue; }
            const auto& list = var.is_2d ? station_vars_2d : station_vars_3d;
            for (int i = 0; i < static_cast<int>(list.size()); ++i) {
                if (list[i] == var.name) { var.comp = i; break; }
            }
            AMREX_ALWAYS_ASSERT(var.comp >= 0);
        }
 
        if (station.has_3d_vars() && station.heights.empty()) {
            Abort("Station '" + station.name + "': erf." + station.name + ".height_agl or erf." +
                  station.name + ".height_abs must be given, since this station requests a 3D "
                  "variable");
        }
    }
 
    resolve_station_positions();
 
    station_sampler->buildColumns();
    station_sampler->setWriteTimestamp(use_datetime, datetime_format);
 
    // Everything the signature describes is now resolved, so a restart that
    // would append to a file describing different columns stops here, before the
    // run does any work.
    station_sampler->checkRestartFiles();
 
    if (verbose > 0) {
        Print() << "Station output: " << stations.size() << " station(s), "
                << station_sampler->numColumns() << " column(s) per output step" << std::endl;
    }
}

Here is the call graph for this function:

init_stuff()

void ERF::init_stuff ( int  lev, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm, amrex::Vector< amrex::MultiFab > &  lev_new, amrex::Vector< amrex::MultiFab > &  lev_old, amrex::MultiFab &  tmp_base_state, std::unique_ptr< amrex::MultiFab > &  tmp_zphys_nd  )

private

{
    // ********************************************************************************************
    // Base state holds r_0, pres_0, pi_0, th_0 (in that order)
    //
    // Here is where we set the number of ghost cells for the base state!
    // ********************************************************************************************
    int ngb = ComputeGhostCells(solverChoice) + 1;
    tmp_base_state.define(ba,dm,BaseState::num_comps,ngb);
    tmp_base_state.setVal(zero);
 
    if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
        base_state_new[lev].define(ba,dm,BaseState::num_comps,base_state[lev].nGrowVect());
        base_state_new[lev].setVal(zero);
    }
 
    // ********************************************************************************************
    // Allocate terrain arrays
    // ********************************************************************************************
 
    BoxArray ba_nd(ba);
    ba_nd.surroundingNodes();
 
    // NOTE: this is where we actually allocate z_phys_nd -- but here it's called "tmp_zphys_nd"
    // We need this to be one greater than the ghost cells to handle levels > 0
 
    int ngrow = ComputeGhostCells(solverChoice) + 2;
    tmp_zphys_nd = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
 
    // Offset z-coordinate for interpolate_1d when plane EB is used.
    Real z_offset = zero;
    if (solverChoice.terrain_type == TerrainType::EB) {
        ParmParse pp_eb2("eb2");
        std::string geometry;
        pp_eb2.queryAdd("geometry", geometry);
        if (geometry == "plane") {
            RealArray plane_point{zero, zero, zero};
            pp_eb2.queryAdd("plane_point", plane_point);
            z_offset = plane_point[2];
        }
    }
 
    z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,2);
    init_default_zphys(lev, geom[lev], *tmp_zphys_nd, *z_phys_cc[lev], z_offset);
 
    if (solverChoice.terrain_type == TerrainType::MovingFittedMesh)
    {
        detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
        detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
 
        ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
        ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
        az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
 
        z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
 
        z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
        z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
        z_phys_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
    }
    else
    {
        z_phys_nd_new[lev] = nullptr;
          detJ_cc_new[lev] = nullptr;
 
        z_phys_nd_src[lev] = nullptr;
        z_phys_cc_src[lev] = nullptr;
          detJ_cc_src[lev] = nullptr;
 
               z_t_rk[lev] = nullptr;
    }
 
    if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
        solverChoice.buildings_type == BuildingsType::ImmersedForcing)
    {
        terrain_blanking[lev] = std::make_unique<MultiFab>(ba,dm,1,ngrow);
        terrain_blanking[lev]->setVal(one);
 
#if USE_FC_FACTORY
        // Face-centered terrain blanking for momentum forcing
        terrain_blanking_xface[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,ngrow);
        terrain_blanking_yface[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,ngrow);
        terrain_blanking_zface[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,ngrow);
        terrain_blanking_xface[lev]->setVal(one);
        terrain_blanking_yface[lev]->setVal(one);
        terrain_blanking_zface[lev]->setVal(one);
#endif
 
        // Initialize planar average storage for immersed forcing
        // Sized to match PlaneAverage output with ghost cells, using same indexing as subsidence
        Box domain = geom[lev].Domain();
        Box tdomain = domain; tdomain.grow(2, 1);  // Grow by 1 ghost cell in z-direction
        r_plane_avg[lev].resize({tdomain.smallEnd(2)}, {tdomain.bigEnd(2)});
        t_plane_avg[lev].resize({tdomain.smallEnd(2)}, {tdomain.bigEnd(2)});
    }
 
    // We use these area arrays regardless of terrain, EB or none of the above
    detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
         ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
         ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
         az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
 
    detJ_cc[lev]->setVal(one);
         ax[lev]->setVal(one);
         ay[lev]->setVal(one);
         az[lev]->setVal(one);
 
    // ********************************************************************************************
    // Create wall distance array for RANS modeling
    // ********************************************************************************************
    if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
        walldist[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
        walldist[lev]->setVal(bogus_large_value);
    } else {
        walldist[lev] = nullptr;
    }
 
    // ********************************************************************************************
    // These are the persistent containers for the old and new data
    // ********************************************************************************************
    int ncomp;
    if (lev > 0) {
        ncomp = vars_new[lev-1][Vars::cons].nComp();
    } else {
        int n_qstate   = micro->Get_Qstate_Size();
        ncomp = NDRY + NSCALARS + n_qstate;
    }
 
    // ********************************************************************************************
    // The number of ghost cells for density must be 1 greater than that for velocity
    //     so that we can go back in forth between velocity and momentum on all faces
    // ********************************************************************************************
    int ngrow_state = ComputeGhostCells(solverChoice) + 1;
    int ngrow_vels  = ComputeGhostCells(solverChoice);
 
    // ********************************************************************************************
    // New solution data containers
    // ********************************************************************************************
    if (solverChoice.terrain_type != TerrainType::EB) {
        lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
        lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
    } else {
        // EB: Define the MultiFabs with the EBFactory
        lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
        lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
    }
 
    // Initialize all components to zero so we don't need to explicitly set
    //     scalars / moisture variables to zero in the initialization
    lev_new[Vars::cons].setVal(zero);
    lev_old[Vars::cons].setVal(zero);
 
    lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
    lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
 
    lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
    lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
 
    // Set these to avoid operations on uninitialized data
    lev_new[Vars::xvel].setVal(bogus_large_value);
    lev_old[Vars::xvel].setVal(bogus_large_value);
    lev_new[Vars::yvel].setVal(bogus_large_value);
    lev_old[Vars::yvel].setVal(bogus_large_value);
 
    // Note that we need the ghost cells in the z-direction if we are doing any
    // kind of domain decomposition in the vertical (at level 0 or above)
    lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
    lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
 
    gradp[lev][GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp[lev][GpVars::gpx].setVal(zero);
    gradp[lev][GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp[lev][GpVars::gpy].setVal(zero);
    gradp[lev][GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp[lev][GpVars::gpz].setVal(zero);
 
    if ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) {
        pp_inc[lev].define(ba, dm, 1, 1);
        pp_inc[lev].setVal(zero);
    }
 
    // We use this in the fast substepping only
    if (solverChoice.anelastic[lev] == 0) {
        lagged_delta_rt[lev].define(ba, dm, 1, 1);
        lagged_delta_rt[lev].setVal(zero);
    }
 
    // We use these for advecting the slow variables, whether anelastic or compressible
    avg_xmom[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, 1);
    avg_ymom[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, 1);
    avg_zmom[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, 1);
    avg_xmom[lev].setVal(0); avg_ymom[lev].setVal(0); avg_zmom[lev].setVal(zero);
 
    // ********************************************************************************************
    // These are just used for scratch in the time integrator but we might as well define them here
    // ********************************************************************************************
    if (solverChoice.terrain_type != TerrainType::EB) {
        rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
        rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
 
        rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
        rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
 
        rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
        rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
    } else {
        // EB: Define the MultiFabs with the EBFactory
        rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
        rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
 
        rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
        rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
 
        rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
        rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
    }
 
    if (lev > 0) {
        //xmom_crse_rhs[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, IntVect{0});
        //ymom_crse_rhs[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, IntVect{0});
        zmom_crse_rhs[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect{0});
    }
 
    // We do this here just so they won't be undefined in the initial FillPatch
    rU_old[lev].setVal(bogus_large_value);
    rV_old[lev].setVal(bogus_large_value);
    rW_old[lev].setVal(bogus_large_value);
    rU_new[lev].setVal(bogus_large_value);
    rV_new[lev].setVal(bogus_large_value);
    rW_new[lev].setVal(bogus_large_value);
 
    // ********************************************************************************************
    // These are just time averaged fields for diagnostics
    // ********************************************************************************************
 
    // NOTE: We are not completing a fillpatch call on the time averaged data;
    //       which would copy on intersection and interpolate from coarse.
    //       Therefore, we are restarting the averaging when the ba changes,
    //       this may give poor statistics for dynamic mesh refinement.
    vel_t_avg[lev] = nullptr;
    if (solverChoice.time_avg_vel) {
        vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
        vel_t_avg[lev]->setVal(zero);
        t_avg_cnt[lev] = zero;
    }
 
    // Components: u, v, w, theta, uu, vv, ww, uw, vw, wtheta.
    // The field storage is rebuilt for this level, but the time-reset state is
    // global and must survive regridding of any level.
    interval_means[lev] = nullptr;
    if (solverChoice.compute_mean_vars) {
        interval_means[lev] = std::make_unique<MultiFab>(ba, dm, 10, 0);
        interval_means[lev]->setVal(zero);
        t_mean_cnt[lev] = 0.0;
    }
 
    // ********************************************************************************************
    // Initialize flux registers whenever we create/re-create a level
    // ********************************************************************************************
    if (solverChoice.coupling_type == CouplingType::TwoWay) {
        if (lev == 0) {
            advflux_reg[0] = nullptr;
        } else {
            int ncomp_reflux = vars_new[0][Vars::cons].nComp();
            advflux_reg[lev] = new YAFluxRegister(ba       , grids[lev-1],
                                                  dm       ,  dmap[lev-1],
                                                  geom[lev],  geom[lev-1],
                                                  ref_ratio[lev-1], lev, ncomp_reflux);
        }
    }
 
    // ********************************************************************************************
    // Define Theta_prim storage if using surface_layer BC
    // ********************************************************************************************
    Theta_prim[lev] = nullptr;
    Qv_prim[lev]    = nullptr;
    Qr_prim[lev]    = nullptr;
 
    for (OrientationIter oit; oit; ++oit) {
        Orientation ori = oit();
        if (phys_bc_type[ori] == ERF_BC::surface_layer) {
            amrex::Print() << " Found MOST at face " << ori << " : Constructing primitive vars for MOST.." << std::endl;
            Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
            if (solverChoice.moisture_type != MoistureType::None) {
                Qv_prim[lev]    = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
                Qr_prim[lev]    = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
            } else {
                Qv_prim[lev]    = nullptr;
                Qr_prim[lev]    = nullptr;
            }
            // these only need to be defined once
            break;
        }
    }
 
    // ********************************************************************************************
    // Build 1D BA and 2D BA
    // ********************************************************************************************
 
    // NOTE: By design, the compressed BAs have their compressed indices set to 0
    //       MFIters that need more detailed box information should be done over 3D MFs
 
    // Build 2D BA
    BoxList bl2d = ba.boxList();
    for (auto& b : bl2d) {
        b.setRange(2,0);
    }
    ba2d[lev]  = BoxArray(std::move(bl2d));
 
    // Build 1D BA
    BoxList bl1d = ba.boxList();
    for (auto& b : bl1d) {
        b.setRange(0,0);
        b.setRange(1,0);
    }
    ba1d[lev]  = BoxArray(std::move(bl1d));
 
    // ********************************************************************************************
    // Vertical extent of the grid column over each (i,j) -- needed by the implicit
    //     vertical diffusion solves, which must treat a column as one tridiagonal system
    // ********************************************************************************************
    define_column_kextent(lev, ba, dm);
 
    // ********************************************************************************************
    // Map factors
    // ********************************************************************************************
    mapfac[lev].resize(MapFacType::num);
    mapfac[lev][MapFacType::m_x] = std::make_unique<MultiFab>(                        ba2d[lev],dm,1,IntVect(3,3,0));
    mapfac[lev][MapFacType::u_x] = std::make_unique<MultiFab>(convert(ba2d[lev],IntVect(1,0,0)),dm,1,IntVect(3,3,0));
    mapfac[lev][MapFacType::v_x] = std::make_unique<MultiFab>(convert(ba2d[lev],IntVect(0,1,0)),dm,1,IntVect(3,3,0));
 
#if 0
    // For now we comment this out to avoid CI failures but we will need to re-enable
    //     this if using non-conformal mappings
    if (MapFacType::m_y != MapFacType::m_x) {
        mapfac[lev][MapFacType::m_y] = std::make_unique<MultiFab>(ba2d[lev],dm,1,IntVect(3,3,0));
    }
    if (MapFacType::u_y != MapFacType::u_x) {
        mapfac[lev][MapFacType::u_y] = std::make_unique<MultiFab>(convert(ba2d[lev],IntVect(1,0,0)),dm,1,IntVect(3,3,0));
    }
    if (MapFacType::v_y != MapFacType::v_x) {
        mapfac[lev][MapFacType::v_y] = std::make_unique<MultiFab>(convert(ba2d[lev],IntVect(0,1,0)),dm,1,IntVect(3,3,0));
    }
#endif
 
    if (solverChoice.test_mapfactor) {
        for (int i = 0; i < 3; i++) {
            mapfac[lev][i]->setVal(myhalf);
        }
        for (int i = 3; i < mapfac[lev].size(); i++) {
            mapfac[lev][i]->setVal(fourth);
        }
    } else {
        for (int i = 0; i < mapfac[lev].size(); i++) {
            mapfac[lev][i]->setVal(one);
        }
    }
 
    if (solverChoice.nudging_from_input_sounding) {
        nudge_data[lev] = std::make_unique<MultiFab>(ba, dm, 4, ngrow_state);
        nudge_data[lev]->setVal(0.0);
    }
 
    if (solverChoice.large_scale_forcing) {
        lsf_data[lev] = std::make_unique<MultiFab>(ba, dm, 8, ngrow_state);
        lsf_data[lev]->setVal(0.0);
    }
 
 
    // ********************************************************************************************
    // Build WRF data structures
    // ********************************************************************************************
    IntVect ng  = vars_new[lev][Vars::cons].nGrowVect();
 
    if (lev == 0) {
        wrf_C1H  = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
        wrf_C2H  = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
        wrf_RDNW = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
        wrf_MUB  = std::make_unique<MultiFab>(ba2d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
        wrf_PHB  = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,IntVect(ngrow+1,ngrow+1,0));
    }
 
    mf_PSFC[lev] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ng);
 
    //*********************************************************
    // Variables for Fitch model for windfarm parametrization
    //*********************************************************
#if defined(ERF_USE_WINDFARM)
    if (solverChoice.windfarm_type == WindFarmType::Fitch){
        vars_windfarm[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
    }
    if (solverChoice.windfarm_type == WindFarmType::EWP){
        vars_windfarm[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
    }
    if (solverChoice.windfarm_type == WindFarmType::SimpleAD) {
        vars_windfarm[lev].define(ba, dm, 2, ngrow_state);// dudt, dvdt
    }
    if (solverChoice.windfarm_type == WindFarmType::GeneralAD) {
        vars_windfarm[lev].define(ba, dm, 3, ngrow_state);// dudt, dvdt, dwdt
    }
        Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
        SMark[lev].define(ba, dm, 2, 1); // Free stream velocity/source term
                                                   // sampling marker in a cell - 2 components
#endif
 
    if(solverChoice.init_type == InitType::HindCast and
        solverChoice.hindcast_lateral_forcing) {
 
        int ncomp_extra = 2;
        int nvars = vars_new[lev].size();
 
        // Resize all containers
        forecast_state_1[lev].resize(nvars + 1);
        forecast_state_2[lev].resize(nvars + 1);
        forecast_state_interp[lev].resize(nvars + 1);
 
        // Define the "normal" components
        for (int comp = 0; comp < nvars; ++comp) {
            const MultiFab& src = vars_new[lev][comp];
            ncomp = src.nComp();
            ngrow = src.nGrow();
 
            forecast_state_1[lev][comp].define(ba, dm, ncomp, ng);
            forecast_state_2[lev][comp].define(ba, dm, ncomp, ng);
            forecast_state_interp[lev][comp].define(ba, dm, ncomp, ng);
        }
 
        // Define the "extra" component (last slot)
        {
            const MultiFab& src0 = vars_new[lev][0];
            ngrow = src0.nGrow();
            int idx = nvars;
 
            forecast_state_1[lev][idx].define(ba, dm, ncomp_extra, ngrow);
            forecast_state_2[lev][idx].define(ba, dm, ncomp_extra, ngrow);
            forecast_state_interp[lev][idx].define(ba, dm, ncomp_extra, ngrow);
        }
        bool regrid_forces_file_read = true;
        WeatherDataInterpolation(lev, t_new[0],z_phys_nd, regrid_forces_file_read);
    }
 
 
    if(solverChoice.init_type == InitType::HindCast and
        solverChoice.hindcast_surface_bcs)
 
    {
        const MultiFab& src = vars_new[lev][0];
        const amrex::DistributionMapping& dm_hc = src.DistributionMap();
 
        surface_state_1[lev].define(ba2d[lev], dm_hc, 2, src.nGrow());
        surface_state_2[lev].define(ba2d[lev], dm_hc, 2, src.nGrow());
        surface_state_interp[lev].define(ba2d[lev], dm_hc, 2, src.nGrow());
 
        bool regrid_forces_file_read = true;
        SurfaceDataInterpolation(lev, t_new[0], z_phys_nd, regrid_forces_file_read);
    }
 
#ifdef ERF_USE_WW3_COUPLING
    // create a new BoxArray and DistributionMapping for a MultiFab with 1 box
    BoxArray ba_onegrid(geom[lev].Domain());
    BoxList bl2d_onegrid = ba_onegrid.boxList();
    for (auto& b : bl2d_onegrid) { b.setRange(2,b.smallEnd(2)); }
    BoxArray ba2d_onegrid(std::move(bl2d_onegrid));
    Vector<int> pmap;
    pmap.resize(1);
    pmap[0]=0;
    DistributionMapping dm_onegrid(ba2d_onegrid);
    dm_onegrid.define(pmap);
 
    Hwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
    Lwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
 
    BoxList bl2d_wave = ba.boxList();
    for (auto& b : bl2d_wave) { b.setRange(2,b.smallEnd(2)); }
    BoxArray ba2d_wave(std::move(bl2d_wave));
 
    Hwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
    Lwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
 
    std::cout<<ba_onegrid<<std::endl;
    std::cout<<ba2d_onegrid<<std::endl;
    std::cout<<dm_onegrid<<std::endl;
#endif
 
 
    //*********************************************************
    // Radiation heating source terms
    //*********************************************************
    // Every radiation model (RRTMGP, Simple, TwoStream) writes the same
    // 2-component (SW, LW) heating rates, so the arrays are shaped the same
    // way whichever one erf.radiation_model selects.
    if (solverChoice.rad_type != RadiationType::None)
    {
        // Allocate with 1 ghost cell for interpolation stencil (cell_cons_interp)
        // and FillBoundary operations (needed for nested patches)
        qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, 1);
        // Level layout (RRTMGP's): index k holds the fluxes at the lower
        // interface of layer k, and the top-of-atmosphere interface sits in
        // the z-ghost cell above the top layer (k = khi + 1), which is why
        // the array carries one ghost cell in z. See ERF.H.
        //
        // The ghost cells in x and y are not part of that layout. They are here so that
        // this array can be the coarse source of an InterpFromCoarseLevel, exactly as
        // qheating_rates is: that overload asserts that the coarse source itself carries
        // the ghost cells the interpolation stencil reads, and its ParallelCopy then
        // reads them. See the note in ERF_AdvanceRadiation.cpp.
        rad_fluxes[lev]     = std::make_unique<MultiFab>(ba, dm, 4, IntVect(1,1,1));
        qheating_rates[lev]->setVal(zero);
        // Zeroing the ghost cells too is load-bearing, not tidiness: the ghost cells that
        // lie outside the physical domain are never written by anything else, and they are
        // read as interpolation-stencil neighbors when this level is a parent.
        rad_fluxes[lev]->setVal(zero);
    }
 
    // Two-stream radiation: the model owns its 2D surface and SEB fields.
    if (solverChoice.rad_type == RadiationType::TwoStream)
    {
        two_stream_rad.define_level(lev, solverChoice.radChoice, solverChoice.rdOcp, ba2d[lev], dm);
    }
 
    //*********************************************************
    // Turbulent perturbation region initialization
    //*********************************************************
    if (solverChoice.use_perturbation(lev))
    {
        turbPert.init_tpi_type(lev, solverChoice.pert_type[lev], max_level);
        turbPert.init_tpi(lev, subdomains[lev], geom[lev].CellSizeArray(),
                          ba, dm, ngrow_state, pp_prefix, refRatio(), max_level);
    }
 
    //
    // Define the land mask here and set it to all land by default
    // NOTE: the logic below will BREAK if we have any grids not touching the bottom boundary
    //
    {
    lmask_lev[lev].resize(1);
    auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0;
    lmask_lev[lev][0] = std::make_unique<iMultiFab>(ba2d[lev],dm,1,ngv);
    lmask_lev[lev][0]->setVal(solverChoice.is_land[lev]);
    lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
 
    land_type_lev[lev].resize(1);
    land_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d[lev],dm,1,ngv);
    land_type_lev[lev][0]->setVal(0);
    land_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
 
    soil_type_lev[lev].resize(1);
    soil_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d[lev],dm,1,ngv);
    soil_type_lev[lev][0]->setVal(0);
    soil_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
 
    urb_frac_lev[lev].resize(1);
    urb_frac_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ngv);
    urb_frac_lev[lev][0]->setVal(one);
    urb_frac_lev[lev][0]->FillBoundary(geom[lev].periodicity());
    }
 
    // Read in tables needed for windfarm simulations
    // fill in Nturb multifab - number of turbines in each mesh cell
    // write out the vtk files for wind turbine location and/or
    // actuator disks
    #ifdef ERF_USE_WINDFARM
        //init_windfarm(lev);
    #endif
 
    if (lev > 0) {
        fine_mask[lev] = std::make_unique<MultiFab>(grids[lev-1], dmap[lev-1], 1, 0);
        build_fine_mask(lev, *fine_mask[lev].get());
    }
 
#ifdef ERF_USE_FFT
    if ( ( (solverChoice.anelastic[lev] == 1)               || (solverChoice.project_initial_velocity[lev] == 1) ) &&
         ( (solverChoice.mesh_type == MeshType::ConstantDz) || (solverChoice.mesh_type == MeshType::StretchedDz) ) ) {
        build_fft_solvers(lev);
    }
#endif
}

Here is the call graph for this function:

init_thin_body()

void ERF::init_thin_body	(	int	lev,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

{
    //********************************************************************************************
    // Thin immersed body
    // *******************************************************************************************
#if 0
    if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
        (solverChoice.advChoice.zero_yflux.size() > 0) ||
        (solverChoice.advChoice.zero_zflux.size() > 0))
    {
        overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
        overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
    }
#endif
 
    if (solverChoice.advChoice.zero_xflux.size() > 0) {
        amrex::Print() << "Setting up thin immersed body for "
            << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
        BoxArray ba_xf(ba);
        ba_xf.surroundingNodes(0);
        thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
        thin_xforce[lev]->setVal(0.0);
        xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
        xflux_imask[lev]->setVal(1);
        for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
        {
            Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
            //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
            Box xbx = mfi.nodaltilebox(0);
            for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
                const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
                if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
                    (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
                    (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
                {
                    imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
                    amrex::AllPrint() << "  mask xface at " << faceidx << std::endl;
                }
            }
        }
    } else {
        thin_xforce[lev] = nullptr;
        xflux_imask[lev] = nullptr;
    }
 
    if (solverChoice.advChoice.zero_yflux.size() > 0) {
        amrex::Print() << "Setting up thin immersed body for "
            << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
        BoxArray ba_yf(ba);
        ba_yf.surroundingNodes(1);
        thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
        thin_yforce[lev]->setVal(0.0);
        yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
        yflux_imask[lev]->setVal(1);
        for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
        {
            Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
            //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
            Box ybx = mfi.nodaltilebox(1);
            for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
                const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
                if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
                    (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
                    (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
                {
                    imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
                    amrex::AllPrint() << "  mask yface at " << faceidx << std::endl;
                }
            }
        }
    } else {
        thin_yforce[lev] = nullptr;
        yflux_imask[lev] = nullptr;
    }
 
    if (solverChoice.advChoice.zero_zflux.size() > 0) {
        amrex::Print() << "Setting up thin immersed body for "
            << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
        BoxArray ba_zf(ba);
        ba_zf.surroundingNodes(2);
        thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
        thin_zforce[lev]->setVal(0.0);
        zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
        zflux_imask[lev]->setVal(1);
        for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
        {
            Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
            //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
            Box zbx = mfi.nodaltilebox(2);
            for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
                const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
                if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
                    (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
                    (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
                {
                    imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
                    amrex::AllPrint() << "  mask zface at " << faceidx << std::endl;
                }
            }
        }
    } else {
        thin_zforce[lev] = nullptr;
        zflux_imask[lev] = nullptr;
    }
}

init_zphys()

void ERF::init_zphys	(	int	lev,
		double	elapsed_time
	)

{
    // For EB, z_phys_nd was already initialized with the correct z_offset by init_default_zphys.
    // The terrain-fitting (BTF) done below is irrelevant for a flat EB mesh and would clobber
    // the offset, so return early here.
    if (solverChoice.terrain_type == TerrainType::EB) {
        Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
        micro->Set_dzmin(lev, dzmin);
        return;
    }
 
    if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
    {
        if (lev > 0) {
            //
            // First interpolate from coarser level if there is one
            // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
            //       have been pre-filled - this includes ghost cells both inside and outside
            //       the domain
            //
            InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
                                  IntVect(0,0,0), // do NOT fill ghost cells outside the domain
                                  *z_phys_nd[lev-1], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &node_bilinear_interp,
                                  domain_bcs_type, BCVars::cons_bc);
        }
 
        //
        // The STF and Sullivan TF transformations are only defined on a level that
        // spans the full column, so on a fine level we build the mesh from the one
        // just interpolated from the coarse level instead.  "interpolate" uses that
        // mesh as-is; "transform" additionally reads the terrain at the fine
        // resolution and blends it in with a correction that decays with height.
        //
        ParmParse pp("erf");
        int terrain_smoothing = 0;
        pp.query("terrain_smoothing", terrain_smoothing);
 
        FineTerrain fine_terrain = (lev > 0 && terrain_smoothing != 0) ? which_fine_terrain()
                                                                      : FineTerrain::None;
 
        //
        // Save the interpolated mesh before the fine terrain overwrites the k=0 slab,
        // since the transform is expressed relative to it
        //
        MultiFab z_phys_interp;
        if (fine_terrain == FineTerrain::Transform) {
            z_phys_interp.define(z_phys_nd[lev]->boxArray(), z_phys_nd[lev]->DistributionMap(),
                                 1, z_phys_nd[lev]->nGrowVect());
            MultiFab::Copy(z_phys_interp, *z_phys_nd[lev], 0, 0, 1, 0); // valid region only
        }
 
        if (fine_terrain != FineTerrain::Interpolate) {
            fill_terrain_surface(lev, *z_phys_nd[lev], elapsed_time);
        }
 
        make_terrain_fitted_coords(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type,
                                   fine_terrain,
                                   (fine_terrain == FineTerrain::Transform) ? &z_phys_interp : nullptr);
 
        z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
 
        if (lev == 0) {
            Real zmax = z_phys_nd[0]->max(0,0,false);
            Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
            if (rel_diff > Real(1.e-8)) {
                amrex::Print() << "max of zphys_nd " << zmax << std::endl;
                amrex::Print() << "max of zlevels  " << zlevels_stag[0][zlevels_stag[0].size()-1] << std::endl;
                amrex::Warning("Terrain is taller than domain top!");
            }
#if 0
            // This remains commented out until we verify that the stretched and variable dz pathways
            //   in fact give the same answer when appropriate
            if (SolverChoice::mesh_type == MeshType::VariableDz)
            {
                check_mesh_type(lev);
            }
#endif
        } // lev == 0
 
    } else {
        // If a WRFInput / met_em file is NOT provided for a finer level, we simply
        // interpolate the terrain from the coarse level. This is necessary since we
        // average_down the terrain (see ERF_MakeNewLevel.cpp).
        //
        // If a file IS present at this level, the terrain has already been built from
        // that file's PH + PHB by init_terrain_from_wrfinput (or its metgrid analogue),
        // which runs BEFORE this routine on both the from-scratch and the from-coarse
        // paths (init_only precedes init_zphys there because level 0 has to read its
        // terrain and its data in the same pass). Interpolating from the coarse level
        // here would therefore throw that away and leave the fine level running on
        // coarse-derived heights -- so we must not do it.
        //
        // NOTE: this is the behavior the comment here has always described; the guard
        //       below is what actually makes it true.
        if ( (lev > 0) && nc_init_file[lev].empty() ) {
            //
            // First interpolate from coarser level if there is one
            // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
            //       have been pre-filled - this includes ghost cells both inside and outside
            //       the domain
            //
            InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
                                  z_phys_nd[lev]->nGrowVect(), // DO fill ghost cells outside the domain
                                  *z_phys_nd[lev-1], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &node_bilinear_interp,
                                  domain_bcs_type, BCVars::cons_bc);
        }
    } // init_type
 
    if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
        solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
        // Read the small_volfrac threshold from eb2 namespace
        Real small_volfrac = 0.005;
        ParmParse pp_eb2("eb2");
        pp_eb2.queryAdd("small_volfrac", small_volfrac);
 
        // Cell-centered terrain blanking
        terrain_blanking[lev]->setVal(one);
        const int ng_sub = std::min(ComputeGhostCells(solverChoice) + 2, EBFactory(lev).getVolFrac().nGrow());
        MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ng_sub);
 
        // Clip small terrain_blanking values (almost fluid cells) using same threshold as eb2.small_volfrac
        if (small_volfrac > zero) {
            for (MFIter mfi(*terrain_blanking[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                auto const& tblank = terrain_blanking[lev]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    if (tblank(i,j,k) < small_volfrac) { tblank(i,j,k) = zero; }
                });
            }
        }
        terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
 
#if USE_FC_FACTORY
        // Face-centered terrain blanking from face-centered EB volume fractions
        terrain_blanking_xface[lev]->setVal(one);
        terrain_blanking_yface[lev]->setVal(one);
        terrain_blanking_zface[lev]->setVal(one);
 
        // Check if face factories are available before using them
        auto const* u_factory = eb[lev]->get_u_const_factory();
        auto const* v_factory = eb[lev]->get_v_const_factory();
        auto const* w_factory = eb[lev]->get_w_const_factory();
 
        if (u_factory && v_factory && w_factory) {
            MultiFab::Subtract(*terrain_blanking_xface[lev], u_factory->getVolFrac(), 0, 0, 1, ng_sub);
            MultiFab::Subtract(*terrain_blanking_yface[lev], v_factory->getVolFrac(), 0, 0, 1, ng_sub);
            MultiFab::Subtract(*terrain_blanking_zface[lev], w_factory->getVolFrac(), 0, 0, 1, ng_sub);
 
            // Clip small terrain_blanking values on faces (almost fluid cells) using same threshold
            if (small_volfrac > zero) {
                for (MFIter mfi(*terrain_blanking_xface[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    const Box& xbx = mfi.tilebox();
                    auto const& tblank_x = terrain_blanking_xface[lev]->array(mfi);
                    ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        if (tblank_x(i,j,k) < small_volfrac) { tblank_x(i,j,k) = zero; }
                    });
                }
                for (MFIter mfi(*terrain_blanking_yface[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    const Box& ybx = mfi.tilebox();
                    auto const& tblank_y = terrain_blanking_yface[lev]->array(mfi);
                    ParallelFor(ybx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        if (tblank_y(i,j,k) < small_volfrac) { tblank_y(i,j,k) = zero; }
                    });
                }
                for (MFIter mfi(*terrain_blanking_zface[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    const Box& zbx = mfi.tilebox();
                    auto const& tblank_z = terrain_blanking_zface[lev]->array(mfi);
                    ParallelFor(zbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        if (tblank_z(i,j,k) < small_volfrac) { tblank_z(i,j,k) = zero; }
                    });
                }
            }
        }
 
        terrain_blanking_xface[lev]->FillBoundary(geom[lev].periodicity());
        terrain_blanking_yface[lev]->FillBoundary(geom[lev].periodicity());
        terrain_blanking_zface[lev]->FillBoundary(geom[lev].periodicity());
#endif
 
        init_immersed_forcing(lev); // needed for real cases
 
        // buildings are landmask = 2
        for (MFIter mfi(*lmask_lev[lev][0]); mfi.isValid(); ++mfi) {
            const Box& bx2d = mfi.growntilebox();
            auto lmask_arr = lmask_lev[lev][0]->array(mfi);
            const auto& t_blank_arr = terrain_blanking[lev]->array(mfi);
 
            amrex::ParallelFor(bx2d, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                // Use k=0 for the terrain_blanking field
                if (t_blank_arr(i, j, 0) > 0.0) {
                    lmask_arr(i, j, k) = 2;
                }
            });
        }
    }
 
    // Compute the min dz and pass to the micro model
    Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
    micro->Set_dzmin(lev, dzmin);
}

Here is the call graph for this function:

InitData()

void ERF::InitData

(

)

{
    BL_PROFILE_VAR("ERF::InitData()", InitData);
    InitData_pre();
    InitData_post();
    BL_PROFILE_VAR_STOP(InitData);
}

Referenced by main().

Here is the caller graph for this function:

InitData_post()

void ERF::InitData_post

(

)

{
    if (solverChoice.advChoice.have_zero_flux_faces)
    {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
            "Thin immersed body with refinement not currently supported.");
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            amrex::Print() << "NOTE: Thin immersed body with non-constant dz has not been tested." << std::endl;
        }
    }
 
    if (!restart_chkfile.empty()) {
        restart();
    }
 
    // Faces of the resolved buildings, from the blanking both paths have built.
    init_ibseb();
 
    // Select 2-D variables after the active LSM has initialized its runtime
    // field inventory, including provider-specific soil layers.
    setPlotVariables2D("plot2d_vars_1", plot2d_var_names_1);
    setPlotVariables2D("plot2d_vars_2", plot2d_var_names_2);
 
    //
    // Make sure that detJ is the average of the data on a finer level if there is one and if two way coupling
    //
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        if (solverChoice.coupling_type == CouplingType::TwoWay) {
        // NOTE: z_phys_cc is deliberately NOT averaged down.  Every level's base state is
        //       built to be in discrete hydrostatic balance against that level's own
        //       cell-centered heights, so replacing the coarse heights with the average of
        //       the fine ones -- after the base states have already been built -- leaves the
        //       coarse base state out of balance with the heights the dycore then uses for
        //       vertical gradients, Rayleigh damping and the sponge zones.  z_phys_cc is also
        //       derived from z_phys_nd, which is not averaged down either, so averaging only
        //       the cell-centered heights made the two disagree inside the refined region.
        //
        //       detJ IS still averaged down: AverageDownTo weights (rho S) by detJ_cc before
        //       averaging and divides by it afterwards, so the coarse detJ must be the average
        //       of the fine detJ for that average-down to telescope and stay conservative.
            for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
                average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
            }
        }
        for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
              detJ_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
            z_phys_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
        }
    }
 
#ifdef ERF_IMPLICIT_W
    if ( (SolverChoice::mesh_type == MeshType::VariableDz) &&
         (solverChoice.implicit_momentum_diffusion) ) {
        for (int lev = 0; lev <= finest_level; lev++) {
            if ( (solverChoice.vert_implicit_fac[lev][0] > 0) ||
                 (solverChoice.vert_implicit_fac[lev][1] > 0) ||
                 (solverChoice.vert_implicit_fac[lev][2] > 0) )
            {
                Warning("Doing implicit solve for u, v, and w with terrain at level " << lev << " -- this has not been tested");
            }
        }
    }
#endif
 
    //
    // Copy vars_new into vars_old, then use vars_old to fill covered cells in vars_new during AverageDown
    //
    if (SolverChoice::terrain_type == TerrainType::EB) {
        for (int lev = 0; lev <= finest_level; lev++) {
            int ncomp_cons = vars_new[lev][Vars::cons].nComp();
            MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,ncomp_cons,vars_new[lev][Vars::cons].nGrowVect());
        }
    }
 
    if (restart_chkfile.empty()) {
        if (solverChoice.coupling_type == CouplingType::TwoWay) {
            AverageDown();
        }
    }
 
#ifdef ERF_USE_PARTICLES
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
        // Promote the Lagrangian PC to the multi-level ParGDB before init so
        // per-level addParticles() can use ParticleBoxArray(lev)/DistributionMap(lev).
        auto* pc_ptr = dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer();
        AMREX_ALWAYS_ASSERT(pc_ptr != nullptr);
        pc_ptr->Define(static_cast<amrex::ParGDBBase*>(GetParGDB()));
 
        if (erf_interval_means::initialization_accumulates_state(
                !restart_chkfile.empty())) {
            if (solverChoice.moisture_tight_coupling) {
                Warning("Tight coupling has not been tested with Lagrangian microphysics");
            }
 
            for (int lev = 0; lev <= finest_level; lev++) {
                dynamic_cast<LagrangianMicrophysics&>(*micro).initParticles(lev, z_phys_nd[lev]);
            }
        }
    }
#endif
 
    if (!restart_chkfile.empty()) { // Restart from a checkpoint
 
        // Create the physbc objects for {cons, u, v, w, base state}
        // We fill the additional base state ghost cells just in case we have read the old format
        for (int lev(0); lev <= finest_level; ++lev) {
            make_physbcs(lev);
            (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
        }
 
        if (solverChoice.do_forest_drag) {
            for (int lev(0); lev <= finest_level; ++lev) {
                m_forest_drag[lev]->define_drag_field(grids[lev], dmap[lev], geom[lev],
                                                      z_phys_cc[lev].get(), z_phys_nd[lev].get(),
                                                      solverChoice.forest_biophysics &&
                                                      solverChoice.forest_biophysics_heat, lev);
            }
        }
 
#ifdef ERF_USE_NETCDF
        //
        // Create the needed bdy_data_xlo etc ... since we don't read it in from checkpoint any more
        // This follows init_from_wrfinput()
        //
        bool use_moist = (solverChoice.moisture_type != MoistureType::None);
        if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
 
            if ( geom[0].isPeriodic(0) || geom[0].isPeriodic(1) ) {
                 amrex::Error("Cannot set periodic lateral boundary conditions when reading in real boundary values");
            }
 
            bdy_time_interval = read_times_from_wrfbdy(nc_bdy_file,
                                                       bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                                       start_bdy_time, final_bdy_time);
 
            double time_since_start_bdy = t_new[0] + start_time - start_bdy_time;
            int n_time_old = static_cast<int>(time_since_start_bdy /  bdy_time_interval);
            MultiFab r_hse(base_state[0], make_alias, BaseState::r0_comp, 1);
            Array<MultiFab*, AMREX_SPACEDIM> area_vec = {ax[0].get(), ay[0].get(), az[0].get()};
            auto repack_runtime_bdy = [&] (const int itime) {
                const bool separate_hydrometeors = solverChoice.use_wrf_bdy_qc_qi &&
                    wrf_bdy_has_separate_hydrometeors(solverChoice.moisture_indices);
                repack_wrfbdy_to_realbdy(bdy_data_xlo[itime], solverChoice.use_wrf_bdy_qc_qi, separate_hydrometeors);
                repack_wrfbdy_to_realbdy(bdy_data_xhi[itime], solverChoice.use_wrf_bdy_qc_qi, separate_hydrometeors);
                repack_wrfbdy_to_realbdy(bdy_data_ylo[itime], solverChoice.use_wrf_bdy_qc_qi, separate_hydrometeors);
                repack_wrfbdy_to_realbdy(bdy_data_yhi[itime], solverChoice.use_wrf_bdy_qc_qi, separate_hydrometeors);
            };
 
            // Need itime=0 for vertical interpolation
            if (n_time_old > 0) {
                int itime = 0;
                bool is_anelastic = (solverChoice.anelastic[0] == 1);
                read_and_convert_from_wrfbdy(itime,nc_bdy_file,
                                             bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                             wrf_MUB, wrf_C1H, wrf_C2H, wrf_RDNW, wrf_PHB, z_phys_cc[0], z_phys_nd[0],
                                             vars_new[0][Vars::xvel], vars_new[0][Vars::yvel], vars_new[0][Vars::cons],
                                             r_hse, area_vec, geom[0], use_moist,
                                             solverChoice.use_wrf_bdy_qc_qi,
                                             solverChoice.moisture_indices.qi >= 0,
                                             solverChoice.use_wrf_bdy_qc_qi &&
                                             wrf_bdy_has_separate_hydrometeors(solverChoice.moisture_indices),
                                             solverChoice.rebalance_wrf_input, domain_bcs_type,
                                             real_width, bdy_time_interval, is_anelastic);
                repack_runtime_bdy(itime);
            }
 
            int ntimes = std::min(n_time_old+3, static_cast<int>(bdy_data_xlo.size()));
 
            for (int itime = n_time_old; itime < ntimes; itime++)
            {
                bool is_anelastic = (solverChoice.anelastic[0] == 1);
                read_and_convert_from_wrfbdy(itime,nc_bdy_file,
                                             bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                             wrf_MUB, wrf_C1H, wrf_C2H, wrf_RDNW, wrf_PHB, z_phys_cc[0], z_phys_nd[0],
                                             vars_new[0][Vars::xvel], vars_new[0][Vars::yvel], vars_new[0][Vars::cons],
                                             r_hse, area_vec, geom[0], use_moist,
                                             solverChoice.use_wrf_bdy_qc_qi,
                                             solverChoice.moisture_indices.qi >= 0,
                                             solverChoice.use_wrf_bdy_qc_qi &&
                                             wrf_bdy_has_separate_hydrometeors(solverChoice.moisture_indices),
                                             solverChoice.rebalance_wrf_input, domain_bcs_type,
                                             real_width, bdy_time_interval, is_anelastic);
                if (itime == static_cast<int>(bdy_data_xlo.size())-1 && itime > 0) {
                    repack_runtime_bdy(itime-1);
                }
                repack_runtime_bdy(itime);
            } // itime
        } // use_real_bcs
 
        if (!nc_low_file.empty())
        {
            low_time_interval = read_times_from_wrflow(nc_low_file, low_data_zlo, start_low_time, final_low_time);
 
            int lev = 0;
            sst_lev[lev].resize(low_data_zlo.size());
            tsk_lev[lev].resize(low_data_zlo.size());
 
            double time_since_start_low = t_new[0] + start_time - start_low_time;
            int n_time_old = static_cast<int>(time_since_start_low /  low_time_interval);
 
            int ntimes = std::min(n_time_old+3, static_cast<int>(low_data_zlo.size()));
 
            for (int itime = n_time_old; itime < ntimes; itime++)
            {
                read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
 
                // Need to read PSFC
                FArrayBox NC_fab_var_file;
                for (int idx = 0; idx < num_boxes_at_level[lev]; idx++) {
                    int success, use_theta_m;
                    read_from_wrfinput(lev, boxes_at_level[lev][idx], nc_init_file[lev][0],
                                       NC_fab_var_file, "PSFC", geom[lev],
                                       use_theta_m, success);
                    auto& var_fab = NC_fab_var_file;
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                    for ( MFIter mfi(*mf_PSFC[lev], false); mfi.isValid(); ++mfi )
                    {
                        FArrayBox &cur_fab = (*mf_PSFC[lev])[mfi];
                        cur_fab.template copy<RunOn::Device>(var_fab, 0, 0, 1);
                    }
                    var_fab.clear();
                }
 
                update_sst_tsk(itime, geom[lev], ba2d[lev],
                               sst_lev[lev], tsk_lev[lev],
                               m_SurfaceLayer[Orientation(Direction::z, Orientation::low)], low_data_zlo,
                               vars_new[lev][Vars::cons], *mf_PSFC[lev],
                               solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
            } // itime
        }
#endif
#ifdef ERF_USE_FFT
        for (int lev = 0; lev <= finest_level; lev++) {
            // rebuild fft solvers here in case mesh type was changed when reading the checkpoint file
            if ( ( (solverChoice.anelastic[lev] == 1)               || (solverChoice.project_initial_velocity[lev] == 1) ) &&
                 ( (solverChoice.mesh_type == MeshType::ConstantDz) || (solverChoice.mesh_type == MeshType::StretchedDz) ) ) {
                build_fft_solvers(lev);
            }
        }
#endif
    } // end restart
 
#ifdef ERF_USE_PARTICLES
    /* If using a Lagrangian microphysics model, its particle container has now been
       constructed and initialized (calls to micro->Init). So, add its pointer to
       ERF::particleData and remove its name from list of unallocated particle containers. */
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
        const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
        const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
        AMREX_ALWAYS_ASSERT(pc_ptr != nullptr);
        particleData.pushBack(pc_name, pc_ptr);
        particleData.getNamesUnalloc().remove(pc_name);
    }
#endif
 
    if (input_bndry_planes) {
        // Read the "time.dat" file to know what data is available
        m_r2d->read_time_file();
 
        // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
        double dt_dummy = 0.0;
        m_r2d->read_input_files(t_new[0]+start_time,dt_dummy,m_bc_extdir_vals);
    }
 
    if (solverChoice.custom_rhotheta_forcing)
    {
        rhotheta_src.resize(max_level+1);
        for (int lev = 0; lev <= finest_level; lev++) {
            BoxList bl_src = vars_new[lev][Vars::cons].boxArray().boxList();
            for (auto& b : bl_src) {
                if (!solverChoice.spatial_rhotheta_forcing)
                {
                    // source is only defined in Z
                    b.setRange(0, 0, 1);
                    b.setRange(1, 0, 1);
                }
            }
            BoxArray ba_src(std::move(bl_src));
            rhotheta_src[lev] = std::make_unique<MultiFab>(ba_src, vars_new[lev][Vars::cons].DistributionMap(), 1, 0);
            rhotheta_src[lev]->setVal(0.);
            prob->update_rhotheta_sources(t_new[0],
                                          rhotheta_src[lev].get(),
                                          geom[lev], z_phys_cc[lev]);
        }
    }
 
    if (solverChoice.have_geo_wind_profile)
    {
        h_u_geos.resize(max_level+1, Vector<Real>(0));
        d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        h_v_geos.resize(max_level+1, Vector<Real>(0));
        d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        for (int lev = 0; lev <= finest_level; lev++) {
            const int domlen = geom[lev].Domain().length(2);
            h_u_geos[lev].resize(domlen, 0.0_rt);
            d_u_geos[lev].resize(domlen, 0.0_rt);
            h_v_geos[lev].resize(domlen, 0.0_rt);
            d_v_geos[lev].resize(domlen, 0.0_rt);
            if (solverChoice.custom_geostrophic_profile) {
                prob->update_geostrophic_profile(t_new[0],
                                                 h_u_geos[lev], d_u_geos[lev],
                                                 h_v_geos[lev], d_v_geos[lev],
                                                 geom[lev], z_phys_cc[lev]);
            } else {
                if (SolverChoice::mesh_type == MeshType::VariableDz) {
                    amrex::Print() << "Note: 1-D geostrophic wind profile input is not defined for real terrain" << std::endl;
                }
                init_geo_wind_profile(solverChoice.abl_geo_wind_table,
                                      h_u_geos[lev], d_u_geos[lev],
                                      h_v_geos[lev], d_v_geos[lev],
                                      geom[lev],
                                      zlevels_stag[0]);
            }
        }
    }
 
    if (solverChoice.custom_moisture_forcing)
    {
        rhoqt_src.resize(max_level+1);
        for (int lev = 0; lev <= finest_level; lev++) {
            BoxList bl_src = vars_new[lev][Vars::cons].boxArray().boxList();
            for (auto& b : bl_src) {
                if (!solverChoice.spatial_moisture_forcing)
                {
                    // source is only defined in Z
                    b.setRange(0, 0, 1);
                    b.setRange(1, 0, 1);
                }
            }
            BoxArray ba_src(std::move(bl_src));
            rhoqt_src[lev] = std::make_unique<MultiFab>(ba_src, vars_new[lev][Vars::cons].DistributionMap(), 1, 0);
            rhoqt_src[lev]->setVal(0.);
            prob->update_rhoqt_sources(t_new[0],
                                       rhoqt_src[lev].get(),
                                       geom[lev], z_phys_cc[lev]);
        }
    }
 
    if (solverChoice.custom_w_subsidence)
    {
        h_w_subsid.resize(max_level+1, Vector<Real>(0));
        d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        for (int lev = 0; lev <= finest_level; lev++) {
            const int domlen = geom[lev].Domain().length(2) + 1; // lives on z-faces
            h_w_subsid[lev].resize(domlen, 0.0_rt);
            d_w_subsid[lev].resize(domlen, 0.0_rt);
            prob->update_w_subsidence(t_new[0],
                                      h_w_subsid[lev], d_w_subsid[lev], base_state[lev],
                                      geom[lev], z_phys_nd[lev]);
        }
    }
 
    if (solverChoice.large_scale_forcing)
    {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!solverChoice.nudging_from_input_sounding || lsf.tau_lsf > 0.0,
                                         "erf.forcing_timescale must be positive when nudging with large-scale forcing");
        if (verbose) {
            lsf.verbose_print = true;
        }
        lsf.read_forcing_file();
        lsf.interp_forcing(geom[0].data(), zlevels_stag[0], input_sounding_data);
    }
 
    if (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
        solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T)
    {
        for (int lev = 0; lev <= finest_level; lev++) {
            initRayleigh_at_level(lev);
        }
        if (solverChoice.init_type == InitType::Input_Sounding)
        {
            // Overwrite ubar, vbar, and thetabar with input profiles;
            // wbar is assumed to be zero Note: the tau coefficient set by
            // prob->erf_init_rayleigh() is still used
            bool restarting = (!restart_chkfile.empty());
            setRayleighRefFromSounding(restarting);
        }
    }
 
    // Read in sponge data from input file
    if(solverChoice.spongeChoice.sponge_type == SpongeType::Input_Sponge)
    {
        initSponge();
        bool restarting = (!restart_chkfile.empty());
        setSpongeRefFromSounding(restarting);
    }
 
    if (solverChoice.any_perturbation()) {
        if (is_it_time_for_action(istep[0], t_new[0], dt[0], pert_interval, -one)) {
            turbPert.debug(t_new[0]);
        }
    }
 
    // We only write the file at level 0 for now
    if (output_bndry_planes)
    {
        // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
        m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
 
        double tot_time = t_new[0]+start_time;
        if (tot_time >= bndry_output_planes_start_time) {
            bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
            m_w2d->write_planes(0, tot_time, vars_new, is_moist);
        }
    }
 
    // Fill boundary conditions in vars_new
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        auto& lev_new = vars_new[lev];
 
        // ***************************************************************************
        // Physical bc's at domain boundary
        // ***************************************************************************
        IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
        IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
 
        int ncomp_cons = lev_new[Vars::cons].nComp();
        bool do_fb     = true;
 
#ifdef ERF_USE_NETCDF
        if (solverChoice.use_real_bcs && (lev==0)) {
            int icomp_cons = 0;
            bool cons_only = false;
            Vector<MultiFab*> mfs_vec = {&lev_new[Vars::cons],&lev_new[Vars::xvel],
                                         &lev_new[Vars::yvel],&lev_new[Vars::zvel]};
            fill_from_realbdy(mfs_vec,t_new[lev],cons_only,icomp_cons,
                              ncomp_cons,ngvect_cons,ngvect_vels);
            do_fb = false;
        }
#endif
 
        (*physbcs_cons[lev])(lev_new[Vars::cons],lev_new[Vars::xvel],lev_new[Vars::yvel],0,ncomp_cons,
                             ngvect_cons,t_new[lev],BCVars::cons_bc,do_fb);
        (   *physbcs_u[lev])(lev_new[Vars::xvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                             ngvect_vels,t_new[lev],BCVars::xvel_bc,do_fb);
        (   *physbcs_v[lev])(lev_new[Vars::yvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                             ngvect_vels,t_new[lev],BCVars::yvel_bc,do_fb);
        (   *physbcs_w[lev])(lev_new[Vars::zvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                             ngvect_vels,t_new[lev],BCVars::zvel_bc,do_fb);
    }
 
    //
    // If we are starting from scratch, we have the option to project the initial velocity field
    //    regardless of how we initialized.  Note that project_initial_velocity operates on vars_new.
    // pp_inc is used as scratch space here; we zero it out after the projection
    //
    if (restart_chkfile == "")
    {
        for (int lev = 0; lev <= finest_level; ++lev)
        {
            if (solverChoice.project_initial_velocity[lev] == 1) {
                double dummy_dt = 1.0;
                if (verbose > 0) {
                    amrex::Print() << "Projecting initial velocity field at level " << lev << std::endl;
                }
 
                project_initial_velocity(lev, t_new[lev], dummy_dt);
 
                pp_inc[lev].setVal(0.);
                gradp[lev][GpVars::gpx].setVal(0.);
                gradp[lev][GpVars::gpy].setVal(0.);
                gradp[lev][GpVars::gpz].setVal(0.);
            } // project
        } // lev
    }
 
    // Copy from new into old just in case (after filling boundary conditions and possibly projecting)
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        int nc = vars_new[lev][Vars::cons].nComp();
 
        MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,nc,vars_new[lev][Vars::cons].nGrowVect());
        MultiFab::Copy(vars_old[lev][Vars::xvel],vars_new[lev][Vars::xvel],0,0, 1,vars_new[lev][Vars::xvel].nGrowVect());
        MultiFab::Copy(vars_old[lev][Vars::yvel],vars_new[lev][Vars::yvel],0,0, 1,vars_new[lev][Vars::yvel].nGrowVect());
        MultiFab::Copy(vars_old[lev][Vars::zvel],vars_new[lev][Vars::zvel],0,0, 1,vars_new[lev][Vars::zvel].nGrowVect());
    }
 
    // Compute the minimum dz in the domain at each level (to be used for setting the timestep)
    dz_min.resize(max_level+1);
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        dz_min[lev] = geom[lev].CellSize(2);
        if ( SolverChoice::mesh_type != MeshType::ConstantDz ) {
            dz_min[lev] *= (*detJ_cc[lev]).min(0);
        }
    }
 
 
    // We don't need to recompute dt[lev] on restart because we read it in from the checkpoint file.
    if (restart_chkfile.empty()) {
        ComputeDt();
    }
 
    // Check the viscous limit
    DiffChoice dc = solverChoice.diffChoice;
    if (dc.molec_diff_type == MolecDiffType::Constant ||
        dc.molec_diff_type == MolecDiffType::ConstantAlpha) {
        Real delta = std::min({geom[finest_level].CellSize(0),
                               geom[finest_level].CellSize(1),
                               dz_min[finest_level]});
        if (dc.dynamic_viscosity == 0) {
            Print() << "Note: Molecular diffusion specified but dynamic_viscosity has not been specified" << std::endl;
        } else {
            Real nu = dc.dynamic_viscosity / dc.rho0_trans;
            Real viscous_limit = two * delta*delta / nu;
            Print() << "smallest grid spacing at level " << finest_level << " = " << delta << std::endl;
            Print() << "dt at level " << finest_level << " = " << dt[finest_level] << std::endl;
            Print() << "Viscous CFL is " << dt[finest_level] / viscous_limit << std::endl;
            if (fixed_dt[finest_level] >= viscous_limit) {
                Warning("Specified fixed_dt is above the viscous limit");
            } else if (dt[finest_level] >= viscous_limit) {
                Warning("Adaptive dt based on convective CFL only is above the viscous limit");
            }
        }
    }
 
    // Fill ghost cells/faces
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        if (lev > 0 && cf_width >= 0) {
            Construct_ERFFillPatchers(lev);
        }
 
        auto& lev_new = vars_new[lev];
 
        //
        // Fill boundary conditions -- not sure why we need this here
        //
        bool fillset = false;
        if (lev == 0) {
            FillPatchCrseLevel(lev, t_new[lev],
                      {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]});
        } else {
            FillPatchFineLevel(lev, t_new[lev],
                      {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
                      {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                      base_state[lev], base_state[lev],
                      fillset);
        }
 
        //
        // We do this here to make sure level (lev-1) boundary conditions are filled
        // before we interpolate to level (lev) ghost cells
        //
        if (lev < finest_level) {
            auto& lev_old = vars_old[lev];
            MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
            MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
            MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
            MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
        }
 
        //
        // We fill the ghost cell values of the base state in case it wasn't done in the initialization
        //
        base_state[lev].FillBoundary(geom[lev].periodicity());
 
        // For moving terrain only
        if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
            MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
            base_state_new[lev].FillBoundary(geom[lev].periodicity());
        }
 
    }
 
    // If lev > 0, we need to fill bc's by interpolation from coarser grid
    for (int lev = 1; lev <= finest_level; ++lev)
    {
        Interp2DArrays(lev,ba2d[lev],dmap[lev]);
    } // lev
 
#ifdef ERF_USE_WW3_COUPLING
    int my_lev = 0;
    amrex::Print() <<  " About to call send_to_ww3 from ERF.cpp" << std::endl;
    send_to_ww3(my_lev);
    amrex::Print() <<  " About to call read_waves from ERF.cpp"  << std::endl;
    read_waves(my_lev);
   // send_to_ww3(my_lev);
#endif
 
    // Create wall distance field for RANS model
    for (int lev = 0; lev <= finest_level; lev++) {
        if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
            // Handle bottom boundary
            poisson_wall_dist(lev);
 
            // Correct the wall distance for immersed bodies
            if (solverChoice.advChoice.have_zero_flux_faces) {
                thinbody_wall_dist(walldist[lev],
                                   solverChoice.advChoice.zero_xflux,
                                   solverChoice.advChoice.zero_yflux,
                                   solverChoice.advChoice.zero_zflux,
                                   geom[lev],
                                   z_phys_cc[lev]);
 
                // The correction is only applied on the valid region
                fill_wall_dist_ghost_cells(*walldist[lev], geom[lev]);
            }
        }
    }
 
    // Configure SurfaceLayer params if used
    // NOTE: we must set up the MOST routine after calling FillPatch
    //       in order to have lateral ghost cells filled (MOST + terrain interp).
    bool updated_prim = false;
    // Count number of surface layer boundaries to determine correct parser prefix
    int n_faces = 0;
    amrex::GpuArray<int, AMREX_SPACEDIM*2> surface_layer_faces{};
    for (OrientationIter oit; oit; ++oit) {
        Orientation ori = oit();
        if (phys_bc_type[ori] == ERF_BC::surface_layer) {
            n_faces += 1;
            surface_layer_faces[static_cast<int>(ori)] = 1;
        }
    }
 
    // With multiple surface-layer faces, face-qualified prefixes are normally
    // required.  Preserve the historical unqualified zlo inputs when a user adds
    // another surface-layer face.  What the user actually wrote is read from the
    // snapshot taken in ReadParameters; see has_surface_layer_inputs for why the
    // live ParmParse table cannot be asked this question here.
    bool use_legacy_zlo_prefix = false;
    if (n_faces > 1 &&
        phys_bc_type[Orientation::zlo()] == ERF_BC::surface_layer) {
        const bool has_legacy_inputs = has_surface_layer_inputs(pp_prefix);
        const bool has_zlo_inputs = has_surface_layer_inputs(
            pp_prefix + "." + BoundaryFaceName[Orientation::zlo()]);
 
        if (has_legacy_inputs && has_zlo_inputs) {
            Abort("Both legacy unqualified and zlo-qualified surface-layer inputs "
                  "are present. Use only erf.zlo.most.* and/or "
                  "erf.zlo.surface_layer.* when multiple surface-layer faces "
                  "are enabled.");
        }
 
        if (has_legacy_inputs) {
            use_legacy_zlo_prefix = true;
            Warning("Multiple surface-layer faces are enabled while using legacy "
                    "unqualified surface-layer inputs. Applying erf.most.* and "
                    "erf.surface_layer.* to zlo; migrate them to erf.zlo.most.* "
                    "and erf.zlo.surface_layer.*.");
        }
    }
 
    for (OrientationIter oit; oit; ++oit) {
        Orientation ori = oit();
        if (phys_bc_type[ori] == ERF_BC::surface_layer) {
            bool has_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
                              (solverChoice.turbChoice[0].les_type  != LESType::None)          ||
                              (solverChoice.turbChoice[0].rans_type != RANSType::None)         ||
                              (solverChoice.turbChoice[0].pbl_type  != PBLType::None) );
            AMREX_ALWAYS_ASSERT(has_diff);
 
            bool rotate = solverChoice.use_rotate_surface_flux;
            if (rotate) {
                Print() << "Using surface layer model with stress rotations" << std::endl;
                AMREX_ALWAYS_ASSERT_WITH_MESSAGE(ori.coordDir() == 2 && ori.faceDir() == Orientation::Side::low,
                                                 "Surface layer with stress rotations can only be enabled for the bottom z face");
            }
 
            //
            // This constructor will make the SurfaceLayer object but not allocate the arrays at each level.
            //
            // Build vector of eb pointers for all levels
            amrex::Vector<const eb_*> eb_ptrs;
            eb_ptrs.resize(finest_level + 1, nullptr);
            if (solverChoice.terrain_type == TerrainType::EB) {
                AMREX_ALWAYS_ASSERT_WITH_MESSAGE(ori.coordDir() == 2 && ori.faceDir() == Orientation::Side::low,
                                                 "Surface layer with EB can only be enabled for the bottom z face");
                for (int lev = 0; lev <= finest_level; lev++) {
                    eb_ptrs[lev] = eb[lev] ? eb[lev].get() : nullptr;
                }
            }
 
            // Keep the historical unqualified prefix for a single zlo face,
            // and for multiple faces when legacy zlo inputs were detected.
            std::string face_pp_prefix(pp_prefix + "." + BoundaryFaceName[ori]);
            if ((n_faces == 1 || use_legacy_zlo_prefix) &&
                static_cast<int>(ori) == Orientation::zlo()) {
                face_pp_prefix = pp_prefix;
            }
            m_SurfaceLayer[ori] = std::make_unique<SurfaceLayer>(ori, geom, rotate, face_pp_prefix, Qv_prim,
                                                                 z_phys_nd,
                                                                 zlevels_stag,
                                                                 solverChoice.mesh_type,
                                                                 solverChoice.terrain_type,
                                                                 solverChoice.turbChoice[finest_level],
                                                                 solverChoice.rdOcp,
#ifdef ERF_USE_NETCDF
                                                                 start_low_time, final_low_time, low_time_interval,
#else
                                                                 zero, zero, zero,
#endif
                                                                 eb_ptrs);
            m_SurfaceLayer[ori]->set_surface_layer_faces(surface_layer_faces);
            m_SurfaceLayer[ori]->set_coupled_sst_active(solverChoice.use_coupled_sst &&
                                                        static_cast<int>(ori) == Orientation::zlo());
            // This call will allocate the arrays at each level. If we regrid later, either changing
            // the number of levels or just the grids at each existing level, we will call an update routine
            // to redefine the internal arrays in m_SurfaceLayer.
            for (int lev = 0; lev <= finest_level; lev++)
            {
                Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
                                             &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
                m_SurfaceLayer[ori]->make_SurfaceLayer_at_level(lev,finest_level+1,
                                                                mfv_old, Theta_prim[lev], Qv_prim[lev],
                                                                Qr_prim[lev], z_phys_nd[lev],
                                                                Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
                                                                lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
                                                                sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
            }
 
            // If initializing from an input_sounding, make sure the surface layer
            // is using the same surface conditions
            // Note: do this only if using a single face on zlo, otherwise this will overwrite user specified wall values
            if (n_faces == 1 && static_cast<int>(ori) == Orientation::zlo()) {
                if (solverChoice.init_type == InitType::Input_Sounding) {
                    const Real theta0 = input_sounding_data.theta_ref_inp_sound;
                    const Real qv0    = input_sounding_data.qv_ref_inp_sound;
                    for (int lev = 0; lev <= finest_level; lev++) {
                        m_SurfaceLayer[ori]->set_t_surf(lev, theta0);
                        m_SurfaceLayer[ori]->set_q_surf(lev, qv0);
                    }
                }
            }
 
            // We now configure ABLMost params here so that we can print the averages at t=0
            // Note we don't fill ghost cells here because this is just for diagnostics
            for (int lev = 0; lev <= finest_level; ++lev)
            {
                IntVect ng = Theta_prim[lev]->nGrowVect();
 
                if (!updated_prim) {
                    // This only needs to be done once (Theta,Qv_prim,Qr_prim should only be calculated once and reused for other faces)
                    MultiFab::Copy(  *Theta_prim[lev], vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, ng);
                    MultiFab::Divide(*Theta_prim[lev], vars_new[lev][Vars::cons],      Rho_comp, 0, 1, ng);
 
                    if (solverChoice.moisture_type != MoistureType::None) {
                        ng = Qv_prim[lev]->nGrowVect();
 
                        MultiFab::Copy(  *Qv_prim[lev], vars_new[lev][Vars::cons], RhoQ1_comp, 0, 1, ng);
                        MultiFab::Divide(*Qv_prim[lev], vars_new[lev][Vars::cons],   Rho_comp, 0, 1, ng);
 
                        int rhoqr_comp = solverChoice.moisture_indices.qr;
                        if (rhoqr_comp > -1) {
                            MultiFab::Copy(  *Qr_prim[lev], vars_new[lev][Vars::cons], rhoqr_comp, 0, 1, ng);
                            MultiFab::Divide(*Qr_prim[lev], vars_new[lev][Vars::cons],   Rho_comp, 0, 1, ng);
                        } else {
                            Qr_prim[lev]->setVal(0.0);
                        }
                    }
                }
                m_SurfaceLayer[ori]->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim, Qr_prim);
 
                if (restart_chkfile == "") {
                    // Only do this if starting from scratch; if restarting, then
                    // we don't want to call update_fluxes multiple times because
                    // it will change u* and theta* from their previous values
                    m_SurfaceLayer[ori]->update_pblh(lev, vars_new, z_phys_cc[lev].get(),
                                                     solverChoice.moisture_indices);
#ifdef ERF_USE_NETCDF
                    double elapsed_time_since_start_low = t_new[lev] + (start_time - start_low_time);
#else
                    double elapsed_time_since_start_low = t_new[lev] + start_time;
#endif
                    m_SurfaceLayer[ori]->update_fluxes(lev, t_new[lev], elapsed_time_since_start_low,
                                                       vars_new[lev][Vars::cons],
                                                       z_phys_nd[lev],
                                                       walldist[lev]);
 
                    if (ori.coordDir() == 2 && ori.faceDir() == Orientation::Side::low) {
                        // Initialize tke(x,y,z) as a function of u*(x,y)
                        if (solverChoice.turbChoice[lev].init_tke_from_ustar) {
                            Real qkefac = one;
                            if (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25 ||
                                solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF)
                            {
                                // https://github.com/NCAR/MYNN-EDMF/blob/90f36c25259ec1960b24325f5b29ac7c5adeac73/module_bl_mynnedmf.F90#L1325-L1333
                                const Real B1 = solverChoice.turbChoice[lev].pbl_mynn.B1;
                                qkefac = Real(1.5) * std::pow(B1, two/three);
                            }
                            m_SurfaceLayer[ori]->init_tke_from_ustar(lev, vars_new[lev][Vars::cons], z_phys_nd[lev], qkefac);
                        }
                    }
                }
            }
            updated_prim = true;
        } else {
            m_SurfaceLayer[ori] = nullptr;
        }
    } // end if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
 
    if (!restart_chkfile.empty()) {
        // All active faces now exist, so restore every surface-layer field once.
        ReadCheckpointFileSurfaceLayer();
    }
 
    // Update micro vars and finish moisture model initializations before first plot file
    if (solverChoice.moisture_type != MoistureType::None) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            const amrex::MultiFab* base = solverChoice.anelastic[lev] ? &base_state[lev] : nullptr;
            micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons], base);
            micro->FinishInit(lev, vars_new[lev][Vars::cons], z_phys_nd);
        }
    }
 
    // Fill time averaged velocities before first plot file
    //
    // NOTE: only when starting from scratch.  On restart the running sum and its
    //       normalizer have just been read back in, and that sum already includes
    //       the state at the checkpoint time (ERF::Advance accumulates after each
    //       step).  Accumulating it again here would count that state twice, so a
    //       restarted run would not match the equivalent continuous run, and the
    //       bias would compound over a chain of restarts.
    if (solverChoice.time_avg_vel && restart_chkfile.empty()) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
                              vars_new[lev][Vars::xvel],
                              vars_new[lev][Vars::yvel],
                              vars_new[lev][Vars::zvel]);
        }
    }
 
    if (solverChoice.compute_mean_vars) {
        if (solverChoice.mean_vars_reset_mode == "time" &&
            static_cast<double>(t_new[0]) >=
            static_cast<double>(solverChoice.mean_vars_reset_time)) {
            if (mean_vars_time_reset_done == 0) {
                ResetIntervalMeans();
                mean_vars_time_reset_done = 1;
            }
        }
        // A restart restores the already accumulated window from the
        // checkpoint.  Sampling the restart state here would count that
        // instant twice; a fresh run still needs its initial sample for the
        // historical t=0 behavior.
        if (restart_chkfile.empty()) {
            for (int lev = 0; lev <= finest_level; ++lev) {
                Accumulate_Interval_Means(dt[lev], t_mean_cnt[lev], interval_means[lev].get(),
                                          vars_new[lev][Vars::xvel],
                                          vars_new[lev][Vars::yvel],
                                          vars_new[lev][Vars::zvel],
                                          vars_new[lev][Vars::cons]);
            }
        }
    }
 
#ifdef ERF_USE_PARTICLES
    // Redistribute particles so the container has valid data at all AMR levels
    // before the initial plotfile write
    if (finest_level > 0) {
        particleData.Redistribute(z_phys_nd);
    }
#endif
 
    // Print max values of lateral gradients of base state pressure at level 0
    if (verbose > 0) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            if (lev == 0) {
                compute_max_pressure_gradient_diagnostic(lev);
                compute_max_buoyancy_gradp_diagnostic(lev);
            }
        }
    }
 
    // check for additional plotting variables that are available after particle containers
    // are setup.
    const std::string& pv3d_1 = "plot_vars_1"  ; appendPlotVariables(pv3d_1,plot3d_var_names_1);
    const std::string& pv3d_2 = "plot_vars_2"  ; appendPlotVariables(pv3d_2,plot3d_var_names_2);
    const std::string& pv2d_1 = "plot2d_vars_1"; appendPlotVariables(pv2d_1,plot2d_var_names_1);
    const std::string& pv2d_2 = "plot2d_vars_2"; appendPlotVariables(pv2d_2,plot2d_var_names_2);
 
    if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > zero) )
    {
        WriteCheckpointFile();
        last_check_file_step = 0;
        if (m_check_per > zero) {last_check_file_time += m_check_per;}
    }
 
    if ( (restart_chkfile.empty()) ||
         (!restart_chkfile.empty() && plot_file_on_restart) )
    {
        int plotfiles_3d_written = 0;
        bool interval_diagnostic_consumed = false;
        if (m_plot3d_int_1 > 0 || m_plot3d_per_1 > zero)
        {
            const bool wrote_plotfile = Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
            if (wrote_plotfile) {
                ++plotfiles_3d_written;
                interval_diagnostic_consumed = interval_diagnostic_consumed ||
                    erf_plotfile::plot3d_selection_has_interval_mean_diagnostic(plot3d_var_names_1);
            }
            if (m_plot3d_per_1 > zero) {last_plot3d_file_time_1 += m_plot3d_per_1;}
            last_plot3d_file_step_1 = istep[0];
        }
        if (m_plot3d_int_2 > 0 || m_plot3d_per_2 > zero)
        {
            const bool wrote_plotfile = Write3DPlotFile(2,plotfile3d_type_2,plot3d_var_names_2);
            if (wrote_plotfile) {
                ++plotfiles_3d_written;
                interval_diagnostic_consumed = interval_diagnostic_consumed ||
                    erf_plotfile::plot3d_selection_has_interval_mean_diagnostic(plot3d_var_names_2);
            }
            if (m_plot3d_per_2 > zero) {last_plot3d_file_time_2 += m_plot3d_per_2;}
            last_plot3d_file_step_2 = istep[0];
        }
        // A restart-only plot is observational: it must not consume the
        // restored interval window before the first post-restart advance.
        ResetIntervalMeansAfter3DPlotfileBatch(
            plotfiles_3d_written,
            erf_interval_means::initialization_plot_consumes_interval(
                !restart_chkfile.empty(), interval_diagnostic_consumed));
        if (m_plot2d_int_1 > 0 || m_plot2d_per_1 > zero)
        {
            Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
            if (m_plot2d_per_1 > zero) {last_plot2d_file_time_1 += m_plot2d_per_1;}
            last_plot2d_file_step_1 = istep[0];
        }
        if (m_plot2d_int_2 > 0 || m_plot2d_per_2 > zero)
        {
            Write2DPlotFile(2,plotfile2d_type_2,plot2d_var_names_2);
            if (m_plot2d_per_2 > zero) {last_plot2d_file_time_2 += m_plot2d_per_2;}
            last_plot2d_file_step_2 = istep[0];
        }
        for (int i = 0; i < m_subvol_int.size(); i++) {
            if (m_subvol_int[i] > 0 || m_subvol_per[i] > zero) {
                WriteSubvolume(i,subvol3d_var_names);
                last_subvol_step[i] = istep[0];
                if (m_subvol_per[i] > zero) {last_subvol_time[i] += m_subvol_per[i];}
            }
        }
    }
 
    // Set these up here because we need to know which MPI rank "cell" is on...
    ParmParse pp("erf");
    if (pp.contains("data_log"))
    {
        int num_datalogs = pp.countval("data_log");
        datalog.resize(num_datalogs);
        datalogname.resize(num_datalogs);
        pp.queryarr("data_log",datalogname,0,num_datalogs);
        for (int i = 0; i < num_datalogs; i++) {
            setRecordDataInfo(i,datalogname[i]);
        }
    }
 
    if (pp.contains("der_data_log"))
    {
        int num_der_datalogs = pp.countval("der_data_log");
        der_datalog.resize(num_der_datalogs);
        der_datalogname.resize(num_der_datalogs);
        pp.queryarr("der_data_log",der_datalogname,0,num_der_datalogs);
        for (int i = 0; i < num_der_datalogs; i++) {
            setRecordDerDataInfo(i,der_datalogname[i]);
        }
    }
 
    if (pp.contains("energy_data_log"))
    {
        int num_energy_datalogs = pp.countval("energy_data_log");
        tot_e_datalog.resize(num_energy_datalogs);
        tot_e_datalogname.resize(num_energy_datalogs);
        pp.queryarr("energy_data_log",tot_e_datalogname,0,num_energy_datalogs);
        for (int i = 0; i < num_energy_datalogs; i++) {
            setRecordEnergyDataInfo(i,tot_e_datalogname[i]);
        }
    }
 
    if (solverChoice.rad_uses_interface())
    {
        // Create data log for radiation model if requested
        rad[0]->setupDataLog();
    }
 
 
    if (restart_chkfile.empty() && profile_int > 0) {
        if (destag_profiles) {
            // all variables cell-centered
            write_1D_profiles(t_new[0]);
        } else {
            // some variables staggered
            write_1D_profiles_stag(t_new[0]);
        }
    }
 
    if (pp.contains("sample_point_log") && pp.contains("sample_point"))
    {
        int lev = 0;
 
        int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
        if (num_samplepts > 0) {
            Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
            samplepoint.resize(num_samplepts);
 
            pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
            for (int i = 0; i < num_samplepts; i++) {
                IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                samplepoint[i] = iv;
            }
        }
 
        int num_sampleptlogs = pp.countval("sample_point_log");
        AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
        if (num_sampleptlogs > 0) {
            sampleptlog.resize(num_sampleptlogs);
            sampleptlogname.resize(num_sampleptlogs);
            pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
 
            for (int i = 0; i < num_sampleptlogs; i++) {
                setRecordSamplePointInfo(i,lev,samplepoint[i],sampleptlogname[i]);
            }
        }
 
    }
 
    bool has_sample_line = pp.contains("sample_line");
    bool has_sample_line_real = pp.contains("sample_line_real");
    if (has_sample_line && has_sample_line_real) {
        Abort("Specify only one of erf.sample_line or erf.sample_line_real");
    }
 
    if (pp.contains("sample_line_log") && (has_sample_line || has_sample_line_real))
    {
        int lev = 0;
 
        int num_samplelines = 0;
        if (has_sample_line) {
            num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
            if (num_samplelines > 0) {
                Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
                sampleline.resize(num_samplelines);
 
                pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
                for (int i = 0; i < num_samplelines; i++) {
                    IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                    sampleline[i] = iv;
                }
            }
        } else {
            int num_real_vals = pp.countval("sample_line_real");
            if (num_real_vals % AMREX_SPACEDIM != 0) {
                Abort("erf.sample_line_real must be specified as (x,y,z) triples");
            }
 
            num_samplelines = num_real_vals / AMREX_SPACEDIM;
            if (num_samplelines > 0) {
                Vector<Real> location; location.resize(num_real_vals);
                sampleline.resize(num_samplelines);
 
                pp.queryarr("sample_line_real",location,0,num_real_vals);
 
                const Box& domain = geom[lev].Domain();
                const auto* prob_lo = geom[lev].ProbLo();
                const auto* prob_hi = geom[lev].ProbHi();
                const auto* dx = geom[lev].CellSize();
 
                for (int i = 0; i < num_samplelines; i++) {
                    Real xloc = location[AMREX_SPACEDIM*i+0];
                    Real yloc = location[AMREX_SPACEDIM*i+1];
                    Real zloc = location[AMREX_SPACEDIM*i+2];
 
                    if (xloc < prob_lo[0] || xloc > prob_hi[0] ||
                        yloc < prob_lo[1] || yloc > prob_hi[1] ||
                        zloc < prob_lo[2] || zloc > prob_hi[2]) {
                        Abort("erf.sample_line_real must lie within the level-0 domain");
                    }
 
                    int i_cell = domain.smallEnd(0) + static_cast<int>(std::floor((xloc - prob_lo[0]) / dx[0]));
                    int j_cell = domain.smallEnd(1) + static_cast<int>(std::floor((yloc - prob_lo[1]) / dx[1]));
                    int k_cell = domain.smallEnd(2) + static_cast<int>(std::floor((zloc - prob_lo[2]) / dx[2]));
 
                    i_cell = std::min(i_cell, domain.bigEnd(0));
                    j_cell = std::min(j_cell, domain.bigEnd(1));
                    k_cell = std::min(k_cell, domain.bigEnd(2));
 
                    sampleline[i] = IntVect(i_cell, j_cell, k_cell);
                }
            }
        }
 
        int num_samplelinelogs = pp.countval("sample_line_log");
        AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
        if (num_samplelinelogs > 0) {
            samplelinelog.resize(num_samplelinelogs);
            samplelinelogname.resize(num_samplelinelogs);
            pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
 
            for (int i = 0; i < num_samplelinelogs; i++) {
                setRecordSampleLineInfo(i,lev,sampleline[i],samplelinelogname[i]);
            }
        }
 
    }
 
    if (is_it_time_for_action(istep[0], t_new[0], dt[0], sum_interval, sum_per)) {
        sum_integrated_quantities(t_new[0]);
        sum_derived_quantities(t_new[0]);
        sum_energy_quantities(t_new[0]);
    }
 
    // Create object to do line and plane sampling if needed
    bool do_line = false; bool do_plane = false;
    pp.queryAdd("do_line_sampling",do_line); pp.queryAdd("do_plane_sampling",do_plane);
    if (do_line) {
        if (line_sampling_interval < 0 && line_sampling_per < 0) {
            Abort("Need to specify line_sampling_interval or line_sampling_per");
        }
        line_sampler = std::make_unique<LineSampler>();
        line_sampler->write_coords(z_phys_cc, geom);
    }
    if (do_plane) {
        if (plane_sampling_interval < 0 && plane_sampling_per < 0) {
            Abort("Need to specify plane_sampling_interval or plane_sampling_per");
        }
        plane_sampler = std::make_unique<PlaneSampler>();
    }
 
    // Create the object that writes station time series if any stations are named
    {
        // Naming stations turns the output on; an explicit erf.do_station_sampling
        // = false turns it back off, so a deck can keep its stations and have the
        // output switched off from the command line.
        bool do_station = (pp.countval("station_names") > 0);
        pp.queryAdd("do_station_sampling", do_station);
        if (do_station && pp.countval("station_names") == 0) {
            Abort("erf.do_station_sampling is true but erf.station_names is empty, "
                  "so there is nothing to sample");
        }
        if (do_station) {
            if (station_sampling_interval < 0 && station_sampling_per < 0) {
                // No default, as for the line and plane samplers above.  A
                // sample costs a fillpatch and a fill of every requested
                // variable over every level that hosts a station, so defaulting
                // to every step would make a run with a one-second time step
                // pay for output nobody asked for, and write a row a second.
                Abort("Need to specify station_sampling_interval or station_sampling_per");
            }
            station_sampler = std::make_unique<StationSampler>(pp_prefix);
            station_sampler->setRestart(!restart_chkfile.empty());
            init_stations();
            // Start the series at the initial condition, as WRF's tslist does.
            // A restart does not repeat it: the row at that time is already in
            // the file the earlier run wrote.
            if (restart_chkfile.empty()) {
                sample_stations(static_cast<Real>(t_new[0]));
            }
        }
    }
 
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing  ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing )
    {
        bool write_eb_surface = false;
        pp.queryAdd("write_eb_surface", write_eb_surface);
        if (write_eb_surface) {
            if (verbose > 0) {
                amrex::Print() << "Writing the geometry to a vtp file.\n" << std::endl;
            }
            WriteEBSurface(grids[finest_level],dmap[finest_level],Geom(finest_level),&EBFactory(finest_level));
        }
    }
}

Here is the call graph for this function:

InitData_pre()

void ERF::InitData_pre

(

)

{
    if (m_driver_has_atm2ocn_coupling && verbose > 0) {
        amrex::Print() << "ERF InitData_pre: driver-managed atm2ocn coupling enabled"
                       << " two_way=" << (m_driver_uses_two_way_coupling ? 1 : 0)
                       << " active_contract="
                       << (m_driver_atm2ocn_uses_state_contract ? "state" : "flux")
                       << "\n";
    }
 
    // Initialize the start time for our CPU-time tracker
    startCPUTime = ParallelDescriptor::second();
 
    // Create the ReadBndryPlanes object so we can read boundary plane data
    // m_r2d is used by init_bcs so we must instantiate this class before
    if (input_bndry_planes) {
        Print() << "Defining r2d for the first time " << std::endl;
        m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
    }
 
    if (restart_chkfile.empty()) {
        // Start simulation from the beginning
        InitFromScratch(zero);
    } else {
        // For initialization this is done in init_only; it is done here for restart
        init_bcs();
    }
 
    solverChoice.check_params(max_level,geom,phys_bc_type);
}

initHSE() [1/2]

void ERF::initHSE ( )

private

Initialize HSE.

Initialize density and pressure base state on every active AMR level.

{
    for (int lev = 0; lev <= finest_level; lev++)
    {
        initHSE(lev);
    }
}

initHSE() [2/2]

void ERF::initHSE ( int  lev )

private

Initialize density and pressure base state in hydrostatic equilibrium.

Parameters

lev	Integer specifying the current level

{
    // This integrates up through column to update p_hse, pi_hse, th_hse;
    // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
 
    MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
    MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
    MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
    MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
 
    bool all_boxes_touch_bottom = true;
    Box domain(geom[lev].Domain());
 
    if (lev == 0) {
        BoxArray ba(base_state[lev].boxArray());
        for (int i = 0; i < ba.size(); i++) {
            if (ba[i].smallEnd(2) != domain.smallEnd(2)) {
                all_boxes_touch_bottom = false;
            }
        }
    }
    else
    {
        //
        // Set the values of the base state inside the domain but outside of the fine region,
        // and in the fine ghost cells; the vertical integration below then overwrites only
        // the fine grids, and for a box whose klo is in the interior it reads the value this
        // leaves in klo-1 as its starting point.
        //
        interp_base_state_from_coarse(lev);
    }
 
    bool is_constant_dz  = (solverChoice.mesh_type == MeshType::ConstantDz);
    bool is_stretched_dz = (solverChoice.mesh_type == MeshType::StretchedDz);
 
    auto initialize_base_state =
        [this, lev, is_constant_dz, is_stretched_dz]
        (MultiFab& dens, MultiFab& pres, MultiFab& pi, MultiFab& theta, MultiFab& qv,
         std::unique_ptr<MultiFab>& z_phys_nd_in, std::unique_ptr<MultiFab>& z_phys_cc_in)
    {
        if (solverChoice.init_type == InitType::MoistBaseState ||
            solverChoice.init_type == InitType::HindCast)
        {
            AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
            prob->erf_init_dens_hse_moist(dens, z_phys_nd_in, geom[lev]);
        }
        else if (solverChoice.init_type == InitType::ConstantDensity)
        {
            prob->erf_init_const_dens_hse(dens);
        }
        else if (solverChoice.init_type == InitType::Uniform)
        {
            AMREX_ALWAYS_ASSERT(!solverChoice.use_gravity || solverChoice.anelastic[lev]);
            prob->erf_init_const_dens_and_th_hse(dens, pres, pi, theta, qv, solverChoice.rdOcp);
        }
        else if (solverChoice.init_type == InitType::ConstantDensityLinearTheta)
        {
            AMREX_ALWAYS_ASSERT(!solverChoice.use_gravity || solverChoice.anelastic[lev]);
            prob->erf_init_const_dens_and_linear_th_hse(dens, pres, pi, theta, qv,
                                                        solverChoice.rdOcp, z_phys_cc_in);
        }
        else
        {
            prob->erf_init_dens_hse_dry(dens, z_phys_nd_in, z_phys_cc_in, geom[lev],
                                        stretched_dz_h[lev], is_constant_dz, is_stretched_dz);
        }
 
        if (solverChoice.init_type != InitType::Uniform &&
            solverChoice.init_type != InitType::ConstantDensityLinearTheta)
        {
            erf_enforce_hse(lev, dens, pres, pi, theta, qv, z_phys_cc_in);
        }
    };
 
    if (all_boxes_touch_bottom || lev > 0) {
 
        initialize_base_state(r_hse, p_hse, pi_hse, th_hse, qv_hse,
                              z_phys_nd[lev], z_phys_cc[lev]);
 
        //
        // Impose physical bc's on the base state
        //
        (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
 
    } else {
 
        BoxArray ba_new(domain);
 
        ChopGrids2D(ba_new, domain, ParallelDescriptor::NProcs());
 
        DistributionMapping dm_new(ba_new);
 
        MultiFab new_base_state(ba_new, dm_new, BaseState::num_comps, base_state[lev].nGrowVect());
        new_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
                                    base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
 
        MultiFab new_r_hse (new_base_state, make_alias, BaseState::r0_comp, 1);
        MultiFab new_p_hse (new_base_state, make_alias, BaseState::p0_comp, 1);
        MultiFab new_pi_hse(new_base_state, make_alias, BaseState::pi0_comp, 1);
        MultiFab new_th_hse(new_base_state, make_alias, BaseState::th0_comp, 1);
        MultiFab new_qv_hse(new_base_state, make_alias, BaseState::qv0_comp, 1);
 
        std::unique_ptr<MultiFab> new_z_phys_cc;
        std::unique_ptr<MultiFab> new_z_phys_nd;
        if (solverChoice.mesh_type != MeshType::ConstantDz ||
            solverChoice.init_type == InitType::ConstantDensityLinearTheta)
        {
            new_z_phys_cc = std::make_unique<MultiFab>(ba_new,dm_new,1,1);
            new_z_phys_cc->ParallelCopy(*z_phys_cc[lev],0,0,1,1,1);
        }
 
        if (solverChoice.mesh_type != MeshType::ConstantDz) {
            BoxArray ba_new_nd(ba_new);
            ba_new_nd.surroundingNodes();
            new_z_phys_nd = std::make_unique<MultiFab>(ba_new_nd,dm_new,1,1);
            new_z_phys_nd->ParallelCopy(*z_phys_nd[lev],0,0,1,1,1);
        }
 
        initialize_base_state(new_r_hse, new_p_hse, new_pi_hse, new_th_hse, new_qv_hse,
                              new_z_phys_nd, new_z_phys_cc);
 
        //
        // Impose physical bc's on the base state (we must make new, temporary bcs object because the z_phys_nd is different)
        //
        ERFPhysBCFunct_base* temp_physbcs_base =
            new ERFPhysBCFunct_base(lev, geom[lev], domain_bcs_type, domain_bcs_type_d, new_z_phys_nd,
                                    (solverChoice.terrain_type == TerrainType::MovingFittedMesh),
                                    (solverChoice.mesh_type != MeshType::ConstantDz),
                                    solverChoice.rdOcp, solverChoice.gravity);
        (*temp_physbcs_base)(new_base_state,0,new_base_state.nComp(),new_base_state.nGrowVect());
        delete temp_physbcs_base;
 
        // Now copy back into the original arrays
        base_state[lev].ParallelCopy(new_base_state,0,0,base_state[lev].nComp(),
                                     base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
    }
 
    //
    // Impose physical bc's on the base state -- the values outside the fine region
    //   but inside the domain have already been filled in the call above to InterpFromCoarseLevel
    //
    (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
}

Here is the call graph for this function:

initialize_integrator()

void ERF::initialize_integrator ( int  lev, amrex::MultiFab &  cons_mf, amrex::MultiFab &  vel_mf  )

private

{
    const BoxArray& ba(cons_mf.boxArray());
    const DistributionMapping& dm(cons_mf.DistributionMap());
 
    int ncomp_cons = cons_mf.nComp();
 
    // Initialize the integrator memory
    Vector<MultiFab> int_state; // integration state data structure example
    int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons));         // cons
    int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
    int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
    int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
 
    mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
    mri_integrator_mem[lev]->setNoSubstepping((solverChoice.substepping_type[lev] == SubsteppingType::None));
    mri_integrator_mem[lev]->setAnelastic(solverChoice.anelastic[lev]);
    mri_integrator_mem[lev]->setAnelasticType(solverChoice.anelastic_type[lev]);
    mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
    mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
}

initializeEB()

void ERF::initializeEB ( )

private

{
    //
    // Construct the EB data structures and store in a separate class
    //
    std::string geometry ="terrain";
    ParmParse pp_eb2("eb2");
    pp_eb2.queryAdd("geometry", geometry);
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing)
    {
        constexpr int ngrow_for_eb = 4;  // This is the default in amrex but we need to explicitly pass it here since
                               // we want to also pass the build_coarse_level_by_coarsening argument
        const bool build_eb_for_multigrid = (solverChoice.terrain_type == TerrainType::EB &&
                                            ((solverChoice.project_initial_velocity[0] == 1) ||
                                            solverChoice.anelastic[0] == 1));
        // Note this just needs to be an integer > number of V-cycles one might use
        const int max_coarsening_level = (build_eb_for_multigrid) ? 100 : 0;
        const bool build_coarse_level_by_coarsening(false);
 
        // Define GeometryShop using the implicit function
        if (geometry == "terrain") {
            // Query building STL parameters upfront
            //
            // Note these live under the "erf" prefix, not "eb2", because "eb2" is amrex's own
            // namespace and amrex already reads eb2.stl_* for its own eb2.geometry = stl path.
            ParmParse pp_erf("erf");
            std::string buildings_stl_file;
            bool has_buildings_stl = pp_erf.query("buildings_stl_file", buildings_stl_file);
 
            Real stl_scale = 1.0;
            Array<Real,3> stl_center = {zero, zero, zero};
            int stl_reverse_normal = 0;
 
            if (has_buildings_stl) {
                pp_erf.query("buildings_stl_scale", stl_scale);
                pp_erf.query("buildings_stl_center", stl_center);
                pp_erf.query("buildings_stl_reverse_normal", stl_reverse_normal);
            }
 
            // Decide whether the terrain surface is part of the implicit function.  By default
            // we include it whenever the user has specified terrain by any of the means that
            // init_terrain_surface understands -- not just erf.terrain_file_name -- but the
            // user may also state the choice outright with erf.buildings_only.
            bool buildings_only = (has_buildings_stl && !prob->terrain_is_specified());
 
            bool buildings_only_in = false;
            if (pp_erf.query("buildings_only", buildings_only_in)) {
                if (buildings_only_in && !has_buildings_stl) {
                    Abort("erf.buildings_only is true but no erf.buildings_stl_file was given");
                }
                buildings_only = buildings_only_in;
            }
 
            // Determine geometry mode and log configuration
            std::string mode_description;
            if (buildings_only) {
                mode_description = "Buildings-only mode (no terrain)";
            } else if (has_buildings_stl) {
                mode_description = "Terrain + 3D buildings";
            } else {
                mode_description = "Terrain-only mode (no buildings)";
            }
 
            Print() << "Building EB geometry: " << mode_description << "\n";
            if (has_buildings_stl) {
                Print() << "  STL file: " << buildings_stl_file << "\n";
                Print() << "  STL scale: " << stl_scale << "\n";
                Print() << "  STL center: " << stl_center[0] << " " << stl_center[1] << " " << stl_center[2] << "\n";
                Print() << "  Reverse normals: " << stl_reverse_normal << "\n";
            }
 
            // Lambda to build EB geometry from shop
            auto build_eb = [&](auto const& gshop) {
                if (build_eb_for_multigrid) {
                    EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                                ngrow_for_eb, build_coarse_level_by_coarsening);
                } else {
                    EB2::Build(gshop, this->Geom(), ngrow_for_eb);
#if USE_FC_FACTORY
                    EB2::BuildFC();
#endif
                }
            };
 
            // Build the appropriate implicit function and geometry
            if (buildings_only) {
                // Buildings-only
                BuildingsIF buildings_if(buildings_stl_file, stl_scale, stl_center,
                                        stl_reverse_normal, geom[max_level]);
                auto gshop = EB2::makeShop(buildings_if);
                build_eb(gshop);
 
            } else {
                // Load terrain (from file or custom init_my_custom_terrain)
                Box terrain_bx(surroundingNodes(geom[max_level].Domain()));
                terrain_bx.grow(3);
                FArrayBox terrain_fab(makeSlab(terrain_bx,2,0),1);
                double dummy_time = 0.0;
                prob->init_terrain_surface(geom[max_level], terrain_fab, dummy_time);
                TerrainIF terrain_if(terrain_fab, geom[max_level], stretched_dz_d[max_level]);
 
                if (has_buildings_stl) {
                    // Terrain + Buildings
                    BuildingsIF buildings_if(buildings_stl_file, stl_scale, stl_center,
                                            stl_reverse_normal, geom[max_level]);
                    auto combined_if = EB2::makeUnion(terrain_if, buildings_if);
                    auto gshop = EB2::makeShop(combined_if);
                    build_eb(gshop);
                } else {
                    // Terrain-only
                    auto gshop = EB2::makeShop(terrain_if);
                    build_eb(gshop);
                }
            }
 
            Print() << "EB geometry built successfully: " << mode_description << ".\n";
        } else if (geometry == "plane") {
            RealArray plane_point{zero, zero, zero};
            RealArray plane_normal{zero, zero, -one}; // pointing into the solid region
            pp_eb2.queryAdd("plane_point", plane_point);
            pp_eb2.queryAdd("plane_normal", plane_normal);
            EB2::PlaneIF implicit_fun(plane_point, plane_normal, true);
            auto gshop = EB2::makeShop(implicit_fun);
            if (build_eb_for_multigrid) {
                EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                            ngrow_for_eb, build_coarse_level_by_coarsening);
            } else {
                EB2::Build(gshop, this->Geom(), ngrow_for_eb);
#if USE_FC_FACTORY
                EB2::BuildFC();
#endif
            }
        } else if (geometry == "box") {
            RealArray box_lo{zero, zero, zero};
            RealArray box_hi{zero, zero, zero};
            pp_eb2.queryAdd("box_lo", box_lo);
            pp_eb2.queryAdd("box_hi", box_hi);
            EB2::BoxIF implicit_fun(box_lo, box_hi, false);
            auto gshop = EB2::makeShop(implicit_fun);
            if (build_eb_for_multigrid) {
                EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                            ngrow_for_eb, build_coarse_level_by_coarsening);
            } else {
                EB2::Build(gshop, this->Geom(), ngrow_for_eb);
#if USE_FC_FACTORY
                EB2::BuildFC();
#endif
            }
        } else if (geometry == "sphere") {
            auto ProbLoArr = geom[max_level].ProbLoArray();
            auto ProbHiArr = geom[max_level].ProbHiArray();
            const Real xcen = myhalf * (ProbLoArr[0] + ProbHiArr[0]);
            const Real ycen = myhalf * (ProbLoArr[1] + ProbHiArr[1]);
            RealArray sphere_center = {xcen, ycen, zero};
            EB2::SphereIF implicit_fun(myhalf, sphere_center, false);
            auto gshop = EB2::makeShop(implicit_fun);
            if (build_eb_for_multigrid) {
                EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                            ngrow_for_eb, build_coarse_level_by_coarsening);
            } else {
                EB2::Build(gshop, this->Geom(), ngrow_for_eb);
#if USE_FC_FACTORY
                EB2::BuildFC();
#endif
            }
        }
    }
 
    if ( solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
        constexpr int ngrow_for_eb = 4;
        if (geometry == "terrain") {
            Box buildings_bx(surroundingNodes(geom[max_level].Domain())); buildings_bx.grow(3);
            FArrayBox buildings_fab(makeSlab(buildings_bx,2,0),1);
            double dummy_time = 0.0;
            prob->init_buildings_surface(geom[max_level], buildings_fab, dummy_time);
            TerrainIF implicit_fun(buildings_fab, geom[max_level], stretched_dz_d[max_level]);
            auto gshop = EB2::makeShop(implicit_fun);
            EB2::Build(gshop, this->Geom(), ngrow_for_eb);
#if USE_FC_FACTORY
            EB2::BuildFC();
#endif
        } else if (geometry == "plane") {
            amrex::Abort("plane geometry is not supported with ImmersedForcing for buildings");
        } else if (geometry == "box") {
            RealArray box_lo{zero, zero, zero};
            RealArray box_hi{zero, zero, zero};
            pp_eb2.queryAdd("box_lo", box_lo);
            pp_eb2.queryAdd("box_hi", box_hi);
            EB2::BoxIF implicit_fun(box_lo, box_hi, false);
            auto gshop = EB2::makeShop(implicit_fun);
            EB2::Build(gshop, this->Geom(), ngrow_for_eb);
#if USE_FC_FACTORY
            EB2::BuildFC();
#endif
        } else if (geometry == "sphere") {
            amrex::Abort("sphere geometry is not supported with ImmersedForcing for buildings");
        }
    }
}

Referenced by ERF_shared().

Here is the caller graph for this function:

InitializeFromFile()

void ERF::InitializeFromFile ( )

private

InitializeLevelFromData()

void ERF::InitializeLevelFromData ( int  lev, const amrex::MultiFab &  initial_data  )

private

initializeMicrophysics()

void ERF::initializeMicrophysics ( const int &  a_nlevsmax )

private

Parameters

a_nlevsmax

number of AMR levels

{
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
 
        micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
 
    } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
#ifdef ERF_USE_PARTICLES
        micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
        /* Lagrangian microphysics models will have a particle container; it needs to be added
           to ERF::particleData */
        const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
        /* The particle container has not yet been constructed and initialized, so just add
           its name here for now (so that functions to set plotting variables can see it). */
        particleData.addName( pc_name );
 
#else
        Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
#endif
    }
 
    qmoist.resize(a_nlevsmax);
    return;
}

Referenced by ERF_shared().

Here is the call graph for this function:

Here is the caller graph for this function:

initRayleigh_at_level()

void ERF::initRayleigh_at_level ( const int &  lev )

private

Initialize Rayleigh damping profiles at a level.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

Parameters

lev	Integer specifying the current level

{
    const int khi = geom[0].Domain().bigEnd(2);
    solverChoice.dampingChoice.rayleigh_ztop = (solverChoice.terrain_type == TerrainType::None) ?  geom[0].ProbHi(2) : zlevels_stag[0][khi+1];
 
    // These have 4 components: ubar, vbar, wbar, thetabar
    h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
    d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
 
    const int zlen_rayleigh = geom[lev].Domain().length(2);
 
    // Allocate space for these 1D vectors
    for (int n = 0; n < Rayleigh::nvars; n++) {
        h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
        d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
    }
 
    h_sinesq_ptrs[lev].resize(zlen_rayleigh);
    d_sinesq_ptrs[lev].resize(zlen_rayleigh);
 
    h_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
    d_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
 
    Real ztop     = solverChoice.dampingChoice.rayleigh_ztop;
    Real zdamp    = solverChoice.dampingChoice.rayleigh_zdamp;
 
    for (int k = 0; k < zlen_rayleigh; k++) {
        Real z = myhalf * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
        if (z > (ztop - zdamp)) {
            Real zfrac = one - (ztop - z) / zdamp;
            Real s = std::sin(PIoTwo*zfrac);
            h_sinesq_ptrs[lev][k] = s*s;
        } else {
            h_sinesq_ptrs[lev][k] = zero;
        }
    }
 
    for (int k = 0; k < zlen_rayleigh+1; k++) {
        Real z = zlevels_stag[lev][k];
        if (z > (ztop - zdamp)) {
            Real zfrac = one - (ztop - z) / zdamp;
            Real s = std::sin(PIoTwo*zfrac);
            h_sinesq_stag_ptrs[lev][k] = s*s;
        } else {
            h_sinesq_stag_ptrs[lev][k] = zero;
        }
    }
 
    // Init the host vectors for the reference states
    prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_nd[lev],
                            solverChoice.dampingChoice.rayleigh_zdamp);
 
    // Copy from host vectors to device vectors
    for (int n = 0; n < Rayleigh::nvars; n++) {
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
                  d_rayleigh_ptrs[lev][n].begin());
    }
    Gpu::copy(Gpu::hostToDevice, h_sinesq_ptrs[lev].begin(), h_sinesq_ptrs[lev].end(), d_sinesq_ptrs[lev].begin());
    Gpu::copy(Gpu::hostToDevice, h_sinesq_stag_ptrs[lev].begin(), h_sinesq_stag_ptrs[lev].end(), d_sinesq_stag_ptrs[lev].begin());
}

initSponge()

void ERF::initSponge ( )

private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

{
    h_sponge_ptrs.resize(max_level+1);
    d_sponge_ptrs.resize(max_level+1);
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        // These have 2 components: ubar, vbar
        h_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
        d_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
 
        const int zlen_sponge = geom[lev].Domain().length(2);
 
        // Allocate space for these 1D vectors
        for (int n = 0; n < Sponge::nvars_sponge; n++) {
            h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
            d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
        }
 
    }
}

input_sponge()

void ERF::input_sponge

(

int

lev

)

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters

lev	Integer specifying the current level

{
    // We only want to read the file once
    if (lev == 0) {
        if (input_sponge_data.input_sponge_file.empty())
            Error("input_sounding file name must be provided via input");
 
        // this will interpolate the input profiles to the nominal height levels
        // (ranging from 0 to the domain top)
        input_sponge_data.read_from_file(geom[lev], zlevels_stag[lev]);
    }
}

Interp2DArrays()

void ERF::Interp2DArrays	(	int	lev,
		const amrex::BoxArray &	my_ba2d,
		const amrex::DistributionMapping &	my_dm
	)

{
    if (lev == 0) { return; }
 
    //
    // These fields have no vertical extent: ba2d collapses z to a single index at both
    // levels, so there is nothing in the vertical for the interpolation to do.  It must
    // therefore be handed a ratio of 1 in z even when the levels really are refined in
    // the vertical -- a ratio greater than 1 makes the interpolator ask for a layer of
    // coarse ghost cells in z that a 2-D MultiFab does not, and cannot, carry.
    //
    const IntVect rr2d(refRatio(lev-1)[0], refRatio(lev-1)[1], 1);
 
    //
    // Map factors, surface pressure, land mask and land / soil types.  Unlike the fields
    // below, these always exist by the time we get here (init_stuff builds them), so there
    // is no "only if it hasn't been made yet" guard -- we must overwrite the placeholder
    // that init_stuff left behind (a map factor of 1, a uniform land mask, an
    // uninitialized surface pressure).  A level that read its own init file already has
    // all of them from that file and must be left alone.
    //
    if (!has_own_init_file(lev)) {
        interp_mapfac_from_coarse(lev);
        interp_psfc_from_coarse(lev);
        interp_land_masks_from_coarse(lev);
    }
 
    if (lon_m[lev-1] && !lon_m[lev]) {
        auto ngv = lon_m[lev-1]->nGrowVect(); ngv[2] = 0;
        lon_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
        InterpFromCoarseLevel(*lon_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                              *lon_m[lev-1], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              rr2d, &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
    }
    if (lat_m[lev-1] && !lat_m[lev]) {
        auto ngv = lat_m[lev-1]->nGrowVect(); ngv[2] = 0;
        lat_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
        InterpFromCoarseLevel(*lat_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                              *lat_m[lev-1], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              rr2d, &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
    }
    //
    // The Coriolis factors.  Unlike lat_m/lon_m above, these are read one cell past the
    // hi domain faces: the momentum sources average them onto the faces of the nodal
    // tilebox (see the var_coriolis branch of ERF_MakeMomSources.cpp), so the x-momentum
    // source at i = nx reads sinPhi/cosPhi at i = nx.  A level that reads its own
    // wrfinput/met_em file fills those ghosts by clamping into the valid region; a level
    // created by refinement must get them from the parent instead, or the Coriolis source
    // on its hi faces is computed from uninitialized memory.
    //
    // NOTE: filling ghost cells outside the domain requires that ALL ghost cells of the
    //       coarse MultiFab have already been filled, inside the domain and out.  That
    //       holds here: init_from_wrfinput/init_from_metgrid fill the grown box at a level
    //       with a file, and this interpolation fills it at a level without one.
    //
    if (sinPhi_m[lev-1] && !sinPhi_m[lev]) {
        auto ngv = sinPhi_m[lev-1]->nGrowVect(); ngv[2] = 0;
        sinPhi_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
        InterpFromCoarseLevel(*sinPhi_m[lev], ngv, ngv, // DO fill ghost cells outside the domain
                              *sinPhi_m[lev-1], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              rr2d, &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
    }
    if (cosPhi_m[lev-1] && !cosPhi_m[lev]) {
        auto ngv = cosPhi_m[lev-1]->nGrowVect(); ngv[2] = 0;
        cosPhi_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
        InterpFromCoarseLevel(*cosPhi_m[lev], ngv, ngv, // DO fill ghost cells outside the domain
                              *cosPhi_m[lev-1], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              rr2d, &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
    }
    if (sst_lev[lev-1][0]) {
        if (sst_lev[lev].size() < sst_lev[lev-1].size()) {
            sst_lev[lev].resize(sst_lev[lev-1].size());
        }
#ifdef ERF_USE_NETCDF
        double time_since_start_low = t_new[0] + start_time - start_low_time;
        int n_time_old = static_cast<int>(time_since_start_low /  low_time_interval);
        int ntimes_to_interp = std::min(n_time_old+3, static_cast<int>(sst_lev[lev-1].size()));
#else
        // TODO: Fix if SST is provided without NETCDF
        int n_time_old       = 0;
        int ntimes_to_interp = 1;
#endif
        auto ngv = sst_lev[lev-1][0]->nGrowVect(); ngv[2] = 0;
 
        for (int n = n_time_old; n < ntimes_to_interp; n++) {
            if (!sst_lev[lev-1][n]) { continue; }
            if (!sst_lev[lev][n]) {
                sst_lev[lev][n] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
                // DO fill the ghost cells outside the domain: SurfaceLayer populates t_surf
                // on grown tileboxes, so it reads the sea surface temperature past a
                // non-periodic domain face.  As for the Coriolis factors above, the coarse
                // level has those ghost cells filled (update_sst_tsk fills the grown box).
                InterpFromCoarseLevel(*sst_lev[lev][n], ngv, ngv,
                                      *sst_lev[lev-1][n], 0, 0, 1,
                                      geom[lev-1], geom[lev],
                                      rr2d, &cell_cons_interp,
                                      domain_bcs_type, BCVars::cons_bc);
            }
        }
    }
    if (tsk_lev[lev-1][0]) {
        if (tsk_lev[lev].size() < tsk_lev[lev-1].size()) {
            tsk_lev[lev].resize(tsk_lev[lev-1].size());
        }
#ifdef ERF_USE_NETCDF
        double time_since_start_low = t_new[0] + start_time - start_low_time;
        int n_time_old = static_cast<int>(time_since_start_low /  low_time_interval);
        int ntimes_to_interp = std::min(n_time_old+3, static_cast<int>(tsk_lev[lev-1].size()));
#else
        // TODO: Fix if TSK is provided without NETCDF
        int n_time_old       = 0;
        int ntimes_to_interp = 1;
#endif
        auto ngv = tsk_lev[lev-1][0]->nGrowVect(); ngv[2] = 0;
 
        for (int n = n_time_old; n < ntimes_to_interp; n++) {
            if (!tsk_lev[lev-1][n]) { continue; }
            if (!tsk_lev[lev][n]) {
                tsk_lev[lev][n] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
                // DO fill the ghost cells outside the domain -- see the note on sst_lev
                InterpFromCoarseLevel(*tsk_lev[lev][n], ngv, ngv,
                                      *tsk_lev[lev-1][n], 0, 0, 1,
                                      geom[lev-1], geom[lev],
                                      rr2d, &cell_cons_interp,
                                      domain_bcs_type, BCVars::cons_bc);
            }
        }
    }
 
    Real time_for_fp = 0.0; // This is not actually used
    Vector<Real> ftime    = {static_cast<Real>(time_for_fp), static_cast<Real>(time_for_fp)};
    Vector<Real> ctime    = {static_cast<Real>(time_for_fp), static_cast<Real>(time_for_fp)};
    if (lat_m[lev]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        Vector<MultiFab*> fmf = {lat_m[lev  ].get(), lat_m[lev  ].get()};
        Vector<MultiFab*> cmf = {lat_m[lev-1].get(), lat_m[lev-1].get()};
        IntVect ngv = lat_m[lev]->nGrowVect(); ngv[2] = 0;
        Interpolater* mapper = &cell_cons_interp;
        FillPatchTwoLevels(*lat_m[lev].get(), ngv, IntVect(0,0,0),
                           time_for_fp, cmf, ctime, fmf, ftime,
                           0, 0, 1, geom[lev-1], geom[lev],
                           rr2d, mapper, domain_bcs_type,
                           BCVars::cons_bc);
    }
    if (lon_m[lev]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        Vector<MultiFab*> fmf = {lon_m[lev  ].get(), lon_m[lev  ].get()};
        Vector<MultiFab*> cmf = {lon_m[lev-1].get(), lon_m[lev-1].get()};
        IntVect ngv = lon_m[lev]->nGrowVect(); ngv[2] = 0;
        Interpolater* mapper = &cell_cons_interp;
        FillPatchTwoLevels(*lon_m[lev].get(), ngv, IntVect(0,0,0),
                           time_for_fp, cmf, ctime, fmf, ftime,
                           0, 0, 1, geom[lev-1], geom[lev],
                           rr2d, mapper, domain_bcs_type,
                           BCVars::cons_bc);
    } // lon_m
    if (sinPhi_m[lev]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        Vector<MultiFab*> fmf = {sinPhi_m[lev  ].get(), sinPhi_m[lev  ].get()};
        Vector<MultiFab*> cmf = {sinPhi_m[lev-1].get(), sinPhi_m[lev-1].get()};
        IntVect ngv = sinPhi_m[lev]->nGrowVect(); ngv[2] = 0;
        Interpolater* mapper = &cell_cons_interp;
        FillPatchTwoLevels(*sinPhi_m[lev].get(), ngv, IntVect(0,0,0),
                           time_for_fp, cmf, ctime, fmf, ftime,
                           0, 0, 1, geom[lev-1], geom[lev],
                           rr2d, mapper, domain_bcs_type,
                           BCVars::cons_bc);
    } // sinPhi
    if (cosPhi_m[lev]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        Vector<MultiFab*> fmf = {cosPhi_m[lev  ].get(), cosPhi_m[lev  ].get()};
        Vector<MultiFab*> cmf = {cosPhi_m[lev-1].get(), cosPhi_m[lev-1].get()};
        IntVect ngv = cosPhi_m[lev]->nGrowVect(); ngv[2] = 0;
        Interpolater* mapper = &cell_cons_interp;
        FillPatchTwoLevels(*cosPhi_m[lev].get(), ngv, IntVect(0,0,0),
                           time_for_fp, cmf, ctime, fmf, ftime,
                           0, 0, 1, geom[lev-1], geom[lev],
                           rr2d, mapper, domain_bcs_type,
                           BCVars::cons_bc);
    } // cosPhi
    if (sst_lev[lev][0]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
#ifdef ERF_USE_NETCDF
        double time_since_start_low = t_new[0] + start_time - start_low_time;
        int n_time_old = static_cast<int>(time_since_start_low /  low_time_interval);
        int ntimes_to_interp = std::min(n_time_old+3, static_cast<int>(sst_lev[lev-1].size()));
#else
        // TODO: Fix if SST is provided without NETCDF
        int n_time_old       = 0;
        int ntimes_to_interp = 1;
#endif
        for (int n = n_time_old; n < ntimes_to_interp; n++) {
            if (!sst_lev[lev][n] || !sst_lev[lev-1][n]) { continue; }
            Vector<MultiFab*> fmf = {sst_lev[lev  ][n].get(), sst_lev[lev  ][n].get()};
            Vector<MultiFab*> cmf = {sst_lev[lev-1][n].get(), sst_lev[lev-1][n].get()};
            IntVect ngv = sst_lev[lev][n]->nGrowVect(); ngv[2] = 0;
            Interpolater* mapper = &cell_cons_interp;
            FillPatchTwoLevels(*sst_lev[lev][n].get(), ngv, IntVect(0,0,0),
                               time_for_fp, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               rr2d, mapper, domain_bcs_type,
                               BCVars::cons_bc);
        } // ntimes
    } // sst_lev
    if (tsk_lev[lev][0]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
#ifdef ERF_USE_NETCDF
        double time_since_start_low = t_new[0] + start_time - start_low_time;
        int n_time_old = static_cast<int>(time_since_start_low /  low_time_interval);
        int ntimes_to_interp = std::min(n_time_old+3, static_cast<int>(tsk_lev[lev-1].size()));
#else
        // TODO: Fix if TSK is provided without NETCDF
        int n_time_old       = 0;
        int ntimes_to_interp = 1;
#endif
        for (int n = n_time_old; n < ntimes_to_interp; n++) {
            if (!tsk_lev[lev][n] || !tsk_lev[lev-1][n]) { continue; }
            Vector<MultiFab*> fmf = {tsk_lev[lev  ][n].get(), tsk_lev[lev  ][n].get()};
            Vector<MultiFab*> cmf = {tsk_lev[lev-1][n].get(), tsk_lev[lev-1][n].get()};
            IntVect ngv = tsk_lev[lev][n]->nGrowVect(); ngv[2] = 0;
            Interpolater* mapper = &cell_cons_interp;
            FillPatchTwoLevels(*tsk_lev[lev][n].get(), ngv, IntVect(0,0,0),
                               time_for_fp, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               rr2d, mapper, domain_bcs_type,
                               BCVars::cons_bc);
        } // ntimes
    } // tsk_lev
}

Here is the call graph for this function:

interp_base_state_from_coarse()

void ERF::interp_base_state_from_coarse ( int  lev )

private

Fill base_state[lev] by conservative interpolation from the next coarser level and then apply the base-state physical boundary conditions.

This is what gives values to the part of a fine level that lies inside the domain but outside the fine grids, and to the fine ghost cells. Any construction that writes only the fine grids – initHSE's vertical integration, or the analytic WRF profile – must run after this, not before.

Parameters

lev	Integer specifying the current level; must be > 0

{
    AMREX_ALWAYS_ASSERT(lev > 0);
 
    int icomp = 0; int ncomp = BaseState::num_comps;
 
    base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
    //
    // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
    //       have been pre-filled - this includes ghost cells both inside and outside
    //       the domain
    //
    InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                          IntVect(0,0,0), // do not fill ghost cells outside the domain
                          base_state[lev-1], icomp, icomp, ncomp,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), &cell_cons_interp,
                          domain_bcs_type, BCVars::cons_bc);
 
    // We need to do this here because the interpolation above may leave corners unfilled
    //    when the corners need to be filled by, for example, reflection of the fine ghost
    //    cell outside the fine region but inside the domain.
    (*physbcs_base[lev])(base_state[lev],icomp,ncomp,base_state[lev].nGrowVect());
}

Here is the call graph for this function:

interp_land_masks_from_coarse()

void ERF::interp_land_masks_from_coarse

(

int

lev

)

{
    AMREX_ALWAYS_ASSERT(lev > 0);
 
    //
    // Only wrfinput fills these from a file.  For every other init_type the values that
    // init_stuff left at this level are the ones that were asked for -- erf.is_land is
    // set per level -- and must not be replaced by the parent's.
    //
    // NOTE: init_from_metgrid also reads LANDMASK, but as a time series (one iMultiFab
    //       per met_em time) rather than the single entry used here, so it needs its own
    //       handling and is left alone.
    //
    if (solverChoice.init_type != InitType::WRFInput) { return; }
 
    auto entry = [] (Vector<Vector<std::unique_ptr<iMultiFab>>>& v, int l) -> iMultiFab*
                 { return (v[l].empty()) ? nullptr : v[l][0].get(); };
 
    Vector<iMultiFab*> fine = {entry(    lmask_lev,lev),
                               entry(land_type_lev,lev),
                               entry(soil_type_lev,lev)};
 
    Vector<iMultiFab*> crse = {entry(    lmask_lev,lev-1),
                               entry(land_type_lev,lev-1),
                               entry(soil_type_lev,lev-1)};
 
    // Ratio of 1 in z: these live on ba2d, which has no vertical extent (see the note
    // in Interp2DArrays)
    const IntVect rr2d(refRatio(lev-1)[0], refRatio(lev-1)[1], 1);
 
    for (int n = 0; n < fine.size(); n++)
    {
        if (!fine[n] || !crse[n]) { continue; }
 
        // The land mask is the first of the three above
        const bool is_lmask = (n == 0);
 
        IntVect ngc = crse[n]->nGrowVect(); ngc[2] = 0;
        IntVect ngf = fine[n]->nGrowVect(); ngf[2] = 0;
 
        MultiFab crse_real(crse[n]->boxArray(), crse[n]->DistributionMap(), 1, ngc);
        MultiFab fine_real(fine[n]->boxArray(), fine[n]->DistributionMap(), 1, ngf);
 
        for (MFIter mfi(crse_real, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
            const Box& gbx = mfi.growntilebox(ngc);
            const Array4<      Real>& dst_arr = crse_real.array(mfi);
            const Array4<const int >& src_arr = crse[n]->const_array(mfi);
            ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                dst_arr(i,j,k) = static_cast<Real>(src_arr(i,j,k));
            });
        }
 
        //
        // DO fill the ghost cells outside the domain: SurfaceLayer::compute_fluxes reads
        // the land mask on grown boxes without clipping into the domain.  The coarse
        // level has those cells filled -- init_from_wrfinput fills the grown box by
        // clamping into the valid region, init_stuff does a whole-array setVal, and this
        // routine fills them at a level with no file of its own.
        //
        InterpFromCoarseLevel(fine_real, ngf, ngf,
                              crse_real, 0, 0, 1,
                              geom[lev-1], geom[lev],
                              rr2d, &pc_interp,
                              domain_bcs_type, BCVars::cons_bc);
 
        for (MFIter mfi(fine_real, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
            const Box& gbx = mfi.growntilebox(ngf);
            const Array4<       int>& dst_arr = fine[n]->array(mfi);
            const Array4<const Real>& src_arr = fine_real.const_array(mfi);
            ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                const int injected = static_cast<int>(std::round(src_arr(i,j,k)));
                //
                // A land mask of 2 means a building, which init_zphys marks from this
                // level's own terrain blanking before we get here (see the ordering in
                // MakeNewLevelFromCoarse).  That is finer information than anything the
                // parent can offer, so keep it.  2 is a legitimate category for the land
                // and soil types, hence the test on which array this is.
                //
                dst_arr(i,j,k) = (is_lmask && dst_arr(i,j,k) == 2) ? 2 : injected;
            });
        }
 
        // Prefer this level's own data in the ghost cells it shares with another fine box
        fine[n]->FillBoundary(geom[lev].periodicity());
    }
}

Here is the call graph for this function:

interp_mapfac_from_coarse()

void ERF::interp_mapfac_from_coarse

(

int

lev

)

{
    AMREX_ALWAYS_ASSERT(lev > 0);
    AMREX_ALWAYS_ASSERT(mapfac[lev].size() == mapfac[lev-1].size());
 
    for (int i = 0; i < mapfac[lev].size(); i++)
    {
        //
        // With the isotropic MapFacType the _y entries alias the _x ones, so this loop
        // visits m_x, u_x and v_x exactly once; with the anisotropic version enabled it
        // visits all six.
        //
        if (!mapfac[lev][i] || !mapfac[lev-1][i]) { continue; }
 
        //
        // m_x is cell-centered, u_x lives on x-faces and v_x on y-faces, so each needs
        // the interpolater that matches its index type.
        //
        const IndexType ixtype = mapfac[lev][i]->boxArray().ixType();
        Interpolater* mapper = &cell_cons_interp;
        int bccomp           = BCVars::cons_bc;
        if (ixtype.nodeCentered(0)) {
            mapper = &face_cons_linear_interp;
            bccomp = BCVars::xvel_bc;
        } else if (ixtype.nodeCentered(1)) {
            mapper = &face_cons_linear_interp;
            bccomp = BCVars::yvel_bc;
        }
 
        //
        // DO fill the ghost cells outside the domain: the coarse level's map factors are
        // defined there (init_from_wrfinput fills the grown box by clamping into the valid
        // region) and the advection routines read the map factors on grown boxes.
        //
        IntVect ngv = mapfac[lev][i]->nGrowVect(); ngv[2] = 0;
 
        // Ratio of 1 in z: the map factors live on ba2d, which has no vertical extent
        // (see the note in Interp2DArrays)
        const IntVect rr2d(refRatio(lev-1)[0], refRatio(lev-1)[1], 1);
 
        // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
        //       have been pre-filled - this includes ghost cells both inside and outside
        //       the domain
        InterpFromCoarseLevel(*mapfac[lev][i], ngv, ngv,
                              *mapfac[lev-1][i], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              rr2d, mapper,
                              domain_bcs_type, bccomp);
 
        // Prefer this level's own data in the ghost cells it shares with another fine box
        mapfac[lev][i]->FillBoundary(geom[lev].periodicity());
    }
}

Here is the call graph for this function:

interp_psfc_from_coarse()

void ERF::interp_psfc_from_coarse

(

int

lev

)

{
    AMREX_ALWAYS_ASSERT(lev > 0);
 
    //
    // PSFC comes only from wrfinput.  On any other pathway it is never filled at any
    // level, so there is nothing to interpolate and interpolating anyway would read
    // uninitialized data.
    //
    if (solverChoice.init_type != InitType::WRFInput) { return; }
 
    if (!mf_PSFC[lev] || !mf_PSFC[lev-1]) { return; }
 
    IntVect ngv = mf_PSFC[lev]->nGrowVect(); ngv[2] = 0;
 
    // Ratio of 1 in z: PSFC lives on ba2d, which has no vertical extent (see the note
    // in Interp2DArrays)
    const IntVect rr2d(refRatio(lev-1)[0], refRatio(lev-1)[1], 1);
 
    //
    // Do NOT fill the ghost cells outside the domain: unlike the fields that
    // init_from_wrfinput fills by clamping into the valid region, PSFC is copied out of
    // the file over the intersection with the domain, so the coarse level has nothing
    // there for us to interpolate.  Nothing reads those cells -- update_sst_tsk clamps
    // its PSFC index into the domain.
    //
    InterpFromCoarseLevel(*mf_PSFC[lev], ngv, IntVect(0,0,0),
                          *mf_PSFC[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          rr2d, &cell_cons_interp,
                          domain_bcs_type, BCVars::cons_bc);
 
    // Prefer this level's own data in the ghost cells it shares with another fine box
    mf_PSFC[lev]->FillBoundary(geom[lev].periodicity());
}

Here is the call graph for this function:

is_it_time_for_action()

bool ERF::is_it_time_for_action ( int  nstep, double  time, double  dtlev, int  action_interval, double  action_per  )

static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters

nstep	Timestep number
time	Current time
dtlev	Timestep for the current level
action_interval	Interval in number of timesteps for taking action
action_per	Interval in simulation time for taking action

{
  bool int_test = (action_interval > 0 && nstep % action_interval == 0);
 
  bool per_test = false;
  if (action_per > zero) {
    const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
    const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
 
    if (num_per_old != num_per_new) {
      per_test = true;
    }
  }
 
  return int_test || per_test;
}

make_eb_box()

void ERF::make_eb_box

(

)

make_eb_regular()

void ERF::make_eb_regular

(

)

make_lsm_at_level()

void ERF::make_lsm_at_level ( int  lev, bool from_regrid]  [[maybe_unused] = false  )

private

{
    int lsm_data_size  = lsm.Get_Data_Size();
    int lsm_flux_size  = lsm.Get_Flux_Size();
    lsm_data[lev].resize(lsm_data_size);
    lsm_data_name.resize(lsm_data_size);
    lsm_flux[lev].resize(lsm_flux_size);
    lsm_flux_name.resize(lsm_flux_size);
    lsm.Define(lev, solverChoice);
 
    // Check if we'll be using surface-only init (atmospheric state comes later from FillCoarsePatch)
    // Only applies to fine levels (lev > 0) with WRFInput during initial level creation.
    // Metgrid files only contain surface data anyway, so this doesn't apply to Metgrid.
    // During regrid (from_regrid=true), always initialize LSM immediately to avoid losing LSM state
    bool will_use_surface_only = false;
#ifdef ERF_USE_NETCDF
    if (!from_regrid && lev > 0 && !nc_init_file[lev].empty() &&
        solverChoice.init_type == InitType::WRFInput) {
        will_use_surface_only = solverChoice.interp_atmos_from_coarse;
    }
#endif
 
    // Only initialize LSM now if we're NOT using surface-only init
    // (for surface-only, LSM init must wait until after FillCoarsePatch provides atmospheric state)
    if (solverChoice.lsm_type != LandSurfaceType::None && !will_use_surface_only) {
        //
        // A level with no land file of its own takes its LSM state from level 0 rather
        // than from its parent (see NOAHMP::interp_from_lev0, which is handed Geom(0)
        // just below), so this must be the ratio between level 0 and this level -- the
        // product across every interface beneath it, not just the ratio across the last
        // one.
        //
        IntVect RefRatio(1);
        for (int l = 0; l < lev; ++l) { RefRatio *= refRatio(l); }
        lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), Geom(0),
                 domain_bcs_type, RefRatio, zero, nc_init_file); // dummy dt value
    }
 
    // Only access LSM data pointers if LSM has been initialized
    // (if using surface-only init, this will be done later after FillCoarsePatch)
    if (solverChoice.lsm_type != LandSurfaceType::None && !will_use_surface_only) {
        for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
            lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
            lsm_data_name[mvar] = lsm.Get_DataName(mvar);
        }
        for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
            lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
            lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
        }
        if (lev>0) {
            lsm.Set_Lev0_Data_Ptr(lev);
            lsm.Set_Lev0_Flux_Ptr(lev);
        }
    }
}

make_physbcs()

void ERF::make_physbcs ( int  lev )

private

{
    if (SolverChoice::mesh_type == MeshType::VariableDz) {
        AMREX_ALWAYS_ASSERT(z_phys_nd[lev] != nullptr);
    }
 
    // Real (wrfbdy/metgrid) boundary data only exists at level 0 -- there is no machinery
    //    to interpolate it onto a refined level.  At lev > 0 we must therefore fall back on
    //    the boundary condition types specified in the inputs file, so that the lateral
    //    ghost cells outside the domain get filled if a fine grid abuts a domain boundary.
    bool l_use_real_bcs = (solverChoice.use_real_bcs && (lev == 0));
 
    physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                               m_bc_extdir_vals, m_bc_neumann_vals,
                                                               z_phys_nd[lev], l_use_real_bcs, th_bc_data[lev].data());
    physbcs_u[lev]    = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                            m_bc_extdir_vals, m_bc_neumann_vals,
                                                            z_phys_nd[lev], l_use_real_bcs, xvel_bc_data[lev].data());
    physbcs_v[lev]    = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                            m_bc_extdir_vals, m_bc_neumann_vals,
                                                            z_phys_nd[lev], l_use_real_bcs, yvel_bc_data[lev].data());
    physbcs_w[lev]    = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                            m_bc_extdir_vals, m_bc_neumann_vals,
                                                            solverChoice.terrain_type, mapfac[lev], z_phys_nd[lev],
                                                            l_use_real_bcs, zvel_bc_data[lev].data());
    physbcs_base[lev] = std::make_unique<ERFPhysBCFunct_base> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d, z_phys_nd[lev],
                                                               (solverChoice.terrain_type == TerrainType::MovingFittedMesh),
                                                               (solverChoice.mesh_type != MeshType::ConstantDz),
                                                               solverChoice.rdOcp, solverChoice.gravity);
}

Here is the call graph for this function:

make_subdomains()

void ERF::make_subdomains	(	const amrex::BoxList &	ba,
		amrex::Vector< amrex::BoxArray > &	bins
	)

Group a list of boxes into subdomains based on adjacency.

Parameters

[in]	bl	List of boxes to partition.
[out]	bins	Vector of BoxArrays containing the grouped subdomains.

{
    Vector<BoxList> bins_bl;
 
    // Clear out any old bins
    bins.clear();
 
    // Iterate over boxes
    for (auto bx : bl)
    {
        Box gbx(bx); gbx.grow(1);
 
        // Find *every* existing bin this box touches.  Stopping at the first match
        // is not enough: a box that bridges two bins connects them, so those bins
        // have to be merged.  Otherwise the two halves remain separate subdomains
        // even though they are now adjacent through bx, which breaks the invariant
        // that no box in a subdomain touches a box in any other subdomain.
        Vector<int> touched;
        for (int j = 0; j < bins_bl.size(); ++j) {
            for (auto const& b : bins_bl[j])
            {
                if (gbx.intersects(b)) {
                    touched.push_back(j);
                    break;
                }
            }
        }
 
        if (touched.empty()) {
            // Box touches nothing seen so far, so it starts a new bin
            BoxList new_bin;
            new_bin.push_back(bx);
            bins_bl.push_back(new_bin);
        } else {
            // Keep the first bin touched and fold the others, plus bx, into it.
            // Erasing from the back leaves the lower indices -- including the one
            // we hold a reference to -- valid.
            BoxList& keep = bins_bl[touched[0]];
            keep.push_back(bx);
            for (int n = int(touched.size())-1; n >= 1; --n) {
                keep.join(bins_bl[touched[n]]);
                bins_bl.erase(bins_bl.begin() + touched[n]);
            }
        }
    }
 
#ifdef AMREX_DEBUG
    // Verify the invariant we just worked to maintain: boxes in different bins
    // must not touch, else the Poisson solve would treat one connected region as
    // two independent problems with bogus conditions at their shared interface.
    for (int i = 0; i < bins_bl.size(); ++i) {
        for (int j = i+1; j < bins_bl.size(); ++j) {
            for (auto const& bi : bins_bl[i]) {
                Box gbi(bi); gbi.grow(1);
                for (auto const& bj : bins_bl[j]) {
                    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!gbi.intersects(bj),
                        "make_subdomains: boxes in different subdomains are adjacent");
                }
            }
        }
    }
#endif
 
    // Convert the BoxLists to BoxArrays
    for (int i = 0; i < bins_bl.size(); ++i) {
        bins.push_back(BoxArray(bins_bl[i]));
    }
}

Here is the call graph for this function:

MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage	(	amrex::Vector< amrex::Real > &	h_havg,
		amrex::MultiFab &	S,
		int	n
	)

{
    // Get the number of cells in z at level 0
    int dir_z = AMREX_SPACEDIM-1;
    auto domain = geom[0].Domain();
    int size_z = domain.length(dir_z);
    int start_z = domain.smallEnd()[dir_z];
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
 
    // resize the level 0 horizontal average vectors
    h_havg.resize(size_z, 0.0_rt);
 
    // Get the cell centered data and construct sums
    //
    // NOTE: deliberately not threaded.  Every iteration does "h_havg[k] += ..." into the
    //       same shared Vector with no atomic and no per-thread partials, so threading it
    //       would be an outright data race on top of an order-dependent sum.  The body is
    //       a pure reduction, so running it serially costs little.
    //
    for (MFIter mfi(S); mfi.isValid(); ++mfi) {
        const Box& box = mfi.validbox();
        const IntVect& se = box.smallEnd();
        const IntVect& be = box.bigEnd();
 
        auto      fab_arr = S[mfi].array();
 
        FArrayBox fab_reduce(box, 1, The_Async_Arena());
        auto arr_reduce   = fab_reduce.array();
 
        ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
            arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
        });
 
        for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
            Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
            h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
        }
    }
 
    // combine sums from different MPI ranks
    ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), static_cast<int>(h_havg.size()));
 
    // divide by the total number of cells we are averaging over
    for (int k = 0; k < size_z; ++k) {
        h_havg[k]     /= area_z;
    }
}

Here is the call graph for this function:

MakeEBGeometry()

void ERF::MakeEBGeometry

(

)

MakeFilename_EyeTracker_latlon()

std::string ERF::MakeFilename_EyeTracker_latlon

(

int

nstep

)

                                             {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/latlon";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_track_latlon" << std::setw(7) << std::setfill('0') << nstep << ".txt";
    return oss.str();
}

MakeFilename_EyeTracker_maxvel()

std::string ERF::MakeFilename_EyeTracker_maxvel

(

int

nstep

)

                                             {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/maxvel";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_maxvel_" << std::setw(7) << std::setfill('0') << nstep << ".txt";
    return oss.str();
}

MakeFilename_EyeTracker_minpressure()

std::string ERF::MakeFilename_EyeTracker_minpressure

(

int

nstep

)

                                                  {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/minpressure";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_minpressure_" << std::setw(7) << std::setfill('0') << nstep << ".txt";
    return oss.str();
}

MakeHorizontalAverages()

void ERF::MakeHorizontalAverages

(

)

{
    int lev = 0;
 
    // First, average down all levels (if doing two-way coupling)
    if (solverChoice.coupling_type == CouplingType::TwoWay) {
        AverageDown();
    }
 
    MultiFab mf(grids[lev], dmap[lev], 5, 0);
 
    int zdir = 2;
    auto domain = geom[0].Domain();
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
    bool is_anelastic = (solverChoice.anelastic[lev] == 1);
 
    for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.validbox();
        auto  fab_arr = mf.array(mfi);
        auto const  hse_arr = base_state[lev].const_array(mfi);
        auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
            Real dens = cons_arr(i, j, k, Rho_comp);
            fab_arr(i, j, k, 0) = dens;
            fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
            if (!use_moisture) {
                if (is_anelastic) {
                    fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
                } else {
                    fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp));
                }
            }
        });
    }
 
    if (use_moisture)
    {
        for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.validbox();
            auto  fab_arr = mf.array(mfi);
            auto const  hse_arr = base_state[lev].const_array(mfi);
            auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
            int ncomp = vars_new[lev][Vars::cons].nComp();
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                Real dens = cons_arr(i, j, k, Rho_comp);
                if (is_anelastic) {
                    fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
                } else {
                    Real qv = cons_arr(i, j, k, RhoQ1_comp) / dens;
                    fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
                }
                fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : zero);
                fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : zero);
            });
        }
 
        Gpu::HostVector<Real> h_avg_qv          = sumToLine(mf,3,1,domain,zdir);
        Gpu::HostVector<Real> h_avg_qc          = sumToLine(mf,4,1,domain,zdir);
    }
 
    // Sum in the horizontal plane
    Gpu::HostVector<Real> h_avg_density     = sumToLine(mf,0,1,domain,zdir);
    Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
    Gpu::HostVector<Real> h_avg_pressure    = sumToLine(mf,2,1,domain,zdir);
 
    // Divide by the total number of cells we are averaging over
    int size_z = domain.length(zdir);
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    int klen = static_cast<int>(h_avg_density.size());
 
    for (int k = 0; k < klen; ++k) {
        h_havg_density[k]     /= area_z;
        h_havg_temperature[k] /= area_z;
        h_havg_pressure[k]    /= area_z;
        if (solverChoice.moisture_type != MoistureType::None)
        {
            h_havg_qc[k]          /= area_z;
            h_havg_qv[k]          /= area_z;
        }
    } // k
 
    // resize device vectors
    d_havg_density.resize(size_z, 0.0_rt);
    d_havg_temperature.resize(size_z, 0.0_rt);
    d_havg_pressure.resize(size_z, 0.0_rt);
 
    // copy host vectors to device vectors
    Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
    Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
    Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
 
    if (solverChoice.moisture_type != MoistureType::None)
    {
        d_havg_qv.resize(size_z, 0.0_rt);
        d_havg_qc.resize(size_z, 0.0_rt);
        Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
        Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
    }
}

Here is the call graph for this function:

MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

override

{
    //
    // Note that "time" here is elapsed time
    //
    AMREX_ALWAYS_ASSERT(lev > 0);
 
    if (verbose) {
        amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
    }
 
    //
    // Grow the subdomains vector and build the subdomains vector at this level
    //
    subdomains.resize(lev+1);
    //
    // Create subdomains at each level within the domain such that
    // 1) all boxes in a given subdomain are "connected"
    // 2) no boxes in a subdomain touch any boxes in any other subdomain
    //
    make_subdomains(ba.simplified_list(), subdomains[lev]);
 
    // ****************************************************************************************
    // Define grids[lev]/dmap[lev] to be the passed-in ba/dm *before* usage.
    //
    // AmrCore::regrid does not call SetBoxArray/SetDistributionMap for this new level
    // until MakeNewLevelFromCoarse returns, so grids[lev]/dmap[lev] are still empty here.
    // Any subsequent calls that utilize grids[lev]/dmap[lev] will then fail (e.g. anelastic
    // velocity projection below).
    // ****************************************************************************************
    SetBoxArray(lev, ba);
    SetDistributionMap(lev, dm);
 
    if (lev == 0) init_bcs();
 
    //********************************************************************************************
    // This allocates all kinds of things, including but not limited to: solution arrays,
    //      terrain arrays, ba2d, metric terms and base state.
    // *******************************************************************************************
    init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev], base_state[lev], z_phys_nd[lev]);
 
    //
    // Note that t_new = time here is elapsed time
    //
    t_new[lev] = time;
    t_old[lev] = time - bogus_large_value;
 
    // ********************************************************************************************
    // Build the data structures for metric quantities used with terrain-fitted coordinates
    // ********************************************************************************************
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing)
    {
        const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
        const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
        if (solverChoice.terrain_type == TerrainType::EB) {
            eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
        } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                   solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
#if USE_FC_FACTORY
            eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
#else
            eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
#endif
        }
    }
    init_zphys(lev, time);
    update_terrain_arrays(lev);
 
    //
    // Make sure that detJ is the average of the data on a finer level if there is one
    //     *and* if there is two-way coupling
    //
    if ( (SolverChoice::mesh_type != MeshType::ConstantDz) && (solverChoice.coupling_type == CouplingType::TwoWay) ) {
        // NOTE: z_phys_cc is deliberately NOT averaged down.  Every level's base state is
        //       built to be in discrete hydrostatic balance against that level's own
        //       cell-centered heights, so replacing the coarse heights with the average of
        //       the fine ones -- after the base states have already been built -- leaves the
        //       coarse base state out of balance with the heights the dycore then uses for
        //       vertical gradients, Rayleigh damping and the sponge zones.  z_phys_cc is also
        //       derived from z_phys_nd, which is not averaged down either, so averaging only
        //       the cell-centered heights made the two disagree inside the refined region.
        //
        //       detJ IS still averaged down: AverageDownTo weights (rho S) by detJ_cc before
        //       averaging and divides by it afterwards, so the coarse detJ must be the average
        //       of the fine detJ for that average-down to telescope and stay conservative.
        for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
            average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
        }
    }
 
    // ********************************************************************************************
    // Build the data structures for canopy model (depends upon z_phys)
    // ********************************************************************************************
    if (solverChoice.do_forest_drag) {
        m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev],
                                              z_phys_cc[lev].get(), z_phys_nd[lev].get(),
                                              solverChoice.forest_biophysics &&
                                              solverChoice.forest_biophysics_heat, lev);
    }
 
    //********************************************************************************************
    // Radiation
    // *******************************************************************************************
    if (solverChoice.rad_uses_interface())
    {
        rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
    }
 
    // *****************************************************************************************************
    // Initialize the boundary conditions (after initializing the terrain but before calling
    //     initHSE or FillCoarsePatch)
    // *****************************************************************************************************
    make_physbcs(lev);
 
    // ********************************************************************************************
    // Update the base state at this level by interpolation from coarser level
    // ********************************************************************************************
    InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                          IntVect(0,0,0), // do not fill ghost cells outside the domain
                          base_state[lev-1], 0, 0, base_state[lev].nComp(),
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), &cell_cons_interp,
                          domain_bcs_type, BCVars::cons_bc);
 
    // Impose bc's outside the domain
    (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
 
    //********************************************************************************************
    // Microphysics
    // *******************************************************************************************
    int q_size  = micro->Get_Qmoist_Size(lev);
    qmoist[lev].resize(q_size);
    micro->Define(lev, solverChoice);
    if (solverChoice.moisture_type != MoistureType::None)
    {
        micro->Init(lev, vars_new[lev][Vars::cons],
                    grids[lev], Geom(lev), zero,
                    z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
        // Refresh the land/water mask pointer. Must be re-issued at every
        // micro->Init site: init_stuff rebuilds lmask_lev[lev][0], so a stale
        // pointer would dangle. No-op for every scheme except WDM6.
        micro->Set_Lmask(lev, (lmask_lev[lev].empty()) ? nullptr : lmask_lev[lev][0].get());
    }
    for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
        qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
    }
 
    // ********************************************************************************************
    // Build the data structures for calculating diffusive/turbulent terms
    // ********************************************************************************************
    update_diffusive_arrays(lev, ba, dm);
 
    // ********************************************************************************************
    // Build the data structures for holding sea surface temps and skin temps
    // ********************************************************************************************
    sst_lev[lev].resize(1);     sst_lev[lev][0] = nullptr;
    tsk_lev[lev].resize(1);     tsk_lev[lev][0] = nullptr;
 
    // ********************************************************************************************
    // Fill data at the new level by interpolation from the coarser level
    // Note that internal to FillCoarsePatch we will convert velocity to momentum,
    //      then interpolate momentum, then convert momentum back to velocity
    // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
    // ********************************************************************************************
 
#ifdef ERF_USE_NETCDF
    if ( ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) ) &&
         !nc_init_file[lev].empty() )
    {
        // Just making sure that ghost cells aren't uninitialized...
        vars_new[lev][Vars::cons].setVal(0.0); vars_old[lev][Vars::cons].setVal(0.0);
        vars_new[lev][Vars::xvel].setVal(0.0); vars_old[lev][Vars::xvel].setVal(0.0);
        vars_new[lev][Vars::yvel].setVal(0.0); vars_old[lev][Vars::yvel].setVal(0.0);
        vars_new[lev][Vars::zvel].setVal(0.0); vars_old[lev][Vars::zvel].setVal(0.0);
 
        AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
 
        //
        // CHOOSE INITIALIZATION PATH:
        // If interp_atmos_from_coarse is enabled for a finer level with WRFInput,
        // read only surface fields from wrfinput and interpolate atmospheric state from coarse.
        // Metgrid files only contain surface data anyway, so use_surface_only doesn't apply.
        //
        bool use_surface_only = solverChoice.interp_atmos_from_coarse && (lev > 0) &&
                                (solverChoice.init_type == InitType::WRFInput);
 
        // Tell advance_radiation whether this level's atmospheric state is about to come from
        // FillCoarsePatch, in which case it must interpolate the radiation fields from the
        // parent for one step rather than run RRTMGP on a state that is not yet consistent.
        if (static_cast<int>(rad_interp_from_coarse_pending.size()) <= lev) {
            rad_interp_from_coarse_pending.resize(lev+1, 0);
        }
        rad_interp_from_coarse_pending[lev] = use_surface_only ? 1 : 0;
 
        // If using surface-only init with LSM, we need to resize the lsm_data/lsm_flux vectors
        // before the deferred lsm.Init() call below. The LSM will read its initial state
        // from nc_init_file when lsm.Init() is called (after FillCoarsePatch provides
        // atmospheric data). Note: Define() is empty for both LSMs.
        if (use_surface_only && solverChoice.lsm_type != LandSurfaceType::None) {
            int lsm_data_size  = lsm.Get_Data_Size();
            int lsm_flux_size  = lsm.Get_Flux_Size();
            lsm_data[lev].resize(lsm_data_size);
            lsm_data_name.resize(lsm_data_size);
            lsm_flux[lev].resize(lsm_flux_size);
            lsm_flux_name.resize(lsm_flux_size);
            lsm.Define(lev, solverChoice);
        }
 
        if (solverChoice.init_type == InitType::Metgrid) {
            init_from_metgrid(lev);
        } else if (solverChoice.init_type == InitType::WRFInput) {
            if (use_surface_only) {
                amrex::Print() << "Using interp_atmos_from_coarse mode at level " << lev << ":\n";
                amrex::Print() << "  - Reading surface fields from wrfinput\n";
                amrex::Print() << "  - Atmospheric state will be interpolated from level " << lev-1 << "\n";
                init_from_wrfinput(lev, *mf_PSFC[lev], /*read_atmos_state*/ false);
            } else {
                init_from_wrfinput(lev, *mf_PSFC[lev]);
            }
        }
        init_zphys(lev, time);
        update_terrain_arrays(lev);
        make_physbcs(lev);
 
        dz_min[lev] = (*detJ_cc[lev]).min(0) * geom[lev].CellSize(2);
 
        //
        // If we used surface-only init, we need to rebuild the base state and
        // interpolate the atmospheric state from coarse (just like a level with no init file)
        //
        if (use_surface_only) {
            // Rebuild base state from wrfinput on top of the fine terrain (must happen before
            // FillCoarsePatch, which interpolates perturbational quantities relative to base state)
            rebuild_base_state_from_wrfinput(lev, base_state[lev]);
            (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
 
            // Interpolate atmospheric state from coarse level.
            // NOTE: This creates thermodynamically inconsistent data because ρ, θ, qv are
            // interpolated independently. We rely on:
            // 1. Skipping radiation on the first timestep (see ERF_AdvanceRadiation.cpp)
            // 2. Letting dynamics equilibrate the state on the first timestep
            FillCoarsePatch(lev, time);
 
            // Now initialize LSM with the properly filled atmospheric state.
            // The LSM will read its soil/vegetation initial state from nc_init_file
            // (e.g., Noah-MP reads via the noahmpio library in Fortran).
            if (solverChoice.lsm_type != LandSurfaceType::None) {
                amrex::Print() << "Initializing LSM at level " << lev << " after FillCoarsePatch\n";
 
                IntVect RefRatio(1);
                for (int l = 0; l < lev; ++l) { RefRatio *= refRatio(l); }
                lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), Geom(0),
                         domain_bcs_type, RefRatio, zero, nc_init_file);
 
                // Set up the LSM data/flux pointers (same as in make_lsm_at_level)
                for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
                    lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
                    lsm_data_name[mvar] = lsm.Get_DataName(mvar);
                }
                for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
                    lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
                    lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
                }
                if (lev>0) {
                    lsm.Set_Lev0_Data_Ptr(lev);
                    lsm.Set_Lev0_Flux_Ptr(lev);
                }
            }
        }
 
    } else {
#endif
 
#ifdef ERF_USE_NETCDF
    //
    // A WRFInput level created by refinement rather than read from a file: build its base
    // state by the same construction level 0 used, on top of the coarse interpolation done
    // above.  This must happen BEFORE FillCoarsePatch, which interpolates perturbational
    // quantities relative to the base state.
    //
    if ( (solverChoice.init_type == InitType::WRFInput) && nc_init_file[lev].empty() ) {
        rebuild_base_state_from_wrfinput(lev, base_state[lev]);
        (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
    }
#endif
 
    //
    // Interpolate the solution data
    //
    FillCoarsePatch(lev, time);
 
    //
    // Interpolate the 2D arrays at the lower boundary
    // Note that ba2d is constructed already in init_stuff, but we have not yet defined dmap[lev]
    // so we must explicitly pass dm.
    Interp2DArrays(lev,ba2d[lev],dm);
 
    // Populate dz_min for dynamically-created fine levels (non-terrain path).
    if (static_cast<int>(dz_min.size()) <= lev) { dz_min.resize(lev+1); }
    dz_min[lev] = geom[lev].CellSize(2);
    if ( SolverChoice::mesh_type != MeshType::ConstantDz && detJ_cc[lev] ) {
        dz_min[lev] *= (*detJ_cc[lev]).min(0);
    }
#ifdef ERF_USE_NETCDF
    }
#endif
 
    // ********************************************************************************************
    // Initialize the integrator class
    // ********************************************************************************************
    dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
    initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
 
    // ********************************************************************************************
    // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
    // ********************************************************************************************
    if (lev > 0 && cf_width >= 0) {
        Construct_ERFFillPatchers(lev);
           Define_ERFFillPatchers(lev);
    }
 
    // ********************************************************************************************
    // For anelastic levels created from coarse (either on restart or during a run), project the
    // interpolated velocity to enforce the divergence-free constraint. This Initializes gradp[lev]
    // via the pressure projection, handling both the pure-anelastic case and the hybrid case
    // (compressible lev-1, anelastic lev) where there is no coarse gradp to interpolate.
    // FillPatchers must be constructed above before this call. pp_inc is scratch; zero afterward.
    // ********************************************************************************************
    if (solverChoice.anelastic[lev]) {
        double dummy_dt = 1.0;
        project_initial_velocity(lev, time, dummy_dt);
        pp_inc[lev].setVal(0.0);
    }
 
    //********************************************************************************************
    // Land Surface Model - setup data structures
    // NOTE: Actual LSM initialization (lsm.Init) is deferred if using interp_atmos_from_coarse
    //       because it needs valid atmospheric state from FillCoarsePatch
    // *******************************************************************************************
    make_lsm_at_level(lev);
 
    // ********************************************************************************************
    // Create the SurfaceLayer arrays at this (new) level
    // ********************************************************************************************
    for (OrientationIter oit; oit; ++oit) {
        Orientation ori = oit();
        if (phys_bc_type[ori] == ERF_BC::surface_layer) {
            Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
                                         &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
            m_SurfaceLayer[ori]->make_SurfaceLayer_at_level(lev,lev+1,
                                                            mfv_old, Theta_prim[lev], Qv_prim[lev],
                                                            Qr_prim[lev], z_phys_nd[lev],
                                                            Hwave[lev].get(), Lwave[lev].get(), eddyDiffs_lev[lev].get(),
                                                            lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
                                                            sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
        }
    }
 
    // ********************************************************************************************
    // Set up the Rayleigh damping vectors at this (new) level
    // ********************************************************************************************
    if (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
        solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T)
    {
        initRayleigh_at_level(lev);
    }
 
}

Here is the call graph for this function:

MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

override

{
    //
    // Note that "time" here is elapsed time
    //
 
    // A level built here gets its atmospheric state from its own initialization -- an input
    // file, a problem setup or a restart -- rather than from FillCoarsePatch, so radiation
    // has a consistent state to work with and nothing is pending for it.  Clearing this
    // explicitly matters because the entry may be left over from an earlier life of this
    // level: erf.interp_atmos_from_coarse is ignored on this path (see the warning below),
    // so a stale flag would skip a step of radiation for no reason.
    if (static_cast<int>(rad_interp_from_coarse_pending.size()) <= lev) {
        rad_interp_from_coarse_pending.resize(lev+1, 0);
    }
    rad_interp_from_coarse_pending[lev] = 0;
 
    BoxArray ba;
    DistributionMapping dm;
    Box domain(Geom(0).Domain());
    if (lev == 0 && restart_chkfile.empty() &&
        (max_grid_size[0][0] >= domain.length(0)) &&
        (max_grid_size[0][1] >= domain.length(1)) &&
        ba_in.size() != ParallelDescriptor::NProcs())
    {
        // We only decompose in z if max_grid_size_z indicates we should,
        //    and if we are allowed to split boxes in z at all
        // (amr.no_box_split_dir = 2, the ERF default, forbids that)
        bool decompose_in_z = (max_grid_size[0][2] < domain.length(2)) &&
                              (no_box_split_dir != 2);
 
        ba = ERFPostProcessBaseGrids(Geom(0).Domain(),decompose_in_z);
        dm = DistributionMapping(ba);
    } else {
        ba = ba_in;
        dm = dm_in;
    }
 
    // ********************************************************************************************
    // Define grids[lev] to be ba
    // ********************************************************************************************
    SetBoxArray(lev, ba);
 
    // ********************************************************************************************
    // Define dmap[lev] to be dm
    // ********************************************************************************************
    SetDistributionMap(lev, dm);
 
    if (verbose) {
        amrex::Print() <<            "BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
        // amrex::Print() <<" SIMPLIFIED BA FROM SCRATCH AT LEVEL " << lev << " " << ba.simplified_list() << std::endl;
    }
 
    subdomains.resize(lev+1);
    //
    // Create subdomains at each level within the domain such that
    // 1) all boxes in a given subdomain are "connected"
    // 2) no boxes in a subdomain touch any boxes in any other subdomain
    //
    make_subdomains(ba.simplified_list(), subdomains[lev]);
 
    if (lev == 0) init_bcs();
 
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing)
    {
        const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
        const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
        if (solverChoice.terrain_type == TerrainType::EB) {
            eb[lev]->make_all_factories(lev, geom[lev], grids[lev], dmap[lev], eb_level);
        } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                   solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
#if USE_FC_FACTORY
            eb[lev]->make_all_factories(lev, geom[lev], grids[lev], dmap[lev], eb_level);
#else
            eb[lev]->make_cc_factory(lev, geom[lev], grids[lev], dmap[lev], eb_level);
#endif
        }
    }
 
    auto& lev_new = vars_new[lev];
    auto& lev_old = vars_old[lev];
 
    //********************************************************************************************
    // This allocates all kinds of things, including but not limited to: solution arrays,
    //      terrain arrays, metric terms and base state.
    // *******************************************************************************************
    init_stuff(lev, ba, dm, lev_new, lev_old, base_state[lev], z_phys_nd[lev]);
 
    //********************************************************************************************
    // Land Surface Model
    // *******************************************************************************************
    // In MakeNewLevelFromScratch, both levels are created at startup and read from their
    // respective wrfinput files, so there's no surface-only initialization. Pass from_regrid=true
    // to prevent deferring LSM initialization (deferred init only applies in MakeNewLevelFromCoarse
    // when interp_atmos_from_coarse is enabled).
    make_lsm_at_level(lev, true);
 
    // ********************************************************************************************
    // Build the data structures for calculating diffusive/turbulent terms
    // ********************************************************************************************
    update_diffusive_arrays(lev, ba, dm);
 
    // ********************************************************************************************
    // Build the data structures for holding sea surface temps and skin temps
    // ********************************************************************************************
    sst_lev[lev].resize(1);     sst_lev[lev][0] = nullptr;
    tsk_lev[lev].resize(1);     tsk_lev[lev][0] = nullptr;
 
    // ********************************************************************************************
    // Thin immersed body
    // *******************************************************************************************
    init_thin_body(lev, ba, dm);
 
    // ********************************************************************************************
    // Initialize the integrator class
    // ********************************************************************************************
    initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
 
    // ********************************************************************************************
    // Initialize the data itself
    // If (init_type == InitType::WRFInput) then we are initializing terrain and the initial data in
    //                                      the same call so we must call init_only before update_terrain_arrays
    // If (init_type != InitType::WRFInput) then we want to initialize the terrain before the initial data
    //                                      since we may need to use the grid information before constructing
    //                                      initial idealized data
    // ********************************************************************************************
    if (restart_chkfile.empty()) {
        if ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) )
        {
            AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
            //
            // Note that "time" here is elapsed time, and start_time is the start_time from wrfinput/metgrid files
            //
            if (nc_init_file[lev].empty()) {
                //
                // A refined level with no file of its own has no PH/PHB to build terrain
                // from, so its terrain must be interpolated from the coarser level FIRST --
                // the base state init_only builds is a function of the cell-centered heights
                // and cannot run before they exist.  (init_zphys interpolates from coarse
                // exactly when this level has no file, which is the same condition.)
                //
                init_zphys(lev, time);
                update_terrain_arrays(lev);
                make_physbcs(lev);
                init_only(lev, time);
            } else {
                //
                // A level that does have a file reads its terrain and its data in the same
                // pass, so init_only must come first here.
                //
                // Check if user requested interp_atmos_from_coarse but we're in MakeNewLevelFromScratch.
                // In this case, both levels are starting at the same time, so the atmospheric states
                // in the files should already be consistent. We ignore the flag and read from the file.
                if (solverChoice.interp_atmos_from_coarse && lev > 0 &&
                    solverChoice.init_type == InitType::WRFInput) {
                    if (ParallelDescriptor::IOProcessor()) {
                        amrex::Warning("erf.interp_atmos_from_coarse = true is set, but both levels are starting "
                                       "from scratch at the same time. The atmospheric state at level " + std::to_string(lev) +
                                       " will be read from the wrfinput file instead of interpolated from coarse. "
                                       "This option is intended for time-mismatched WRF input files or regridding, "
                                       "not for initial startup with consistent files.");
                    }
                }
                init_only(lev, time);
                init_zphys(lev, time);
                update_terrain_arrays(lev);
                make_physbcs(lev);
            }
        } else {
            //
            // Note that "time" here is elapsed time, and start_time = 0 when not using wrfinput/metgrid
            //
            init_zphys(lev, time);
            update_terrain_arrays(lev);
            // Note that for init_type != InitType::WRFInput and != InitType::Metgrid,
            // make_physbcs is called inside init_only
            init_only(lev, time);
        }
    } else {
        // If restarting from an input-sounding run, load the sounding again. It is
        // needed for nudging, and InitData_post also takes the surface temperature
        // and moisture of the surface layer from its reference values; without this
        // read those are zero on a restart, the surface layer sees a 0 K surface,
        // and the restarted run drifts from the uninterrupted one.
        if (lev == 0 && solverChoice.init_type == InitType::Input_Sounding)
        {
            if (input_sounding_data.input_sounding_file.empty()) {
                Error("input_sounding file name must be provided via input");
            }
 
            input_sounding_data.resize_arrays();
 
            // this will interpolate the input profiles to the nominal height levels
            // (ranging from 0 to the domain top)
            bool is_moist = (solverChoice.moisture_type != MoistureType::None);
            for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
                input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n, is_moist);
            }
 
            input_sounding_data.set_start_time(start_time);
 
            // this will calculate the hydrostatically balanced density and pressure
            // profiles following WRF ideal.exe
            if (solverChoice.sounding_type == SoundingType::Ideal) {
                input_sounding_data.calc_rho_p(0);
            } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
                       solverChoice.sounding_type == SoundingType::DryIsentropic) {
                input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
                input_sounding_data.calc_rho_p_isentropic(0);
            }
        }
 
        // We re-create terrain_blanking on restart rather than storing it in the checkpoint
        if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
            solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
            int ngrow = ComputeGhostCells(solverChoice) + 2;
            terrain_blanking[lev]->setVal(1.0);
            MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
            terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
        }
    }
 
     // Read in tables needed for windfarm simulations
    // fill in Nturb multifab - number of turbines in each mesh cell
    // write out the vtk files for wind turbine location and/or
    // actuator disks
    #ifdef ERF_USE_WINDFARM
        init_windfarm(lev);
    #endif
 
    // ********************************************************************************************
    // Build the data structures for canopy model (depends upon z_phys)
    // ********************************************************************************************
    if (restart_chkfile.empty()) {
        if (solverChoice.do_forest_drag) {
            m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev],
                                                  z_phys_cc[lev].get(), z_phys_nd[lev].get(),
                                                  solverChoice.forest_biophysics &&
                                                  solverChoice.forest_biophysics_heat, lev);
        }
    }
 
    //********************************************************************************************
    // Microphysics
    // *******************************************************************************************
    int q_size  = micro->Get_Qmoist_Size(lev);
    qmoist[lev].resize(q_size);
    micro->Define(lev, solverChoice);
    if (solverChoice.moisture_type != MoistureType::None)
    {
        micro->Init(lev, vars_new[lev][Vars::cons],
                    grids[lev], Geom(lev), zero,
                    z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
        // Refresh the land/water mask pointer. Must be re-issued at every
        // micro->Init site: init_stuff rebuilds lmask_lev[lev][0], so a stale
        // pointer would dangle. No-op for every scheme except WDM6.
        micro->Set_Lmask(lev, (lmask_lev[lev].empty()) ? nullptr : lmask_lev[lev][0].get());
    }
    for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
        qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
    }
 
    //********************************************************************************************
    // Radiation
    // *******************************************************************************************
    if (solverChoice.rad_uses_interface())
    {
        rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
    }
 
    // ********************************************************************************************
    // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
    // ********************************************************************************************
    if (lev > 0 && cf_width >= 0) {
        Construct_ERFFillPatchers(lev);
           Define_ERFFillPatchers(lev);
    }
 
#ifdef ERF_USE_PARTICLES
    if (restart_chkfile.empty()) {
        if (lev == 0) {
            initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,time);
        }
        // For lev > 0: particle redistribute is handled in timeStep() AFTER
        // regrid() completes, not here inside MakeNewLevelFromCoarse.
    }
#endif
}

Here is the call graph for this function:

MakeVTKFilename()

std::string ERF::MakeVTKFilename

(

int

nstep

)

                              {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker";
    if (!fs::exists(dir)) {
        fs::create_directory(dir);
    }
 
    std::ostringstream oss;
    oss << dir << "/storm_track_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
    return oss.str();
}

MakeVTKFilename_EyeTracker_xy()

std::string ERF::MakeVTKFilename_EyeTracker_xy

(

int

nstep

)

                                            {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/xy";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_track_xy_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
    return oss.str();
}

MakeVTKFilename_TrackerCircle()

std::string ERF::MakeVTKFilename_TrackerCircle

(

int

nstep

)

                                            {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/tracker_circle";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_tracker_circle_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
    return oss.str();
}

nghost_eb_basic()

static int ERF::nghost_eb_basic ( )

inlinestaticprivate

    { return 5; }

nghost_eb_full()

static int ERF::nghost_eb_full ( )

inlinestaticprivate

    { return 4; }

nghost_eb_volume()

static int ERF::nghost_eb_volume ( )

inlinestaticprivate

    { return 5; }

NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )

inlineprivatenoexcept

    {
        return static_cast<int>(datalog.size());
    }

NumDerDataLogs()

AMREX_FORCE_INLINE int ERF::NumDerDataLogs ( )

inlineprivatenoexcept

    {
        return static_cast<int>(der_datalog.size());
    }

NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )

inlineprivatenoexcept

    {
        return static_cast<int>(samplelinelog.size());
    }

NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )

inlineprivatenoexcept

    {
        return static_cast<int>(sampleline.size());
    }

NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )

inlineprivatenoexcept

    {
        return static_cast<int>(sampleptlog.size());
    }

NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )

inlineprivatenoexcept

    {
        return static_cast<int>(samplepoint.size());
    }

operator=() [1/2]

ERF& ERF::operator= ( const ERF &  other )

delete

operator=() [2/2]

ERF& ERF::operator= ( ERF &&  other )

deletenoexcept

PackAtmosphericStates()

void ERF::PackAtmosphericStates	(	amrex::Vector< amrex::MultiFab * > &	states,
		double	time,
		const amrex::Vector< const amrex::MultiFab * > &	weight_mf,
		const amrex::Vector< const amrex::iMultiFab * > &	index_mf,
		int	max_stencil_size,
		const amrex::MultiFab *	dst_mskr = nullptr,
		const amrex::MultiFab *	dst_msku = nullptr,
		const amrex::MultiFab *	dst_mskv = nullptr
	)

{
    using namespace amrex;
 
    // WeightFamily slot order, mirrored from ERFRemoraMultiBlockContainer.H:
    // {SCALAR_CENTERED, TAUX_FACE, TAUY_FACE, WIND_FACE_TO_CENTER}.
    constexpr int kScalarFamily = 0, kTauXFamily = 1, kTauYFamily = 2, kWindFamily = 3;
    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
        weight_mf_by_family.size() == 4 && index_mf_by_family.size() == 4,
        "PackAtmosphericStates requires 4 WeightFamily-keyed weight/index pairs.");
    for (int f = 0; f < 4; ++f) {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            weight_mf_by_family[f] != nullptr && index_mf_by_family[f] != nullptr,
            "PackAtmosphericStates requires valid driver remap stencil components for every WeightFamily slot.");
        // The stencil loops read max_stencil_size weights and 3*max_stencil_size
        // index components. Nothing else checks that the driver allocated that
        // many, so a driver/submodule version skew would read past the component
        // count and silently return garbage rather than failing.
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            weight_mf_by_family[f]->nComp() >= max_stencil_size &&
            index_mf_by_family[f]->nComp() >= 3 * max_stencil_size,
            "PackAtmosphericStates: driver remap stencil arrays have too few components for max_stencil_size.");
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
            weight_mf_by_family[f]->boxArray() == index_mf_by_family[f]->boxArray() &&
            weight_mf_by_family[f]->DistributionMap() == index_mf_by_family[f]->DistributionMap(),
            "PackAtmosphericStates: weight and index arrays for a WeightFamily slot disagree on layout.");
    }
    const amrex::MultiFab* weight_mf = weight_mf_by_family[kScalarFamily];
    const amrex::iMultiFab* index_mf = index_mf_by_family[kScalarFamily];
 
    // Values for destination cells no source cell overlaps (see
    // ApplyConservativeRemap). These are standard-atmosphere placeholders, not
    // physics: they exist so the receiving model's flux formulas stay in their
    // valid domain over cells the atmosphere grid never covered. Wind is the one
    // field where zero is genuinely meaningful (calm), so it needs no special
    // value. Units are the driver contract's: Pa and K, converted downstream.
    constexpr Real fallback_pressure_pa = Real(101325.0);   // sea-level standard
    constexpr Real fallback_temp_k      = Real(288.15);     // 15 C, standard
    constexpr Real fallback_wind_ms     = Real(0.0);        // calm
 
    const int lev   = 0;
 
    auto& cons = vars_new[lev][Vars::cons];
    auto& xvel = vars_new[lev][Vars::xvel]; // XFace
    auto& yvel = vars_new[lev][Vars::yvel]; // YFace
 
    const bool has_moisture  = (solverChoice.moisture_type != MoistureType::None);
    // Level 0 only, matching HasRadiation(); lev is 0 throughout this routine.
    const bool has_radiation = HasRadiation();
 
    amrex::ignore_unused(has_moisture, has_radiation);
 
    const auto& ba = cons.boxArray();
    const auto& dm = cons.DistributionMap();
    const auto& ba2d_lev = ba2d[lev];
    const auto& xba2d_lev = amrex::convert(ba2d_lev, IntVect(1,0,0));
    const auto& yba2d_lev = amrex::convert(ba2d_lev, IntVect(0,1,0));
    const int klo = ba.minimalBox().smallEnd(2);
 
    const bool flux_mode = (states.size() == nFluxLanes);
    if (flux_mode) {
        if (verbose) {
            amrex::Print() << "ERF flux-pack: states.size()=" << states.size()
                           << " has_radiation=" << has_radiation
                           << " tau13_ptr=" << (Tau[lev][TauType::tau13] != nullptr)
                           << " tau23_ptr=" << (Tau[lev][TauType::tau23] != nullptr)
                           << " sfs_hfx3_ptr="
                           << (!SFS_hfx3_lev.empty() && SFS_hfx3_lev[lev] != nullptr)
                           << " sfs_q1fx3_ptr="
                           << (!SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev] != nullptr)
                           << " rad_fluxes_ptr="
                           << (!rad_fluxes.empty() && rad_fluxes[lev] != nullptr)
                           << "\n";
            if (Tau[lev][TauType::tau13] != nullptr) {
                PrintFluxLaneStats("tau13_src", *Tau[lev][TauType::tau13]);
            }
            if (Tau[lev][TauType::tau23] != nullptr) {
                PrintFluxLaneStats("tau23_src", *Tau[lev][TauType::tau23]);
            }
            if (!SFS_hfx3_lev.empty() && SFS_hfx3_lev[lev] != nullptr) {
                PrintFluxLaneStats("SFS_hfx3_src", *SFS_hfx3_lev[lev]);
            }
            if (!SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev] != nullptr) {
                PrintFluxLaneStats("SFS_q1fx3_src", *SFS_q1fx3_lev[lev]);
            }
            if (has_radiation) {
                PrintFluxLaneStats("rad_fluxes_swup_src", *rad_fluxes[lev], 0);
                PrintFluxLaneStats("rad_fluxes_swdn_src", *rad_fluxes[lev], 1);
                PrintFluxLaneStats("rad_fluxes_lwup_src", *rad_fluxes[lev], 2);
                PrintFluxLaneStats("rad_fluxes_lwdn_src", *rad_fluxes[lev], 3);
            }
        }
 
        //
        // NOTE: Tau tau13/tau23 already hold CONSERVATIVE stress in [N m-2] --
        //       compute_u_flux in ERF_MOSTStress.H returns rho*<u'w'>, and
        //       ComputeStress_*_N/T scale the strain by rho_bar*mu_eff or by
        //       mu_turb (= rho*K).  They must therefore be exported as-is,
        //       without another factor of density.  This also matches the
        //       SH/LH lanes below, which apply only Cp_d / L_v.
        //
        // NOTE: Tau is only allocated when diffusion is active, so the pointers
        //       have to be checked before they are dereferenced.
        //
 
        // TAU LANE STAGGER: unresolved, see the parked block below.
        //
        // ERF tau13/tau23 live on x-/y-faces and REMORA vec_sustr/vec_svstr live on
        // u-/v-faces, so the physically meaningful transfer is face -> face: a
        // stagger-matched transfer with no averaging on a conformal grid. That is
        // what the live code below does, and what the driver's WeightFamily comment
        // describes.
        //
        // The defect is on the weight side, not here: the driver builds its remap
        // stencils from source *cell* polygons and stores ERF *cell* indices for
        // every family (ERFRemoraMultiBlockContainer.cpp, the ExtractCellQuadFromCoords
        // / CellBoundsFromGeom calls in BuildAtmosToOceanWeights). So the index this
        // face-staggered source is read with was computed for a cell, displacing tau
        // by ~dx/2 and leaving the outermost face column unreachable.
        //
        // The parked block below "fixed" that by averaging tau to cell centres so the
        // source would match the cell-based weights. That is the wrong direction: it
        // smooths the stress twice (face -> cell, then cell -> face via area weights)
        // and throws away the face alignment. Keeping it only as a record of the
        // attempt. The correct fix is to build ERF *face* control-volume source
        // polygons for the TAUX/TAUY families in the driver, so the weights match the
        // face source that is already being passed here.
#if 0
        if (iTauX < static_cast<int>(states.size()) && states[iTauX] != nullptr) {
            MultiFab tmp(ba2d_lev, dm, 1, 0);
            for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                Box bx = mfi.tilebox();
                auto const& tau13 = Tau[lev][TauType::tau13]->const_array(mfi);
                auto const& c = cons.const_array(mfi);
                auto t = tmp.array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    t(i,j,k) = c(i,j,klo,Rho_comp)
                             * Real(0.5) * (tau13(i,j,klo) + tau13(i+1,j,klo));
                });
            }
            ApplyConservativeRemap(tmp, *states[iTauX], *weight_mf_by_family[kTauXFamily],
                                   *index_mf_by_family[kTauXFamily], max_stencil_size, dst_msku);
        }
        if (iTauY < static_cast<int>(states.size()) && states[iTauY] != nullptr) {
            MultiFab tmp(ba2d_lev, dm, 1, 0);
            for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                Box bx = mfi.tilebox();
                auto const& tau23 = Tau[lev][TauType::tau23]->const_array(mfi);
                auto const& c = cons.const_array(mfi);
                auto t = tmp.array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    t(i,j,k) = c(i,j,klo,Rho_comp)
                             * Real(0.5) * (tau23(i,j,klo) + tau23(i,j+1,klo));
                });
            }
            ApplyConservativeRemap(tmp, *states[iTauY], *weight_mf_by_family[kTauYFamily],
                                   *index_mf_by_family[kTauYFamily], max_stencil_size, dst_mskv);
        }
#endif
 
        if (iTauX < static_cast<int>(states.size()) && states[iTauX] != nullptr) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(Tau[lev][TauType::tau13] != nullptr,
                "Flux-mode coupling of tau_x requires Tau; enable diffusion or a surface_layer bc");
            MultiFab tmp(xba2d_lev, dm, 1, 0);
            for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                Box bx = mfi.tilebox();
                auto const& tau13 = Tau[lev][TauType::tau13]->const_array(mfi);
                auto t = tmp.array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    t(i,j,k) = tau13(i,j,klo);
                });
            }
            ApplyConservativeRemap(tmp, *states[iTauX], *weight_mf_by_family[kTauXFamily],
                                   *index_mf_by_family[kTauXFamily], max_stencil_size, dst_msku);
            if (verbose) { PrintFluxLaneStats("tau_x_lane", *states[iTauX]); }
        }
 
        if (iTauY < static_cast<int>(states.size()) && states[iTauY] != nullptr) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(Tau[lev][TauType::tau23] != nullptr,
                "Flux-mode coupling of tau_y requires Tau; enable diffusion or a surface_layer bc");
            MultiFab tmp(yba2d_lev, dm, 1, 0);
            for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                Box bx = mfi.tilebox();
                auto const& tau23 = Tau[lev][TauType::tau23]->const_array(mfi);
                auto t = tmp.array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    t(i,j,k) = tau23(i,j,klo);
                });
            }
            ApplyConservativeRemap(tmp, *states[iTauY], *weight_mf_by_family[kTauYFamily],
                                   *index_mf_by_family[kTauYFamily], max_stencil_size, dst_mskv);
            if (verbose) { PrintFluxLaneStats("tau_y_lane", *states[iTauY]); }
        }
 
        if (iSHflux < static_cast<int>(states.size()) && states[iSHflux] != nullptr &&
            !SFS_hfx3_lev.empty() && SFS_hfx3_lev[lev] != nullptr) {
            MultiFab tmp(ba2d_lev, dm, 1, 0);
            for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                Box bx = mfi.tilebox();
                auto const& hfx = SFS_hfx3_lev[lev]->const_array(mfi);
                auto t = tmp.array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    t(i,j,k) = surface_flux_diagnostics::sensible_heat_flux_wm2_from_rhotheta_flux(
                        hfx(i,j,klo));
                });
            }
            ApplyConservativeRemap(tmp, *states[iSHflux], *weight_mf, *index_mf, max_stencil_size, dst_mskr);
            if (verbose) { PrintFluxLaneStats("SHflux_lane", *states[iSHflux]); }
        }
 
        if (iLHflux < static_cast<int>(states.size()) && states[iLHflux] != nullptr &&
            !SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev] != nullptr) {
            MultiFab tmp(ba2d_lev, dm, 1, 0);
            for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                Box bx = mfi.tilebox();
                auto const& qfx = SFS_q1fx3_lev[lev]->const_array(mfi);
                auto t = tmp.array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    t(i,j,k) = surface_flux_diagnostics::latent_heat_flux_wm2_from_rhoqv_flux(
                        qfx(i,j,klo));
                });
            }
            ApplyConservativeRemap(tmp, *states[iLHflux], *weight_mf, *index_mf, max_stencil_size, dst_mskr);
            if (verbose) { PrintFluxLaneStats("LHflux_lane", *states[iLHflux]); }
        }
 
        if (has_radiation) {
            if (iFluxSWrad < static_cast<int>(states.size()) && states[iFluxSWrad] != nullptr) {
                MultiFab tmp(ba2d_lev, dm, 1, 0);
                tmp.ParallelCopy(*rad_fluxes[lev], 1, 0, 1);
                ApplyConservativeRemap(tmp, *states[iFluxSWrad], *weight_mf, *index_mf, max_stencil_size, dst_mskr);
                if (verbose) { PrintFluxLaneStats("SWrad_lane", *states[iFluxSWrad]); }
            }
            if (iFluxLWrad < static_cast<int>(states.size()) && states[iFluxLWrad] != nullptr) {
                MultiFab tmp(ba2d_lev, dm, 1, 0);
                for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    Box bx = mfi.tilebox();
                    auto const& rad_flux = rad_fluxes[lev]->const_array(mfi);
                    auto t = tmp.array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        t(i,j,k) = rad_flux(i,j,k,3) - rad_flux(i,j,k,2);
                    });
                }
                ApplyConservativeRemap(tmp, *states[iFluxLWrad], *weight_mf, *index_mf, max_stencil_size, dst_mskr);
                if (verbose) { PrintFluxLaneStats("LWrad_lane", *states[iFluxLWrad]); }
            }
        }
 
        if (iFluxEvap < static_cast<int>(states.size()) && states[iFluxEvap] != nullptr &&
            !SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev] != nullptr) {
            MultiFab tmp(ba2d_lev, dm, 1, 0);
            for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                Box bx = mfi.tilebox();
                auto const& qfx = SFS_q1fx3_lev[lev]->const_array(mfi);
                auto t = tmp.array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    t(i,j,k) = qfx(i,j,klo);
                });
            }
            ApplyConservativeRemap(tmp, *states[iFluxEvap], *weight_mf, *index_mf, max_stencil_size, dst_mskr);
            if (verbose) { PrintFluxLaneStats("evap_lane", *states[iFluxEvap]); }
        }
 
#if 0
        // Extract Rain Flux using surface qr and a proxy terminal velocity
        if (has_moisture && iFluxRain < static_cast<int>(states.size()) && states[iFluxRain] != nullptr) {
            int qr_idx = solverChoice.moisture_indices.qr;
            if (qr_idx != -1) {
                MultiFab tmp(ba2d_lev, dm, 1, 0);
                for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    Box bx = mfi.tilebox();
                    auto const& c = cons.const_array(mfi);
                    auto t = tmp.array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        // Rain flux proxy: (rho * qr) * V_terminal (approx 5.0 m/s for rain) -> kg m^-2 s^-1
                        t(i,j,k) = c(i,j,klo,qr_idx) * Real(5.0);
                    });
                }
                ApplyConservativeRemap(tmp, *states[iFluxRain], *weight_mf, *index_mf, max_stencil_size, dst_mskr);
                if (verbose) { PrintFluxLaneStats("rain_lane", *states[iFluxRain]); }
            }
        }
#endif
        amrex::ignore_unused(iFluxRain);
        return;
    }
 
    // --- Uwind + Vwind ---
    if ((iUwind < static_cast<int>(states.size()) && states[iUwind] != nullptr) ||
        (iVwind < static_cast<int>(states.size()) && states[iVwind] != nullptr)) {
 
        auto& zvel = vars_new[lev][Vars::zvel];
        MultiFab cc_vel(ba, dm, AMREX_SPACEDIM, 0);
        amrex::average_face_to_cellcenter(cc_vel, 0,
            Array<const MultiFab*, AMREX_SPACEDIM>{&xvel, &yvel, &zvel});
 
        // Collapse to 2D slab
        MultiFab uv_slab(ba2d_lev, dm, 2, 0);  // comp0=u, comp1=v
        uv_slab.ParallelCopy(cc_vel, 0, 0, 2);
 
        if (iUwind < static_cast<int>(states.size()) && states[iUwind] != nullptr) {
            MultiFab u_alias(uv_slab, amrex::make_alias, 0, 1); // alias u component
            ApplyConservativeRemap(u_alias, *states[iUwind], *weight_mf_by_family[kWindFamily],
                                   *index_mf_by_family[kWindFamily], max_stencil_size,
                                   nullptr, fallback_wind_ms);
        }
 
        if (iVwind < static_cast<int>(states.size()) && states[iVwind] != nullptr) {
            MultiFab v_alias(uv_slab, amrex::make_alias, 1, 1); // alias v component
            ApplyConservativeRemap(v_alias, *states[iVwind], *weight_mf_by_family[kWindFamily],
                                   *index_mf_by_family[kWindFamily], max_stencil_size,
                                   nullptr, fallback_wind_ms);
        }
    }
 
    // --- Patm ---
    if (iPatm < static_cast<int>(states.size()) && states[iPatm] != nullptr) {
        MultiFab tmp(ba2d_lev, dm, 1, 0);
        for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
            Box bx = mfi.tilebox();
            auto const& c = cons.const_array(mfi);
            auto         t = tmp.array(mfi);
            if (has_moisture) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    const Real qv = c(i,j,k,RhoQ1_comp) / c(i,j,k,Rho_comp);
                    t(i,j,k) = getPgivenRTh(c(i,j,k,RhoTheta_comp), qv);
                });
            } else {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    t(i,j,k) = getPgivenRTh(c(i,j,k,RhoTheta_comp));
                });
            }
        }
        ApplyConservativeRemap(tmp, *states[iPatm], *weight_mf, *index_mf, max_stencil_size, dst_mskr,
                               fallback_pressure_pa);
    }
 
    // --- Tair ---
    if (iTair < static_cast<int>(states.size()) && states[iTair] != nullptr) {
        MultiFab tmp(ba2d_lev, dm, 1, 0);
        for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
            Box bx = mfi.tilebox();
            auto const& c = cons.const_array(mfi);
            auto         t = tmp.array(mfi);
            if (has_moisture) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    const Real qv = c(i,j,k,RhoQ1_comp) / c(i,j,k,Rho_comp);
                    t(i,j,k) = getTgivenRandRTh(c(i,j,k,Rho_comp), c(i,j,k,RhoTheta_comp), qv);
                });
            } else {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    t(i,j,k) = getTgivenRandRTh(c(i,j,k,Rho_comp), c(i,j,k,RhoTheta_comp));
                });
            }
        }
        ApplyConservativeRemap(tmp, *states[iTair], *weight_mf, *index_mf, max_stencil_size, dst_mskr,
                               fallback_temp_k);
    }
 
    // --- Humidity/Cloud/Rain ---
    if (has_moisture) {
#if 0
        if (iRH < static_cast<int>(states.size()) && states[iRH] != nullptr) {
            MultiFab tmp(ba2d_lev, dm, 1, 0);
            for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                Box bx = mfi.tilebox();
                auto const& c = cons.const_array(mfi);
                auto         t = tmp.array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    const Real qv = c(i,j,k,RhoQ1_comp) / c(i,j,k,Rho_comp);
                    const Real p_pa = getPgivenRTh(c(i,j,k,RhoTheta_comp), qv);
                    const Real temp = getTgivenRandRTh(c(i,j,k,Rho_comp), c(i,j,k,RhoTheta_comp), qv);
                    Real qsat = Real(0.0);
                    erf_qsatw(temp, p_pa * Real(0.01), qsat);
                    t(i,j,k) = amrex::max(Real(0.0), amrex::min(Real(1.0),
                                                                 qv / amrex::max(qsat, Real(1.0e-12))));
                });
            }
            IntVect ratio = ba2d_lev.minimalBox().length() / states[iRH]->boxArray().minimalBox().length();
            amrex::average_down(tmp, *states[iRH], 0, 1, ratio);
        }
#else
        amrex::ignore_unused(iRH);
        // Avoiding unexercised moist assumptions: retain the driver-prefilled
        // humidity lane until live ERF->REMORA Qair semantics are made explicit.
#endif
        if (iCloud < static_cast<int>(states.size()) && states[iCloud] != nullptr) {
            const int qc_idx = solverChoice.moisture_indices.qc;
            const int qi_idx = solverChoice.moisture_indices.qi;
            if (qc_idx != -1 || qi_idx != -1) {
                MultiFab tmp(ba2d_lev, dm, 1, 0);
                const Real cf = amrex::max(Real(0.0), amrex::min(Real(1.0), cloud_fraction(0.0)));
                amrex::ignore_unused(cf); // keep diagnostic computation active for consistency with ERF scalar stats
                for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    Box bx = mfi.tilebox();
                    auto const& c = cons.const_array(mfi);
                    auto t = tmp.array(mfi);
                    const int khi = ba.minimalBox().bigEnd(2);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        int cloudy = 0;
                        for (int kk = klo; kk <= khi; ++kk) {
                            const Real rho = c(i,j,kk,Rho_comp);
                            const Real qc = (qc_idx != -1) ? c(i,j,kk,qc_idx) / rho : Real(0.0);
                            const Real qi = (qi_idx != -1) ? c(i,j,kk,qi_idx) / rho : Real(0.0);
                            if (qc + qi > Real(0.0)) { cloudy = 1; break; }
                        }
                        t(i,j,k) = static_cast<Real>(cloudy);
                    });
                }
                ApplyConservativeRemap(tmp, *states[iCloud], *weight_mf, *index_mf, max_stencil_size, dst_mskr);
            }
        }
#if 0
        if (iRain < static_cast<int>(states.size()) && states[iRain] != nullptr) {
            int qr_idx = solverChoice.moisture_indices.qr;
            if (qr_idx != -1) {
                MultiFab tmp(ba2d_lev, dm, 1, 0);
                for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    Box bx = mfi.tilebox();
                    auto const& c = cons.const_array(mfi);
                    auto         t = tmp.array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        t(i,j,k) = c(i,j,k,qr_idx) / c(i,j,k,Rho_comp);
                    });
                }
                IntVect ratio = ba2d_lev.minimalBox().length() / states[iRain]->boxArray().minimalBox().length();
                amrex::average_down(tmp, *states[iRain], 0, 1, ratio);
            }
        }
#else
        amrex::ignore_unused(iRain);
        // Avoiding unexercised moist assumptions: retain the driver-prefilled
        // rain lane until live ERF->REMORA rain semantics are made explicit.
#endif
    }
    // No moisture: leave RH/Cloud/Rain slabs at their driver-pre-filled values.
 
    // --- Radiation ---
    if (has_radiation) {
        if (iSWrad < static_cast<int>(states.size()) && states[iSWrad] != nullptr) {
            MultiFab tmp(ba2d_lev, dm, 1, 0);
            tmp.ParallelCopy(*rad_fluxes[lev], 1, 0, 1);
            ApplyConservativeRemap(tmp, *states[iSWrad], *weight_mf, *index_mf, max_stencil_size, dst_mskr);
        }
        if (iLWrad < static_cast<int>(states.size()) && states[iLWrad] != nullptr) {
            MultiFab tmp(ba2d_lev, dm, 1, 0);
            tmp.ParallelCopy(*rad_fluxes[lev], 3, 0, 1);
            ApplyConservativeRemap(tmp, *states[iLWrad], *weight_mf, *index_mf, max_stencil_size, dst_mskr);
        }
    }
}

Here is the call graph for this function:

ParameterSanityChecks()

void ERF::ParameterSanityChecks ( )

private

{
    AMREX_ALWAYS_ASSERT(cfl > zero || fixed_dt[0] > zero);
 
    // We don't allow use_real_bcs to be true if init_type is not either InitType::WRFInput or InitType::Metgrid
    AMREX_ALWAYS_ASSERT( !solverChoice.use_real_bcs ||
                        ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) );
 
    if (solverChoice.use_wrf_bdy_density) {
        if (!solverChoice.use_real_bcs || solverChoice.init_type != InitType::WRFInput) {
            Abort("erf.use_wrf_bdy_density requires standard WRFInput real boundary conditions");
        }
        if (nc_bdy_file.empty()) {
            Abort("erf.use_wrf_bdy_density requires nc_bdy_file");
        }
        if (!solverChoice.anelastic.empty() && solverChoice.anelastic[0] != 0) {
            Abort("erf.use_wrf_bdy_density is not supported for anelastic simulations");
        }
        if (input_bndry_planes) {
            Abort("erf.use_wrf_bdy_density is not supported with generic boundary-plane input");
        }
        const Real rho_factor = (solverChoice.bdy_rho_nudge_factor > zero)
                              ? solverChoice.bdy_rho_nudge_factor
                              : solverChoice.bdy_nudge_factor;
        if (rho_factor <= zero) {
            Abort("erf.bdy_rho_nudge_factor or erf.bdy_nudge_factor must be positive");
        }
        Print() << "WRF lateral boundary density forcing: enabled\n"
                << "WRF boundary density nudge factor: " << rho_factor << std::endl;
    } else {
        Print() << "WRF lateral boundary density forcing: disabled" << std::endl;
    }
 
    if (solverChoice.use_wrf_bdy_qc_qi) {
        if (!solverChoice.use_real_bcs || solverChoice.init_type != InitType::WRFInput) {
            Abort("erf.use_wrf_bdy_qc_qi requires standard WRFInput real boundary conditions");
        }
        if (nc_bdy_file.empty()) {
            Abort("erf.use_wrf_bdy_qc_qi requires erf.nc_bdy_file");
        }
        if (input_bndry_planes) {
            Abort("erf.use_wrf_bdy_qc_qi is not supported with generic boundary-plane input");
        }
        Print() << "WRF lateral boundary cloud-water/cloud-ice forcing: enabled" << std::endl;
    } else {
        Print() << "WRF lateral boundary cloud-water/cloud-ice forcing: disabled" << std::endl;
    }
 
    AMREX_ALWAYS_ASSERT(real_width >= 0);
 
    if (cf_set_width != 0) {
        Abort("You must set cf_set_width == 0");
    }
    if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
        Abort("You must set cf_width >= cf_set_width >= 0");
    }
    if (max_level > 0 && cf_set_width > 0) {
        for (int lev = 1; lev <= max_level; lev++) {
            if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
                cf_set_width%ref_ratio[lev-1][1] != 0 ||
                cf_set_width%ref_ratio[lev-1][2] != 0 ) {
                Abort("You must set cf_width to be a multiple of ref_ratio");
            }
        }
    }
 
    // If fixed_mri_dt_ratio is set, it must be even
    if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
    {
        Abort("If you specify fixed_mri_dt_ratio, it must be even");
    }
 
    for (int lev = 0; lev <= max_level; lev++)
    {
        // We ignore fixed_fast_dt if not substepping
        if (solverChoice.substepping_type[lev] == SubsteppingType::None) {
            fixed_fast_dt[lev] = -one;
        }
 
        // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
        if (fixed_dt[lev] > zero && fixed_fast_dt[lev] > zero && fixed_mri_dt_ratio <= 0)
        {
            Real eps = Real(1.e-12);
            int ratio = static_cast<int>( ( (one+eps) * fixed_dt[lev] ) / fixed_fast_dt[lev] );
            if (fixed_dt[lev] / fixed_fast_dt[lev] != ratio)
            {
                Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
            }
        }
 
        // If all three are specified, they must be consistent
        if (fixed_dt[lev] > zero && fixed_fast_dt[lev] > zero &&  fixed_mri_dt_ratio > 0)
        {
            if (fixed_dt[lev] / fixed_fast_dt[lev] != fixed_mri_dt_ratio)
            {
                Abort("Dt is over-specfied");
            }
        }
    } // lev
 
    if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
        Abort("For two-way coupling you must set cf_width = 0");
    }
}

Here is the call graph for this function:

PerformDataAssimilation()

void ERF::PerformDataAssimilation

(

int

da_iter

)

PlotFileName()

std::string ERF::PlotFileName ( int  lev ) const

private

PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string >  plot_var_names )

staticprivate

{
    Vector<std::string> names;
 
    names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
 
    return names;
 
}

poisson_wall_dist()

void ERF::poisson_wall_dist

(

int

lev

)

Calculate wall distances using the Poisson equation

The zlo boundary is assumed to correspond to the land surface. If there are no boundary walls, then the other use case is to calculate wall distances for immersed boundaries (embedded or thin body).

See Tucker, P. G. (2003). Differential equation-based wall distance computation for DES and RANS. Journal of Computational Physics, 190(1), 229–Real(248.) https://doi.org/Real(10.1016)/S0021-9991(03)00272-9

Parameters

lev	Level index for the wall-distance solve

{
    BL_PROFILE("ERF::poisson_wall_dist()");
 
    bool havewall{false};
    Orientation zlo(Direction::z, Orientation::low);
    if ( ( phys_bc_type[zlo] == ERF_BC::surface_layer                      ) ||
         ( phys_bc_type[zlo] == ERF_BC::no_slip_wall                       ) )/*||
         ((phys_bc_type[zlo] == ERF_BC::slip_wall) && (dom_hi.z > dom_lo.z)) )*/
    {
        havewall = true;
    }
 
    auto const& geomdata = geom[lev];
    auto const& dxinv    = geomdata.InvCellSizeArray();
 
    auto const& zphys_arr = z_phys_nd[lev]->const_arrays();
 
    if (havewall) {
#if 1
        // Bypass wall dist calc in the trivial cases
 
        if (solverChoice.mesh_type == MeshType::ConstantDz) {
            Print() << "Directly calculating direct wall distance for constant dz" << std::endl;
            const auto prob_lo = geomdata.ProbLoArray();
            const auto dx = geomdata.CellSizeArray();
            for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.validbox();
                auto dist_arr = walldist[lev]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    dist_arr(i, j, k) = prob_lo[2] + (k + myhalf) * dx[2];
                });
            }
            fill_wall_dist_ghost_cells(*walldist[lev], geom[lev]);
            return;
        }
 
        if (solverChoice.mesh_type == MeshType::StretchedDz) {
            Print() << "Directly calculating direct wall distance for stretched dz" << std::endl;
            for (MFIter mfi(*walldist[lev],TileNoZ()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.validbox();
                auto dist_arr = walldist[lev]->array(mfi);
                const auto zcc_arr = z_phys_cc[lev]->const_array(mfi);
                const auto znd_arr = z_phys_nd[lev]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    dist_arr(i, j, k) = zcc_arr(i, j, k) - znd_arr(i, j, 0);
                });
            }
            fill_wall_dist_ghost_cells(*walldist[lev], geom[lev]);
            return;
        }
#endif
    }
    else
    {
        Error("No solid boundaries in the computational domain");
    }
 
    if (havewall && solverChoice.wall_dist_type == "terrain_height") {
        // Height above the local surface projected on the surface normal:
        // d = (z_cc - z_surf) / sqrt(1 + h_xi^2 + h_eta^2), exact for a
        // plane, within a few percent of the true distance for hills with
        // slopes below about 0.3, and free of any linear solve (cf. the
        // terrain height used by the amr-wind immersed terrain and Kynema).
        Print() << "Calculating wall distance from the terrain height (normal-projected)" << std::endl;
        const int klo = geomdata.Domain().smallEnd(2);
 
        // The surface nodes z_nd(:,:,klo) live only in the boxes that touch the surface;
        // when the BoxArray is split in z the boxes above hold nodes from their own k
        // range, so gather the surface slab onto every box (the same 2D footprint, at klo).
        BoxList bl_surf = z_phys_nd[lev]->boxArray().boxList();
        for (auto& b : bl_surf) { b.setRange(2,klo); }
        BoxArray ba_surf(std::move(bl_surf));
        IntVect ng_surf = z_phys_nd[lev]->nGrowVect(); ng_surf[2] = 0;
        MultiFab znd_surf(ba_surf, z_phys_nd[lev]->DistributionMap(), 1, ng_surf);
        znd_surf.setVal(bogus_large_value);
        znd_surf.ParallelCopy(*z_phys_nd[lev], 0, 0, 1, ng_surf, ng_surf, geom[lev].periodicity());
        // Every node the stencil below reads (valid plus the x/y ghosts) must
        // have been gathered; the slab has no z ghosts, so reduce over ng_surf
        // rather than a scalar ghost count.
        Real znd_max = ReduceMax(znd_surf, ng_surf,
            [=] AMREX_GPU_HOST_DEVICE (Box const& bx, Array4<Real const> const& a) -> Real
            {
                Real m = std::numeric_limits<Real>::lowest();
                amrex::Loop(bx, [&] (int i, int j, int k) { m = amrex::max(m, a(i,j,k)); });
                return m;
            });
        ParallelAllReduce::Max(znd_max, ParallelContext::CommunicatorSub());
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(znd_max < bogus_large_value,
            "poisson_wall_dist: the surface nodes were not gathered onto every box");
 
        for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.validbox();
            auto dist_arr = walldist[lev]->array(mfi);
            const auto zcc_arr = z_phys_cc[lev]->const_array(mfi);
            const auto znd_arr = znd_surf.const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                Real z_surf = fourth * ( znd_arr(i,j,klo) + znd_arr(i+1,j,klo)
                                       + znd_arr(i,j+1,klo) + znd_arr(i+1,j+1,klo) );
                Real h_xi  = Compute_h_xi_AtKface (i, j, klo, dxinv, znd_arr);
                Real h_eta = Compute_h_eta_AtKface(i, j, klo, dxinv, znd_arr);
                Real dz_loc = zcc_arr(i,j,k) - z_surf;
                dist_arr(i, j, k) = amrex::max(dz_loc / std::sqrt(one + h_xi*h_xi + h_eta*h_eta),
                                               std::numeric_limits<Real>::epsilon());
            });
        }
        fill_wall_dist_ghost_cells(*walldist[lev], geom[lev]);
        return;
    }
 
    Print() << "Calculating Poisson wall distance for general terrain" << std::endl;
 
    // Make sure the solver only sees the levels over which we are solving
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(walldist[lev]->boxArray());
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(walldist[lev]->DistributionMap());
 
    Vector<MultiFab> rhs;
    Vector<MultiFab> phi;
 
    if (solverChoice.terrain_type == TerrainType::EB) {
        amrex::Error("Wall dist calc not implemented for EB");
    } else {
        rhs.resize(1);   rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
        phi.resize(1);   phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
    }
 
    rhs[0].setVal(1.0);
 
    auto const dom_lo = lbound(geom[lev].Domain());
    auto const dom_hi = ubound(geom[lev].Domain());
 
    // ****************************************************************************
    // Initialize phi
    // (It is essential that we do this in order to fill the corners; this is
    // used if we include blanking.)
    // ****************************************************************************
    phi[0].setVal(0.0);
 
    // ****************************************************************************
    // Interior boundaries are marked with phi=0
    // ****************************************************************************
#if 0
    // Define an overset mask (0 or 1) to set dirichlet nodes on walls
    // 1 means the node is an unknown. 0 means it's known.
    iMultiFab mask(ba_tmp[0], dm_tmp[0], 1, 0);
    Vector<const iMultiFab*> overset_mask = {&mask};
 
    mask.setVal(1);
    if (solverChoice.advChoice.have_zero_flux_faces) {
        Warning("Poisson distance is inaccurate for bodies in open domains that are small compared to the domain size, skipping");
        return;
 
        Gpu::DeviceVector<IntVect> xfacelist, yfacelist, zfacelist;
 
        xfacelist.resize(solverChoice.advChoice.zero_xflux.size());
        yfacelist.resize(solverChoice.advChoice.zero_yflux.size());
        zfacelist.resize(solverChoice.advChoice.zero_zflux.size());
 
        if (xfacelist.size() > 0) {
            Gpu::copy(amrex::Gpu::hostToDevice,
                      solverChoice.advChoice.zero_xflux.begin(),
                      solverChoice.advChoice.zero_xflux.end(),
                      xfacelist.begin());
            Print() << "  masking interior xfaces" << std::endl;
        }
        if (yfacelist.size() > 0) {
            Gpu::copy(amrex::Gpu::hostToDevice,
                      solverChoice.advChoice.zero_yflux.begin(),
                      solverChoice.advChoice.zero_yflux.end(),
                      yfacelist.begin());
            Print() << "  masking interior yfaces" << std::endl;
        }
        if (zfacelist.size() > 0) {
            Gpu::copy(amrex::Gpu::hostToDevice,
                      solverChoice.advChoice.zero_zflux.begin(),
                      solverChoice.advChoice.zero_zflux.end(),
                      zfacelist.begin());
            Print() << "  masking interior zfaces" << std::endl;
        }
 
        for (MFIter mfi(phi[0]); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.validbox();
 
            auto phi_arr  = phi[0].array(mfi);
            auto mask_arr = mask.array(mfi);
 
            if (xfacelist.size() > 0) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    for (int iface=0; iface < xfacelist.size(); ++iface) {
                        if ((i == xfacelist[iface][0]) &&
                            (j == xfacelist[iface][1]) &&
                            (k == xfacelist[iface][2]))
                        {
                            mask_arr(i, j  , k  ) = 0;
                            mask_arr(i, j  , k+1) = 0;
                            mask_arr(i, j+1, k  ) = 0;
                            mask_arr(i, j+1, k+1) = 0;
                        }
                    }
                });
            }
 
            if (yfacelist.size() > 0) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    for (int iface=0; iface < yfacelist.size(); ++iface) {
                        if ((i == yfacelist[iface][0]) &&
                            (j == yfacelist[iface][1]) &&
                            (k == yfacelist[iface][2]))
                        {
                            mask_arr(i  , j, k  ) = 0;
                            mask_arr(i  , j, k+1) = 0;
                            mask_arr(i+1, j, k  ) = 0;
                            mask_arr(i+1, j, k+1) = 0;
                        }
                    }
                });
            }
 
            if (zfacelist.size() > 0) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    for (int iface=0; iface < zfacelist.size(); ++iface) {
                        if ((i == xfacelist[iface][0]) &&
                            (j == xfacelist[iface][1]) &&
                            (k == xfacelist[iface][2]))
                        {
                            mask_arr(i  , j  , k) = 0;
                            mask_arr(i  , j+1, k) = 0;
                            mask_arr(i+1, j  , k) = 0;
                            mask_arr(i+1, j+1, k) = 0;
                        }
                    }
                });
            }
        }
    }
#endif
 
    // ****************************************************************************
    // Setup BCs, with solid domain boundaries being dirichlet
    // ****************************************************************************
    amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bc3d_lo, bc3d_hi;
    for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
        if (geom[lev].isPeriodic(dir)) {
            bc3d_lo[dir] = LinOpBCType::Periodic;
            bc3d_hi[dir] = LinOpBCType::Periodic;
        } else {
            bc3d_lo[dir] = LinOpBCType::Neumann;
            bc3d_hi[dir] = LinOpBCType::Neumann;
        }
    }
    if (havewall) {
        Print() << "  Poisson zlo BC is dirichlet" << std::endl;
        bc3d_lo[2] = LinOpBCType::Dirichlet;
    }
    Print() << "  bc lo : " << bc3d_lo << std::endl;
    Print() << "  bc hi : " << bc3d_hi << std::endl;
 
    if (!solverChoice.advChoice.have_zero_flux_faces && !havewall) {
        Error("No solid boundaries in the computational domain");
    }
 
    LPInfo info; // defaults
 
/* Nodal solver cannot have hidden dimensions */
#if 0
    // Allow a hidden direction if the domain is one cell wide
    if (dom_lo.x == dom_hi.x) {
        info.setHiddenDirection(0);
        Print() << "  domain is 2D in yz" << std::endl;
    } else if (dom_lo.y == dom_hi.y) {
        info.setHiddenDirection(1);
        Print() << "  domain is 2D in xz" << std::endl;
    } else if (dom_lo.z == dom_hi.z) {
        info.setHiddenDirection(2);
        Print() << "  domain is 2D in xy" << std::endl;
    }
#endif
 
#if 0
    Vector<EBFArrayBoxFactory const*> factory_vec;
    factory_vec.push_back(static_cast<FabFactory<FArrayBox> const*>(&EBFactory(lev));
#endif
 
    // ****************************************************************************
    // Setup Poisson problem
    // (A \alpha - B \nabla \cdot \beta \nabla ) \phi = f
    //
    // In physical space:
    //   \nabla \cdot \nabla \phi = -1
    //
    // In computational space:
    //   grad(phi) = T^T \nabla \phi
    // and
    //   \nabla \cdot (h_zeta T (T^T \nabla \phi)) = -h_zeta
    // where T = inv(J), T^T is the transpose of inv(J)
    //
    // Posed as -div(beta grad phi) = +h_zeta, i.e. B = +1, the positive
    // definite form MLABecLaplacian documents (B = -1 with f = -h_zeta is
    // the same equation and gave the same iterates). Note: on a 3D fitted
    // mesh with dx != dz this multigrid diverges (residual 18x after the
    // first cycle, 1e10 by iteration 100) with or without semi-coarsening
    // and independent of the box layout. Use erf.wall_dist_type =
    // terrain_height there.
    // ****************************************************************************
    constexpr Real constA = zero;
    constexpr Real constB = one;
 
    MLABecLaplacian mlabec(geom_tmp, ba_tmp, dm_tmp, info);
 
    mlabec.setScalars(constA, constB);
    mlabec.setACoeffs(0, zero);
#if 1
    // Set beta coefficients at faces
    Array<MultiFab, AMREX_SPACEDIM> beta;
 
    for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
        BoxArray ba_face = ba_tmp[0];
        ba_face.surroundingNodes(idim);  // Convert to face-centered in direction idim
        beta[idim].define(ba_face, dm_tmp[0], 1, 0);
    }
 
    auto beta0_arr = beta[0].arrays();
    auto beta1_arr = beta[1].arrays();
    auto beta2_arr = beta[2].arrays();
 
    // Note: This ignores the off-diagonal components of (h_zeta T T^T), which
    //       is equivalent to assuming that h_xi and h_eta are small.
 
    ParallelFor(beta[0], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
        beta0_arr[b](i, j, k) = Compute_h_zeta_AtIface(i, j, k, dxinv, zphys_arr[b]);;
    });
    ParallelFor(beta[1], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
        beta1_arr[b](i, j, k) = Compute_h_zeta_AtJface(i, j, k, dxinv, zphys_arr[b]);;
    });
    ParallelFor(beta[2], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
        Real inv_h_zeta = one / Compute_h_zeta_AtKface(i, j, k, dxinv, zphys_arr[b]);
        Real h_xi = Compute_h_xi_AtKface(i, j, k, dxinv, zphys_arr[b]);
        Real h_eta = Compute_h_eta_AtKface(i, j, k, dxinv, zphys_arr[b]);
        beta2_arr[b](i, j, k) = inv_h_zeta * (1 + h_xi*h_xi + h_eta*h_eta);
    });
 
    mlabec.setBCoeffs(0, GetArrOfConstPtrs(beta));
 
    // Set RHS := +h_zeta (for -div(beta grad phi) = h_zeta)
    auto rhs_arr = rhs[0].arrays();
    ParallelFor(rhs[0], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
        rhs_arr[b](i, j, k) = Compute_h_zeta_AtCellCenter(i, j, k, dxinv, zphys_arr[b]);
    });
#else
    mlabec.setBCoeffs(0, one);
#endif
 
    mlabec.setDomainBC(bc3d_lo, bc3d_hi);
 
    if (lev > 0) {
        mlabec.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
    }
 
    // If we have inhomogeneous BCs -- do this after setCoarseFineBC
    mlabec.setLevelBC(0, nullptr);
 
    // ****************************************************************************
    // Solve Poisson problem with MLMG
    // ****************************************************************************
    const Real reltol = solverChoice.poisson_reltol;
    const Real abstol = solverChoice.poisson_abstol;
    const int n_corr = solverChoice.ncorr;
    constexpr int max_iter = 100;
 
    MLMG mlmg(mlabec);
    mlmg.setMaxIter(max_iter);
    mlmg.setVerbose(mg_verbose);
    mlmg.setBottomVerbose(0);
 
    for (int icorr=0; icorr <= n_corr; ++icorr) {
        Print()<< "Solving wall distance poisson, icorr=" << icorr << std::endl;
 
        mlmg.solve(GetVecOfPtrs(phi),
                   GetVecOfConstPtrs(rhs),
                   reltol, abstol);
 
        // ****************************************************************************
        // Apply BCs: dirichlet (odd) on zlo, neumann (even) / periodic elsewhere
        // ****************************************************************************
 
        // Overwrite with periodic fill outside domain and fine-fine fill inside
        phi[0].FillBoundary(geom[lev].periodicity());
 
        if (!geom[lev].isPeriodic(0)) {
            for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
            {
                Box bx = mfi.tilebox();
                const Array4<Real>& phi_arr = phi[0].array(mfi);
                if (bx.smallEnd(0) <= dom_lo.x) {
                    ParallelFor(makeSlab(bx,0,dom_lo.x),
                    [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        phi_arr(i-1,j,k) =  phi_arr(i,j,k); // even BC
                    });
                } // lo x
                if (bx.bigEnd(0) >= dom_hi.x) {
                    ParallelFor(makeSlab(bx,0,dom_hi.x),
                    [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        phi_arr(i+1,j,k) =  phi_arr(i,j,k); // even BC
                    });
                } // hi x
            } // mfi
        } // not periodic in x
 
        if (!geom[lev].isPeriodic(1)) {
            for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
            {
                Box bx = mfi.tilebox();
                Box bx2(bx); bx2.grow(0,1);
                const Array4<Real>& phi_arr = phi[0].array(mfi);
                if (bx.smallEnd(1) <= dom_lo.y) {
                    ParallelFor(makeSlab(bx2,1,dom_lo.y),
                    [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        phi_arr(i,j-1,k) =  phi_arr(i,j,k); // even BC
                    });
                } // lo y
                if (bx.bigEnd(1) >= dom_hi.y) {
                    ParallelFor(makeSlab(bx2,1,dom_hi.y),
                    [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        phi_arr(i,j+1,k) =  phi_arr(i,j,k); // even BC
                    });
                } // hi y
 
            } // mfi
        } // not periodic in y
 
        for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
        {
            Box bx = mfi.tilebox();
            Box bx3(bx); bx3.grow(0,1); bx3.grow(1,1);
            const Array4<Real>& phi_arr = phi[0].array(mfi);
            if (bx.smallEnd(2) <= dom_lo.z) {
                ParallelFor(makeSlab(bx3,2,dom_lo.z),
                [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    phi_arr(i,j,k-1) = -phi_arr(i,j,k); // ODD BC
                });
            } // lo z
            if (bx.bigEnd(2) >= dom_hi.z) {
                ParallelFor(makeSlab(bx3,2,dom_hi.z),
                [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    phi_arr(i,j,k+1) =  phi_arr(i,j,k); // even BC
                });
            } // hi z
        } // mfi
 
        // ****************************************************************************
        // Compute grad(phi) to get distances
        // ****************************************************************************
        auto const& phi_arr = phi[0].const_arrays();
        //auto rhs_arr = rhs[0].arrays();
        auto dist_arr = walldist[lev]->arrays();
 
        ParallelFor(*walldist[lev], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
            // Cell-centred gradient of phi in physical space: centred
            // differences in computational space with the cell-centre
            // terrain metrics (chain rule for a mesh deformed in z only).
            // The face fluxes used before sat half a cell below and beside
            // the centre, which overstated |grad phi| by dz/2 and shortened
            // every distance by z dz / (2H) on a flat mesh (0.8 % for 64
            // cells). The ghost cells of phi carry the Dirichlet (odd) value
            // below the wall and even values elsewhere, so the stencil needs
            // nothing beyond one ghost cell and the cell's own eight nodes.
            const auto& p  = phi_arr[b];
            const auto& zp = zphys_arr[b];
            Real dpdxi   = myhalf * (p(i+1,j,k) - p(i-1,j,k)) * dxinv[0];
            Real dpdeta  = myhalf * (p(i,j+1,k) - p(i,j-1,k)) * dxinv[1];
            Real dpdzeta = myhalf * (p(i,j,k+1) - p(i,j,k-1)) * dxinv[2];
            Real h_zeta  = Compute_h_zeta_AtCellCenter(i, j, k, dxinv, zp);
            Real h_xi    = Compute_h_xi_AtCellCenter  (i, j, k, dxinv, zp);
            Real h_eta   = Compute_h_eta_AtCellCenter (i, j, k, dxinv, zp);
            Real dpdz = dpdzeta / h_zeta;
            Real dpdx = dpdxi  - h_xi  * dpdz;
            Real dpdy = dpdeta - h_eta * dpdz;
 
            Real magsqr_dphi = dpdx*dpdx + dpdy*dpdy + dpdz*dpdz;
            Real mag_dphi = std::sqrt(magsqr_dphi);
#if 1
            // Tucker 2003 Eqn 2
            dist_arr[b](i, j, k) = -mag_dphi + std::sqrt(magsqr_dphi + 2*phi_arr[b](i, j, k));
#else
            // DEBUG: output phi instead
            if (i==0 && j==0) AllPrint() << "walldist"<<IntVect(i,j,k) << " = " << dist_arr[b](i,j,k) << std::endl;
            dist_arr[b](i, j, k) = phi_arr[b](i, j, k);
#endif
            // Update RHS source term to explicitly include cross-terms
            if (n_corr > 0) {
                // d/dxi ( h_xi * dphi/dzeta )
                Real phi_zeta_xlo = fourth * dxinv[2] * ( phi_arr[b](i  , j, k+1) - phi_arr[b](i  , j, k-1)
                                                      + phi_arr[b](i-1, j, k+1) - phi_arr[b](i-1, j, k-1) );
                Real phi_zeta_xhi = fourth * dxinv[2] * ( phi_arr[b](i  , j, k+1) - phi_arr[b](i  , j, k-1)
                                                      + phi_arr[b](i+1, j, k+1) - phi_arr[b](i+1, j, k-1) );
                Real h_xi_xlo = Compute_h_xi_AtIface(i  , j, k, dxinv, zphys_arr[b]);
                Real h_xi_xhi = Compute_h_xi_AtIface(i+1, j, k, dxinv, zphys_arr[b]);
 
                // d/deta ( h_eta * dphi/dzeta )
                Real phi_zeta_ylo = fourth * dxinv[2] * ( phi_arr[b](i, j  , k+1) - phi_arr[b](i, j  , k-1)
                                                      + phi_arr[b](i, j-1, k+1) - phi_arr[b](i, j-1, k-1) );
                Real phi_zeta_yhi = fourth * dxinv[2] * ( phi_arr[b](i, j  , k+1) - phi_arr[b](i, j  , k-1)
                                                      + phi_arr[b](i, j+1, k+1) - phi_arr[b](i, j+1, k-1) );
                Real h_eta_ylo = Compute_h_eta_AtJface(i, j  , k, dxinv, zphys_arr[b]);
                Real h_eta_yhi = Compute_h_eta_AtJface(i, j+1, k, dxinv, zphys_arr[b]);
 
                // d/dzeta ( h_xi * dphi/dxi )
                Real phi_xi_zlo = fourth * dxinv[0] * ( phi_arr[b](i+1, j, k  ) - phi_arr[b](i-1, j, k  )
                                                    + phi_arr[b](i+1, j, k-1) - phi_arr[b](i-1, j, k-1) );
                Real phi_xi_zhi = fourth * dxinv[0] * ( phi_arr[b](i+1, j, k  ) - phi_arr[b](i-1, j, k  )
                                                    + phi_arr[b](i+1, j, k+1) - phi_arr[b](i-1, j, k+1) );
                Real h_xi_zlo = Compute_h_xi_AtKface(i, j, k  , dxinv, zphys_arr[b]);
                Real h_xi_zhi = Compute_h_xi_AtKface(i, j, k+1, dxinv, zphys_arr[b]);
 
                // d/dzeta ( h_eta * dphi/deta )
                Real phi_eta_zlo = fourth * dxinv[1] * ( phi_arr[b](i, j+1, k  ) - phi_arr[b](i, j-1, k  )
                                                     + phi_arr[b](i, j+1, k-1) - phi_arr[b](i, j-1, k-1) );
                Real phi_eta_zhi = fourth * dxinv[1] * ( phi_arr[b](i, j+1, k  ) - phi_arr[b](i, j-1, k  )
                                                     + phi_arr[b](i, j+1, k+1) - phi_arr[b](i, j-1, k+1) );
                Real h_eta_zlo = Compute_h_eta_AtKface(i, j, k  , dxinv, zphys_arr[b]);
                Real h_eta_zhi = Compute_h_eta_AtKface(i, j, k+1, dxinv, zphys_arr[b]);
 
                Real detJ = Compute_h_zeta_AtCellCenter(i, j, k, dxinv, zphys_arr[b]);
 
                // same sign convention as the predictor: -div(beta grad phi) = detJ - cross terms
                rhs_arr[b](i, j, k) = detJ
                                    - dxinv[0] * ( h_xi_xhi * phi_zeta_xhi - h_xi_xlo * phi_zeta_xlo)
                                    - dxinv[1] * ( h_eta_yhi * phi_zeta_yhi - h_eta_ylo * phi_zeta_ylo)
                                    - dxinv[2] * ( h_xi_zhi * phi_xi_zhi - h_xi_zlo * phi_xi_zlo
                                                 + h_eta_zhi * phi_eta_zhi - h_eta_zlo * phi_eta_zlo);
            }
        });
    } // corrector loop
 
    // The solve only fills the valid region, so fill the ghost cells here
    fill_wall_dist_ghost_cells(*walldist[lev], geom[lev]);
}

Here is the call graph for this function:

post_timestep()

void ERF::post_timestep	(	int	nstep,
		double	time,
		double	dt_lev
	)

{
    BL_PROFILE("ERF::post_timestep()");
 
    // nstep is the 0-based index of the step just completed; the balance
    // reports by the number of completed steps, the plotfiles' numbering.
    ibseb_report(nstep + 1, time);
 
    if (cloud_chamber_budget) {
        cloud_chamber_budget->report(
            nstep + 1, time, vars_new[0][Vars::cons], geom[0],
            solverChoice.moisture_type == MoistureType::SatAdj);
    }
 
#ifdef ERF_USE_PARTICLES
    particleData.Redistribute(z_phys_nd);
#endif
 
    if (solverChoice.coupling_type == CouplingType::TwoWay)
    {
        int ncomp = vars_new[0][Vars::cons].nComp();
        for (int lev = finest_level-1; lev >= 0; lev--)
        {
            // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
            // m is the map scale factor at cell centers
            // Here we pre-divide (rho S) by m^2 before refluxing
            for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<      Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
                const Array4<const Real>  mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<const Real>  mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                if (SolverChoice::mesh_type == MeshType::ConstantDz) {
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                    });
                } else {
                    const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                    });
                }
            } // mfi
 
            // This call refluxes all "slow" cell-centered variables
            // (i.e. not density or (rho theta) or velocities) from the lev/lev+1 interface onto lev
            getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 2, 2, ncomp-2);
 
            // Here we multiply (rho S) by m^2 after refluxing
            for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
                const Array4<const Real>  mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<const Real>  mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                if (SolverChoice::mesh_type == MeshType::ConstantDz) {
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                    });
                } else {
                    const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
                    });
                }
            } // mfi
 
            // We need to do this before anything else because refluxing changes the
            // values of coarse cells underneath fine grids with the assumption they'll
            // be over-written by averaging down
            int src_comp;
            if (solverChoice.anelastic[lev]) {
                src_comp = 1;
            } else {
                src_comp = 0;
            }
            int num_comp = ncomp - src_comp;
            AverageDownTo(lev,src_comp,num_comp);
        }
    }
 
    if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
        sum_integrated_quantities(time);
        sum_derived_quantities(time);
        sum_energy_quantities(time);
    }
 
    if (solverChoice.any_perturbation()) {
        if (is_it_time_for_action(nstep, time, dt_lev0, pert_interval, -one)) {
            turbPert.debug(time);
        }
    }
 
    if (profile_int > 0 && (nstep+1) % profile_int == 0) {
        if (destag_profiles) {
            // all variables cell-centered
            write_1D_profiles(time);
        } else {
            // some variables staggered
            write_1D_profiles_stag(time);
        }
    }
 
    if (solverChoice.rad_uses_interface())
    {
        if ( rad_datalog_int > 0 &&
             (((nstep+1) % rad_datalog_int == 0) || (nstep==0)) ) {
            if (rad[0]->hasDatalog()) {
                rad[0]->WriteDataLog(time + start_time);
            }
        }
    }
 
    if (output_1d_column) {
#ifdef ERF_USE_NETCDF
      if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
      {
         int lev_column = 0;
         for (int lev = finest_level; lev >= 0; lev--)
         {
            const Box& domain_lev = geom[lev].Domain();
            Real dx_lev = geom[lev].CellSize(0);
            Real dy_lev = geom[lev].CellSize(1);
            // Cell containing (column_loc_x,column_loc_y) -- note that these locations are
            //     measured from ProbLo, which need not be at the origin
            int i_lev = domain_lev.smallEnd(0) +
                static_cast<int>(std::floor((column_loc_x - geom[lev].ProbLo(0)) / dx_lev));
            int j_lev = domain_lev.smallEnd(1) +
                static_cast<int>(std::floor((column_loc_y - geom[lev].ProbLo(1)) / dy_lev));
            // Loop runs from finest to coarsest, so stop at the first (finest) level
            //     that contains the column
            if (grids[lev].contains(IntVect(i_lev,j_lev,domain_lev.smallEnd(2)))) {
                lev_column = lev; break;
            }
         }
         writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
      }
#else
      Abort("To output 1D column files ERF must be compiled with NetCDF");
#endif
    }
 
    if (output_bndry_planes)
    {
      if (is_it_time_for_action(istep[0], time, dt_lev0, bndry_output_planes_interval, bndry_output_planes_per) &&
          time >= bndry_output_planes_start_time)
      {
         bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
         m_w2d->write_planes(istep[0], time+start_time, vars_new, is_moist);
      }
    }
 
    // Write plane/line sampler data
    if (line_sampler && is_it_time_for_action(nstep+1, time, dt_lev0, line_sampling_interval, line_sampling_per)) {
        const int nlev = static_cast<int>(vars_new.size());
        Vector<MultiFab*> tau13_lev(nlev, nullptr);
        Vector<MultiFab*> tau23_lev(nlev, nullptr);
        Vector<MultiFab*> hfx3_lev(nlev, nullptr);
        Vector<int> prognostic_tke_available(nlev, 0);
        for (int lev = 0; lev < nlev; ++lev) {
            prognostic_tke_available[lev] = solverChoice.turbChoice[lev].use_tke ? 1 : 0;
            if (lev < static_cast<int>(Tau.size())) {
                if (Tau[lev].size() > TauType::tau13) {
                    tau13_lev[lev] = Tau[lev][TauType::tau13].get();
                }
                if (Tau[lev].size() > TauType::tau23) {
                    tau23_lev[lev] = Tau[lev][TauType::tau23].get();
                }
            }
            if (lev < static_cast<int>(SFS_hfx3_lev.size())) {
                hfx3_lev[lev] = SFS_hfx3_lev[lev].get();
            }
        }
        line_sampler->get_sample_data(geom, vars_new, tau13_lev, tau23_lev, hfx3_lev,
                                      prognostic_tke_available);
        line_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
    }
    if (plane_sampler && is_it_time_for_action(nstep+1, time, dt_lev0, plane_sampling_interval, plane_sampling_per)) {
        plane_sampler->get_sample_data(geom, vars_new);
        plane_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
    }
 
    // Write station time series
    if (station_sampler &&
        is_it_time_for_action(nstep+1, time, dt_lev0, station_sampling_interval, station_sampling_per)) {
        sample_stations(static_cast<Real>(time));
    }
 
    // Moving terrain
    if ( solverChoice.terrain_type == TerrainType::MovingFittedMesh )
    {
      for (int lev = finest_level; lev >= 0; lev--)
      {
        // Copy z_phs_nd and detJ_cc at end of timestep
        MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
        MultiFab::Copy(  *detJ_cc[lev],   *detJ_cc_new[lev], 0, 0, 1,   detJ_cc[lev]->nGrowVect());
        MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
 
        make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
      }
    }
 
    if (solverChoice.io_hurricane_eye_tracker and
         (nstep == 0 or (m_plot3d_int_1 > 0 and (nstep+1)%m_plot3d_int_1 == 0))) {
 
        int levc=finest_level;
 
        HurricaneEyeTracker(solverChoice);
 
        MultiFab& U_new = vars_new[levc][Vars::xvel];
        MultiFab& V_new = vars_new[levc][Vars::yvel];
        MultiFab& W_new = vars_new[levc][Vars::zvel];
 
        MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
        average_face_to_cellcenter(mf_cc_vel,0,{AMREX_D_DECL(&U_new,&V_new,&W_new)},0);
 
        HurricaneMaxVelTracker(geom[levc],
                               mf_cc_vel,
                               t_new[0]);
 
        HurricaneMinPressureTracker(solverChoice.moisture_type,
                                    geom[levc],
                                    vars_new[levc][Vars::cons],
                                    t_new[0]);
 
        std::string filename_tracker = MakeVTKFilename_TrackerCircle(nstep);
        std::string filename_xy      = MakeVTKFilename_EyeTracker_xy(nstep);
        std::string filename_latlon  = MakeFilename_EyeTracker_latlon(nstep);
        std::string filename_maxvel  = MakeFilename_EyeTracker_maxvel(nstep);
        std::string filename_minpressure  = MakeFilename_EyeTracker_minpressure(nstep);
 
        if (ParallelDescriptor::IOProcessor()) {
            WriteVTKPolyline(filename_tracker, hurricane_tracker_circle);
            WriteVTKPolyline(filename_xy, hurricane_eye_track_xy);
            WriteLinePlot(filename_latlon, hurricane_eye_track_latlon);
            WriteLinePlot(filename_maxvel, hurricane_maxvel_vs_time);
            WriteLinePlot(filename_minpressure, hurricane_minpressure_vs_time);
        }
    }
 
} // post_timestep

Referenced by EvolveOneStep().

Here is the call graph for this function:

Here is the caller graph for this function:

post_update()

void ERF::post_update ( amrex::MultiFab &  state_mf, double  time, const amrex::Geometry &  geom  )

private

print_banner()

void ERF::print_banner ( MPI_Comm  comm, std::ostream &  out  )

static

                     : " << msg << std::endl;
}
 
void ERF::print_banner (MPI_Comm comm, std::ostream& out)
{
#ifdef AMREX_USE_MPI
    int irank = 0;
    int num_ranks = 1;
    MPI_Comm_size(comm, &num_ranks);
    MPI_Comm_rank(comm, &irank);
 
    // Only root process does the printing
    if (irank != 0) return;
#else
    amrex::ignore_unused(comm);
#endif
 
    auto etime = std::chrono::system_clock::now();
    auto etimet = std::chrono::system_clock::to_time_t(etime);
#ifndef _WIN32
    char time_buf[64];
    ctime_r(&etimet, time_buf);
    const std::string tstamp(time_buf);
#else
    char* time_buf = new char[64];
    ctime_s(time_buf, 64, &etimet);
    const std::string tstamp(time_buf);
#endif
 
    const char* githash1 = amrex::buildInfoGetGitHash(1);
    const char* githash2 = amrex::buildInfoGetGitHash(2);
 
    // clang-format off
    out << dbl_line
        << "                ERF (https://github.com/erf-model/ERF)"
        << std::endl << std::endl
        << "  ERF Git SHA      :: " << githash1 << std::endl
        << "  AMReX Git SHA    :: " << githash2 << std::endl
        << "  AMReX version    :: " << amrex::Version() << std::endl << std::endl
        << "  Exec. time       :: " << tstamp
        << "  Build time       :: " << amrex::buildInfoGetBuildDate() << std::endl
        << "  C++ compiler     :: " << amrex::buildInfoGetComp()
        << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
        << "  MPI              :: "
#ifdef AMREX_USE_MPI
        << "ON    (Num. ranks = " << num_ranks << ")" << std::endl
#else
        << "OFF " << std::endl
#endif
        << "  GPU              :: "
#ifdef AMREX_USE_GPU
        << "ON    "
#if defined(AMREX_USE_CUDA)
        << "(Backend: CUDA)"
#elif defined(AMREX_USE_HIP)
        << "(Backend: HIP)"
#elif defined(AMREX_USE_SYCL)
        << "(Backend: SYCL)"
#endif
        << std::endl
#else
        << "OFF" << std::endl
#endif
        << "  OpenMP           :: "
#ifdef AMREX_USE_OMP
        << "ON    (Num. threads = " << omp_get_max_threads() << ")" << std::endl
#else
        << "OFF" << std::endl
#endif
        << std::endl;
 

Referenced by main().

Here is the caller graph for this function:

print_error()

void ERF::print_error ( MPI_Comm  comm, const std::string &  msg  )

static

        :
    input_file   : Input file with simulation settings
 
Optional:
    param=value  : Overrides for parameters during runtime
)doc" << std::endl;
}
 
void ERF::print_error (MPI_Comm comm, const std::string& msg)
{
#ifdef AMREX_USE_MPI
    int irank = 0;
    int num_ranks = 1;
    MPI_Comm_size(comm, &num_ranks);
    MPI_Comm_rank(comm, &irank);
 

Referenced by main().

Here is the caller graph for this function:

print_summary()

static void ERF::print_summary ( std::ostream &  )

static

print_tpls()

void ERF::print_tpls ( std::ostream &  out )

static

                      ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
        << dash_line << std::endl;
    // clang-format on
}
 
void ERF::print_tpls (std::ostream& out)
{
    amrex::Vector<std::string> tpls;
 
#ifdef ERF_USE_NETCDF
    tpls.push_back(std::string("NetCDF    ") + NC_VERSION);
#endif
#ifdef AMREX_USE_SUNDIALS
    tpls.push_back(std::string("SUNDIALS     ") + SUNDIALS_VERSION);
#endif
 
    if (!tpls.empty()) {
        out << "  Enabled third-party libraries: ";
        for (const auto& val : tpls) {

print_usage()

void ERF::print_usage ( MPI_Comm  comm, std::ostream &  out  )

static

{
#ifdef AMREX_USE_MPI
    int irank = 0;
    int num_ranks = 1;
    MPI_Comm_size(comm, &num_ranks);
    MPI_Comm_rank(comm, &irank);
 
    // Only root process does the printing
    if (irank != 0) return;
#else
    amrex::ignore_unused(comm);
#endif
 
    out << R"doc(Usage:
    ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

project_initial_velocity()

void ERF::project_initial_velocity	(	int	lev,
		double	time,
		double	l_dt
	)

Project the single-level velocity field to enforce the anelastic constraint Note that the level may or may not be level zero

Parameters

lev	Level index for the velocity projection
time	Time at which coarse data are registered
l_dt	Time step used for coarse data registration

{
    BL_PROFILE("ERF::project_initial_velocity()");
    // Impose FillBoundary on density since we use it in the conversion of velocity to momentum
    vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
 
    const MultiFab* c_vfrac = nullptr;
    if (solverChoice.terrain_type == TerrainType::EB) {
        c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
    }
 
    VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
                       vars_new[lev][Vars::yvel], IntVect{0},
                       vars_new[lev][Vars::zvel], IntVect{0},
                       vars_new[lev][Vars::cons],
                       rU_new[lev], rV_new[lev], rW_new[lev],
                       Geom(lev).Domain(), domain_bcs_type, c_vfrac);
 
    Vector<MultiFab> tmp_mom;
 
    tmp_mom.push_back(MultiFab(vars_new[lev][Vars::cons],make_alias,0,1));
    tmp_mom.push_back(MultiFab(rU_new[lev],make_alias,0,1));
    tmp_mom.push_back(MultiFab(rV_new[lev],make_alias,0,1));
    tmp_mom.push_back(MultiFab(rW_new[lev],make_alias,0,1));
 
    // If at lev > 0 we must first fill the velocities at the c/f interface -- this must
    //    be done *after* the projection at lev-1
    if (lev > 0) {
        int levc = lev-1;
 
        const MultiFab* c_vfrac_crse = nullptr;
        if (solverChoice.terrain_type == TerrainType::EB) {
            c_vfrac_crse = &((get_eb(levc).get_const_factory())->getVolFrac());
        }
 
        MultiFab& S_new_crse = vars_new[levc][Vars::cons];
        MultiFab& U_new_crse = vars_new[levc][Vars::xvel];
        MultiFab& V_new_crse = vars_new[levc][Vars::yvel];
        MultiFab& W_new_crse = vars_new[levc][Vars::zvel];
 
        VelocityToMomentum(U_new_crse, IntVect{0}, V_new_crse, IntVect{0}, W_new_crse, IntVect{0}, S_new_crse,
                           rU_new[levc], rV_new[levc], rW_new[levc],
                           Geom(levc).Domain(), domain_bcs_type, c_vfrac_crse);
 
        rU_new[levc].FillBoundary(geom[levc].periodicity());
        FPr_u[levc].RegisterCoarseData({&rU_new[levc], &rU_new[levc]}, {time, time+l_dt});
 
        rV_new[levc].FillBoundary(geom[levc].periodicity());
        FPr_v[levc].RegisterCoarseData({&rV_new[levc], &rV_new[levc]}, {time, time+l_dt});
 
        rW_new[levc].FillBoundary(geom[levc].periodicity());
        FPr_w[levc].RegisterCoarseData({&rW_new[levc], &rW_new[levc]}, {time, time+l_dt});
    }
 
    // Use the same time that was registered in the FillPatcher above so that the
    // FillSet assertion (time >= crse_times[0] && time <= crse_times[1]) is satisfied
    // when called at non-zero simulation time (restart or mid-run regrid).
    project_momenta(lev, time, l_dt, tmp_mom);
 
    MomentumToVelocity(vars_new[lev][Vars::xvel],
                       vars_new[lev][Vars::yvel],
                       vars_new[lev][Vars::zvel],
                       vars_new[lev][Vars::cons],
                       rU_new[lev], rV_new[lev], rW_new[lev],
                       Geom(lev).Domain(), domain_bcs_type, c_vfrac);
 }

Here is the call graph for this function:

project_momenta()

void ERF::project_momenta	(	int	lev,
		double	l_time,
		double	l_dt,
		amrex::Vector< amrex::MultiFab > &	vars
	)

Project the single-level momenta to enforce the anelastic constraint Note that the level may or may not be level zero

Parameters

lev	Level index for the momentum projection
l_time	Time used for coarse-fine momentum fills
l_dt	Time step used in the projection update
vars	Conserved density and face-centered momenta to project

{
    BL_PROFILE("ERF::project_momenta()");
    Real l_dt = static_cast<Real>(l_dt_d);
    //
    // If at lev > 0 we must first fill the momenta at the c/f interface with interpolated coarse values
    //
    if (lev > 0) {
        PhysBCFunctNoOp null_bc;
        FPr_u[lev-1].FillSet(mom_mf[IntVars::xmom], l_time, null_bc, domain_bcs_type);
        FPr_v[lev-1].FillSet(mom_mf[IntVars::ymom], l_time, null_bc, domain_bcs_type);
        FPr_w[lev-1].FillSet(mom_mf[IntVars::zmom], l_time, null_bc, domain_bcs_type);
    }
 
    // Make sure the solver only sees the levels over which we are solving
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(mom_mf[Vars::cons].boxArray());
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap());
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
 
    Box domain = geom[lev].Domain();
 
    MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
 
    Vector<MultiFab> rhs;
    Vector<MultiFab> phi;
 
    if (solverChoice.terrain_type == TerrainType::EB)
    {
        rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
        phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1, MFInfo(), EBFactory(lev));
    } else {
        rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
        phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
    }
 
    MultiFab rhs_lev(rhs[0], make_alias, 0, 1);
    MultiFab phi_lev(phi[0], make_alias, 0, 1);
 
    auto dx    = geom[lev].CellSizeArray();
    auto dxInv = geom[lev].InvCellSizeArray();
 
    // Inflow on an x-face -- note only the normal velocity is used in the projection
    if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow") {
        (*physbcs_u[lev])(vars_new[lev][Vars::xvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                        IntVect{1,0,0},t_new[lev],BCVars::xvel_bc,false);
    }
 
    // Inflow on a  y-face -- note only the normal velocity is used in the projection
    if (domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
        (*physbcs_v[lev])(vars_new[lev][Vars::yvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                          IntVect{0,1,0},t_new[lev],BCVars::yvel_bc,false);
    }
 
    if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow" ||
        domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
 
        const MultiFab* c_vfrac = nullptr;
        if (solverChoice.terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
        }
 
        VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
                           vars_new[lev][Vars::yvel], IntVect{0},
                           vars_new[lev][Vars::zvel], IntVect{0},
                           vars_new[lev][Vars::cons],
                           mom_mf[IntVars::xmom],
                           mom_mf[IntVars::ymom],
                           mom_mf[IntVars::zmom],
                           Geom(lev).Domain(),
                           domain_bcs_type, c_vfrac);
    }
 
    // If !fixed_density, we must convert (rho u) which came in
    // to (rho0 u) which is what we will project
    if (!solverChoice.fixed_density[lev]) {
        ConvertForProjection(mom_mf[Vars::cons], r_hse,
                             mom_mf[IntVars::xmom],
                             mom_mf[IntVars::ymom],
                             mom_mf[IntVars::zmom],
                             Geom(lev).Domain(),
                             domain_bcs_type);
    }
 
    //
    // ****************************************************************************
    // Now convert the rho0w MultiFab to hold Omega rather than rhow
    // ****************************************************************************
    //
    if (solverChoice.mesh_type == MeshType::VariableDz)
    {
        // OmegaFromW below averages (rho0 u) and (rho0 v) over the faces below and above
        // each w-face, so at the lowest and highest w-face of a box it reads one face in the
        // z-ghost layer. Where a box face lies inside the domain (a BoxArray split in z) that
        // ghost face must hold the neighbouring box's momentum, in the same rho0 scaling as
        // the valid faces; VelocityToMomentum and ConvertForProjection both write the valid
        // faces only. Fill the z-ghost layer here, after the conversion, so no caller has to.
        // At the domain bottom the w-face k = 0 is set to zero and nothing below it is read;
        // at the domain top the ghost face is outside the domain, which FillBoundary leaves
        // to the extrapolation in VelocityToMomentum (or the boundary fill in the time step).
        AMREX_ALWAYS_ASSERT(mom_mf[IntVars::xmom].nGrowVect()[2] >= 1 &&
                            mom_mf[IntVars::ymom].nGrowVect()[2] >= 1);
        mom_mf[IntVars::xmom].FillBoundary(IntVect(0,0,1), geom[lev].periodicity());
        mom_mf[IntVars::ymom].FillBoundary(IntVect(0,0,1), geom[lev].periodicity());
 
        for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Array4<Real const>& rho0u_arr = mom_mf[IntVars::xmom].const_array(mfi);
            const Array4<Real const>& rho0v_arr = mom_mf[IntVars::ymom].const_array(mfi);
            const Array4<Real      >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
 
            const Array4<Real const>&     z_nd = z_phys_nd[lev]->const_array(mfi);
            const Array4<Real const>&     mf_u =  mapfac[lev][MapFacType::u_x]->const_array(mfi);
            const Array4<Real const>&     mf_v =  mapfac[lev][MapFacType::v_y]->const_array(mfi);
 
            //
            // Define Omega from (rho0 W) but store it in the same array
            //
            Box tbz = mfi.nodaltilebox(2);
            ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                if (k == 0) {
                    rho0w_arr(i,j,k) = zero;
                } else {
                    Real rho0w = rho0w_arr(i,j,k);
                    rho0w_arr(i,j,k) = OmegaFromW(i,j,k,rho0w,
                                                  rho0u_arr,rho0v_arr,
                                                  mf_u,mf_v,z_nd,dxInv);
                }
            });
        } // mfi
    }
 
    // ****************************************************************************
    // Allocate fluxes
    // ****************************************************************************
    Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
    fluxes.resize(1);
    for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
        if (solverChoice.terrain_type == TerrainType::EB) {
            fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
        } else {
            fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
        }
        //
        // A subdomain whose RHS is already below poisson_abstol is skipped below, so its
        // fluxes are never written by a solve. They are still added to the momenta after
        // the loop, so they must start at zero (FArrayBox data is uninitialized unless
        // AMREX_DEBUG is set).
        //
        fluxes[0][idim].setVal(0.0);
    }
 
    // ****************************************************************************
    // Initialize phi to 0
    // (It is essential that we do this in order to fill the corners; these are never
    //  used but the Saxpy requires the values to be initialized.)
    // ****************************************************************************
    phi_lev.setVal(0.0);
 
    // ****************************************************************************
    // Break into subdomains
    // ****************************************************************************
 
    std::map<int,int> index_map;
 
    BoxArray ba(grids[lev]);
 
    Vector<MultiFab> rhs_sub; rhs_sub.resize(1);
    Vector<MultiFab> phi_sub; phi_sub.resize(1);
    Vector<Array<MultiFab,AMREX_SPACEDIM>> fluxes_sub; fluxes_sub.resize(1);
 
    MultiFab ax_sub, ay_sub, az_sub, dJ_sub, znd_sub;
    MultiFab mfmx_sub, mfmy_sub, mfvx_sub, mfuy_sub;
 
    Array<MultiFab,AMREX_SPACEDIM> rho0_u_sub;
    Array<MultiFab const*, AMREX_SPACEDIM> rho0_u_const;
 
    // If we are going to solve with MLMG then we do not need to break this into subdomains
    bool will_solve_with_mlmg = false;
    if (solverChoice.mesh_type == MeshType::ConstantDz) {
        will_solve_with_mlmg = true;
#ifdef ERF_USE_FFT
        if (use_fft) {
            bool all_boxes_ok = true;
            for (int isub = 0; isub < subdomains[lev].size(); ++isub) {
                Box my_region(subdomains[lev][isub].minimalBox());
                bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
                if (!boxes_make_rectangle) {
                    all_boxes_ok = false;
                }
            } // isub
            if (all_boxes_ok) {
                will_solve_with_mlmg = false;
            }
        } // use_fft
#else
        if (use_fft) {
            amrex::Warning("You set use_fft=true but didn't build with USE_FFT = TRUE; defaulting to MLMG");
        }
#endif
    } // No terrain or grid stretching
 
    for (int isub = 0; isub < subdomains[lev].size(); ++isub)
    {
        BoxList bl_sub;
        Vector<int> dm_sub;
 
        for (int j = 0; j < ba.size(); j++)
        {
            if (subdomains[lev][isub].intersects(ba[j]))
            {
                //
                // Note that bl_sub.size() is effectively a counter which is
                // incremented above
                //
                // if (ParallelDescriptor::MyProc() == j) {
                // }
                index_map[bl_sub.size()] = j;
 
                bl_sub.push_back(grids[lev][j]);
                dm_sub.push_back(dmap[lev][j]);
            } // intersects
        } // loop over ba (j)
 
        BoxArray ba_sub(bl_sub);
 
        BoxList bl2d_sub = ba_sub.boxList();
        for (auto& b : bl2d_sub) {
            b.setRange(2,0);
        }
        BoxArray ba2d_sub(std::move(bl2d_sub));
 
        // Define MultiFabs that hold only the data in this particular subdomain
        if (solverChoice.terrain_type == TerrainType::EB) {
            if (ba_sub != ba) {
                amrex::Print() << "EB Solves with multiple regions is not yet supported" << std::endl;
            }
            rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs_lev.nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi_lev.nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
 
            mfmx_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_x]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            mfmy_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_y]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            mfvx_sub.define(convert(ba2d_sub, IntVect(0,1,0)), DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::v_x]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            mfuy_sub.define(convert(ba2d_sub, IntVect(1,0,0)), DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::u_y]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
              dJ_sub.define(ba_sub, DistributionMapping(dm_sub), 1, detJ_cc[lev]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
 
            for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
                fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
                                                   IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            }
            rho0_u_sub[0].define(convert(ba_sub, IntVect::TheDimensionVector(0)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::xmom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            rho0_u_sub[1].define(convert(ba_sub, IntVect::TheDimensionVector(1)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::ymom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            rho0_u_sub[2].define(convert(ba_sub, IntVect::TheDimensionVector(2)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::zmom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
        } else {
            rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
            phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
 
            mfmx_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_x]->nGrowVect(), MFInfo{}.SetAlloc(false));
            mfmy_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_y]->nGrowVect(), MFInfo{}.SetAlloc(false));
            mfvx_sub.define(convert(ba2d_sub, IntVect(0,1,0)), DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::v_x]->nGrowVect(), MFInfo{}.SetAlloc(false));
            mfuy_sub.define(convert(ba2d_sub, IntVect(1,0,0)), DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::u_y]->nGrowVect(), MFInfo{}.SetAlloc(false));
              dJ_sub.define(ba_sub, DistributionMapping(dm_sub), 1, detJ_cc[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
 
            for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
                fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
                                                   IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false));
            }
            rho0_u_sub[0].define(convert(ba_sub, IntVect::TheDimensionVector(0)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::xmom].nGrowVect(), MFInfo{}.SetAlloc(false));
            rho0_u_sub[1].define(convert(ba_sub, IntVect::TheDimensionVector(1)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::ymom].nGrowVect(), MFInfo{}.SetAlloc(false));
            rho0_u_sub[2].define(convert(ba_sub, IntVect::TheDimensionVector(2)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::zmom].nGrowVect(), MFInfo{}.SetAlloc(false));
        }
 
        // Link the new MultiFabs to the FABs in the original MultiFabs (no copy required)
        for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
        {
            int orig_index = index_map[mfi.index()];
            rhs_sub[0].setFab(mfi, FArrayBox(rhs_lev[orig_index], amrex::make_alias, 0, 1));
            phi_sub[0].setFab(mfi, FArrayBox(phi_lev[orig_index], amrex::make_alias, 0, 1));
 
            mfmx_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::m_x])[orig_index], amrex::make_alias, 0, 1));
            mfmy_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::m_y])[orig_index], amrex::make_alias, 0, 1));
            mfvx_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::v_x])[orig_index], amrex::make_alias, 0, 1));
            mfuy_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::u_y])[orig_index], amrex::make_alias, 0, 1));
 
            fluxes_sub[0][0].setFab(mfi,FArrayBox(fluxes[0][0][orig_index], amrex::make_alias, 0, 1));
            fluxes_sub[0][1].setFab(mfi,FArrayBox(fluxes[0][1][orig_index], amrex::make_alias, 0, 1));
            fluxes_sub[0][2].setFab(mfi,FArrayBox(fluxes[0][2][orig_index], amrex::make_alias, 0, 1));
 
            rho0_u_sub[0].setFab(mfi,FArrayBox(mom_mf[IntVars::xmom][orig_index], amrex::make_alias, 0, 1));
            rho0_u_sub[1].setFab(mfi,FArrayBox(mom_mf[IntVars::ymom][orig_index], amrex::make_alias, 0, 1));
            rho0_u_sub[2].setFab(mfi,FArrayBox(mom_mf[IntVars::zmom][orig_index], amrex::make_alias, 0, 1));
        }
 
        rho0_u_const[0] = &rho0_u_sub[0];
        rho0_u_const[1] = &rho0_u_sub[1];
        rho0_u_const[2] = &rho0_u_sub[2];
 
        if (solverChoice.mesh_type != MeshType::ConstantDz) {
            ax_sub.define(convert(ba_sub,IntVect(1,0,0)), DistributionMapping(dm_sub), 1,
                          ax[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
            ay_sub.define(convert(ba_sub,IntVect(0,1,0)), DistributionMapping(dm_sub), 1,
                          ay[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
            az_sub.define(convert(ba_sub,IntVect(0,0,1)), DistributionMapping(dm_sub), 1,
                          az[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
            znd_sub.define(convert(ba_sub,IntVect(1,1,1)), DistributionMapping(dm_sub), 1,
                           z_phys_nd[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
 
            for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
                int orig_index = index_map[mfi.index()];
                ax_sub.setFab(mfi, FArrayBox((*ax[lev])[orig_index], amrex::make_alias, 0, 1));
                ay_sub.setFab(mfi, FArrayBox((*ay[lev])[orig_index], amrex::make_alias, 0, 1));
                az_sub.setFab(mfi, FArrayBox((*az[lev])[orig_index], amrex::make_alias, 0, 1));
                znd_sub.setFab(mfi, FArrayBox((*z_phys_nd[lev])[orig_index], amrex::make_alias, 0, 1));
                dJ_sub.setFab(mfi, FArrayBox((*detJ_cc[lev])[orig_index], amrex::make_alias, 0, 1));
            }
        }
 
        if (solverChoice.terrain_type == TerrainType::EB) {
            for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
                int orig_index = index_map[mfi.index()];
                dJ_sub.setFab(mfi, FArrayBox((*detJ_cc[lev])[orig_index], amrex::make_alias, 0, 1));
            }
        }
 
        // ****************************************************************************
        // Compute divergence which will form RHS
        // Note that we replace "rho0w" with the contravariant momentum, Omega
        // ****************************************************************************
 
        compute_divergence(lev, rhs_sub[0], rho0_u_const, mfmx_sub, mfmy_sub, mfvx_sub, mfuy_sub,
                           ax_sub, ay_sub, dJ_sub, geom_tmp[0]);
 
        Real rhsnorm;
 
        // Max norm over the entire MultiFab
        rhsnorm = rhs_sub[0].norm0();
 
        if (mg_verbose > 0) {
            bool local = false;
            Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
            Print() << "Max/L2 norm of divergence before solve in subdomain " << isub << " at level " << lev << " : " << rhsnorm << " " <<
                        rhs_sub[0].norm2() << " and volume-weighted sum " << sum << std::endl;
        }
 
        if (lev == 0 && solverChoice.use_real_bcs)
        {
            // We always use VariableDz if use_real_bcs is true
            AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::VariableDz);
 
            // Note that we always impose the projections one level at a time so this will always be a vector of length 1
            Array<MultiFab*, AMREX_SPACEDIM> rho0_u_vec =
               {&mom_mf[IntVars::xmom], &mom_mf[IntVars::ymom], &mom_mf[IntVars::zmom]};
            Array<MultiFab*, AMREX_SPACEDIM> area_vec = {ax[lev].get(), ay[lev].get(), az[lev].get()};
            //
            // Modify ax,ay,ax to include the map factors as used in the divergence calculation
            // We do this here so that it is seen in the call to enforceInOutSolvability
            //
            for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
            {
                Box xbx = mfi.nodaltilebox(0);
                Box ybx = mfi.nodaltilebox(1);
                Box zbx = mfi.nodaltilebox(2);
                const Array4<Real      >& ax_ar = ax[lev]->array(mfi);
                const Array4<Real      >& ay_ar = ay[lev]->array(mfi);
                const Array4<Real      >& az_ar = az[lev]->array(mfi);
                const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(xbx,ybx,zbx,
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ax_ar(i,j,k) /= mf_uy(i,j,0);
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ay_ar(i,j,k) /= mf_vx(i,j,0);
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
                });
            } // mfi
 
            if (mg_verbose > 0) {
                Print() << "Calling enforceInOutSolvability" << std::endl;
            }
            enforceInOutSolvability(lev, rho0_u_vec, area_vec, geom[lev]);
 
            //
            // Return ax,ay,ax to their original definition
            //
            for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
            {
                Box xbx = mfi.nodaltilebox(0);
                Box ybx = mfi.nodaltilebox(1);
                Box zbx = mfi.nodaltilebox(2);
                const Array4<Real      >& ax_ar = ax[lev]->array(mfi);
                const Array4<Real      >& ay_ar = ay[lev]->array(mfi);
                const Array4<Real      >& az_ar = az[lev]->array(mfi);
                const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(xbx,ybx,zbx,
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ax_ar(i,j,k) *= mf_uy(i,j,0);
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ay_ar(i,j,k) *= mf_vx(i,j,0);
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
                });
            } // mfi
 
            compute_divergence(lev, rhs_lev, rho0_u_const, *mapfac[lev][MapFacType::m_x],
                               *mapfac[lev][MapFacType::m_y], *mapfac[lev][MapFacType::v_x],
                               *mapfac[lev][MapFacType::u_y], *ax[lev], *ay[lev],
                               *detJ_cc[lev], geom_tmp[0]);
 
            // Re-define max norm over the entire MultiFab
            rhsnorm = rhs_lev.norm0();
 
            if (mg_verbose > 0)
            {
                bool local = false;
                Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
                Print() << "Max/L2 norm of divergence before solve at level " << lev << " : " << rhsnorm << " " <<
                            rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
            }
        } // lev 0 && use_real_bcs
 
        // *******************************************************************************************
        // Enforce solvability if the problem is singular (i.e all sides Neumann or periodic)
        // Note that solves at lev > 0 are always singular because we impose Neumann bc's on all sides
        // *******************************************************************************************
        bool is_singular = true;
        if (lev == 0) {
            if ( (domain_bc_type[0] == "Outflow" || domain_bc_type[0] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
            if ( (domain_bc_type[1] == "Outflow" || domain_bc_type[1] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
            if ( (domain_bc_type[3] == "Outflow" || domain_bc_type[3] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
            if ( (domain_bc_type[4] == "Outflow" || domain_bc_type[4] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
            if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open")                               ) is_singular = false;
        } else {
            Box my_region(subdomains[lev][isub].minimalBox());
            if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open") && (my_region.bigEnd(2) == domain.bigEnd(2)) ) is_singular = false;
        }
 
        if (is_singular)
        {
            bool local = false;
            Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
 
            Real vol;
            if (solverChoice.mesh_type == MeshType::ConstantDz) {
                vol = rhs_sub[0].boxArray().numPts();
            } else {
                vol = dJ_sub.sum();
            }
 
            sum /= (vol * dx[0] * dx[1] * dx[2]);
 
            for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
            {
                rhs_sub[0][mfi.index()].template minus<RunOn::Device>(sum);
            }
            if (mg_verbose > 0) {
                amrex::Print() << " Subtracting " << sum << " from rhs in subdomain " << isub << std::endl;
 
                sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
                Print() << "Sum after subtraction " << sum << " in subdomain " << isub << std::endl;
            }
 
        } // if is_singular
 
        rhsnorm = rhs_sub[0].norm0();
 
        // ****************************************************************************
        // No need to build the solver if RHS == 0
        // ****************************************************************************
        if (rhsnorm <= solverChoice.poisson_abstol) continue; // this subdomain only
 
        double start_step = ParallelDescriptor::second();
 
        if (mg_verbose > 0) {
            amrex::Print() << " Solving in subdomain " << isub << " of " << subdomains[lev].size() << " bins at level " << lev << std::endl;
        }
 
        if (solverChoice.mesh_type == MeshType::VariableDz) {
            //
            // Modify ax,ay,ax to include the map factors as used in the divergence calculation
            // We do this here to set the coefficients used in the stencil -- the extra factor
            // of the mapfac comes from the gradient
            //
            for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
            {
                Box xbx = mfi.nodaltilebox(0);
                Box ybx = mfi.nodaltilebox(1);
                Box zbx = mfi.nodaltilebox(2);
                const Array4<Real      >& ax_ar = ax_sub.array(mfi);
                const Array4<Real      >& ay_ar = ay_sub.array(mfi);
                const Array4<Real      >& az_ar = az_sub.array(mfi);
                const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
                const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
                const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(xbx,ybx,zbx,
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ax_ar(i,j,k) *= (mf_ux(i,j,0) / mf_uy(i,j,0));
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ay_ar(i,j,k) *= (mf_vy(i,j,0) / mf_vx(i,j,0));
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
                });
            } // mfi
        }
 
        if (solverChoice.terrain_type != TerrainType::EB) {
 
#ifdef ERF_USE_FFT
        Box my_region(subdomains[lev][isub].minimalBox());
#endif
 
        // ****************************************************************************
        // No terrain or grid stretching
        // ****************************************************************************
        if (solverChoice.mesh_type == MeshType::ConstantDz) {
            if (will_solve_with_mlmg) {
                solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub, geom[lev], ref_ratio, domain_bc_type,
                                mg_verbose, solverChoice.poisson_reltol, solverChoice.poisson_abstol);
            } else {
#ifdef ERF_USE_FFT
                solve_with_fft(lev, isub, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
#endif
            }
        } // No terrain or grid stretching
        // ****************************************************************************
        // Grid stretching (flat terrain)
        // ****************************************************************************
        else if (solverChoice.mesh_type == MeshType::StretchedDz) {
#ifndef ERF_USE_FFT
            amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT solver");
#else
            bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
            if (!boxes_make_rectangle) {
                amrex::Abort("FFT won't work unless the union of boxes is rectangular");
            } else {
                if (!use_fft) {
                    amrex::Warning("Using FFT even though you didn't set use_fft to true; it's the best choice");
                }
                solve_with_fft(lev, isub, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
            }
#endif
        } // grid stretching
 
        // ****************************************************************************
        // General terrain
        // ****************************************************************************
        else if (solverChoice.mesh_type == MeshType::VariableDz) {
#ifdef ERF_USE_FFT
            bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
            if (!boxes_make_rectangle) {
                amrex::Abort("FFT preconditioner for GMRES won't work unless the union of boxes is rectangular");
            } else {
                solve_with_gmres(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0], ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
            }
#else
            amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT preconditioner for GMRES");
#endif
 
            //
            // Restore ax,ay,ax to their original definitions
            //
            for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
            {
                Box xbx = mfi.nodaltilebox(0);
                Box ybx = mfi.nodaltilebox(1);
                Box zbx = mfi.nodaltilebox(2);
                const Array4<Real      >& ax_ar = ax_sub.array(mfi);
                const Array4<Real      >& ay_ar = ay_sub.array(mfi);
                const Array4<Real      >& az_ar = az_sub.array(mfi);
                const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
                const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
                const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(xbx,ybx,zbx,
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ax_ar(i,j,k) *= (mf_uy(i,j,0) / mf_ux(i,j,0));
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ay_ar(i,j,k) *= (mf_vx(i,j,0) / mf_vy(i,j,0));
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
                });
            } // mfi
 
        } // MeshType::VariableDz
 
        // ****************************************************************************
        // Print time in solve
        // ****************************************************************************
        double end_step = ParallelDescriptor::second();
        if (mg_verbose > 0) {
            amrex::Print() << "Time in solve " << end_step - start_step << std::endl;
        }
 
        } // not EB
    } // loop over subdomains (i)
 
    // ****************************************************************************
    // When using multigrid we can solve for all of the level at once, even if there
    //      are disjoint regions
    // ****************************************************************************
    if (solverChoice.terrain_type == TerrainType::EB) {
        double start_step_eb = ParallelDescriptor::second();
        solve_with_EB_mlmg(lev, rhs_sub, phi_sub, fluxes_sub,
                           *(get_eb(lev).get_const_factory()),
                           *(get_eb(lev).get_u_const_factory()),
                           *(get_eb(lev).get_v_const_factory()),
                           *(get_eb(lev).get_w_const_factory()),
                           geom[lev], ref_ratio, domain_bc_type,
                           mg_verbose, solverChoice.poisson_reltol, solverChoice.poisson_abstol);
        double end_step_eb = ParallelDescriptor::second();
        if (mg_verbose > 0) {
            amrex::Print() << "Time in solve " << end_step_eb - start_step_eb << std::endl;
        }
    }
 
    // ****************************************************************************
    // Subtract dt grad(phi) from the momenta (rho0u, rho0v, Omega)
    // ****************************************************************************
    MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0);
    MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0);
    MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0);
 
    // ****************************************************************************
    // Define gradp from fluxes -- note that fluxes is dt * change in Gp
    //   (weighted by map factor!)
    // ****************************************************************************
    MultiFab::Saxpy(gradp[lev][GpVars::gpx],-one/l_dt,fluxes[0][0],0,0,1,0);
    MultiFab::Saxpy(gradp[lev][GpVars::gpy],-one/l_dt,fluxes[0][1],0,0,1,0);
    MultiFab::Saxpy(gradp[lev][GpVars::gpz],-one/l_dt,fluxes[0][2],0,0,1,0);
 
    gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
    gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
    gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
 
    //
    // This call is only to verify the divergence after the solve
    // It is important we do this before computing the rho0w_arr from Omega back to rho0w
    //
    // ****************************************************************************
    // THIS IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE
    // ****************************************************************************
    //
    if (mg_verbose > 0)
    {
        rho0_u_const[0] = &mom_mf[IntVars::xmom];
        rho0_u_const[1] = &mom_mf[IntVars::ymom];
        rho0_u_const[2] = &mom_mf[IntVars::zmom];
 
        compute_divergence(lev, rhs_lev, rho0_u_const, *mapfac[lev][MapFacType::m_x],
                           *mapfac[lev][MapFacType::m_y], *mapfac[lev][MapFacType::v_x],
                           *mapfac[lev][MapFacType::u_y], *ax[lev], *ay[lev],
                           *detJ_cc[lev], geom_tmp[0]);
 
        bool local = false;
        Real sum = volWgtSumMF(lev,rhs_lev,0,*detJ_cc[lev],*mapfac[lev][MapFacType::m_x],*mapfac[lev][MapFacType::m_y],false,local);
 
        if (mg_verbose > 0) {
            Print() << "Max/L2 norm of divergence after  solve at level " << lev << " : " << rhs_lev.norm0() << " " <<
                        rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
        }
 
#if 0
         // FOR DEBUGGING ONLY
         for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
            const Array4<Real const>& rhs_arr = rhs_lev.const_array(mfi);
            Box bx = mfi.validbox();
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                if (std::abs(rhs_arr(i,j,k)) > Real(1.e-10)) {
                    amrex::AllPrint() << "RHS after solve at " <<
                                          IntVect(i,j,k) << " " << rhs_arr(i,j,k) << std::endl;
                }
            });
         } // mfi
#endif
 
    } // mg_verbose
 
    //
    // ****************************************************************************
    // Now convert the rho0w MultiFab back to holding (rho0w) rather than Omega
    // ****************************************************************************
    //
    if (solverChoice.mesh_type == MeshType::VariableDz)
    {
        // WFromOmega reads the same z-ghost faces of (rho0 u) and (rho0 v) as OmegaFromW above,
        // but the fluxes were added to the valid faces only. Fill the ghost faces again, or the
        // lowest and highest w-face of a box inside the domain (a BoxArray split in z) would be
        // converted back with the horizontal momenta from before the projection.
        mom_mf[IntVars::xmom].FillBoundary(IntVect(0,0,1), geom[lev].periodicity());
        mom_mf[IntVars::ymom].FillBoundary(IntVect(0,0,1), geom[lev].periodicity());
 
        for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
             Box tbz = mfi.nodaltilebox(2);
             const Array4<Real      >& rho0u_arr = mom_mf[IntVars::xmom].array(mfi);
             const Array4<Real      >& rho0v_arr = mom_mf[IntVars::ymom].array(mfi);
             const Array4<Real      >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
             const Array4<Real const>&      z_nd = z_phys_nd[lev]->const_array(mfi);
             const Array4<Real const>&      mf_u =  mapfac[lev][MapFacType::u_x]->const_array(mfi);
             const Array4<Real const>&      mf_v =  mapfac[lev][MapFacType::v_y]->const_array(mfi);
             ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                 Real omega = rho0w_arr(i,j,k);
                 rho0w_arr(i,j,k) = WFromOmega(i,j,k,omega,
                                               rho0u_arr,rho0v_arr,
                                               mf_u,mf_v,z_nd,dxInv);
             });
        } // mfi
    }
 
    // If !fixed_density, we must convert (rho0 u) back
    // to (rho0 u) which is what we will pass back out
    if (!solverChoice.fixed_density[lev]) {
        ConvertForProjection(r_hse, mom_mf[Vars::cons],
                             mom_mf[IntVars::xmom],
                             mom_mf[IntVars::ymom],
                             mom_mf[IntVars::zmom],
                             Geom(lev).Domain(),
                             domain_bcs_type);
    }
 
    // ****************************************************************************
    // Update pressure variable with phi -- note that phi is dt * change in pressure
    // ****************************************************************************
    MultiFab::Saxpy(pp_inc[lev], one/l_dt, phi_lev,0,0,1,1);
}

Here is the call graph for this function:

project_velocity_tb()

void ERF::project_velocity_tb	(	int	lev,
		double	dt,
		amrex::Vector< amrex::MultiFab > &	vars
	)

Project the single-level velocity field to enforce incompressibility with a thin body

Parameters

lev	Level index for the thin-body projection
dt	Time step used in the projection update
vars	State MultiFabs containing the velocity fields to project

{
    BL_PROFILE("ERF::project_velocity_tb()");
    Real l_dt = static_cast<Real>(l_dt_d);
    AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
 
    // Make sure the solver only sees the levels over which we are solving
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(vmf[Vars::cons].boxArray());
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(vmf[Vars::cons].DistributionMap());
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
 
    // Use the default settings
    LPInfo info;
    std::unique_ptr<MLPoisson> p_mlpoisson;
#if 0
    if (overset_imask[0]) {
        // Add overset mask around thin body
        p_mlpoisson = std::make_unique<MLPoisson>(geom, grids, dmap, GetVecOfConstPtrs(overset_imask), info);
    }
    else
#endif
    {
        // Use the default settings
        p_mlpoisson = std::make_unique<MLPoisson>(geom_tmp, ba_tmp, dm_tmp, info);
    }
 
    auto bclo = get_lo_projection_bc(geom[lev],domain_bc_type);
    auto bchi = get_hi_projection_bc(geom[lev],domain_bc_type);
 
    bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true;
    p_mlpoisson->setDomainBC(bclo, bchi);
 
    if (lev > 0) {
        p_mlpoisson->setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
    }
 
    p_mlpoisson->setLevelBC(0, nullptr);
 
    Vector<MultiFab> rhs;
    Vector<MultiFab> phi;
    Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
    Vector<Array<MultiFab,AMREX_SPACEDIM> > deltaf; // f^* - f^{n-1}
    Vector<Array<MultiFab,AMREX_SPACEDIM> > u_plus_dtdf; // u + dt*deltaf
 
    // Used to pass array of const MFs to ComputeDivergence
    Array<MultiFab const*, AMREX_SPACEDIM> u;
 
    rhs.resize(1);
    phi.resize(1);
    fluxes.resize(1);
    deltaf.resize(1);
    u_plus_dtdf.resize(1);
 
    rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
    phi[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
    rhs[0].setVal(0.0);
    phi[0].setVal(0.0);
 
    for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
             fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
        u_plus_dtdf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
 
             deltaf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
             deltaf[0][idim].setVal(0.0); // start with f^* == f^{n-1}
    }
 
#if 0
    // DEBUG
    u[0] = &(vmf[Vars::xvel]);
    u[1] = &(vmf[Vars::yvel]);
    u[2] = &(vmf[Vars::zvel]);
    computeDivergence(rhs[0], u, geom[0]);
    Print() << "Max norm of divergence before solve at level 0 : " << rhs[0].norm0() << std::endl;
#endif
 
    for (int itp = 0; itp < solverChoice.ncorr; ++itp)
    {
        // Calculate u + dt*deltaf
        for (int idim = 0; idim < 3; ++idim) {
            MultiFab::Copy(u_plus_dtdf[0][idim], deltaf[0][idim], 0, 0, 1, 0);
            u_plus_dtdf[0][idim].mult(-l_dt,0,1,0);
        }
        MultiFab::Add(u_plus_dtdf[0][0], vmf[Vars::xvel], 0, 0, 1, 0);
        MultiFab::Add(u_plus_dtdf[0][1], vmf[Vars::yvel], 0, 0, 1, 0);
        MultiFab::Add(u_plus_dtdf[0][2], vmf[Vars::zvel], 0, 0, 1, 0);
 
        u[0] = &(u_plus_dtdf[0][0]);
        u[1] = &(u_plus_dtdf[0][1]);
        u[2] = &(u_plus_dtdf[0][2]);
        computeDivergence(rhs[0], u, geom_tmp[0]);
 
#if 0
        // DEBUG
        if (itp==0) {
            for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real const>& divU = rhs[0].const_array(mfi);
                const Array4<Real const>& uarr = vmf[Vars::xvel].const_array(mfi);
                const Array4<Real const>& varr = vmf[Vars::yvel].const_array(mfi);
                const Array4<Real const>& warr = vmf[Vars::zvel].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    if ((i>=120) && (i<=139) && (j==0) && ((k>=127)&&(k<=128))) {
                        amrex::AllPrint() << "before project div"<<IntVect(i,j,k)<<" = "<< divU(i,j,k)
                            << " u: " << uarr(i,j,k) << " " << uarr(i+1,j,k)
                            << " v: " << varr(i,j,k) << " " << varr(i,j+1,k)
                            << " w: " << warr(i,j,k) << " " << warr(i,j,k+1)
                            << std::endl;
                    }
                });
            }
        }
#endif
 
        // If all Neumann BCs, adjust RHS to make sure we can converge
        if (need_adjust_rhs) {
            bool local = false;
            Real offset = volWgtSumMF(lev,rhs[0],0,*detJ_cc[lev],*mapfac[lev][MapFacType::m_x],*mapfac[lev][MapFacType::m_y],false,local);
            // amrex::Print() << "Poisson solvability offset = " << offset << std::endl;
            rhs[0].plus(-offset, 0, 1);
        }
 
        // Initialize phi to 0
        phi[0].setVal(0.0);
 
        MLMG mlmg(*p_mlpoisson);
        int max_iter = 100;
        mlmg.setMaxIter(max_iter);
 
        mlmg.setVerbose(mg_verbose);
        //mlmg.setBottomVerbose(mg_verbose);
 
        // solve for dt*p
        mlmg.solve(GetVecOfPtrs(phi),
                   GetVecOfConstPtrs(rhs),
                   solverChoice.poisson_reltol,
                   solverChoice.poisson_abstol);
 
        mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
 
        // Calculate new intermediate body force with updated gradp
        if (thin_xforce[lev]) {
            MultiFab::Copy(   deltaf[0][0], fluxes[0][0], 0, 0, 1, 0);
            ApplyInvertedMask(deltaf[0][0], *xflux_imask[0]);
        }
        if (thin_yforce[lev]) {
            MultiFab::Copy(   deltaf[0][1], fluxes[0][1], 0, 0, 1, 0);
            ApplyInvertedMask(deltaf[0][1], *yflux_imask[0]);
        }
        if (thin_zforce[lev]) {
            MultiFab::Copy(   deltaf[0][2], fluxes[0][2], 0, 0, 1, 0);
            ApplyInvertedMask(deltaf[0][2], *zflux_imask[0]);
        }
 
        // DEBUG
        //        for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
        //        {
        //            const Box& bx = mfi.tilebox();
        //            const Array4<Real const>& dfz_arr = deltaf[0][2].const_array(mfi);
        //            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        //            {
        //                if ((i>=120) && (i<=139) && (j==0) && (k==128)) {
        //                    amrex::AllPrint()
        //                        << " piter" << itp
        //                        << " dfz"<<IntVect(i,j,k)<<" = "<< dfz_arr(i,j,k)
        //                        << std::endl;
        //                }
        //            });
        //        }
 
        // Update pressure variable with phi -- note that phi is change in pressure, not the full pressure
        MultiFab::Saxpy(pp_inc[lev], one, phi[0],0,0,1,0);
 
        // Subtract grad(phi) from the velocity components
        Real beta = one;
        MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
        MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
        MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
        if (thin_xforce[lev]) {
            ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
        }
        if (thin_yforce[lev]) {
            ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
        }
        if (thin_zforce[lev]) {
            ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
        }
    } // itp: pressure-force iterations
 
    // ****************************************************************************
    // Define gradp from fluxes -- note that fluxes is dt * change in Gp
    // ****************************************************************************
    MultiFab::Saxpy(gradp[lev][GpVars::gpx],-one/l_dt,fluxes[0][0],0,0,1,0);
    MultiFab::Saxpy(gradp[lev][GpVars::gpy],-one/l_dt,fluxes[0][1],0,0,1,0);
    MultiFab::Saxpy(gradp[lev][GpVars::gpz],-one/l_dt,fluxes[0][2],0,0,1,0);
 
    gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
    gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
    gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
 
    // Subtract grad(phi) from the velocity components
//    Real beta = one;
//    for (int ilev = lev_min; ilev <= lev_max; ++ilev) {
//        MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
//        MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
//        MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
//        if (thin_xforce[lev]) {
//            ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
//        }
//        if (thin_yforce[lev]) {
//            ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
//        }
//        if (thin_zforce[lev]) {
//            ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
//        }
//    }
 
#if 0
    // Confirm that the velocity is now divergence free
    u[0] = &(vmf[Vars::xvel]);
    u[1] = &(vmf[Vars::yvel]);
    u[2] = &(vmf[Vars::zvel]);
    computeDivergence(rhs[0], u, geom_tmp[0]);
    Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
 
#endif
}

Here is the call graph for this function:

read_box_for_refinement()

void ERF::read_box_for_refinement	(	std::string &	ref_prefix,
		int &	lev_for_box,
		amrex::RealBox &	real_box
	)

Read the refinement box from the inputs file.

Parameters

[in]	ref_prefix	Prefix for parsing input parameters.
[out]	lev_for_box	Level associated with the refinement box.
[out]	real_box	Real-space coordinates of the refinement box.

{
    ParmParse ppr(ref_prefix);
 
    int num_real_lo      = ppr.countval("in_box_lo");
    int num_real_hi      = ppr.countval("in_box_hi");
 
    int num_indx_lo      = ppr.countval("in_box_lo_indices");
    int num_indx_hi      = ppr.countval("in_box_hi_indices");
 
    int num_indx_lo_crse = ppr.countval("in_box_lo_indices_crse");
    int num_indx_hi_crse = ppr.countval("in_box_hi_indices_crse");
 
    AMREX_ALWAYS_ASSERT( (num_real_lo      == num_real_hi)      && (num_real_lo      == 0 || num_real_lo      >= 2) );
    AMREX_ALWAYS_ASSERT( (num_indx_lo      == num_indx_hi)      && (num_indx_lo      == 0 || num_indx_lo      >= 2) );
    AMREX_ALWAYS_ASSERT( (num_indx_lo_crse == num_indx_hi_crse) && (num_indx_lo_crse == 0 || num_indx_lo_crse >= 2) );
 
    // Problem low and high (in real not index space) are the same at all levels
    const Real* plo = geom[0].ProbLo();
    const Real* phi = geom[0].ProbHi();
    if ( !((num_real_lo >= AMREX_SPACEDIM-1 && num_indx_lo == 0 && num_indx_lo_crse == 0) ||
           (num_indx_lo >= AMREX_SPACEDIM-1 && num_real_lo == 0 && num_indx_lo_crse == 0) ||
           (num_indx_lo ==              0   && num_real_lo == 0 && num_indx_lo_crse == 0) ||
           (num_indx_lo_crse >= AMREX_SPACEDIM-1 && num_real_lo == 0 && num_indx_lo == 0)
        ) )
    {
        amrex::Abort("Must only specify box for refinement using real OR index space with fine/coarse grid indices");
    }
 
    lev_for_box = max_level;
    if (num_real_lo > 0) {
        ppr.queryAdd("max_level",lev_for_box);
    } else if (num_indx_lo > 0 || num_indx_lo_crse > 0) {
        ppr.get("max_level",lev_for_box);
    }
 
    // If this indicator doesn't specify a box at all then there is nothing to do here --
    //    in particular we must not touch num_boxes_at_level / boxes_at_level, since
    //    lev_for_box has just defaulted to max_level and we would wipe out the boxes
    //    registered by a previously processed indicator
    if (num_real_lo == 0 && num_indx_lo == 0 && num_indx_lo_crse == 0) {
        return;
    }
 
    // Validate lev_for_box *before* using it to index the per-level containers, which
    //    are only sized (max_level+1).  We tolerate an out-of-range level -- e.g. an
    //    inputs file written for more levels than we are currently running with --
    //    but we must not read or write out of bounds, so we simply ignore this box.
    if (lev_for_box < 1 || lev_for_box > max_level) {
        Print() << "Refinement indicator " << ref_prefix << " specifies max_level = " << lev_for_box
                << " which is outside the range of refined levels [1," << max_level
                << "] in this run -- ignoring the box specified there" << std::endl;
        return;
    }
 
    // Clear stale data at this level (this matters when we are moving boxes and
    //    therefore re-reading them at every regrid)
    num_boxes_at_level[lev_for_box] = 0;
    boxes_at_level[lev_for_box].clear();
 
    if (num_real_lo > 0) {
 
        std::vector<Real> rbox_lo(3), rbox_hi(3);
        if (lev_for_box > 0 && lev_for_box <= max_level)
        {
            if (n_error_buf[0] != IntVect::TheZeroVector()) {
                amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
            }
 
            ppr.getarr("in_box_lo",rbox_lo,0,num_real_lo);
            ppr.getarr("in_box_hi",rbox_hi,0,num_real_hi);
 
            if (rbox_lo[0] < plo[0]) rbox_lo[0] = plo[0];
            if (rbox_lo[1] < plo[1]) rbox_lo[1] = plo[1];
            if (rbox_hi[0] > phi[0]) rbox_hi[0] = phi[0];
            if (rbox_hi[1] > phi[1]) rbox_hi[1] = phi[1];
            if (num_real_lo < AMREX_SPACEDIM) {
                rbox_lo[2] = plo[2];
                rbox_hi[2] = phi[2];
            }
 
            const Box& domain = geom[lev_for_box].Domain();
 
            real_box = RealBox(&(rbox_lo[0]),&(rbox_hi[0]));
 
            Print() << "Realbox read in and intersected laterally with domain is " << real_box << std::endl;
 
            num_boxes_at_level[lev_for_box] += 1;
 
            int ilo, jlo, klo;
            int ihi, jhi, khi;
            const auto* dx  = geom[lev_for_box].CellSize();
            ilo = static_cast<int>((rbox_lo[0] - plo[0])/dx[0]);
            jlo = static_cast<int>((rbox_lo[1] - plo[1])/dx[1]);
            ihi = static_cast<int>((rbox_hi[0] - plo[0])/dx[0]-1);
            jhi = static_cast<int>((rbox_hi[1] - plo[1])/dx[1]-1);
            if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                // Search for k indices corresponding to nominal grid
                // AGL heights
                klo = domain.smallEnd(2) - 1;
                khi = domain.smallEnd(2) - 1;
 
                if (rbox_lo[2] <= zlevels_stag[lev_for_box][domain.smallEnd(2)])
                {
                    klo = domain.smallEnd(2);
                }
                else
                {
                    for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
                        if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
                            klo = k-1;
                            break;
                        }
                    }
                }
                AMREX_ASSERT(klo >= domain.smallEnd(2));
 
                if (rbox_hi[2] >= zlevels_stag[lev_for_box][domain.bigEnd(2)+1])
                {
                    khi = domain.bigEnd(2);
                }
                else
                {
                    for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
                        if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
                            khi = k-1;
                            break;
                        }
                    }
                }
                AMREX_ASSERT((khi <= domain.bigEnd(2)) && (khi > klo));
 
                // Need to update real_box because tagging is based on
                // the initial _un_deformed grid
                real_box = RealBox(plo[0]+ ilo   *dx[0], plo[1]+ jlo   *dx[1], plo[2]+ klo   *dx[2],
                                   plo[0]+(ihi+1)*dx[0], plo[1]+(jhi+1)*dx[1], plo[2]+(khi+1)*dx[2]);
            } else {
                klo = static_cast<int>((rbox_lo[2] - plo[2])/dx[2]);
                khi = static_cast<int>((rbox_hi[2] - plo[2])/dx[2]-1);
            }
 
            // Snap box indices to ref_ratio alignment (round lo down, hi up)
            {
                const auto& rr = ref_ratio[lev_for_box-1];
                auto snap_lo = [](int idx, int r) { return idx - (idx % r + r) % r; };
                auto snap_hi = [](int idx_p1, int r) { // idx_p1 = ihi+1
                    int rem = idx_p1 % r;
                    return (rem == 0) ? idx_p1 - 1 : idx_p1 + (r - rem) - 1;
                };
                int ilo_old = ilo, jlo_old = jlo, klo_old = klo;
                int ihi_old = ihi, jhi_old = jhi, khi_old = khi;
                ilo = snap_lo(ilo, rr[0]);
                jlo = snap_lo(jlo, rr[1]);
                klo = snap_lo(klo, rr[2]);
                ihi = snap_hi(ihi+1, rr[0]);
                jhi = snap_hi(jhi+1, rr[1]);
                khi = snap_hi(khi+1, rr[2]);
                if (ilo != ilo_old || ihi != ihi_old ||
                    jlo != jlo_old || jhi != jhi_old ||
                    klo != klo_old || khi != khi_old) {
                    amrex::Print() << "Refinement box indices snapped to ref_ratio alignment:\n"
                                   << "  ilo: " << ilo_old << " -> " << ilo
                                   << "  ihi: " << ihi_old << " -> " << ihi
                                   << "  jlo: " << jlo_old << " -> " << jlo
                                   << "  jhi: " << jhi_old << " -> " << jhi
                                   << "  klo: " << klo_old << " -> " << klo
                                   << "  khi: " << khi_old << " -> " << khi << "\n";
                }
            }
 
            Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
 
            bool using_pbl = (solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYJ      ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNN25   ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNNEDMF ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::YSU      ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MRF);
 
            if ( using_pbl && ( (rbox_lo[2] > plo[2]) || (rbox_hi[2] < phi[2]) ) ) {
                amrex::Print() << "PBL models need refinement boxes that go from the bottom to the top of the domain for calculation of PBLH" << std::endl;
                amrex::Print() << "Please set in_box_lo to geometry.prob_lo in z and in_box_hi to geometry.prob_hi in z and try again" << std::endl;
                amrex::Abort();
            }
 
            boxes_at_level[lev_for_box].push_back(bx);
            Print() << "Saving in 'boxes at level' as " << bx << std::endl;
        } // (lev_for_box > 0 && lev_for_box <= max_level)
 
        if (solverChoice.init_type == InitType::WRFInput) {
            amrex::Print() << "Size of num_boxes " << num_boxes_at_level.size() << std::endl;
            amrex::Print() << "Size of num_files " << num_files_at_level.size() << std::endl;
            amrex::Print() << "Consider lev_for_box = " << lev_for_box << std::endl;
            amrex::Print() << "Number of boxes at level  " << num_boxes_at_level[lev_for_box] << std::endl;
            amrex::Print() << "Number of available files " << num_files_at_level[lev_for_box] << std::endl;
            if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                amrex::Print() << "Will need to rely on refinement criteria from inputs file" << std::endl;
            }
        }
 
    } else if (num_indx_lo > 0) {
 
        std::vector<int> box_lo(3), box_hi(3);
        if (lev_for_box > 0 && lev_for_box <= max_level)
        {
            if (n_error_buf[0] != IntVect::TheZeroVector()) {
                amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
            }
 
            ppr.getarr("in_box_lo_indices",box_lo,0,num_indx_lo);
            ppr.getarr("in_box_hi_indices",box_hi,0,num_indx_hi);
 
            if (num_indx_lo < AMREX_SPACEDIM) {
                box_lo[2] = geom[lev_for_box].Domain().smallEnd(2);
                box_hi[2] = geom[lev_for_box].Domain().bigEnd(2);
            }
 
            Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
            const Box& domain = geom[lev_for_box].Domain();
 
            if (!domain.contains(bx)) {
                amrex::Print() << "\n";
                amrex::Print() << "Box specified       is " << bx << std::endl;
                amrex::Print() << "But domain at level is " << domain << std::endl;
                amrex::Error("Specified box doesn't fit in the domain");
            }
 
            const auto* dx  = geom[lev_for_box].CellSize();
            real_box = RealBox(plo[0]+ box_lo[0]   *dx[0], plo[1]+ box_lo[1]   *dx[1], plo[2]+ box_lo[2]   *dx[2],
                               plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
 
            Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
            num_boxes_at_level[lev_for_box] += 1;
 
            // Snap box indices to ref_ratio alignment (round lo down, hi up)
            {
                const auto& rr = ref_ratio[lev_for_box-1];
                auto snap_lo_fn = [](int idx, int r) { return idx - (idx % r + r) % r; };
                auto snap_hi_fn = [](int idx_p1, int r) {
                    int rem = idx_p1 % r;
                    return (rem == 0) ? idx_p1 - 1 : idx_p1 + (r - rem) - 1;
                };
                int lo_old[3] = {box_lo[0], box_lo[1], box_lo[2]};
                int hi_old[3] = {box_hi[0], box_hi[1], box_hi[2]};
                box_lo[0] = snap_lo_fn(box_lo[0], rr[0]);
                box_lo[1] = snap_lo_fn(box_lo[1], rr[1]);
                box_lo[2] = snap_lo_fn(box_lo[2], rr[2]);
                box_hi[0] = snap_hi_fn(box_hi[0]+1, rr[0]);
                box_hi[1] = snap_hi_fn(box_hi[1]+1, rr[1]);
                box_hi[2] = snap_hi_fn(box_hi[2]+1, rr[2]);
                if (box_lo[0] != lo_old[0] || box_hi[0] != hi_old[0] ||
                    box_lo[1] != lo_old[1] || box_hi[1] != hi_old[1] ||
                    box_lo[2] != lo_old[2] || box_hi[2] != hi_old[2]) {
                    amrex::Print() << "Refinement box indices snapped to ref_ratio alignment:\n"
                                   << "  ilo: " << lo_old[0] << " -> " << box_lo[0]
                                   << "  ihi: " << hi_old[0] << " -> " << box_hi[0]
                                   << "  jlo: " << lo_old[1] << " -> " << box_lo[1]
                                   << "  jhi: " << hi_old[1] << " -> " << box_hi[1]
                                   << "  klo: " << lo_old[2] << " -> " << box_lo[2]
                                   << "  khi: " << hi_old[2] << " -> " << box_hi[2] << "\n";
                }
                bx = Box(IntVect(box_lo[0],box_lo[1],box_lo[2]),
                         IntVect(box_hi[0],box_hi[1],box_hi[2]));
            }
 
            bool using_pbl = (solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYJ      ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNN25   ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNNEDMF ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::YSU      ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MRF);
 
            if ( using_pbl && ( (box_lo[2] > 0) || (box_hi[2] < domain.bigEnd(2)) ) ) {
                amrex::Print() << "PBL models need refinement boxes that go from the bottom to the top of the domain for calculation of PBLH" << std::endl;
                amrex::Print() << "Please set in_box_lo_indices to 0 in z and in_box_hi_indices to amr.n_cell-1 in z and try again" << std::endl;
                amrex::Abort();
            }
 
            boxes_at_level[lev_for_box].push_back(bx);
            Print() << "Saving in 'boxes at level' as " << bx << std::endl;
        } // lev
 
        if (solverChoice.init_type == InitType::WRFInput) {
            if ( (num_files_at_level[lev_for_box] > 0) &&
                 (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) ) {
                amrex::Error("Number of boxes doesn't match number of input files");
 
            }
        }
 
    } else if (num_indx_lo_crse > 0) {
 
        std::vector<int> box_lo(3), box_hi(3);
        if (lev_for_box > 0 && lev_for_box <= max_level)
        {
            if (n_error_buf[0] != IntVect::TheZeroVector()) {
                amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
            }
 
            ppr.getarr("in_box_lo_indices_crse",box_lo,0,num_indx_lo_crse);
            ppr.getarr("in_box_hi_indices_crse",box_hi,0,num_indx_hi_crse);
 
            if (num_indx_lo_crse < AMREX_SPACEDIM) {
                box_lo[2] = geom[lev_for_box-1].Domain().smallEnd(2);
                box_hi[2] = geom[lev_for_box-1].Domain().bigEnd(2);
            }
 
            Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
 
            if (!geom[lev_for_box-1].Domain().contains(bx)) {
                amrex::Print() << "\n";
                amrex::Print() << "(Coarse) Box specified       is " << bx << std::endl;
                amrex::Print() << "But (coarse) domain at level is " << geom[lev_for_box-1].Domain() << std::endl;
                amrex::Error("Specified box doesn't fit in the domain");
            }
 
            bx.refine(ref_ratio[lev_for_box-1]);
 
            const auto* dx  = geom[lev_for_box-1].CellSize();
 
            real_box = RealBox(plo[0]+ box_lo[0]   *dx[0], plo[1]+ box_lo[1]   *dx[1], plo[2]+ box_lo[2]   *dx[2],
                               plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
 
            Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
            num_boxes_at_level[lev_for_box] += 1;
            bool using_pbl = (solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYJ      ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNN25   ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNNEDMF ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::YSU      ||
                              solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MRF);
 
            const Box& domain = geom[lev_for_box-1].Domain();
            if ( using_pbl && ( (box_lo[2] > 0) || (box_hi[2] < domain.bigEnd(2)) ) ) {
                amrex::Print() << "PBL models need refinement boxes that go from the bottom to the top of the domain for calculation of PBLH" << std::endl;
                amrex::Print() << "Please set in_box_lo_indices_crse to 0 in z and in_box_hi_indices_crse  to amr.n_cell-1 in z and try again" << std::endl;
                amrex::Abort();
            }
 
            boxes_at_level[lev_for_box].push_back(bx);
            Print() << "Saving in 'boxes at level' as " << bx << std::endl;
        } // lev
 
        if (solverChoice.init_type == InitType::WRFInput) {
            if ( (num_files_at_level[lev_for_box] > 0) &&
                 (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) ) {
                amrex::Error("Number of boxes doesn't match number of input files");
 
            }
        }
    }
}

Here is the call graph for this function:

ReadCheckpointFile()

void ERF::ReadCheckpointFile

(

)

ERF function for reading data from a checkpoint file during restart.

{
    Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
 
    const auto provenance_result =
        erf_provenance::read_job_info_file(restart_chkfile + "/job_info");
    if (provenance_result.valid() &&
        provenance_result.record.artifact.artifact_type == erf_provenance::ArtifactType::Checkpoint) {
        execution_provenance = erf_provenance::make_restart_provenance(
            execution_provenance, provenance_result.record, restart_chkfile);
    } else {
        // Provenance is auxiliary metadata. A missing or invalid record must not make
        // a valid physical checkpoint unreadable.
        if (ParallelDescriptor::IOProcessor() && !provenance_warning_emitted) {
            provenance_warning_emitted = true;
            const std::string reason = provenance_result.valid()
                ? "the provenance record has artifact_type=" +
                  std::string(erf_provenance::artifact_type_token(
                      provenance_result.record.artifact.artifact_type)) +
                  ", not checkpoint"
                : provenance_result.diagnostic;
            Warning("Cannot recover provenance from native checkpoint '" +
                    restart_chkfile + "': " + reason +
                    ". ERF will continue with incomplete provenance.");
        }
        const auto failure_status = provenance_result.valid()
            ? erf_provenance::ProvenanceReadStatus::ArtifactTypeMismatch
            : provenance_result.status;
        execution_provenance = erf_provenance::make_incomplete_restart_provenance(
            execution_provenance, failure_status, restart_chkfile);
    }
 
    // Header
    std::string File(restart_chkfile + "/Header");
 
    VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
 
    Vector<char> fileCharPtr;
    ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
    std::string fileCharPtrString(fileCharPtr.dataPtr());
    std::istringstream is(fileCharPtrString, std::istringstream::in);
 
    std::string line, word;
 
    int chk_ncomp_cons, chk_ncomp;
 
    // read in title line
    std::getline(is, line);
 
    // read in finest_level
    is >> finest_level;
    GotoNextLine(is);
 
    validate_surface_temperature_contract(restart_chkfile, finest_level);
 
    // read the number of components
    // for each variable we store
 
    // conservative, cell-centered vars
    is >> chk_ncomp_cons;
    GotoNextLine(is);
 
    // x-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // y-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // z-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // read in array of istep
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            istep[i++] = std::stoi(word);
        }
    }
 
    // read in array of dt
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            dt[i++] = std::stod(word);
        }
    }
 
    // read in array of t_new
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            t_new[i++] = std::stod(word);
        }
    }
 
    // Read zlevels from its own directory
    // NOTE: This read should occur before MakeNewLevelFromScratch
    {
        std::string ZLevelsFile(restart_chkfile + "/zlevels");
        if (amrex::FileExists(ZLevelsFile)) {
            Vector<char> ZLevelsfileCharPtr;
            ParallelDescriptor::ReadAndBcastFile(ZLevelsFile, ZLevelsfileCharPtr);
            std::string ZLevelsfileCharPtrString(ZLevelsfileCharPtr.dataPtr());
            std::istringstream isz(ZLevelsfileCharPtrString, std::istringstream::in);
 
            int nlevs_in_chk = 0;
            isz >> nlevs_in_chk;
 
            for (int lev = 0; lev < nlevs_in_chk; ++lev) {
                int nz_stag = 0;
                isz >> nz_stag;
 
                Vector<Real> zlevels_from_chk(nz_stag);
                for (int k = 0; k < nz_stag; ++k) {
                    isz >> zlevels_from_chk[k];
                }
 
                // A checkpoint from a run with more levels than we hold: read past it
                if (lev >= static_cast<int>(zlevels_stag.size())) { continue; }
 
                if (static_cast<int>(zlevels_stag[lev].size()) != nz_stag) {
                    Print() << "Checkpoint holds " << nz_stag << " staggered z levels at level "
                            << lev << " but this run expects "
                            << zlevels_stag[lev].size() << std::endl;
                    Abort("Cannot restart with a different number of cells in z");
                }
 
                zlevels_stag[lev] = zlevels_from_chk;
 
                // Keep the cell heights consistent with the levels we just read in
                update_stretched_dz(lev, zlevels_stag, stretched_dz_h, stretched_dz_d);
            }
        } // zlevels file exists
    }
 
    for (int lev = 0; lev <= finest_level; ++lev) {
        // read in level 'lev' BoxArray from Header
        BoxArray ba;
        ba.readFrom(is);
        GotoNextLine(is);
 
        // create a distribution mapping
        DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
 
        MakeNewLevelFromScratch (lev, static_cast<Real>(t_new[lev]), ba, dm);
    }
 
    // read in array of t_avg_cnt
    //
    // NOTE: this must come after the loop above, because MakeNewLevelFromScratch ->
    //       init_stuff zeroes the time-averaging state for each level it builds.
    //       An older checkpoint has nothing here, in which case we leave those zeros
    //       in place and the average simply starts over.
    if (solverChoice.time_avg_vel) {
        std::getline(is, line);
        std::istringstream lis(line);
        int i = 0;
        while ((i < t_avg_cnt.size()) && (lis >> word)) {
            t_avg_cnt[i++] = std::stod(word);
        }
        if (i == 0) {
            amrex::Print() << "NOTE: this checkpoint predates time-averaged velocity being "
                              "checkpointed; the running average of velocity will start over"
                           << std::endl;
        }
    }
 
    if (solverChoice.compute_mean_vars) {
        const std::string metadata_name(restart_chkfile + "/IntervalMeansHeader");
        if (!amrex::FileExists(metadata_name)) {
            for (int lev = 0; lev <= finest_level; ++lev) {
                AMREX_ALWAYS_ASSERT(interval_means[lev] != nullptr);
                interval_means[lev]->setVal(zero);
                t_mean_cnt[lev] = 0.0;
            }
            // A legacy checkpoint has no interval-mean metadata.  Keep the
            // safe empty-window fallback, but do not schedule a time reset a
            // second time when the checkpoint was written after its threshold.
            mean_vars_time_reset_done = erf_interval_means::legacy_reset_done(
                solverChoice.mean_vars_reset_mode == "time", t_new[0],
                static_cast<double>(solverChoice.mean_vars_reset_time));
            if (ParallelDescriptor::IOProcessor()) {
                amrex::Print() << "WARNING: legacy checkpoint without interval-mean state; "
                                  "the averaging window starts empty.\n";
            }
        } else {
            Vector<char> metadata_chars;
            ParallelDescriptor::ReadAndBcastFile(metadata_name, metadata_chars);
            std::istringstream metadata(std::string(metadata_chars.dataPtr()),
                                        std::istringstream::in);
            erf_interval_means::Metadata parsed_metadata;
            std::string metadata_error;
            if (!erf_interval_means::parse_metadata(
                    metadata, parsed_metadata, metadata_error)) {
                Abort("Invalid interval-mean checkpoint metadata in '" + metadata_name +
                      "': " + metadata_error);
            }
            const std::string validation_error = erf_interval_means::validate_metadata(
                parsed_metadata, finest_level + 1, 10, grids);
            if (!validation_error.empty()) {
                Abort(validation_error + " in '" + metadata_name + "'");
            }
 
            mean_vars_time_reset_done = erf_interval_means::global_reset_done(parsed_metadata);
            for (int lev = 0; lev <= finest_level; ++lev) {
                const auto& level_metadata = parsed_metadata.level[lev];
 
                const std::string mf_name =
                    MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "IntervalMeans");
                if (!amrex::FileExists(mf_name + "_H")) {
                    Abort("Interval-mean metadata exists but data is missing for level " +
                          std::to_string(lev));
                }
                AMREX_ALWAYS_ASSERT(interval_means[lev] != nullptr);
                VisMF::Read(*interval_means[lev], mf_name);
                t_mean_cnt[lev] = level_metadata.accumulation_count;
            }
        }
    }
 
    // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
    // NOTE: Data is written over ncomp, so check that we match the header file
    int ncomp_cons = vars_new[0][Vars::cons].nComp();
 
    // NOTE: QKE was removed so this is for backward compatibility
    AMREX_ASSERT((chk_ncomp_cons==ncomp_cons) || ((chk_ncomp_cons-1)==ncomp_cons));
    //
    // See if we have a written separate file that tells how many components and how many ghost cells
    // we have of the base state
    //
    // If we can't find the file, then set the number of components to the original number = 3
    //
    int ncomp_base_to_read = 3;
    IntVect ng_base = IntVect{1};
    {
         std::string BaseStateFile(restart_chkfile + "/num_base_state_comps");
 
         if (amrex::FileExists(BaseStateFile))
         {
             Vector<char> BaseStatefileCharPtr;
             ParallelDescriptor::ReadAndBcastFile(BaseStateFile, BaseStatefileCharPtr);
             std::string BaseStatefileCharPtrString(BaseStatefileCharPtr.dataPtr());
 
             // We set this to the default value of 3 but allow it be larger if th0 and qv0 were written
             std::istringstream isb(BaseStatefileCharPtrString, std::istringstream::in);
             isb >> ncomp_base_to_read;
             isb >> ng_base;
         }
    }
 
    // read in the MultiFab data
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        // NOTE: For backward compatibility (chk file has QKE)
        if ((chk_ncomp_cons-1)==ncomp_cons) {
            MultiFab cons(grids[lev],dmap[lev],chk_ncomp_cons,0);
            VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
 
            // Copy up to RhoKE_comp
            MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,(RhoKE_comp+1),0);
 
            // Only if we have a PBL model do we need to copy QKE is src to KE in dst
            if ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
                 (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ) {
                MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+1),RhoKE_comp,1,0);
                vars_new[lev][Vars::cons].mult(myhalf,RhoKE_comp,1,0);
            }
 
            // Copy other components
            int ncomp_remainder = ncomp_cons - (RhoKE_comp + 1);
            MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+2),(RhoKE_comp+1),ncomp_remainder,0);
 
            vars_new[lev][Vars::cons].setBndry(bogus_large_value);
        } else {
            MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
            VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
            MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
            vars_new[lev][Vars::cons].setBndry(bogus_large_value);
        }
 
        MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
        VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
        MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
        vars_new[lev][Vars::xvel].setBndry(bogus_large_value);
 
        MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
        VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
        MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
        vars_new[lev][Vars::yvel].setBndry(bogus_large_value);
 
        MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
        VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
        MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
        vars_new[lev][Vars::zvel].setBndry(bogus_large_value);
 
        if (solverChoice.anelastic[lev] == 1) {
            MultiFab ppinc(grids[lev],dmap[lev],1,0);
            VisMF::Read(ppinc, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PP_Inc"));
            MultiFab::Copy(pp_inc[lev],ppinc,0,0,1,0);
            pp_inc[lev].FillBoundary(geom[lev].periodicity());
 
            MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
            VisMF::Read(gpx, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpx"));
            MultiFab::Copy(gradp[lev][GpVars::gpx],gpx,0,0,1,0);
            gradp[lev][GpVars::gpx].FillBoundary(geom[lev].periodicity());
 
            MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
            VisMF::Read(gpy, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpy"));
            MultiFab::Copy(gradp[lev][GpVars::gpy],gpy,0,0,1,0);
            gradp[lev][GpVars::gpy].FillBoundary(geom[lev].periodicity());
 
            MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
            VisMF::Read(gpz, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpz"));
            MultiFab::Copy(gradp[lev][GpVars::gpz],gpz,0,0,1,0);
            gradp[lev][GpVars::gpz].FillBoundary(geom[lev].periodicity());
        }
 
        // Restore the surface fluxes an ocean model is driven by in flux-passing
        // mode. Without this they are zero until the first advance, and the
        // coupler packs them before that happens. See the matching write above.
        //
        // Absent from checkpoints written before this landed, and from any
        // written by an uncoupled run, so each is probed rather than assumed.
        // Missing means the ocean gets zero forcing for its first step, which is
        // silent and wrong, so say so rather than leaving it to be discovered in
        // the results.
        if (m_driver_has_atm2ocn_coupling) {
            bool restored_any = false;
            bool missing_any  = false;
 
            auto read_if_present = [&] (MultiFab* dst, const char* tag, const IntVect& ixtype)
            {
                if (dst == nullptr) { return; }
                const std::string name =
                    MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", tag);
                if (amrex::FileExists(name + "_H")) {
                    MultiFab tmp(convert(grids[lev],ixtype),dmap[lev],1,0);
                    VisMF::Read(tmp, name);
                    MultiFab::Copy(*dst,tmp,0,0,1,0);
                    dst->FillBoundary(geom[lev].periodicity());
                    restored_any = true;
                } else {
                    missing_any = true;
                }
            };
 
            read_if_present(Tau.empty() ? nullptr : Tau[lev][TauType::tau13].get(),
                            "Tau13", IntVect(1,0,1));
            read_if_present(Tau.empty() ? nullptr : Tau[lev][TauType::tau23].get(),
                            "Tau23", IntVect(0,1,1));
            read_if_present(SFS_hfx3_lev.empty() ? nullptr : SFS_hfx3_lev[lev].get(),
                            "SFS_hfx3", IntVect(0,0,1));
            read_if_present(SFS_q1fx3_lev.empty() ? nullptr : SFS_q1fx3_lev[lev].get(),
                            "SFS_q1fx3", IntVect(0,0,1));
 
            if (missing_any && !restored_any) {
                amrex::Print() << "WARNING: restart checkpoint carries no surface fluxes "
                               << "for level " << lev << ". In flux-passing coupled mode "
                               << "the ocean will be driven by zero wind stress and zero "
                               << "heat flux until the first atmosphere advance. Re-run "
                               << "from a checkpoint written with flux output, or expect "
                               << "the first ocean step to be unforced." << std::endl;
            }
        }
 
        // Restore the running sum of the time-averaged velocity (issue 3654).  Older
        // checkpoints do not carry it; in that case keep the zeros that init_stuff set
        // and drop this level's counter to match, so the average restarts consistently.
        if (solverChoice.time_avg_vel) {
            AMREX_ALWAYS_ASSERT(vel_t_avg[lev] != nullptr);
            const std::string vta_name =
                MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "VelTimeAvg");
            if (amrex::FileExists(vta_name + "_H")) {
                VisMF::Read(*vel_t_avg[lev], vta_name);
            } else {
                vel_t_avg[lev]->setVal(0.0);
                t_avg_cnt[lev] = 0.0;
            }
        }
 
        // Note that we read the ghost cells of the base state (unlike above)
 
        // The original base state only had 3 components and 1 ghost cell -- we read this
        // here to be consistent with the old style
        MultiFab base(grids[lev],dmap[lev],ncomp_base_to_read,ng_base);
        VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
 
        MultiFab::Copy(base_state[lev],base,0,0,ncomp_base_to_read,ng_base);
 
        // Create theta0 from p0, rh0
        if (ncomp_base_to_read < 4) {
            for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // We only compute theta_0 on valid cells since we will impose domain BC's after restart
                const Box& bx = mfi.tilebox();
                Array4<Real> const& fab = base_state[lev].array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    fab(i,j,k,BaseState::th0_comp) = getRhoThetagivenP(fab(i,j,k,BaseState::p0_comp))
                                                     / fab(i,j,k,BaseState::r0_comp);
                });
            }
        }
        // Default theta0 to 0
        if (ncomp_base_to_read < 5) {
            for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // We only compute theta_0 on valid cells since we will impose domain BC's after restart
                const Box& bx = mfi.tilebox();
                Array4<Real> const& fab = base_state[lev].array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    fab(i,j,k,BaseState::qv0_comp) = zero;
                });
            }
        }
        base_state[lev].FillBoundary(geom[lev].periodicity());
 
        if (SolverChoice::mesh_type != MeshType::ConstantDz)  {
            // Note that we also read the ghost cells of z_phys_nd
            IntVect ng = z_phys_nd[lev]->nGrowVect();
            MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
            VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
            MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
            update_terrain_arrays(lev);
 
#if 0
            if ( (solverChoice.init_type != InitType::WRFInput) && (solverChoice.init_type != InitType::Metgrid) ) {
                check_mesh_type(lev);
            }
#endif
        }
 
        // The min dz the microphysics sizes its sedimentation substeps with, for
        // every mesh type as init_zphys does on a fresh start. Set only on fitted
        // meshes before, so a restart on a constant-dz mesh left Kessler's (and
        // SAM's, Morrison's) dzmin uninitialised and the substep count unbounded:
        // the first restarted step of a raining run never finished.
        {
            Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
            micro->Set_dzmin(lev, dzmin);
        }
 
        // Read in the moisture model restart variables
        std::vector<int> qmoist_indices;
        std::vector<std::string> qmoist_names;
        micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
        int qmoist_nvar = static_cast<int>(qmoist_indices.size());
        for (int var = 0; var < qmoist_nvar; var++) {
            const int ncomp  = 1;
            IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
            MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
            VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", qmoist_names[var]));
            MultiFab::Copy(*(qmoist[lev][qmoist_indices[var]]),moist_vars,0,0,ncomp,ng_moist);
        }
 
#if defined(ERF_USE_WINDFARM)
        if(solverChoice.windfarm_type == WindFarmType::Fitch or
           solverChoice.windfarm_type == WindFarmType::EWP or
           solverChoice.windfarm_type == WindFarmType::SimpleAD){
            IntVect ng = Nturb[lev].nGrowVect();
            MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
            VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
            MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
        }
#endif
 
        // Read the LSM data
        if (solverChoice.lsm_type != LandSurfaceType::None) {
            amrex::Print() << "Reading LSM variables" << std::endl;
            for (int ivar(0); ivar<lsm_data[lev].size(); ++ivar) {
                BoxArray ba = lsm_data[lev][ivar]->boxArray();
                DistributionMapping dm = lsm_data[lev][ivar]->DistributionMap();
                IntVect ng = lsm_data[lev][ivar]->nGrowVect();
                int nvar   = lsm_data[lev][ivar]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmData" + std::to_string(ivar)));
                MultiFab::Copy(*(lsm_data[lev][ivar]),lsm_vars,0,0,nvar,ng);
            }
            for (int iflux(0); iflux<lsm_flux[lev].size(); ++iflux) {
                BoxArray ba = lsm_flux[lev][iflux]->boxArray();
                DistributionMapping dm = lsm_flux[lev][iflux]->DistributionMap();
                IntVect ng = lsm_flux[lev][iflux]->nGrowVect();
                int nvar   = lsm_flux[lev][iflux]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmFlux" + std::to_string(iflux)));
                MultiFab::Copy(*(lsm_flux[lev][iflux]),lsm_vars,0,0,nvar,ng);
            }
        }
 
        // Two-stream radiation: the force-restore surface state
        if (solverChoice.rad_type == RadiationType::TwoStream) {
            two_stream_rad.read_checkpoint(lev, restart_chkfile);
        }
 
        // Read the radiation heating rates
        std::string RadFileName = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qrad_H");
        if ((solverChoice.rad_type != RadiationType::None) && amrex::FileExists(RadFileName)) {
            amrex::Print() << "Reading radiation heating rates" << std::endl;
            int nrad = qheating_rates[lev]->nComp();
            MultiFab mf_rad(grids[lev],dmap[lev],nrad,0);
            VisMF::Read(mf_rad, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qrad"));
            MultiFab::Copy(*qheating_rates[lev],mf_rad,0,0,nrad,0);
        }
 
        IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
        MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
 
        std::string MapFacMFileName = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_mx_H");
        if (amrex::FileExists(MapFacMFileName)) {
            VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_mx"));
        } else {
            VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
        }
        MultiFab::Copy(*mapfac[lev][MapFacType::m_x],mf_m,0,0,1,ng);
 
#if 0
        if (MapFacType::m_x != MapFacType::m_y) {
            VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_my"));
            MultiFab::Copy(*mapfac[lev][MapFacType::m_y],mf_m,0,0,1,ng);
        }
#endif
 
        ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
        MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
 
        std::string MapFacUFileName = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_ux_H");
        if (amrex::FileExists(MapFacUFileName)) {
            VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_ux"));
        } else {
            VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
        }
        MultiFab::Copy(*mapfac[lev][MapFacType::u_x],mf_u,0,0,1,ng);
 
#if 0
        if (MapFacType::u_x != MapFacType::u_y) {
            VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_uy"));
            MultiFab::Copy(*mapfac[lev][MapFacType::u_y],mf_u,0,0,1,ng);
        }
#endif
 
        ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
        MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
 
        std::string MapFacVFileName = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vx_H");
        if (amrex::FileExists(MapFacVFileName)) {
            VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vx"));
        } else {
            VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
        }
        MultiFab::Copy(*mapfac[lev][MapFacType::v_x],mf_v,0,0,1,ng);
 
#if 0
        if (MapFacType::v_x != MapFacType::v_y) {
            VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vy"));
            MultiFab::Copy(*mapfac[lev][MapFacType::v_y],mf_v,0,0,1,ng);
        }
#endif
 
 
        // NOTE: We read MOST data in ReadCheckpointFileMOST (see below)!
 
        // See if we wrote out SST data
        std::string FirstSSTFileName = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SST_0_H");
        if (amrex::FileExists(FirstSSTFileName))
        {
            amrex::Print() << "Reading SST data" << std::endl;
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
            sst_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                VisMF::Read(sst_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
                                                             "SST_" + std::to_string(nt)));
                MultiFab::Copy(*sst_lev[lev][nt],sst_at_t,0,0,1,ng);
            }
        }
 
        // See if we wrote out TSK data
        std::string FirstTSKFileName = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "TSK_0_H");
        if (amrex::FileExists(FirstTSKFileName))
        {
            amrex::Print() << "Reading TSK data" << std::endl;
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
            tsk_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                VisMF::Read(tsk_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
                                                             "TSK_" + std::to_string(nt)));
                MultiFab::Copy(*tsk_lev[lev][nt],tsk_at_t,0,0,1,ng);
            }
        }
 
        // See if we wrote out LMASK data
        std::string LMaskFileName = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LMASK_0_H");
        if (amrex::FileExists(LMaskFileName))
        {
            amrex::Print() << "Reading LMASK data" << std::endl;
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                VisMF::Read(lmask_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
                                                              "LMASK_" + std::to_string(nt)));
                for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
                    const Box& bx = mfi.growntilebox();
                    Array4<int>  const& dst_arr = lmask_lev[lev][nt]->array(mfi);
                    Array4<Real> const& src_arr = lmask_at_t.array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        dst_arr(i,j,k) = int(src_arr(i,j,k));
                    });
                }
            }
        } else {
            lmask_lev[lev][0]->setVal(solverChoice.is_land[lev]);
            lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
        }
 
        IntVect ngv = vars_new[lev][Vars::cons].nGrowVect(); ngv[2] = 0;
 
        // Read lat/lon if it exists
        std::string LatFileName = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LAT_H");
        if (amrex::FileExists(LatFileName)) {
            amrex::Print() << "Reading Lat/Lon variables" << std::endl;
            MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
            MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
            VisMF::Read(lat, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LAT"));
            VisMF::Read(lon, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LON"));
            lat_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
            lon_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
            MultiFab::Copy(*lat_m[lev],lat,0,0,1,ngv);
            MultiFab::Copy(*lon_m[lev],lon,0,0,1,ngv);
        }
 
#ifdef ERF_USE_NETCDF
        // Read sinPhi and cosPhi if it exists
        std::string VarCorFileName = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SinPhi_H");
        if (amrex::FileExists(VarCorFileName)) {
            amrex::Print() << "Reading Coriolis factors" << std::endl;
            MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
            MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
            VisMF::Read(sphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SinPhi"));
            VisMF::Read(cphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "CosPhi"));
            sinPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
            cosPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
            MultiFab::Copy(*sinPhi_m[lev],sphi,0,0,1,ngv);
            MultiFab::Copy(*cosPhi_m[lev],cphi,0,0,1,ngv);
        }
 
        if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
            if (lev == 0) {
                amrex::Print() << "Reading C1H/C2H/RDNW/MUB/PHB variables at level " << lev << std::endl;
                MultiFab tmp1d(ba1d[0],dmap[0],1,0);
 
                VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C1H"));
                MultiFab::Copy(*wrf_C1H,tmp1d,0,0,1,0);
 
                VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C2H"));
                MultiFab::Copy(*wrf_C2H,tmp1d,0,0,1,0);
 
                VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "RDNW"));
                MultiFab::Copy(*wrf_RDNW,tmp1d,0,0,1,0);
 
                MultiFab tmp2d(ba2d[0],dmap[0],1,wrf_MUB->nGrowVect());
 
                VisMF::Read(tmp2d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MUB"));
                MultiFab::Copy(*wrf_MUB,tmp2d,0,0,1,wrf_MUB->nGrowVect());
 
                MultiFab tmp3d(convert(grids[0],IntVect(0,0,1)),dmap[0],1,wrf_PHB->nGrowVect());
 
                VisMF::Read(tmp3d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PHB"));
                MultiFab::Copy(*wrf_PHB,tmp3d,0,0,1,wrf_PHB->nGrowVect());
            }
        }
#endif
    } // for lev
 
    // Restore the LSM scalar step counter (e.g. NoahMP itimestep) so the
    // substepping schedule survives restart. lsm.Init() during
    // MakeNewLevelFromScratch has already reset itimestep to 0 (NoahmpIOVarInit);
    // override it here from the checkpoint. Older checkpoints without this file
    // restart with the legacy reset-to-zero behavior.
    if (solverChoice.lsm_type != LandSurfaceType::None) {
        std::string LsmStepFileName(restart_chkfile + "/lsm_step");
        if (amrex::FileExists(LsmStepFileName)) {
            Vector<char> LsmStepCharPtr;
            ParallelDescriptor::ReadAndBcastFile(LsmStepFileName, LsmStepCharPtr);
            std::string LsmStepStr(LsmStepCharPtr.dataPtr());
            std::istringstream lsm_is(LsmStepStr, std::istringstream::in);
            int step_val = 0;
            for (int lev = 0; lev <= finest_level; ++lev) {
                if (lsm_is >> step_val) {
                    lsm.Set_LSM_Step(lev, step_val);
                    Print() << "Restored LSM step counter at level " << lev
                            << " to " << step_val << std::endl;
                }
            }
        } else {
            Print() << "Warning: legacy checkpoint without lsm_step file; "
                    << "LSM substep schedule will reset (may break bitwise reproducibility)."
                    << std::endl;
        }
 
        if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
            // Restore the full LSM prognostic state (e.g. NoahMP soil/snow/canopy)
            // from chk*/noahmp_restart. lsm.Init() during MakeNewLevelFromScratch
            // has already cold-initialized this state from wrfinput/tables; this
            // overwrites it with the checkpoint, and NoahMP's per-step In-transfer
            // pulls it into the physics state on the first Advance (issue #3255).
            // Legacy checkpoints without this directory fall back to cold-init.
            std::string LsmRestartDir(restart_chkfile + "/noahmp_restart");
            for (int lev = 0; lev <= finest_level; ++lev) {
                std::string LsmRestartFile = LsmRestartDir + "/Level_" + std::to_string(lev) + ".nc";
                if (amrex::FileExists(LsmRestartFile)) {
                    lsm.Read_Lsm_Restart(lev, LsmRestartDir);
                    Print() << "Restored full NoahMP prognostic state from "
                            << LsmRestartFile << " at level " << lev << std::endl;
                } else {
                    Print() << "NoahMP restart file " << LsmRestartFile << " is not present at level " << lev << std::endl;
                    if (lev == 0) {
                        Print() << "WARNING: NoahMP will cold-initialize from wrfinput at level " << lev << std::endl;
                    } else {
                        Print() << "NOTE: NoahMP will cold-initialize if a wrfinput file is present at level " << lev
                                << " or it will interpolate from coarse if the file does not exist." << std::endl;
                    }
                }
            } // lev
        } // NoahMP
    } // has LSM
 
#ifdef ERF_USE_PARTICLES
    restartTracers((ParGDBBase*)GetParGDB(),restart_chkfile);
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
        dynamic_cast<LagrangianMicrophysics&>(*micro).restartParticles((ParGDBBase*)GetParGDB(),restart_chkfile);
    }
#endif
 
#if 0
#ifdef ERF_USE_NETCDF
    // Read bdy_data files
    if ( ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
         solverChoice.use_real_bcs )
    {
        int ioproc = ParallelDescriptor::IOProcessorNumber();  // I/O rank
        int num_time;
        int num_var;
        Vector<Box> bx_v;
        if (ParallelDescriptor::IOProcessor()) {
            // Open header file and read from it
            std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
            bdy_h_file >> num_time;
            bdy_h_file >> num_var;
            bdy_h_file >> start_bdy_time;
            bdy_h_file >> bdy_time_interval;
            bdy_h_file >> real_width;
            bx_v.resize(4*num_var);
            for (int ivar(0); ivar<num_var; ++ivar) {
                bdy_h_file >> bx_v[4*ivar  ];
                bdy_h_file >> bx_v[4*ivar+1];
                bdy_h_file >> bx_v[4*ivar+2];
                bdy_h_file >> bx_v[4*ivar+3];
            }
 
            // IO size the FABs
            bdy_data_xlo.resize(num_time);
            bdy_data_xhi.resize(num_time);
            bdy_data_ylo.resize(num_time);
            bdy_data_yhi.resize(num_time);
            for (int itime(0); itime<num_time; ++itime) {
                bdy_data_xlo[itime].resize(num_var);
                bdy_data_xhi[itime].resize(num_var);
                bdy_data_ylo[itime].resize(num_var);
                bdy_data_yhi[itime].resize(num_var);
                for (int ivar(0); ivar<num_var; ++ivar) {
                    bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
                    bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
                    bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
                    bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
                }
            }
 
            // Open data file and read from it
            std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
            for (int itime(0); itime<num_time; ++itime) {
                for (int ivar(0); ivar<num_var; ++ivar) {
                    bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
                }
            }
        } // IO
 
        // Broadcast the data
        ParallelDescriptor::Barrier();
        ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
        ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
        ParallelDescriptor::Bcast(&real_width,1,ioproc);
        ParallelDescriptor::Bcast(&num_time,1,ioproc);
        ParallelDescriptor::Bcast(&num_var,1,ioproc);
 
        // Everyone size their boxes
        bx_v.resize(4*num_var);
 
        ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
 
        // Everyone but IO size their FABs
        if (!ParallelDescriptor::IOProcessor()) {
          bdy_data_xlo.resize(num_time);
          bdy_data_xhi.resize(num_time);
          bdy_data_ylo.resize(num_time);
          bdy_data_yhi.resize(num_time);
          for (int itime(0); itime<num_time; ++itime) {
            bdy_data_xlo[itime].resize(num_var);
            bdy_data_xhi[itime].resize(num_var);
            bdy_data_ylo[itime].resize(num_var);
            bdy_data_yhi[itime].resize(num_var);
            for (int ivar(0); ivar<num_var; ++ivar) {
              bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
              bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
              bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
              bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
            }
          }
        }
 
        for (int itime(0); itime<num_time; ++itime) {
            for (int ivar(0); ivar<num_var; ++ivar) {
                ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
            }
        }
    } // init_type == WRFInput or Metgrid
#endif
#endif
 
#ifdef ERF_USE_NETCDF
    // Load boundary data from erfbdy during restart for metgrid or wrfinput.
    if (((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) &&
        solverChoice.use_real_bcs) {
 
        // Check for erfbdy file.
        std::string erfbdy_header = erfbdy_file + "/Header";
 
        // For now we disable this for InitType::WRFInput because it failed the WPS_Test_restart regression test
        use_erfbdy = ( (solverChoice.init_type == InitType::Metgrid) && FileSystem::Exists(erfbdy_header) );
 
        if (solverChoice.init_type == InitType::Metgrid) {
            if (!use_erfbdy) {
                Abort("Restart with init_type=metgrid requires erfbdy file: " + erfbdy_file);
            }
        }
 
        // Load from erfbdy if it exists.
        if (use_erfbdy) {
            Print() << "Restart: Loading boundary data from erfbdy file: " << erfbdy_file << std::endl;
 
            int ntimes_erfbdy;
            Vector<double> bdy_times;
            bdy_time_interval = read_times_from_erfbdy(erfbdy_file,
                                                       ntimes_erfbdy, nvars_erfbdy, real_width,
                                                       bdy_times, start_bdy_time, final_bdy_time);
 
            Print() << "Restart: erfbdy file contains " << ntimes_erfbdy << " times" << std::endl;
 
            bdy_data_xlo.resize(ntimes_erfbdy);
            bdy_data_xhi.resize(ntimes_erfbdy);
            bdy_data_ylo.resize(ntimes_erfbdy);
            bdy_data_yhi.resize(ntimes_erfbdy);
 
            // Determine which times we need based on current simulation time.
            double time_since_start_bdy = t_new[0] + start_time - start_bdy_time;
            int n_time_old = std::min(static_cast<int>(time_since_start_bdy / bdy_time_interval), ntimes_erfbdy-1);
            int n_time_new = n_time_old + 1;
 
            // Read the necessary times into memory.
            for (int itime = n_time_old; itime <= std::min(n_time_new + 1, ntimes_erfbdy - 1); ++itime) {
                read_from_erfbdy(itime, erfbdy_file,
                                 bdy_data_xlo, bdy_data_xhi,
                                 bdy_data_ylo, bdy_data_yhi,
                                 nvars_erfbdy, real_width);
                Print() << "Restart: Loaded erfbdy time index " << itime << std::endl;
            }
        }
    }
#endif
}

Here is the call graph for this function:

ReadCheckpointFileSurfaceLayer()

void ERF::ReadCheckpointFileSurfaceLayer

(

)

ERF function for reading additional data for MOST from a checkpoint file during restart.

This is called after the ABLMost object is instantiated.

{
    // Count number of surface layer boundaries to determine face prefix
    int n_faces = 0;
    for (OrientationIter oit; oit; ++oit) {
        Orientation ori = oit();
        if (phys_bc_type[ori] == ERF_BC::surface_layer) {
            n_faces += 1;
        }
    }
 
    for (OrientationIter oit; oit; ++oit) {
        Orientation ori = oit();
        if (m_SurfaceLayer[ori])  {
            std::string face = "_" + std::to_string(ori);
            // if we only use a single surface layer, don't add the face prefix
            if (n_faces == 1 && static_cast<int>(ori) == Orientation::zlo()) {
                face = "";
            }
 
            //
            // State of the exponential time filter (erf.most.time_average), if this run uses it.
            // A checkpoint written before this was persisted has no such file, in which case we
            // leave the filter uninitialized and it starts over -- the behavior those files
            // already had.  Anything unexpected in the file is likewise treated as "start over"
            // rather than blending stale state into the average.
            //
            const bool most_time_avg = m_SurfaceLayer[ori]->mac_avg_is_time_averaged();
            Vector<int>          most_avg_init;
            Vector<Vector<Real>> most_plane_avg;
            if (most_time_avg) {
                const std::string MostAvgFile(restart_chkfile + "/most_time_average" + face);
                if (amrex::FileExists(MostAvgFile)) {
                    Vector<char> fileCharPtr;
                    ParallelDescriptor::ReadAndBcastFile(MostAvgFile, fileCharPtr);
                    std::string fileCharPtrString(fileCharPtr.dataPtr());
                    std::istringstream ism(fileCharPtrString, std::istringstream::in);
 
                    int navg_chk = 0;
                    ism >> navg_chk;
                    if (navg_chk == m_SurfaceLayer[ori]->get_num_mac_avg()) {
                        for (int lev = 0; lev <= finest_level; ++lev) {
                            int is_init = 0, n_pavg = 0;
                            if (!(ism >> is_init >> n_pavg)) { break; }
                            Vector<Real> pavg(n_pavg);
                            bool line_ok = true;
                            for (int iavg(0); iavg < n_pavg; ++iavg) {
                                if (!(ism >> pavg[iavg])) { line_ok = false; break; }
                            }
                            if (!line_ok) { break; }
                            most_avg_init.push_back(is_init);
                            most_plane_avg.push_back(pavg);
                        }
                    } else {
                        amrex::Print() << "NOTE: checkpoint holds " << navg_chk << " surface-layer "
                                          "averages but this run expects "
                                       << m_SurfaceLayer[ori]->get_num_mac_avg()
                                       << "; the time filter will start over" << std::endl;
                    }
                } else {
                    amrex::Print() << "NOTE: this checkpoint does not carry the surface-layer time "
                                      "filter state; the filtered averages will start over"
                                   << std::endl;
                }
            }
 
            for (int lev = 0; lev <= finest_level; ++lev)
            {
                amrex::Print() << "Reading MOST variables for face " << ori << std::endl;
 
                auto read_most_var = [&] (const std::string& name, MultiFab* dst) {
                    const std::string mf_name = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", name);
                    if (amrex::FileExists(mf_name + "_H")) {
                        MultiFab m_var;
                        VisMF::Read(m_var, mf_name);
                        // The number of ghost cells depends on whether these live on a 2D or a 3D
                        // BoxArray (see WriteCheckpointFile), and a checkpoint written before the
                        // fix for issue #3560 may have fewer than the destination holds, so only
                        // fill as many ghost cells as both sides have
                        IntVect ng = amrex::min(m_var.nGrowVect(), dst->nGrowVect());
                        dst->ParallelCopy(m_var, 0, 0, 1, ng, ng, geom[lev].periodicity());
                        // The file's ghost cells may be stale (never filled before the
                        // write). As copy sources they reach valid cells through periodic
                        // images, so copy again from the valid cells alone; only
                        // domain-boundary ghosts keep the file's values.
                        dst->ParallelCopy(m_var, 0, 0, 1, IntVect(0), ng, geom[lev].periodicity());
                        return true;
                    }
                    return false;
                };
 
                // U*
                read_most_var("Ustar" + face, m_SurfaceLayer[ori]->get_u_star(lev));
 
                // W*
                read_most_var("Wstar" + face, m_SurfaceLayer[ori]->get_w_star(lev));
 
                // T*
                read_most_var("Tstar" + face, m_SurfaceLayer[ori]->get_t_star(lev));
 
                // Q*
                read_most_var("Qstar" + face, m_SurfaceLayer[ori]->get_q_star(lev));
 
                // Olen
                read_most_var("Olen" + face, m_SurfaceLayer[ori]->get_olen(lev));
 
                // T*
                read_most_var("Tsurf" + face, m_SurfaceLayer[ori]->get_t_surf(lev));
 
                // Qsurf
                read_most_var("Qsurf" + face, m_SurfaceLayer[ori]->get_q_surf(lev));
 
                // PBLH
                read_most_var("PBLH" + face, m_SurfaceLayer[ori]->get_pblh(lev));
 
                // Z0
                read_most_var("Z0" + face, m_SurfaceLayer[ori]->get_z0(lev));
 
                // The exponentially filtered averages.  We only mark the filter as initialized
                // if every piece of its state came back, so that a partial restore degrades to
                // "start the average over" instead of blending in whatever the containers hold.
                if (most_time_avg && (lev < static_cast<int>(most_avg_init.size())) && most_avg_init[lev]) {
                    bool restored_all = m_SurfaceLayer[ori]->set_mac_plane_avg(lev, most_plane_avg[lev]);
                    for (int iavg(0); iavg < m_SurfaceLayer[ori]->get_num_mac_avg(); ++iavg) {
                        restored_all = read_most_var("MOSTAvg" + std::to_string(iavg) + face,
                                                     m_SurfaceLayer[ori]->get_mac_avg_ptr(lev,iavg)) && restored_all;
                    }
                    if (restored_all) {
                        m_SurfaceLayer[ori]->set_mac_avg_initialized(lev);
                    } else {
                        amrex::Print() << "NOTE: the surface-layer time filter state at level " << lev
                                       << " is incomplete in this checkpoint; the filtered averages "
                                          "will start over" << std::endl;
                    }
                }
            }
        }
    }
}

ReadParameters()

void ERF::ReadParameters ( )

private

{
    // Record the inputs table before anything below can modify it.  Every query
    // here and in the classes built afterwards is a queryAdd, which inserts the
    // default it was handed when the key is absent, so this is the only point at
    // which "the key is in the table" still means "the user wrote it".
    user_specified_inputs = ParmParse::getEntries(pp_prefix);
 
    std::string prob_name = "Undefined";
    ParmParse pp_pn("erf");
    pp_pn.queryAdd("prob_name", prob_name);
    ibseb_params.init_params();
    Print() << "Problem name (from inputs file) is: "
            << " \"" << prob_name << "\" " << std::endl;
 
    ParmParse pp(pp_prefix);
    ParmParse pp_amr("amr");
    {
        pp.queryAdd("regrid_level_0_on_restart", regrid_level_0_on_restart);
        pp.queryAdd("regrid_int", regrid_int);
        pp.queryAdd("check_file", check_file);
 
        // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
        //    for those or "erf.restart" / "erf.m_check_int". The amr.* values are
        //    queried after the erf.* values and therefore take precedence if both
        //    are specified.
        pp.queryAdd("check_int", m_check_int);
        pp.queryAdd("check_per", m_check_per);
        pp_amr.queryAdd("check_int", m_check_int);
        pp_amr.queryAdd("check_per", m_check_per);
 
        pp.queryAdd("restart", restart_chkfile);
        pp_amr.queryAdd("restart", restart_chkfile);
 
        // Verbosity
        pp.queryAdd("v", verbose);
        pp.queryAdd("mg_v", mg_verbose);
        pp.queryAdd("use_fft", use_fft);
#ifndef ERF_USE_FFT
        if (use_fft) {
            Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
        }
#endif
 
        // Check for NaNs?
        pp.queryAdd("check_for_nans", check_for_nans);
 
        // Frequency of diagnostic output
        pp.queryAdd("sum_interval", sum_interval);
        pp.queryAdd("sum_period"  , sum_per);
 
        pp.queryAdd("pert_interval", pert_interval);
 
        // Time step controls
        pp.queryAdd("cfl", cfl);
        pp.queryAdd("substepping_cfl", sub_cfl);
        pp.queryAdd("init_shrink", init_shrink);
        pp.queryAdd("change_max", change_max);
        pp.queryAdd("dt_max_initial", dt_max_initial);
        pp.queryAdd("dt_max", dt_max);
 
        fixed_dt.resize(max_level+1,-one);
        fixed_fast_dt.resize(max_level+1,-one);
 
        pp.queryAdd("fixed_dt", fixed_dt[0]);
        pp.queryAdd("fixed_fast_dt", fixed_fast_dt[0]);
 
        int nlevs_max = max_level + 1;
        istep.resize(nlevs_max, 0);
        nsubsteps.resize(nlevs_max, 1);
        rad_interp_from_coarse_pending.resize(nlevs_max, 0);
        // This is the default
        for (int lev = 1; lev <= max_level; ++lev) {
            nsubsteps[lev] = MaxRefRatio(lev-1);
        }
 
        if (max_level > 0) {
            ParmParse pp_erf("erf");
            int count = pp_erf.countval("dt_ref_ratio");
            if (count > 0) {
                Vector<int> nsub;
                nsub.resize(nlevs_max, 0);
                if (count == 1) {
                    pp_erf.queryarr("dt_ref_ratio", nsub, 0, 1);
                    for (int lev = 1; lev <= max_level; ++lev) {
                        nsubsteps[lev] = nsub[0];
                    }
                } else {
                    pp_erf.queryarr("dt_ref_ratio", nsub, 0, max_level);
                    for (int lev = 1; lev <= max_level; ++lev) {
                        nsubsteps[lev] = nsub[lev-1];
                    }
                }
            }
        }
 
        // Make sure we do this after we have defined nsubsteps above
        for (int lev = 1; lev <= max_level; lev++)
        {
            fixed_dt[lev]      = fixed_dt[lev-1]      / static_cast<Real>(nsubsteps[lev]);
            fixed_fast_dt[lev] = fixed_fast_dt[lev-1] / static_cast<Real>(nsubsteps[lev]);
        }
 
        pp.queryAdd("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
 
        // We use this to keep track of how many boxes we read in from WRF initialization
        num_files_at_level.resize(max_level+1,0);
 
        // We use this to keep track of how many boxes are specified thru the refinement indicators
        num_boxes_at_level.resize(max_level+1,0);
            boxes_at_level.resize(max_level+1);
 
        // We always have exactly one file at level 0
        num_boxes_at_level[0] = 1;
        boxes_at_level[0].resize(1);
        boxes_at_level[0][0] = geom[0].Domain();
 
#ifdef ERF_USE_NETCDF
        nc_init_file.resize(max_level+1);
        have_read_nc_init_file.resize(max_level+1);
 
        // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
        //        but we always have exactly one file at level 0
        for (int lev = 0; lev <= max_level; lev++) {
            const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
            if (pp.contains(nc_file_names.c_str())) {
                int num_files = pp.countval(nc_file_names.c_str());
                num_files_at_level[lev] = num_files;
                nc_init_file[lev].resize(num_files);
                have_read_nc_init_file[lev].resize(num_files);
                pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
                for (int j = 0; j < num_files; j++) {
                    Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
                    have_read_nc_init_file[lev][j] = 0;
                } // j
            } // if pp.contains
        } // lev
 
        // NetCDF wrfbdy lateral boundary file
        if (pp.queryAdd("nc_bdy_file", nc_bdy_file)) {
            Print() << "Reading NC bdy file name " << nc_bdy_file << std::endl;
        }
 
        // NetCDF wrflow lateral boundary file
        if (pp.queryAdd("nc_low_file", nc_low_file)) {
            Print() << "Reading NC low file name " << nc_low_file << std::endl;
        }
 
#endif
 
        // Options for vertical interpolation of met_em*.nc data.
        pp.queryAdd("metgrid_debug_quiescent",  metgrid_debug_quiescent);
        pp.queryAdd("metgrid_debug_isothermal", metgrid_debug_isothermal);
        pp.queryAdd("metgrid_debug_dry",        metgrid_debug_dry);
        pp.queryAdd("metgrid_debug_psfc",       metgrid_debug_psfc);
        pp.queryAdd("metgrid_debug_msf",        metgrid_debug_msf);
        pp.queryAdd("metgrid_interp_theta",     metgrid_interp_theta);
        pp.queryAdd("metgrid_basic_linear",     metgrid_basic_linear);
        pp.queryAdd("metgrid_use_below_sfc",    metgrid_use_below_sfc);
        pp.queryAdd("metgrid_use_sfc",          metgrid_use_sfc);
        pp.queryAdd("metgrid_retain_sfc",       metgrid_retain_sfc);
        pp.queryAdd("metgrid_proximity",        metgrid_proximity);
        pp.queryAdd("metgrid_order",            metgrid_order);
        pp.queryAdd("metgrid_force_sfc_k",      metgrid_force_sfc_k);
 
        // Options for boundary file.
        // NOTE: write_erfbdy is deliberately NOT parsed here.  Its default depends on
        //       solverChoice.init_type, which is not known until init_params() runs
        //       below, so it is parsed further down once that default has been chosen.
        pp.queryAdd("erfbdy_file",              erfbdy_file);
 
        // Set default to FullState for now ... later we will try Perturbation
        interpolation_type = StateInterpType::FullState;
        pp.query_enum_case_insensitive("interpolation_type"  ,interpolation_type);
 
        PlotFileType plotfile3d_type_temp = PlotFileType::None;
        pp.query_enum_case_insensitive("plotfile_type"  ,plotfile3d_type_temp);
        pp.query_enum_case_insensitive("plotfile_type_1",plotfile3d_type_1);
        pp.query_enum_case_insensitive("plotfile_type_2",plotfile3d_type_2);
 
        PlotFileType plotfile2d_type_temp = PlotFileType::None;
        pp.query_enum_case_insensitive("plotfile2d_type"  ,plotfile2d_type_temp);
        pp.query_enum_case_insensitive("plotfile2d_type_1",plotfile2d_type_1);
        pp.query_enum_case_insensitive("plotfile2d_type_2",plotfile2d_type_2);
        //
        // This option is for backward consistency -- if only plotfile_type is set,
        //     then it will be used for both 1 and 2 if and only if they are not set
        //
        // Default is native amrex if no type is specified
        //
        if (plotfile3d_type_temp == PlotFileType::None) {
            if (plotfile3d_type_1 == PlotFileType::None) {
                plotfile3d_type_1  = PlotFileType::Amrex;
            }
            if (plotfile3d_type_2 == PlotFileType::None) {
                plotfile3d_type_2  = PlotFileType::Amrex;
            }
        } else {
            if (plotfile3d_type_1 == PlotFileType::None) {
                plotfile3d_type_1  = plotfile3d_type_temp;
            } else {
                Abort("You must set either plotfile_type or plotfile_type_1, not both");
            }
            if (plotfile3d_type_2 == PlotFileType::None) {
                plotfile3d_type_2  = plotfile3d_type_temp;
            } else {
                Abort("You must set either plotfile_type or plotfile_type_2, not both");
            }
        }
        if (plotfile2d_type_temp == PlotFileType::None) {
            if (plotfile2d_type_1 == PlotFileType::None) {
                plotfile2d_type_1  = PlotFileType::Amrex;
            }
            if (plotfile2d_type_2 == PlotFileType::None) {
                plotfile2d_type_2  = PlotFileType::Amrex;
            }
        } else {
            if (plotfile2d_type_1 == PlotFileType::None) {
                plotfile2d_type_1  = plotfile2d_type_temp;
            } else {
                Abort("You must set either plotfile2d_type or plotfile2d_type_1, not both");
            }
            if (plotfile2d_type_2 == PlotFileType::None) {
                plotfile2d_type_2  = plotfile2d_type_temp;
            } else {
                Abort("You must set either plotfile2d_type or plotfile2d_type_2, not both");
            }
        }
#ifndef ERF_USE_NETCDF
        if (plotfile3d_type_1 == PlotFileType::Netcdf ||
            plotfile3d_type_2 == PlotFileType::Netcdf ||
            plotfile2d_type_1 == PlotFileType::Netcdf ||
            plotfile2d_type_2 == PlotFileType::Netcdf) {
            Abort("Plotfile type = Netcdf is not allowed without USE_NETCDF = TRUE");
        }
#endif
 
        pp.queryAdd("plot_file_1"  ,   plot3d_file_1);
        pp.queryAdd("plot_file_2"  ,   plot3d_file_2);
        pp.queryAdd("plot2d_file_1",   plot2d_file_1);
        pp.queryAdd("plot2d_file_2",   plot2d_file_2);
 
        pp.queryAdd("plot_int_1"   , m_plot3d_int_1);
        pp.queryAdd("plot_int_2"   , m_plot3d_int_2);
        pp.queryAdd("plot_per_1"   , m_plot3d_per_1);
        pp.queryAdd("plot_per_2"   , m_plot3d_per_2);
 
        pp.queryAdd("plot2d_int_1" , m_plot2d_int_1);
        pp.queryAdd("plot2d_int_2" , m_plot2d_int_2);
        pp.queryAdd("plot2d_per_1",  m_plot2d_per_1);
        pp.queryAdd("plot2d_per_2",  m_plot2d_per_2);
 
        pp.queryAdd("subvol_file",   subvol_file);
 
        // Should we use format like plt1970-01-01_00:00:Real(00.000000) (if true) or plt00001 (if false)
        pp.queryAdd("use_real_time_in_pltname", use_real_time_in_pltname);
 
        // If use_real_time_in_pltname is false, how many digits should we use for the timestep?
        pp.queryAdd("file_name_digits", file_name_digits);
 
        // Default if subvol_int not specified
        m_subvol_int.resize(1); m_subvol_int[0] = -1;
        m_subvol_per.resize(1); m_subvol_per[0] = -one;
        last_subvol_step.resize(1);
        last_subvol_time.resize(1);
 
        int nsi = pp.countval("subvol_int");
        int nsr = pp.countval("subvol_per");
 
        // We must specify only subvol_int OR subvol_per
        AMREX_ALWAYS_ASSERT (!(nsi > 0 && nsr > 0));
 
        int nsub = -1;
        if (nsi > 0 || nsr > 0) {
            ParmParse pp_sv("erf.subvol");
            int n1 = pp_sv.countval("origin"); int n2 = pp_sv.countval("nxnynz"); int n3 = pp_sv.countval("dxdydz");
            if (n1 != n2 || n1 != n3 || n2 != n3) {
                Abort("WriteSubvolume: must have same number of entries in origin, nxnynz, and dxdydz.");
            }
            if ( n1%AMREX_SPACEDIM != 0) {
                Abort("WriteSubvolume: origin, nxnynz, and dxdydz must have multiples of AMReX_SPACEDIM");
            }
            nsub = n1/AMREX_SPACEDIM;
            last_subvol_step.resize(nsub);
            last_subvol_time.resize(nsub);
            m_subvol_int.resize(nsub);
            m_subvol_per.resize(nsub);
        }
 
        if (nsi > 0) {
            for (int i = 0; i < nsub; i++) m_subvol_per[i] = -one;
            if ( nsi == 1) {
                m_subvol_int[0] = -1;
                pp.get("subvol_int" , m_subvol_int[0]);
                // A single value applies to every subdomain
                for (int i = 1; i < nsub; i++) m_subvol_int[i] = m_subvol_int[0];
            } else if ( nsi == nsub) {
                pp.getarr("subvol_int" , m_subvol_int);
            } else {
                Abort("There must either be a single value of subvol_int or one for every subdomain");
            }
        }
 
        if (nsr > 0) {
            for (int i = 0; i < nsub; i++) m_subvol_int[i] = -1;
            if ( nsr == 1) {
                m_subvol_per[0] = -one;
                pp.get("subvol_per" , m_subvol_per[0]);
                // A single value applies to every subdomain
                for (int i = 1; i < nsub; i++) m_subvol_per[i] = m_subvol_per[0];
            } else if ( nsr == nsub) {
                pp.getarr("subvol_per" , m_subvol_per);
            } else {
                Abort("There must either be a single value of subvol_per or one for every subdomain");
            }
        }
 
        // NOTE: the subvolume variable list is *not* selected here.  Choosing which
        //       "rhoQn" components exist needs the microphysics interface, and that
        //       is constructed after ReadParameters() returns, so setSubVolVariables
        //       is called alongside setPlotVariables in the ERF constructor.
 
        pp.queryAdd("expand_plotvars_to_unif_rr",m_expand_plotvars_to_unif_rr);
 
        pp.queryAdd("plot_face_vels",m_plot_face_vels);
        pp.queryAdd("plot_face_terrain_blanking",m_plot_face_terrain_blanking);
 
        if ( (m_plot3d_int_1 > 0 && m_plot3d_per_1 > 0) ||
             (m_plot3d_int_2 > 0 && m_plot3d_per_2 > zero) ) {
            Abort("Must choose only one of plot_int or plot_per");
        }
        if ( (m_plot2d_int_1 > 0 && m_plot2d_per_1 > 0) ||
             (m_plot2d_int_2 > 0 && m_plot2d_per_2 > zero) ) {
            Abort("Must choose only one of plot_int or plot_per");
        }
 
        pp.queryAdd("profile_int", profile_int);
        pp.queryAdd("destag_profiles", destag_profiles);
 
        pp.queryAdd("plot_lsm", plot_lsm);
#ifdef ERF_USE_RRTMGP
        pp.queryAdd("plot_rad", plot_rad);
#endif
        pp.queryAdd("profile_rad_int", rad_datalog_int);
 
        pp.queryAdd("output_1d_column", output_1d_column);
        pp.queryAdd("column_per", column_per);
        pp.queryAdd("column_interval", column_interval);
        pp.queryAdd("column_loc_x", column_loc_x);
        pp.queryAdd("column_loc_y", column_loc_y);
        pp.queryAdd("column_file_name", column_file_name);
 
        // Sampler output frequency
        pp.queryAdd("line_sampling_per", line_sampling_per);
        pp.queryAdd("line_sampling_interval", line_sampling_interval);
        pp.queryAdd("plane_sampling_per", plane_sampling_per);
        pp.queryAdd("plane_sampling_interval", plane_sampling_interval);
        pp.queryAdd("station_sampling_per", station_sampling_per);
        pp.queryAdd("station_sampling_interval", station_sampling_interval);
 
        // Specify information about outputting planes of data
        pp.queryAdd("output_bndry_planes", output_bndry_planes);
        pp.queryAdd("bndry_output_planes_interval", bndry_output_planes_interval);
        pp.queryAdd("bndry_output_planes_per", bndry_output_planes_per);
        pp.queryAdd("bndry_output_start_time", bndry_output_planes_start_time);
 
        // Specify whether ingest boundary planes of data
        pp.queryAdd("input_bndry_planes", input_bndry_planes);
 
        // Query the total width for wrfbdy interior ghost cells
        pp.queryAdd("real_width", real_width);
 
        // If using real boundaries, do we extrapolate w (or set to 0)
        pp.queryAdd("real_extrap_w", real_extrap_w);
 
        // Query the set and total widths for crse-fine interior ghost cells
        pp.queryAdd("cf_width", cf_width);
        pp.queryAdd("cf_set_width", cf_set_width);
 
        // AmrMesh iterate on grids?
        bool iterate(true);
        pp_amr.queryAdd("iterate_grids",iterate);
        if (!iterate) SetIterateToFalse();
    }
 
#ifdef ERF_USE_PARTICLES
    readTracersParams();
#endif
 
    solverChoice.init_params(max_level,pp_prefix);
 
    // Implicit acoustic substepping inverts one tridiagonal system per column, so it is
    // only well posed if no column is chopped between boxes.  That does not require one
    // box per column: a level may have several boxes over the same (i,j) -- as it does
    // where the refined region is a staircase in z, or covers two separate layers -- as
    // long as they do not touch, so that each contiguous run of cells in the column is
    // solved by itself.  What must not happen is two boxes sharing a face normal to z,
    // which would split one column into pieces solved separately, with spurious internal
    // boundaries at the seam and an answer that depends on the decomposition.
    // amr.no_box_split_dir = 2 -- the ERF default, set in add_par -- is what rules that
    // out: the grid generator merges the boxes it makes along z, so no two of them share
    // a face normal to z.  Refuse to run with any other value while a level substeps
    // implicitly.
    for (int lev = 0; lev <= max_level; lev++) {
        if ( (solverChoice.substepping_type[lev] == SubsteppingType::Implicit) &&
             (no_box_split_dir != 2) )
        {
            Abort("erf.substepping_type = Implicit at level " + std::to_string(lev) +
                  " requires amr.no_box_split_dir = 2 (the ERF default), so that no two grids "
                  "share a face normal to z: the implicit substep solve inverts one tridiagonal "
                  "system per column, and a column chopped at such a seam would instead be "
                  "solved in pieces, giving an answer that depends on the grid decomposition.  "
                  "(Boxes stacked over the same column are fine as long as they do not touch.)  "
                  "Either remove amr.no_box_split_dir = " +
                  std::to_string(no_box_split_dir) + " from the inputs file, or set "
                  "erf.substepping_type = None.");
        }
    }
 
    // Set a default value for write_erfbdy following these rules.
    // Prioritize write_erfbdy provided by user.
    // write_erfbdy must be false for restarts.
    // write_erfbdy defaults to true for clean starts of the metgrid or wrfinput pathways.
    //
    // The context-dependent default is chosen FIRST and the user's value is parsed on
    // top of it, so that "did the user set this" never has to be asked.  It must not be
    // asked: this used to test ParmParse::contains("write_erfbdy") after the key had
    // already been parsed with queryAdd up in the metgrid block, and queryAdd inserts
    // the default into the global table on a miss.  contains() was therefore always
    // true, the metgrid/WRFInput default below never fired, no erfbdy was written on a
    // clean start, and the following restart aborted in ReadCheckpointFile.
    {
        ParmParse pp_erfbdy(pp_prefix);
        const bool is_restart = !restart_chkfile.empty();
        if (!is_restart &&
            ((solverChoice.init_type == InitType::Metgrid) || (solverChoice.init_type == InitType::WRFInput))) {
            write_erfbdy = true;
        }
 
        // A value given in the inputs file overrides the default chosen above.
        pp_erfbdy.queryAdd("write_erfbdy", write_erfbdy);
 
        if (is_restart && write_erfbdy) {
            Abort("Cannot set erf.write_erfbdy = true during restart. erfbdy should only be written during initial runs.");
        }
    }
 
    {
        ParmParse pp_no_prefix;  // Traditionally, max_step and stop_time do not have prefix.
        pp_no_prefix.queryAdd("max_step", max_step);
        if (max_step < 0) {
            max_step = std::numeric_limits<int>::max();
        }
 
        std::string start_datetime, stop_datetime;
        if (pp_no_prefix.queryAdd("start_datetime", start_datetime)) {
            if (start_datetime.length() == 16) { // YYYY-MM-DD HH:MM
                start_datetime += ":00"; // add seconds
            }
            if (start_datetime.length() != 19) {
                Print() << "Got start_datetime = \"" << start_datetime
                    << "\", format should be " << datetime_format << std::endl;
                exit(0);
            }
            start_time = static_cast<double>(getEpochTime(start_datetime, datetime_format));
 
#ifdef ERF_USE_NETCDF
            if (solverChoice.init_type == InitType::WRFInput) {
                // This is the start time as written in the wrfinput file
                double start_time_from_wrfinput = read_start_time_from_wrfinput(0, nc_init_file[0][0]);
                if (start_time != start_time_from_wrfinput) {
                    amrex::Print() << "start_datetime from inputs file = "       << start_time <<
                                      " does not match SIMULATION START DATE from wrfinput = " <<
                                       start_time_from_wrfinput << std::endl;
                    amrex::Abort();
                }
            } else if (solverChoice.init_type == InitType::Metgrid) {
                // This is the start time as written in the metgrid file
                double start_time_from_metgrid = read_start_time_from_metgrid(0, nc_init_file[0][0]);
                if (start_time != start_time_from_metgrid) {
                    amrex::Print() << "start_datetime from inputs file = "       << start_time <<
                                      " does not match SIMULATION START DATE from metgrid = " <<
                                       start_time_from_metgrid << std::endl;
                    amrex::Abort();
                }
            }
#endif
            Print() << "Start datetime   : " << start_datetime << std::endl;
 
            use_datetime = true;
 
        } else {
 
#ifdef ERF_USE_NETCDF
            if (solverChoice.init_type == InitType::WRFInput) {
                // This is the start time as written in the wrfinput file
                double start_time_from_wrfinput = read_start_time_from_wrfinput(0, nc_init_file[0][0]);
                start_time = start_time_from_wrfinput;
 
                use_datetime = true;
 
                if (pp_no_prefix.query("start_time", start_time)) {
                    amrex::Print() << "start_time should not be set from inputs file; we are reading SIMULATION START DATE from wrfinput" << std::endl;
                    amrex::Abort();
                }
            } else if (solverChoice.init_type == InitType::Metgrid) {
                // This is the start time as written in the metgrid file
                double start_time_from_metgrid = read_start_time_from_metgrid(0, nc_init_file[0][0]);
                start_time = start_time_from_metgrid;
 
                use_datetime = true;
 
                if (pp_no_prefix.query("start_time", start_time)) {
                    amrex::Print() << "start_time should not be set from inputs file; we are reading SIMULATION START DATE from metgrid" << std::endl;
                    amrex::Abort();
                }
            }
#endif
        }
 
        if (pp_no_prefix.queryAdd("stop_datetime", stop_datetime)) {
            if (stop_datetime.length() == 16) { // YYYY-MM-DD HH:MM
                stop_datetime += ":00"; // add seconds
            }
            if (stop_datetime.length() != 19) {
                Print() << "Got stop_datetime = \"" << stop_datetime
                    << "\", format should be " << datetime_format << std::endl;
                exit(0);
            }
 
            stop_time = static_cast<double>(getEpochTime(stop_datetime, datetime_format));
            Print() << "Stop  datetime : " << start_datetime << std::endl;
 
        } else {
 
            // stop_time already defaults to numeric_limits<double>::max(), which no user
            // would type, so it serves as its own sentinel and we can test the value
            // instead of the queryAdd return value (which only reports whether the key
            // existed before this call).
            pp_no_prefix.queryAdd("stop_time", stop_time);
            if (stop_time < std::numeric_limits<double>::max()) {
                Print() << "Maximum simulation length based on stop_time: " << stop_time << " s (elapsed) " << std::endl;
                amrex::Print() <<" Adding stop time " << stop_time << " to start_time " << start_time << std::endl;
                stop_time += start_time;
            }
        }
    }
 
#ifndef ERF_USE_NETCDF
    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(( (solverChoice.init_type != InitType::WRFInput) &&
                                       (solverChoice.init_type != InitType::Metgrid ) &&
                                       (solverChoice.init_type != InitType::NCFile  )  ),
                                     "init_type cannot be 'WRFInput', 'Metgrid' or 'NCFile' if we don't build with netcdf!");
#endif
 
    // Query the canopy model parameters
    {
        std::string forestfile;
        std::string forest_lai_file, forest_height_file, forest_cd_file;
 
        bool requested_forest_drag = false;
        bool has_forest_drag_switch = pp.query("do_forest_drag", requested_forest_drag);
 
        bool has_forest_file   = pp.query("forest_file",        forestfile);
        bool has_forest_lai    = pp.query("forest_lai_file",    forest_lai_file);
        bool has_forest_height = pp.query("forest_height_file", forest_height_file);
        bool has_forest_cd     = pp.query("forest_cd_file",     forest_cd_file);
 
        // Optional constant drag coefficient (alternative to forest_cd_file)
        Real forest_cd_const = -1.0;
        bool has_forest_cd_const = pp.query("forest_cd", forest_cd_const);
 
        if (has_forest_cd && has_forest_cd_const) {
            Abort("Cannot specify both 'forest_cd_file' and 'forest_cd'. Choose one.");
        }
 
        int  forest_tree_type = 1;
        Real forest_laimax    = 0.8;
        pp.query("forest_tree_type", forest_tree_type);
        pp.query("forest_laimax",    forest_laimax);
 
        if (has_forest_file && (has_forest_lai || has_forest_height || has_forest_cd || has_forest_cd_const)) {
            Abort("Cannot specify both 'forest_file' and gridded forest options. Choose one mode.");
        }
 
        if (has_forest_file) {
            solverChoice.do_forest_drag = true;
            for (int lev = 0; lev <= max_level; ++lev) {
                m_forest_drag[lev] = std::make_unique<ForestDrag>(forestfile);
            }
            Print() << "ForestDrag: Using discrete patch mode with file: " << forestfile << "\n";
 
        } else if (has_forest_lai && has_forest_height && has_forest_cd) {
            // Gridded NC mode with Cd from file
            solverChoice.do_forest_drag = true;
            for (int lev = 0; lev <= max_level; ++lev) {
                m_forest_drag[lev] = std::make_unique<ForestDrag>(
                    forest_lai_file, forest_height_file, forest_cd_file,
                    forest_tree_type, forest_laimax);
            }
            Print() << "ForestDrag: Using gridded NetCDF mode\n"
                    << "  LAI file: " << forest_lai_file << "\n"
                    << "  Height file: " << forest_height_file << "\n"
                    << "  Cd file: " << forest_cd_file << "\n"
                    << "  Tree type: " << forest_tree_type << "\n"
                    << "  LAImax: " << forest_laimax << "\n";
 
        } else if (has_forest_lai && has_forest_height && has_forest_cd_const) {
            // Gridded NC mode with constant Cd
            solverChoice.do_forest_drag = true;
            for (int lev = 0; lev <= max_level; ++lev) {
                m_forest_drag[lev] = std::make_unique<ForestDrag>(
                    forest_lai_file, forest_height_file, forest_cd_const,
                    forest_tree_type, forest_laimax);
            }
            Print() << "ForestDrag: Using gridded NetCDF mode with constant Cd=" << forest_cd_const << "\n"
                    << "  LAI file: " << forest_lai_file << "\n"
                    << "  Height file: " << forest_height_file << "\n"
                    << "  Tree type: " << forest_tree_type << "\n"
                    << "  LAImax: " << forest_laimax << "\n";
 
        } else if (has_forest_lai || has_forest_height || has_forest_cd || has_forest_cd_const) {
            Abort("Gridded forest mode requires forest_lai_file, forest_height_file, "
                  "and either forest_cd_file or forest_cd.");
        }
 
        if (has_forest_drag_switch && requested_forest_drag != solverChoice.do_forest_drag) {
            Abort("'do_forest_drag' must be true when a complete forest configuration is "
                  "provided, and false when no forest configuration is provided.");
        }
 
        if ((solverChoice.forest_biophysics || solverChoice.forest_biophysics_heat) &&
            !solverChoice.do_forest_drag) {
            Abort("Forest biophysics was requested (forest_biophysics=" +
                  std::string(solverChoice.forest_biophysics ? "true" : "false") +
                  ", forest_biophysics_heat=" +
                  std::string(solverChoice.forest_biophysics_heat ? "true" : "false") +
                  ") but no complete forest configuration was supplied. Provide "
                  "forest_lai_file, forest_height_file, and forest_cd or forest_cd_file.");
        }
 
    }
 
    // If init from WRFInput or Metgrid make sure a valid file name is present at level zero
    // We allow for the possibility that finer levels may use native refinement rather than reading from a file
    if ((solverChoice.init_type == InitType::WRFInput) ||
        (solverChoice.init_type == InitType::Metgrid)  ||
        (solverChoice.init_type == InitType::NCFile) ) {
        int num_files = static_cast<int>(nc_init_file[0].size());
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(num_files>0, "A file name must be present at level 0 for init type WRFInput, Metgrid or NCFile.");
        for (int j = 0; j < num_files; j++) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!nc_init_file[0][j].empty(), "Valid file name must be present at level 0 for init type WRFInput, Metgrid or NCFile.");
        } //j
    } // InitType
 
    // What type of land surface model to use
    // NOTE: Must be checked after init_params
    if (solverChoice.lsm_type == LandSurfaceType::SLM) {
        lsm.SetModel<SLM>();
        Print() << "SLM land surface model!\n";
#ifdef ERF_USE_NOAHMP
    } else if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
        lsm.SetModel<NOAHMP>();
        Print() << "Noah-MP land surface model!\n";
#endif
    } else if (solverChoice.lsm_type == LandSurfaceType::None) {
        lsm.SetModel<NullSurf>();
        Print() << "Null land surface model!\n";
    } else {
        Abort("Dont know this LandSurfaceType!") ;
    }
 
    if (verbose > 0) {
        solverChoice.display(max_level,pp_prefix);
    }
 
    ParameterSanityChecks();
}

Referenced by ERF_shared().

Here is the call graph for this function:

Here is the caller graph for this function:

ReadStormTrackerRestart()

void ERF::ReadStormTrackerRestart

(

)

Read hurricane tracking history from restart files.

{
    hurricane_eye_track_xy.clear();
    hurricane_eye_track_latlon.clear();
    hurricane_maxvel_vs_time.clear();
    hurricane_minpressure_vs_time.clear();
 
    const fs::path base_dir("Output_StormTracker");
 
    // Nothing to do for a fresh run.
    if (!fs::exists(base_dir)) {
        return;
    }
    //
    // Return the alphabetically last file in a directory.
    // Since the filenames are zero-padded, this is also the newest output.
    //
    auto last_file = [](const fs::path& dir) -> fs::path
    {
        std::vector<fs::path> files;
 
        for (const auto& entry : fs::directory_iterator(dir)) {
            if (entry.is_regular_file()) {
                files.push_back(entry.path());
            }
        }
 
        if (files.empty()) {
            return fs::path{};
        }
 
        std::sort(files.begin(), files.end());
 
        return files.back();
    };
 
    //==========================================================
    // Read lat/lon file
    //==========================================================
 
    {
        fs::path file = last_file(base_dir / "latlon");
 
        if (!file.empty())
        {
            std::ifstream ifs(file);
 
            if (!ifs.is_open()) {
                Abort("Could not open " + file.string());
            }
 
            std::string line;
 
            // Skip the header line.
            std::getline(ifs, line);
 
            Real lat, lon;
 
            while (ifs >> lat >> lon)
            {
                hurricane_eye_track_latlon.push_back({lat, lon});
            }
        }
    }
 
    //==========================================================
    // Read maxvel tracker file
    //==========================================================
 
    {
        fs::path file = last_file(base_dir / "maxvel");
 
        if (!file.empty())
        {
            std::ifstream ifs(file);
 
            if (!ifs.is_open()) {
                Abort("Could not open " + file.string());
            }
 
            std::string line;
 
            // Skip the header line.
            std::getline(ifs, line);
 
            amrex::Real val1, val2;
 
            while (ifs >> val1 >> val2)
            {
                hurricane_maxvel_vs_time.push_back({val1, val2});
            }
        }
    }
 
    //==========================================================
    // Read minpressure tracker file
    //==========================================================
 
    {
        fs::path file = last_file(base_dir / "minpressure");
 
        if (!file.empty())
        {
            std::ifstream ifs(file);
 
            if (!ifs.is_open()) {
                Abort("Could not open " + file.string());
            }
 
            std::string line;
 
            // Skip the header line.
            std::getline(ifs, line);
 
            amrex::Real val1, val2;
 
            while (ifs >> val1 >> val2)
            {
                hurricane_minpressure_vs_time.push_back({val1, val2});
            }
        }
    }
 
    //==========================================================
    // Read XY VTK file
    //==========================================================
 
    {
        fs::path file = last_file(base_dir / "xy");
 
        if (!file.empty())
        {
            std::ifstream ifs(file);
            std::string line;
 
            // Skip the first four header lines.
            for (int i = 0; i < 4; ++i) {
                std::getline(ifs, line);
            }
 
 
            std::getline(ifs, line);
            std::istringstream iss(line);
            std::string keyword, datatype;
            int npoints;
 
            iss >> keyword >> npoints >> datatype;
            hurricane_eye_track_xy.reserve(npoints);
 
            for (int i = 0; i < npoints; ++i)
            {
                Real x, y, z;
                ifs >> x >> y >> z;
                hurricane_eye_track_xy.push_back({x, y});
            }
        }
    }
}

ReadVelsOnlyFromCheckpointFile()

void ERF::ReadVelsOnlyFromCheckpointFile	(	int	lev_to_fill,
		std::string &	chkfile_for_vels
	)

ERF function for reading data from a checkpoint file during restart.

{
    Print() << "Reading vels only from native checkpoint " << chkfile_for_vels << " at level " << lev_to_fill << "\n";
 
    // Header
    std::string File(chkfile_for_vels + "/Header");
 
    VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
 
    Vector<char> fileCharPtr;
    ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
    std::string fileCharPtrString(fileCharPtr.dataPtr());
    std::istringstream is(fileCharPtrString, std::istringstream::in);
 
    AMREX_ALWAYS_ASSERT(lev_to_fill >= 0 && lev_to_fill <= finest_level);
 
    int lev = lev_to_fill;
 
    MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
    VisMF::Read(xvel, MultiFabFileFullPrefix(lev, chkfile_for_vels, "Level_", "XFace"));
    MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
    vars_new[lev][Vars::xvel].setBndry(bogus_large_value);
 
    MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
    VisMF::Read(yvel, MultiFabFileFullPrefix(lev, chkfile_for_vels, "Level_", "YFace"));
    MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
    vars_new[lev][Vars::yvel].setBndry(bogus_large_value);
 
    MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
    VisMF::Read(zvel, MultiFabFileFullPrefix(lev, chkfile_for_vels, "Level_", "ZFace"));
    MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
    vars_new[lev][Vars::zvel].setBndry(bogus_large_value);
}

Here is the call graph for this function:

refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )

private

Function to define the refinement criteria based on user input

{
    if (max_level > 0)
    {
        ParmParse pp(pp_prefix);
        Vector<std::string> refinement_indicators;
        pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
 
        for (int i=0; i<refinement_indicators.size(); ++i)
        {
            std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
 
            ParmParse ppr(ref_prefix);
            RealBox real_box;
            int lev_for_box;
 
            read_box_for_refinement(ref_prefix, lev_for_box, real_box);
 
            AMRErrorTagInfo info;
 
            if (real_box.ok()) {
                info.SetRealBox(real_box);
            }
 
            if (ppr.countval("start_time") > 0) {
                double ref_min_time; ppr.get("start_time",ref_min_time);
                info.SetMinTime(static_cast<Real>(ref_min_time));
            }
 
            if (ppr.countval("end_time") > 0) {
                double ref_max_time; ppr.get("end_time",ref_max_time);
                info.SetMaxTime(static_cast<Real>(ref_max_time));
            }
 
            if (ppr.countval("max_level") > 0) {
                int ref_max_level; ppr.get("max_level",ref_max_level);
                info.SetMaxLevel(ref_max_level);
            }
 
            // Read field_name once and validate moisture-field requests against
            // the active moisture model so an unsupported field aborts at setup
            // rather than after the first regrid.
            std::string field;
            if (ppr.countval("field_name") > 0) {
                ppr.get("field_name", field);
                auto is_moist_field = [](const std::string& f) {
                    return f == "qv" || f == "qc" || f == "qi" ||
                           f == "qr" || f == "qs" || f == "qg" || f == "qt";
                };
                if (is_moist_field(field)) {
                    const auto& mi = solverChoice.moisture_indices;
                    int comp = -1;
                    if      (field == "qv") { comp = mi.qv; }
                    else if (field == "qc") { comp = mi.qc; }
                    else if (field == "qi") { comp = mi.qi; }
                    else if (field == "qr") { comp = mi.qr; }
                    else if (field == "qs") { comp = mi.qs; }
                    else if (field == "qg") { comp = mi.qg; }
                    else if (field == "qt") {
                        comp = (mi.qc >= 0 || mi.qi >= 0 || mi.qr >= 0 ||
                                mi.qs >= 0 || mi.qg >= 0) ? 0 : -1;
                    }
                    if (comp < 0) {
                        amrex::Abort("Refinement field_name '" + field +
                                     "' is not available for the configured moisture model");
                    }
                }
            }
 
            // Every push_back into ref_tags must be matched by a push_back into
            // ref_tag_indicator_names so that ErrorEst can recover the ParmParse
            // prefix of the indicator that created each tag.  Indicators such as
            // "storm_tracker" create no ref_tag at all, so the index into ref_tags
            // is not in general the index into refinement_indicators.
            if (ppr.countval("value_greater")) {
                int num_val = ppr.countval("value_greater");
                Vector<Real> value(num_val);
                ppr.getarr("value_greater",value,0,num_val);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
                ref_tag_indicator_names.push_back(refinement_indicators[i]);
            }
            else if (ppr.countval("value_less"))
            {
                int num_val = ppr.countval("value_less");
                Vector<Real> value(num_val);
                ppr.getarr("value_less",value,0,num_val);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
                ref_tag_indicator_names.push_back(refinement_indicators[i]);
            }
            else if (ppr.countval("adjacent_difference_greater"))
            {
                int num_val = ppr.countval("adjacent_difference_greater");
                Vector<Real> value(num_val);
                ppr.getarr("adjacent_difference_greater",value,0,num_val);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
                ref_tag_indicator_names.push_back(refinement_indicators[i]);
            }
            else if (real_box.ok())
            {
                ref_tags.push_back(AMRErrorTag(info));
                ref_tag_indicator_names.push_back(refinement_indicators[i]);
            }
            else if ( (lev_for_box > 0) && (refinement_indicators[i] != "storm_tracker") )
            {
                Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
            }
            AMREX_ALWAYS_ASSERT(ref_tags.size() == ref_tag_indicator_names.size());
        } // loop over criteria
    } // if max_level > 0
}

Referenced by ERF_shared().

Here is the call graph for this function:

Here is the caller graph for this function:

remake_zphys()

void ERF::remake_zphys	(	int	lev,
		amrex::Real	time,
		std::unique_ptr< amrex::MultiFab > &	temp_zphys_nd
	)

{
    if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
    {
        if (lev == 0) {
            temp_zphys_nd->ParallelCopy(*z_phys_nd[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect(), z_phys_nd[lev]->nGrowVect());
        } else {
            //
            // First interpolate from coarser level
            // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
            //       have been pre-filled - this includes ghost cells both inside and outside
            //       the domain
            //
            InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
                                  IntVect(0,0,0), // do NOT fill ghost cells outside the domain
                                  *z_phys_nd[lev-1], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &node_bilinear_interp,
                                  domain_bcs_type, BCVars::cons_bc);
 
            //
            // On a fine level with STF or Sullivan TF we have to rebuild the mesh the
            // same way init_zphys() did, since make_terrain_fitted_coords() cannot
            // apply the transformation to a level that does not span the full column
            //
            ParmParse pp("erf");
            int terrain_smoothing = 0;
            pp.query("terrain_smoothing", terrain_smoothing);
 
            FineTerrain fine_terrain = (terrain_smoothing != 0) ? which_fine_terrain()
                                                                : FineTerrain::None;
 
            MultiFab z_phys_interp;
            if (fine_terrain == FineTerrain::Transform) {
                z_phys_interp.define(temp_zphys_nd->boxArray(), temp_zphys_nd->DistributionMap(),
                                     1, temp_zphys_nd->nGrowVect());
                MultiFab::Copy(z_phys_interp, *temp_zphys_nd, 0, 0, 1, 0); // valid region only
                fill_terrain_surface(lev, *temp_zphys_nd, time);
            }
 
            // This recomputes the fine values using the bottom terrain at the fine resolution,
            //    and also fills values of z_phys_nd outside the domain
            make_terrain_fitted_coords(lev,geom[lev],*temp_zphys_nd,zlevels_stag[lev],phys_bc_type,
                                       fine_terrain,
                                       (fine_terrain == FineTerrain::Transform) ? &z_phys_interp : nullptr);
 
        } // lev > 0
 
        std::swap(temp_zphys_nd, z_phys_nd[lev]);
 
    } else {
        if (lev == 0) {
            temp_zphys_nd->ParallelCopy(*z_phys_nd[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect(), z_phys_nd[lev]->nGrowVect());
        } else {
            //
            // First interpolate from coarser level
            // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
            //       have been pre-filled - this includes ghost cells both inside and outside
            //       the domain
            //
            InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
                                  z_phys_nd[lev]->nGrowVect(), // DO fill ghost cells outside the domain
                                  *z_phys_nd[lev-1], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &node_bilinear_interp,
                                  domain_bcs_type, BCVars::cons_bc);
 
            //
            // If this level has its own wrfinput / met_em file then its terrain was built
            // from that file's PH + PHB, and the coarse interpolation above is only the
            // right answer in the parts of the NEW grids that the OLD grids did not cover.
            // ParallelCopy writes only where source and destination overlap, so this keeps
            // the file-derived terrain wherever we already had it and leaves the
            // coarse-interpolated values everywhere else.
            //
            // Without this, every regrid would quietly demote a nested level back onto
            // coarse-interpolated heights -- which is the same thing the guard in
            // init_zphys above exists to prevent at initialization.
            //
            if (!nc_init_file[lev].empty()) {
                temp_zphys_nd->ParallelCopy(*z_phys_nd[lev], 0, 0, 1,
                                            z_phys_nd[lev]->nGrowVect(), z_phys_nd[lev]->nGrowVect());
            }
 
        } // lev > 0
 
        std::swap(temp_zphys_nd, z_phys_nd[lev]);
    }
 
    if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
        solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
        //
        // This assumes we have already remade the EBGeometry
        //
        // Read the small_volfrac threshold from eb2 namespace
        Real small_volfrac = 0.005;
        ParmParse pp_eb2("eb2");
        pp_eb2.queryAdd("small_volfrac", small_volfrac);
 
        terrain_blanking[lev]->setVal(one);
        MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, z_phys_nd[lev]->nGrowVect());
 
        // Clip small terrain_blanking values (almost fluid cells) using same threshold as eb2.small_volfrac
        if (small_volfrac > zero) {
            for (MFIter mfi(*terrain_blanking[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                auto const& tblank = terrain_blanking[lev]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    if (tblank(i,j,k) < small_volfrac) {
                        tblank(i,j,k) = zero;
                    }
                });
            }
        }
 
#if USE_FC_FACTORY
    // Face-centered terrain blanking from face-centered EB volume fractions
        const int ng_sub = std::min(ComputeGhostCells(solverChoice) + 2, EBFactory(lev).getVolFrac().nGrow());
 
        terrain_blanking_xface[lev]->setVal(one);
        terrain_blanking_yface[lev]->setVal(one);
        terrain_blanking_zface[lev]->setVal(one);
        // Check if face factories are available before using them
        auto const* u_factory = eb[lev]->get_u_const_factory();
        auto const* v_factory = eb[lev]->get_v_const_factory();
        auto const* w_factory = eb[lev]->get_w_const_factory();
 
        if (u_factory && v_factory && w_factory) {
            MultiFab::Subtract(*terrain_blanking_xface[lev],
                               u_factory->getVolFrac(), 0, 0, 1, ng_sub);
            MultiFab::Subtract(*terrain_blanking_yface[lev],
                               v_factory->getVolFrac(), 0, 0, 1, ng_sub);
            MultiFab::Subtract(*terrain_blanking_zface[lev],
                               w_factory->getVolFrac(), 0, 0, 1, ng_sub);
 
            // Clip small terrain_blanking values on faces (almost fluid cells) using same threshold
            if (small_volfrac > zero) {
                for (MFIter mfi(*terrain_blanking_xface[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    const Box& xbx = mfi.tilebox();
                    auto const& tblank_x = terrain_blanking_xface[lev]->array(mfi);
                    ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        if (tblank_x(i,j,k) < small_volfrac) {
                            tblank_x(i,j,k) = zero;
                        }
                    });
                }
                for (MFIter mfi(*terrain_blanking_yface[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    const Box& ybx = mfi.tilebox();
                    auto const& tblank_y = terrain_blanking_yface[lev]->array(mfi);
                    ParallelFor(ybx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        if (tblank_y(i,j,k) < small_volfrac) {
                            tblank_y(i,j,k) = zero;
                        }
                    });
                }
                for (MFIter mfi(*terrain_blanking_zface[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    const Box& zbx = mfi.tilebox();
                    auto const& tblank_z = terrain_blanking_zface[lev]->array(mfi);
                    ParallelFor(zbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                        if (tblank_z(i,j,k) < small_volfrac) {
                            tblank_z(i,j,k) = zero;
                        }
                    });
                }
            }
        }
#endif
    }
 
    // Compute the min dz and pass to the micro model
    Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
    micro->Set_dzmin(lev, dzmin);
}

Here is the call graph for this function:

RemakeLevel()

void ERF::RemakeLevel ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

override

{
    //
    // Note that "time" here is elapsed time
    //
    if (verbose) {
        amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
    }
 
    AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::MovingFittedMesh);
 
    BoxArray            ba_old(vars_new[lev][Vars::cons].boxArray());
    DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
 
    if (verbose) {
        amrex::Print() <<"               OLD BA AT LEVEL " << lev << " " << ba_old << std::endl;
    }
 
    //
    // Re-define subdomain at this level within the domain such that
    // 1) all boxes in a given subdomain are "connected"
    // 2) no boxes in a subdomain touch any boxes in any other subdomain
    //
    subdomains[lev].clear();
    make_subdomains(ba.simplified_list(), subdomains[lev]);
 
    // ********************************************************************************************
    // Define grids[lev]/dmap[lev] to be the passed-in ba/dm *now*, not on the way out.
    //
    // AmrCore::regrid does not call SetBoxArray/SetDistributionMap for this level until this
    // function returns, so for the whole body below grids[lev]/dmap[lev] otherwise still
    // describe the PRE-regrid decomposition.  Anything that reads them instead of the ba/dm
    // passed in silently builds itself on the old grids. This is idempotent with the calls
    // AmrCore makes after we return, and is the only place it happens for lev == 0.
    // ********************************************************************************************
    SetBoxArray(lev, ba);
    SetDistributionMap(lev, dm);
 
    int     ncomp_cons  = vars_new[lev][Vars::cons].nComp();
    IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
 
    int ngrow_vels = ComputeGhostCells(solverChoice);
 
    Vector<MultiFab> temp_lev_new(Vars::NumTypes);
    Vector<MultiFab> temp_lev_old(Vars::NumTypes);
    MultiFab temp_base_state;
 
    std::unique_ptr<MultiFab> temp_zphys_nd;
 
    //
    // init_stuff below rebuilds mapfac[lev] on the new grids and resets it to 1, and nothing
    // downstream of a regrid reads the map factors back out of a file, so hold on to the
    // pre-regrid arrays here and restore them afterwards.  Without this a level-0 regrid
    // (which restart() does whenever the checkpoint has fewer grids than ranks) silently
    // threw away the MAPFAC_M/U/V read from wrfinput_d01.
    //
    Vector<std::unique_ptr<MultiFab>> old_mapfac(mapfac[lev].size());
    for (int i = 0; i < mapfac[lev].size(); i++) {
        old_mapfac[i] = std::move(mapfac[lev][i]);
    }
 
    //
    // The surface pressure and the land mask / land type / soil type are in exactly the
    // same position: init_stuff rebuilds them on the new grids and leaves a placeholder
    // (an uninitialized PSFC, a uniform land mask and type 0), only init_from_wrfinput
    // ever puts the real PSFC / LANDMASK / IVGTYP / ISLTYP there, and a regrid does not
    // re-read the file.  So retain them here too.
    //
    // Retaining the land mask matters off the WRF pathways as well: init_zphys marks the
    // cells under a building with a land mask of 2, and a regrid goes through
    // remake_zphys instead, which does not redo that marking.
    //
    auto retain = [] (Vector<Vector<std::unique_ptr<iMultiFab>>>& v, int l)
                  { return (v[l].empty()) ? nullptr : std::move(v[l][0]); };
 
    std::unique_ptr<MultiFab>  old_psfc      = std::move(mf_PSFC[lev]);
    std::unique_ptr<iMultiFab> old_lmask     = retain(    lmask_lev,lev);
    std::unique_ptr<iMultiFab> old_land_type = retain(land_type_lev,lev);
    std::unique_ptr<iMultiFab> old_soil_type = retain(soil_type_lev,lev);
 
    //********************************************************************************************
    // This allocates all kinds of things, including but not limited to: solution arrays,
    //      terrain arrays and metrics, and base state.
    // *******************************************************************************************
    init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old, temp_base_state, temp_zphys_nd);
 
    //
    // Restore the map factors onto the new grids.  At lev > 0 we interpolate from the parent
    // first so that cells the new grids added -- which the pre-regrid arrays never covered --
    // are filled as well, then copy the retained values on top wherever we still have them.
    // (For a level with no init file of its own the Interp2DArrays call further down repeats
    // that interpolation, which is what defines its map factors in the first place.)
    //
    if (lev > 0) { interp_mapfac_from_coarse(lev); }
    for (int i = 0; i < mapfac[lev].size(); i++) {
        if (!mapfac[lev][i] || !old_mapfac[i]) { continue; }
        IntVect ngv = mapfac[lev][i]->nGrowVect(); ngv[2] = 0;
        mapfac[lev][i]->ParallelCopy(*old_mapfac[i], 0, 0, 1, ngv, ngv, geom[lev].periodicity());
        mapfac[lev][i]->FillBoundary(geom[lev].periodicity());
    }
 
    //
    // Restore the surface pressure and the land arrays the same way.  Both interpolation
    // helpers return without doing anything off the wrfinput pathway, since there these
    // arrays hold what was asked for at this level rather than anything derived from the
    // parent; the copy below is what carries them across the regrid in that case.
    //
    if (lev > 0) {
        interp_psfc_from_coarse(lev);
        interp_land_masks_from_coarse(lev);
    }
    if (mf_PSFC[lev] && old_psfc) {
        IntVect ngv = mf_PSFC[lev]->nGrowVect(); ngv[2] = 0;
        mf_PSFC[lev]->ParallelCopy(*old_psfc, 0, 0, 1, ngv, ngv, geom[lev].periodicity());
        mf_PSFC[lev]->FillBoundary(geom[lev].periodicity());
    }
    {
        auto entry = [] (Vector<Vector<std::unique_ptr<iMultiFab>>>& v, int l) -> iMultiFab*
                     { return (v[l].empty()) ? nullptr : v[l][0].get(); };
 
        Vector<iMultiFab*> new_land = {entry(    lmask_lev,lev),
                                       entry(land_type_lev,lev),
                                       entry(soil_type_lev,lev)};
        Vector<iMultiFab*> old_land = {old_lmask.get(), old_land_type.get(), old_soil_type.get()};
        for (int i = 0; i < new_land.size(); i++) {
            if (!new_land[i] || !old_land[i]) { continue; }
            IntVect ngv = new_land[i]->nGrowVect(); ngv[2] = 0;
            new_land[i]->ParallelCopy(*old_land[i], 0, 0, 1, ngv, ngv, geom[lev].periodicity());
            new_land[i]->FillBoundary(geom[lev].periodicity());
        }
    }
 
    // ********************************************************************************************
    // Build the data structures for terrain-related quantities
    // ********************************************************************************************
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing)
    {
        const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
        const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
        if (solverChoice.terrain_type == TerrainType::EB) {
            eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
        } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                   solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
#if USE_FC_FACTORY
            eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
#else
            eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
#endif
        }
    }
    remake_zphys(lev, time, temp_zphys_nd);
    update_terrain_arrays(lev);
 
    // ********************************************************************************************
    // Make sure that detJ is the average of the data on a finer level if there is one
    // Note that this shouldn't be necessary because the fine grid is created by interpolation
    // from the coarse ... but just in case ...
    // ********************************************************************************************
    if ( (SolverChoice::mesh_type != MeshType::ConstantDz) && (solverChoice.coupling_type == CouplingType::TwoWay) ) {
        // NOTE: z_phys_cc is deliberately NOT averaged down.  Every level's base state is
        //       built to be in discrete hydrostatic balance against that level's own
        //       cell-centered heights, so replacing the coarse heights with the average of
        //       the fine ones -- after the base states have already been built -- leaves the
        //       coarse base state out of balance with the heights the dycore then uses for
        //       vertical gradients, Rayleigh damping and the sponge zones.  z_phys_cc is also
        //       derived from z_phys_nd, which is not averaged down either, so averaging only
        //       the cell-centered heights made the two disagree inside the refined region.
        //
        //       detJ IS still averaged down: AverageDownTo weights (rho S) by detJ_cc before
        //       averaging and divides by it afterwards, so the coarse detJ must be the average
        //       of the fine detJ for that average-down to telescope and stay conservative.
        for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
            average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
        }
    }
 
    //
    // Keep the vertical-spacing minimum in step with the new grids, as MakeNewLevelFromCoarse
    // does.  Without this the dt constraint keeps its pre-regrid value on a stretched or
    // terrain-fitted mesh, where the fine detJ minimum moves with the grids.
    //
    if (static_cast<int>(dz_min.size()) <= lev) { dz_min.resize(lev+1); }
    dz_min[lev] = geom[lev].CellSize(2);
    if ( (SolverChoice::mesh_type != MeshType::ConstantDz) && detJ_cc[lev] ) {
        dz_min[lev] *= (*detJ_cc[lev]).min(0);
    }
 
    // ********************************************************************************************
    // Build the data structures for canopy model (depends upon z_phys)
    // ********************************************************************************************
    if (solverChoice.do_forest_drag) {
        m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev],
                                              z_phys_cc[lev].get(), z_phys_nd[lev].get(),
                                              solverChoice.forest_biophysics &&
                                              solverChoice.forest_biophysics_heat, lev);
    }
 
    // *****************************************************************************************************
    // Create the physbcs objects (after initializing the terrain but before calling FillCoarsePatch
    // *****************************************************************************************************
    make_physbcs(lev);
 
    // ********************************************************************************************
    // Update the base state at this level by interpolation from coarser level AND copy
    //    from previous (pre-regrid) base_state array
    // ********************************************************************************************
    if (lev > 0) {
        Interpolater* mapper = &cell_cons_interp;
 
        Vector<MultiFab*> fmf = {&base_state[lev  ], &base_state[lev  ]};
        Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
        Vector<Real> ftime    = {time, time};
        Vector<Real> ctime    = {time, time};
 
        // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
        FillPatchTwoLevels(temp_base_state, temp_base_state.nGrowVect(), IntVect(0,0,0),
                           time, cmf, ctime, fmf, ftime,
                           0, 0, temp_base_state.nComp(), geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper, domain_bcs_type,
                           BaseBCVars::rho0_bc_comp);
 
        // Impose bc's outside the domain
        (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
 
#ifdef ERF_USE_NETCDF
        // *************************************************************************************************
        // For a WRFInput run, rebuild the base state on the new grids by the SAME construction
        // used at initialization -- the analytic reference profile from the level-0 parameters,
        // evaluated at this level's heights, followed by the discrete hydrostatic rebalance --
        // rather than leaving it as the conservative interpolation of the parent's base state.
        //
        // Without this, the first regrid silently replaces a base state that is discretely
        // hydrostatic on this level's mesh with one that is not, and the level stops agreeing
        // with its parent in the way it did at t = 0.
        //
        // Note this must come AFTER the FillPatchTwoLevels above (which fills the region of
        // this level outside the fine grids, and the ghost cells) and BEFORE the
        // FillPatchFineLevel below, which interpolates perturbational quantities relative to
        // the new base state.
        // *************************************************************************************************
        if (solverChoice.init_type == InitType::WRFInput) {
            rebuild_base_state_from_wrfinput(lev, temp_base_state);
            (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
        }
#endif
 
        // *************************************************************************************************
        // This will fill the temporary MultiFabs with data from vars_new
        // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
        // NOTE: we must create the new base state before calling FillPatch because we will
        //       interpolate perturbational quantities
        // *************************************************************************************************
        FillPatchFineLevel(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
                           &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
                          {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                           base_state[lev], temp_base_state, false);
    } else {
        temp_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
                                     base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
        temp_lev_new[Vars::cons].ParallelCopy(vars_new[lev][Vars::cons],0,0,ncomp_cons,ngrow_state,ngrow_state);
        temp_lev_new[Vars::xvel].ParallelCopy(vars_new[lev][Vars::xvel],0,0,         1,ngrow_vels,ngrow_vels);
        temp_lev_new[Vars::yvel].ParallelCopy(vars_new[lev][Vars::yvel],0,0,         1,ngrow_vels,ngrow_vels);
 
        temp_lev_new[Vars::zvel].setVal(0.);
        temp_lev_new[Vars::zvel].ParallelCopy(vars_new[lev][Vars::zvel],0,0,         1,
                                              IntVect(ngrow_vels,ngrow_vels,0),IntVect(ngrow_vels,ngrow_vels,0));
    }
 
    // Now swap the pointers since we needed both old and new in the FillPatch
    std::swap(temp_base_state, base_state[lev]);
 
    // ********************************************************************************************
    // Copy from new into old just in case
    // ********************************************************************************************
    MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
    MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0,         1,ngrow_vels);
    MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0,         1,ngrow_vels);
    MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0,         1,IntVect(ngrow_vels,ngrow_vels,0));
 
    // ********************************************************************************************
    // Now swap the pointers
    // ********************************************************************************************
    for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
        std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
        std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
    }
 
    //
    // Note that t_new = time here is elapsed time
    //
    t_new[lev] = time;
    t_old[lev] = time - bogus_large_value;
 
    // ********************************************************************************************
    // Build the data structures for calculating diffusive/turbulent terms
    // ********************************************************************************************
    update_diffusive_arrays(lev, ba, dm);
 
    // ********************************************************************************************
    // Thin immersed body -- thin_[xyz]force and [xyz]flux_imask must be re-defined on the new
    //                       (ba,dm), otherwise they still live on the pre-regrid grids
    // ********************************************************************************************
    init_thin_body(lev, ba, dm);
 
    //********************************************************************************************
    // Microphysics
    // *******************************************************************************************
    int q_size = micro->Get_Qmoist_Size(lev);
    qmoist[lev].resize(q_size);
    micro->Define(lev, solverChoice);
    if (solverChoice.moisture_type != MoistureType::None)
    {
        micro->Init(lev, vars_new[lev][Vars::cons],
                    grids[lev], Geom(lev), zero,
                    z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
        // Refresh the land/water mask pointer. Must be re-issued at every
        // micro->Init site: init_stuff rebuilds lmask_lev[lev][0], so a stale
        // pointer would dangle. No-op for every scheme except WDM6.
        micro->Set_Lmask(lev, (lmask_lev[lev].empty()) ? nullptr : lmask_lev[lev][0].get());
    }
    for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
        qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
    }
 
    //********************************************************************************************
    // Radiation
    // *******************************************************************************************
    if (solverChoice.rad_uses_interface())
    {
        rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
    }
 
    // ********************************************************************************************
    // Initialize the integrator class
    // ********************************************************************************************
    initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
 
    // ********************************************************************************************
    // Re-define every FillPatcher that references this level's grids.
    //
    // FPr_*[lev-1] caches this level as the FINE side; FPr_*[lev] caches it as the COARSE side
    // (see Construct_ERFFillPatchers).  A regrid that moves this level invalidates both, but
    // only the first was being redefined -- so a finer level whose own grids happened not to
    // change kept a coarse-fine fill built against this level's pre-regrid layout, which then
    // faults on its first step.
    //
    // AmrCore::regrid walks coarse to fine and does not clear removed levels until its loop
    // finishes, so vars_new[lev+1] is still valid here.  If level lev+1 is itself remade
    // afterwards it just redefines its own patcher again; Define is idempotent.
    // ********************************************************************************************
    if ( (cf_width >= 0) && ((ba != ba_old) || (dm != dm_old)) ) {
        if (lev > 0) { Define_ERFFillPatchers(lev); }
        if ( (lev+1 <= finest_level) && (lev < static_cast<int>(FPr_c.size())) ) {
            Define_ERFFillPatchers(lev+1);
        }
    }
 
    // ********************************************************************************************
    // Re-establish the pressure gradient on an anelastic level.
    //
    // init_stuff has just re-defined gradp[lev] on the new grids and zeroed it.
    // MakeNewLevelFromCoarse recovers it with a projection; without the same call here every
    // regrid silently discards the pressure gradient.  Must follow the FillPatcher definition
    // above, which the projection's coarse-fine fill depends on.
    // ********************************************************************************************
    if (solverChoice.anelastic[lev]) {
        Real dummy_dt = Real(1.0);
        project_initial_velocity(lev, time, dummy_dt);
        pp_inc[lev].setVal(0.0);
    }
 
    // ********************************************************************************************
    // Initialize the 2D data structures
    // ********************************************************************************************
    // NOTE: 2D MFs must be filled before SurfaceLayer is defined since SL class uses sst/tsk
    // Clear the 2D arrays
    if (sst_lev[lev][0]) {
        for (int n = 0; n < sst_lev[lev].size(); n++) {
            sst_lev[lev][n].reset();
        }
    }
    if (tsk_lev[lev][0]) {
        for (int n = 0; n < tsk_lev[lev].size(); n++) {
            tsk_lev[lev][n].reset();
        }
    }
    if (lat_m[lev]) {
        lat_m[lev].reset();
    }
    if (lon_m[lev]) {
        lon_m[lev].reset();
    }
    if (sinPhi_m[lev]) {
        sinPhi_m[lev].reset();
    }
    if (cosPhi_m[lev]) {
        cosPhi_m[lev].reset();
    }
 
    //
    // Interpolate the 2D arrays at the lower boundary. We assume that since we created
    //     them by interpolation it is ok just to recreate them by interpolation.
    // Note that ba2d is constructed already in init_stuff, but we have not yet defined dmap[lev]
    //     so we must explicitly pass dm.
    Interp2DArrays(lev,ba2d[lev],dm);
 
    // ********************************************************************************************
    // Land Surface Model
    //
    // Must precede the SurfaceLayer update below, which caches lsm_data[lev]/lsm_flux[lev] as
    // raw MultiFab*.  Left out, those pointers survive the regrid on the OLD (ba,dm) and are
    // then indexed with an MFIter over the new grids -- an out-of-bounds device read on the
    // level's first step after the regrid.
    //
    // At lev > 0 this is cheap and lossless: NOAHMP::Advance calls interp_from_lev0 every
    // step, so a fine level carries no prognostic state of its own.  At level 0 the state
    // lives in the per-box Fortran NoahmpIO_type objects, which Init() rebuilds from the land
    // file and for which no redistribution onto a new decomposition exists -- so refuse
    // rather than silently cold-start the soil column.  (Level 0 is only ever remade from
    // ERF::restart, not from regrid, which starts at lbase+1.)
    // ********************************************************************************************
    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
        !((lev == 0) && (solverChoice.lsm_type == LandSurfaceType::NOAHMP)),
        "RemakeLevel at level 0 would cold-start the Noah-MP soil state: "
        "NoahmpIO_type redistribution onto a new DistributionMapping is not implemented");
 
    make_lsm_at_level(lev, true); // from_regrid=true: always initialize LSM during regrid
 
    //
    // A level-0 remake replaces the MultiFabs that every finer level's model caches for
    // interp_from_lev0, so re-issue those pointers all the way up.
    //
    if (lev == 0) {
        for (int k = 1; k <= finest_level; ++k) {
            lsm.Set_Lev0_Data_Ptr(k);
            lsm.Set_Lev0_Flux_Ptr(k);
        }
    }
 
    // ********************************************************************************************
    // Update the SurfaceLayer arrays at this level
    // ********************************************************************************************
    for (OrientationIter oit; oit; ++oit) {
        Orientation ori = oit();
        if (m_SurfaceLayer[ori] != nullptr) {
            if (phys_bc_type[ori] == ERF_BC::surface_layer) {
                int nlevs = finest_level+1;
                Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
                                             &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
                m_SurfaceLayer[ori]->make_SurfaceLayer_at_level(lev,nlevs,
                                                                mfv_old, Theta_prim[lev], Qv_prim[lev],
                                                                Qr_prim[lev], z_phys_nd[lev],
                                                                Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
                                                                lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
                                                                sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
            }
        }
    }
 
    // ********************************************************************************************
    // Set up the Rayleigh damping vectors at this (new) level
    // ********************************************************************************************
    if (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
        solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T)
    {
        initRayleigh_at_level(lev);
    }
 
    // Particle redistribute handled in timeStep() after regrid() completes.
    // Calling it here causes stale-grid crashes.
}

Here is the call graph for this function:

ResetIntervalMeans()

void ERF::ResetIntervalMeans

(

)

{
    for (int lev = 0; lev <= finest_level; ++lev) {
        if (interval_means[lev] != nullptr) {
            interval_means[lev]->setVal(zero);
            t_mean_cnt[lev] = 0.0;
        }
    }
}

ResetIntervalMeansAfter3DPlotfileBatch()

void ERF::ResetIntervalMeansAfter3DPlotfileBatch	(	int	plotfiles_written,
		bool	interval_diagnostic_consumed
	)

{
    if (!erf_plotfile::plot3d_batch_resets_interval_means(
            plotfiles_written, interval_diagnostic_consumed,
            solverChoice.compute_mean_vars,
            solverChoice.mean_vars_reset_mode)) {
        return;
    }
 
    ResetIntervalMeans();
}

Here is the call graph for this function:

resolve_station_positions()

void ERF::resolve_station_positions

(

)

{
    auto& stations = station_sampler->stations();
 
    bool any_latlon = false;
    for (const auto& station : stations) {
        for (const auto& loc : station.locs) { any_latlon = any_latlon || loc.use_latlon; }
    }
 
    const auto  problo = geom[0].ProbLoArray();
    const auto  probhi = geom[0].ProbHiArray();
    const auto  dx0    = geom[0].CellSizeArray();
    const Box&  dom0   = geom[0].Domain();
 
    if (any_latlon)
    {
        if (lat_m[0] == nullptr || lon_m[0] == nullptr) {
            Abort("Station output: .lat/.long need a run with latitude/longitude arrays "
                  "(a WRF or metgrid initialization, or a restart from one).  Place the "
                  "stations with .x and .y in domain coordinates instead");
        }
 
        // Gather the level-0 mass-point latitude and longitude onto the IO rank.
        // This is a setup-time, level-0, 2D array, so the gather is affordable and
        // keeps the search and the inverse map as plain host code.
        //
        // The ceiling is one rank holding 2 Reals per level-0 cell -- 100 MB or so
        // for the largest domains ERF is run on, a few hundred million cells -- and
        // a search of that array per station location, once.  If either ever
        // matters, the search is what to distribute: the array is gathered once
        // for all the locations, but each location scans all of it.
        //
        // NOTE: every init path fills lat_m / lon_m with mass-point values --
        //       init_from_wrfinput reads WRF's XLAT / XLONG and init_from_metgrid
        //       reads XLAT_M / XLONG_M -- so no averaging is needed here.
        MultiFab latlon(ba2d[0], dmap[0], 2, 0);
        for (MFIter mfi(latlon, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.tilebox();
            const Array4<Real>&       ll  = latlon.array(mfi);
            const Array4<const Real>& lat = lat_m[0]->const_array(mfi);
            const Array4<const Real>& lon = lon_m[0]->const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                ll(i,j,k,0) = lat(i,j,0);
                ll(i,j,k,1) = lon(i,j,0);
            });
        }
 
        Box dom2d(dom0); dom2d.setRange(2,0);
        BoxArray ba_one(dom2d);
        Vector<int> pmap(1, ParallelDescriptor::IOProcessorNumber());
        DistributionMapping dm_one(pmap);
        MultiFab latlon_all(ba_one, dm_one, 2, 0, MFInfo().SetArena(The_Pinned_Arena()));
        latlon_all.ParallelCopy(latlon, 0, 0, 2);
 
        // The copy is device-side in a GPU build; the search below is host code
        Gpu::streamSynchronize();
 
        const int ilo = dom0.smallEnd(0), ihi = dom0.bigEnd(0);
        const int jlo = dom0.smallEnd(1), jhi = dom0.bigEnd(1);
 
        for (auto& station : stations)
        {
            for (auto& loc : station.locs)
            {
                if (!loc.use_latlon) { continue; }
 
                Real out[4] = {Real(0.0), Real(0.0), Real(0.0), Real(0.0)};
 
                if (ParallelDescriptor::IOProcessor())
                {
                    const Array4<const Real>& ll = latlon_all.const_array(0);
 
                    auto cosfac = std::cos(loc.req_lat * PI / Real(180.0));
                    auto dist2  = [&](int i, int j) {
                        const Real dlat = ll(i,j,0,0) - loc.req_lat;
                        const Real dlon = (ll(i,j,0,1) - loc.req_lon) * cosfac;
                        return dlat*dlat + dlon*dlon;
                    };
 
                    int  bi = ilo, bj = jlo;
                    Real best = dist2(ilo,jlo);
                    for (int j = jlo; j <= jhi; ++j) {
                        for (int i = ilo; i <= ihi; ++i) {
                            const Real d = dist2(i,j);
                            if (d < best) { best = d; bi = i; bj = j; }
                        }
                    }
 
                    // One linear solve in index space inverts the map: over a cell
                    // the projection is linear to well below a cell width.
                    const int im = std::max(bi-1, ilo), ip = std::min(bi+1, ihi);
                    const int jm = std::max(bj-1, jlo), jp = std::min(bj+1, jhi);
                    const Real inv_di = (ip > im) ? Real(1.0)/Real(ip-im) : Real(0.0);
                    const Real inv_dj = (jp > jm) ? Real(1.0)/Real(jp-jm) : Real(0.0);
 
                    const Real dlat_di = (ll(ip,bj,0,0) - ll(im,bj,0,0)) * inv_di;
                    const Real dlon_di = (ll(ip,bj,0,1) - ll(im,bj,0,1)) * inv_di;
                    const Real dlat_dj = (ll(bi,jp,0,0) - ll(bi,jm,0,0)) * inv_dj;
                    const Real dlon_dj = (ll(bi,jp,0,1) - ll(bi,jm,0,1)) * inv_dj;
 
                    const Real rlat = loc.req_lat - ll(bi,bj,0,0);
                    const Real rlon = loc.req_lon - ll(bi,bj,0,1);
                    Real di = Real(0.0), dj = Real(0.0);
                    if (!invert_latlon_offset(rlat, rlon, dlat_di, dlon_di, dlat_dj, dlon_dj, di, dj)) {
                        Abort("Station '" + station.name + "': the latitude/longitude arrays are "
                              "degenerate near the requested point, so it cannot be inverted");
                    }
 
                    // The solve places the station relative to the nearest grid
                    // point, so more than a cell away means the request cannot be
                    // trusted: either it is outside the domain, or the lat/lon map
                    // is too poorly conditioned near it for one linear solve.
                    if (std::abs(di) > Real(1.0) || std::abs(dj) > Real(1.0)) {
                        Abort("Station '" + station.name + "': requested lat=" +
                              std::to_string(loc.req_lat) + " long=" + std::to_string(loc.req_lon) +
                              " resolved to a point more than one cell from the nearest grid "
                              "point (lat=" + std::to_string(ll(bi,bj,0,0)) + " long=" +
                              std::to_string(ll(bi,bj,0,1)) + "), which means it is outside the "
                              "domain, or that the latitude/longitude arrays are too distorted "
                              "near it to invert");
                    }
 
                    out[0] = problo[0] + (Real(bi) + Real(0.5) + di) * dx0[0];
                    out[1] = problo[1] + (Real(bj) + Real(0.5) + dj) * dx0[1];
                    out[2] = ll(bi,bj,0,0) + dlat_di*di + dlat_dj*dj;
                    out[3] = ll(bi,bj,0,1) + dlon_di*di + dlon_dj*dj;
                }
 
                ParallelDescriptor::Bcast(out, 4, ParallelDescriptor::IOProcessorNumber());
                loc.x = out[0]; loc.y = out[1]; loc.got_lat = out[2]; loc.got_lon = out[3];
            }
        }
    }
 
    for (const auto& station : stations) {
        for (const auto& loc : station.locs) {
            if (loc.x < problo[0] || loc.x > probhi[0] ||
                loc.y < problo[1] || loc.y > probhi[1]) {
                Abort("Station '" + station.name + "': x=" + std::to_string(loc.x) +
                      " y=" + std::to_string(loc.y) + " is outside the problem domain");
            }
        }
    }
}

Here is the call graph for this function:

resolve_station_stencils()

void ERF::resolve_station_stencils

(

)

{
    // What this computes -- which level supplies each location, which four cells
    // its horizontal stencil reads, and how far up the column -- is a function of
    // the grids and of the terrain under the station, and it costs a ParallelCopy,
    // a stream synchronization and a broadcast to compute.  Neither input changes
    // between samples unless the grids do, so the answer already in the locations
    // stands and none of that work is needed.
    //
    // The exception is a mesh whose terrain moves: there the column heights change
    // under a fixed grid, so the level a height falls in can change with the grids
    // unchanged and the stencils have to be resolved every time.
    const bool terrain_moves = (solverChoice.terrain_type == TerrainType::MovingFittedMesh);
    if (!terrain_moves && station_sampler->stencilsValidFor(grids, finest_level)) { return; }
 
    auto& stations = station_sampler->stations();
 
    // Flat list of (station, location) pairs, so a stencil and the location it
    // belongs to share an index.
    Vector<std::pair<int,int>> all_locs;
    for (int is = 0; is < static_cast<int>(stations.size()); ++is) {
        for (int il = 0; il < static_cast<int>(stations[is].locs.size()); ++il) {
            all_locs.emplace_back(is, il);
        }
    }
    const int nloc = static_cast<int>(all_locs.size());
 
    // ---- the horizontal stencil, and the coverage above it, on every level ----
    Vector<Vector<StationLevelStencil>> cand(nloc, Vector<StationLevelStencil>(finest_level+1));
 
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        const auto  problo = geom[lev].ProbLoArray();
        const auto  dx     = geom[lev].CellSizeArray();
        const Box&  dom    = geom[lev].Domain();
 
        const int ilo = dom.smallEnd(0), ihi = dom.bigEnd(0);
        const int jlo = dom.smallEnd(1), jhi = dom.bigEnd(1);
        const int klo = dom.smallEnd(2), khi = dom.bigEnd(2);
 
        // How far up this level's valid region covers the column at (i,j) --
        // after wrapping, where the direction is periodic -- counting only
        // coverage that is contiguous from the bottom of the domain.  Returns
        // klo-1 when even the bottom cell is not covered.
        auto column_top = [&](int i, int j)
        {
            int ii = i, jj = j;
            if (geom[lev].isPeriodic(0)) {
                const int nx = dom.length(0);
                ii = ilo + ((i - ilo) % nx + nx) % nx;
            }
            if (geom[lev].isPeriodic(1)) {
                const int ny = dom.length(1);
                jj = jlo + ((j - jlo) % ny + ny) % ny;
            }
            if (ii < ilo || ii > ihi || jj < jlo || jj > jhi) { return klo-1; }
 
            std::vector<std::pair<int,Box>> isects;
            grids[lev].intersections(Box(IntVect(ii,jj,klo), IntVect(ii,jj,khi)), isects);
 
            // The pieces of the column come back in no particular order, so grow
            // the covered span from the bottom until nothing else abuts it.
            int ktop = klo-1;
            bool grew = true;
            while (grew) {
                grew = false;
                for (const auto& is : isects) {
                    if (is.second.smallEnd(2) <= ktop+1 && is.second.bigEnd(2) > ktop) {
                        ktop = is.second.bigEnd(2);
                        grew = true;
                    }
                }
            }
            return ktop;
        };
 
        for (int ip = 0; ip < nloc; ++ip)
        {
            const auto& loc = stations[all_locs[ip].first].locs[all_locs[ip].second];
            auto& c = cand[ip][lev];
 
            int ic = ilo + static_cast<int>(std::floor((loc.x - problo[0]) / dx[0]));
            int jc = jlo + static_cast<int>(std::floor((loc.y - problo[1]) / dx[1]));
            ic = std::min(std::max(ic, ilo), ihi);
            jc = std::min(std::max(jc, jlo), jhi);
 
            const Real tx = (loc.x - (problo[0] + (Real(ic-ilo) + Real(0.5))*dx[0])) / dx[0];
            const Real ty = (loc.y - (problo[1] + (Real(jc-jlo) + Real(0.5))*dx[1])) / dx[1];
 
            int  i0 = (tx >= Real(0.0)) ? ic : ic-1;
            int  j0 = (ty >= Real(0.0)) ? jc : jc-1;
            Real wx = (tx >= Real(0.0)) ? tx : tx + Real(1.0);
            Real wy = (ty >= Real(0.0)) ? ty : ty + Real(1.0);
            int  i1 = i0 + 1;
            int  j1 = j0 + 1;
 
            // Within the outer half cell of a non-periodic boundary there is
            // no second cell to interpolate from, so the stencil collapses to
            // the edge cell.
            if (!geom[lev].isPeriodic(0)) {
                if (i0 <  ilo) { i0 = i1 = ilo; wx = Real(0.0); }
                if (i1 >  ihi) { i0 = i1 = ihi; wx = Real(0.0); }
            }
            if (!geom[lev].isPeriodic(1)) {
                if (j0 <  jlo) { j0 = j1 = jlo; wy = Real(0.0); }
                if (j1 >  jhi) { j0 = j1 = jhi; wy = Real(0.0); }
            }
 
            c.ic = ic; c.jc = jc;
            c.i0 = i0; c.j0 = j0;
            c.i1 = i1; c.j1 = j1;
            c.wx = wx; c.wy = wy;
            c.kcov = std::min(std::min(column_top(i0,j0), column_top(i1,j0)),
                              std::min(column_top(i0,j1), column_top(i1,j1)));
        }
    }
 
    // ---- the cell centre heights of the candidate columns, on the IO rank ----
    //
    // Which cells the vertical interpolation reads depends on the terrain under
    // the station, so choosing the level needs the heights themselves.  They are
    // one component of a 2x2 column, so they are gathered where the choice is
    // made, on the same rank that later interpolates.
    //
    const int io_rank = ParallelDescriptor::IOProcessorNumber();
 
    Vector<Vector<int>> lev_cand(finest_level+1);      // ip of each gathered box, in box order
    Vector<MultiFab> zc(finest_level+1), znd(finest_level+1);
 
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        const Box& dom = geom[lev].Domain();
        const int  klo = dom.smallEnd(2);
 
        for (int ip = 0; ip < nloc; ++ip) {
            if (cand[ip][lev].kcov >= klo) { lev_cand[lev].push_back(ip); }
        }
        // Terrain gives both arrays or neither: z_of_k below reads z_phys_cc and
        // falls back to a uniform dz without it, while z_surf reads z_phys_nd, so
        // a level with one and not the other would measure a height from the
        // terrain and then look it up on a flat column.  No init path produces
        // that, and this says so rather than leaving it to be discovered.
        AMREX_ALWAYS_ASSERT((z_phys_cc[lev] != nullptr) == (z_phys_nd[lev] != nullptr));
        if (lev_cand[lev].empty() || !z_phys_cc[lev]) { continue; }
 
        // Periodicity restricted to the horizontal: the columns start at the
        // bottom of the domain, so there is nothing to wrap in z.
        const Periodicity period(IntVect(geom[lev].isPeriodic(0) ? dom.length(0) : 0,
                                         geom[lev].isPeriodic(1) ? dom.length(1) : 0,
                                         0));
 
        BoxList bl_cc, bl_nd;
        for (const int ip : lev_cand[lev]) {
            const auto& c = cand[ip][lev];
            bl_cc.push_back(Box(IntVect(c.i0,c.j0,klo), IntVect(c.i1,c.j1,c.kcov)));
            bl_nd.push_back(Box(IntVect(c.i0,c.j0,klo), IntVect(c.i1+1,c.j1+1,klo),
                                IntVect::TheNodeVector()));
        }
 
        Vector<int> pmap(static_cast<int>(bl_cc.size()), io_rank);
        DistributionMapping dm_io(pmap);
        MFInfo pinned = MFInfo().SetArena(The_Pinned_Arena());
 
        if (z_phys_cc[lev]) {
            zc[lev].define(BoxArray(bl_cc), dm_io, 1, 0, pinned);
            zc[lev].setVal(Real(0.0));
            zc[lev].ParallelCopy(*z_phys_cc[lev], 0, 0, 1, IntVect(0), IntVect(0), period);
        }
        if (z_phys_nd[lev]) {
            znd[lev].define(BoxArray(bl_nd), dm_io, 1, 0, pinned);
            znd[lev].setVal(Real(0.0));
            znd[lev].ParallelCopy(*z_phys_nd[lev], 0, 0, 1, IntVect(0), IntVect(0), period);
        }
    }
 
    // The gathers above are device-side in a GPU build, and the choice below
    // reads the pinned destinations on the host.
    Gpu::streamSynchronize();
 
    // ---- choose the level, on the IO rank, and tell everyone ----
    //
    // Only the level and the top of the column have to be broadcast: every rank
    // computed the same stencils above.
    //
    Vector<int> chosen(2*nloc, -1);
 
    if (ParallelDescriptor::IOProcessor())
    {
        for (int ip = 0; ip < nloc; ++ip)
        {
            const Station& station = stations[all_locs[ip].first];
 
            // A station of 2D variables only never leaves the bottom of the column
            const bool reads_column = station.has_3d_vars() && !station.heights.empty();
 
            for (int lev = finest_level; lev >= 0 && chosen[2*ip] < 0; --lev)
            {
                const auto& c = cand[ip][lev];
 
                const auto problo = geom[lev].ProbLoArray();
                const auto dx     = geom[lev].CellSizeArray();
                const Box& dom    = geom[lev].Domain();
                const int  klo    = dom.smallEnd(2);
                const int  khi    = dom.bigEnd(2);
 
                if (c.kcov < klo) { continue; }
 
                if (!reads_column) {
                    chosen[2*ip] = lev; chosen[2*ip+1] = klo;
                    break;
                }
 
                const int ib = static_cast<int>(
                    std::find(lev_cand[lev].begin(), lev_cand[lev].end(), ip) - lev_cand[lev].begin());
 
                auto bilinear = [&](const Array4<const Real>& a, int k)
                {
                    return (Real(1.0)-c.wy) * ( (Real(1.0)-c.wx)*a(c.i0,c.j0,k) + c.wx*a(c.i1,c.j0,k) )
                         +            c.wy  * ( (Real(1.0)-c.wx)*a(c.i0,c.j1,k) + c.wx*a(c.i1,c.j1,k) );
                };
 
                Array4<const Real> zc_arr;
                if (z_phys_cc[lev]) { zc_arr = zc[lev].const_array(ib); }
                auto z_of_k = [&](int k)
                {
                    if (zc_arr) { return bilinear(zc_arr, k); }
                    return problo[2] + (Real(k-klo) + Real(0.5)) * dx[2];
                };
 
                // Terrain elevation under the station, which is where an AGL
                // height is measured from
                Real z_surf = problo[2];
                if (z_phys_nd[lev]) {
                    const Array4<const Real>& nd = znd[lev].const_array(ib);
                    auto corner_avg = [&](int i, int j) {
                        return Real(0.25) * (nd(i,j,klo) + nd(i+1,j,klo) + nd(i,j+1,klo) + nd(i+1,j+1,klo));
                    };
                    z_surf = (Real(1.0)-c.wy) * ( (Real(1.0)-c.wx)*corner_avg(c.i0,c.j0) + c.wx*corner_avg(c.i1,c.j0) )
                           +            c.wy  * ( (Real(1.0)-c.wx)*corner_avg(c.i0,c.j1) + c.wx*corner_avg(c.i1,c.j1) );
                }
 
                // Every requested height must be bracketed within the covered
                // part of the column.  Running out of coverage is fatal to the
                // level unless the coverage reaches the top of the domain, where
                // the interpolation legitimately clamps.
                // The zero of the requested heights: the terrain under the
                // station, or the bottom of the domain.
                const Real z_zero = station.heights_are_agl ? z_surf : Real(0.0);
 
                int  ktop = klo;
                bool usable = true;
                bool below_first_cell = false;
                for (const Real height : station.heights)
                {
                    const Real z_target = z_zero + height;
                    if (z_of_k(c.kcov) < z_target && c.kcov < khi) { usable = false; break; }
 
                    if (z_target < z_of_k(klo)) { below_first_cell = true; }
 
                    int kk = klo;
                    while (kk < c.kcov && z_of_k(kk+1) < z_target) { ++kk; }
                    ktop = std::max(ktop, std::min(kk+1, c.kcov));
                }
 
                if (usable) {
                    chosen[2*ip] = lev;
                    chosen[2*ip+1] = ktop;
 
                    // Nothing in the model lives between the terrain and the first
                    // cell centre, so a height there is the first cell centre's
                    // value.  That is a trap for exactly the heights an observation
                    // comparison asks for, so say so, once.
                    if (below_first_cell && !station_sampler->lowHeightWarned()) {
                        station_sampler->setLowHeightWarned();
                        Warning("Station '" + station.name + "': a requested height is below the "
                                "first cell centre of the level it is sampled from, so it is "
                                "reported as the value at that cell centre -- there is no "
                                "similarity extrapolation to the requested height.  For 2 m or "
                                "10 m quantities, request the 2D diagnostics (temperature_2m, "
                                "water_vapor_mixing_ratio_2m, and the surface-layer diagnostics) "
                                "instead of a 3D variable at that height");
                    }
                }
            }
        }
    }
 
    ParallelDescriptor::Bcast(chosen.data(), chosen.size(), io_rank);
 
    for (int ip = 0; ip < nloc; ++ip)
    {
        // Level 0 covers the whole domain from the ground up, so a station
        // inside the domain always resolves.
        AMREX_ALWAYS_ASSERT(chosen[2*ip] >= 0);
 
        auto& loc     = stations[all_locs[ip].first].locs[all_locs[ip].second];
        const auto& c = cand[ip][chosen[2*ip]];
 
        loc.lev = chosen[2*ip];
        loc.ic = c.ic; loc.jc = c.jc;
        loc.i0 = c.i0; loc.j0 = c.j0;
        loc.i1 = c.i1; loc.j1 = c.j1;
        loc.wx = c.wx; loc.wy = c.wy;
        loc.ktop = chosen[2*ip+1];
    }
 
    station_sampler->rememberStencilGrids(grids, finest_level);
 
    // Which level a station is sampled from follows from the grids rather than
    // from anything in the inputs file, so report it once.
    if (verbose > 0 && !station_sampler->levelsReported())
    {
        station_sampler->setLevelsReported();
        for (int ip = 0; ip < nloc; ++ip) {
            const Station&    station = stations[all_locs[ip].first];
            const StationLoc& loc     = station.locs[all_locs[ip].second];
            Print() << "Station output: '" << station.name << "' at x=" << loc.x
                    << " y=" << loc.y << " is sampled from level " << loc.lev
                    << " (cell " << loc.ic << "," << loc.jc
                    << ", column through k=" << loc.ktop << ")" << std::endl;
        }
    }
}

Here is the call graph for this function:

restart()

void ERF::restart

(

)

{
    auto dRestartTime0 = amrex::second();
 
    ReadCheckpointFile();
 
#ifdef ERF_USE_NETCDF
    //
    // The checkpoint carries every level's base state, but not the reference parameters
    // it was built from.  Recover them now, before anything can regrid: the first regrid
    // that creates or remakes a refined level rebuilds that level's base state from them.
    //
    restore_base_state_params_on_restart();
#endif
 
    // Force regrid on level 0 if more procs than boxes are requested
    regrid_level_0_on_restart = ( regrid_level_0_on_restart ||
                                  grids[0].size() < ParallelDescriptor::NProcs() );
 
    if (regrid_level_0_on_restart) {
        //
        // Coarsening before we split the grids ensures that each resulting
        // grid will have an even number of cells in each direction.
        //
        BoxArray new_ba(amrex::coarsen(Geom(0).Domain(),2));
        //
        // Now split up into list of grids within max_grid_size[0] limit.
        //
        new_ba.maxSize(max_grid_size[0]/2);
        //
        // Now refine these boxes back to level zero
        //
        new_ba.refine(2);
 
        if (refine_grid_layout) {
            ChopGrids(0, new_ba, ParallelDescriptor::NProcs());
        }
 
        if (new_ba != grids[0]) {
            DistributionMapping new_dm(new_ba);
            RemakeLevel(0,static_cast<Real>(t_new[0]),new_ba,new_dm);
        }
    }
 
#ifdef ERF_USE_PARTICLES
    // We call this here without knowing whether the particles have already been initialized or not
    initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,t_new[0]);
#endif
 
    double cur_time = t_new[0];
    if (m_check_per    > zero) {last_check_file_time    = std::floor(cur_time/m_check_per   ) * m_check_per;}
    if (m_plot2d_per_1 > zero) {last_plot2d_file_time_1 = std::floor(cur_time/m_plot2d_per_1) * m_plot2d_per_1;}
    if (m_plot2d_per_2 > zero) {last_plot2d_file_time_2 = std::floor(cur_time/m_plot2d_per_2) * m_plot2d_per_2;}
    if (m_plot3d_per_1 > zero) {last_plot3d_file_time_1 = std::floor(cur_time/m_plot3d_per_1) * m_plot3d_per_1;}
    if (m_plot3d_per_2 > zero) {last_plot3d_file_time_2 = std::floor(cur_time/m_plot3d_per_2) * m_plot3d_per_2;}
 
    for (int i = 0; i < m_subvol_per.size(); i++) {
        if (m_subvol_per[i] > zero) {last_subvol_time[i] = std::floor(cur_time/m_subvol_per[i]) * m_subvol_per[i];}
    }
 
    if (m_check_int    > zero) {last_check_file_step    = istep[0];}
    if (m_plot2d_int_1 > zero) {last_plot2d_file_step_1 = istep[0];}
    if (m_plot2d_int_2 > zero) {last_plot2d_file_step_2 = istep[0];}
    if (m_plot3d_int_1 > zero) {last_plot3d_file_step_1 = istep[0];}
    if (m_plot3d_int_2 > zero) {last_plot3d_file_step_2 = istep[0];}
 
    for (int i = 0; i < m_subvol_int.size(); i++) {
        if (m_subvol_int[i] > 0) {last_subvol_step[i] = istep[0];}
    }
 
    if (verbose > 0)
    {
        auto dRestartTime = amrex::second() - dRestartTime0;
        ParallelDescriptor::ReduceRealMax(dRestartTime,ParallelDescriptor::IOProcessorNumber());
        amrex::Print() << "Restart time = " << dRestartTime << " seconds." << '\n';
    }
}

sample_lines()

void ERF::sample_lines	(	int	lev,
		double	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters

lev	Current level
time	Current time
cell	IntVect containing the x,y-dimension indices to sample along z
mf	MultiFab from which we sample the data

{
    int ifile = 0;
 
    const int ncomp = mf.nComp(); // cell-centered state vars
 
    MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
    average_face_to_cellcenter(mf_vels, 0,
                               Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
 
    //
    // Sample the data at a line (in direction "dir") in space
    // In this case we sample in the vertical direction so dir = 2
    // The "k" value of "cell" is ignored
    //
    int dir = 2;
    MultiFab my_line       = get_line_data(mf,              dir, cell);
    MultiFab my_line_vels  = get_line_data(mf_vels,         dir, cell);
    MultiFab my_line_tau11 = get_line_data(*Tau[lev][TauType::tau11], dir, cell);
    MultiFab my_line_tau12 = get_line_data(*Tau[lev][TauType::tau12], dir, cell);
    MultiFab my_line_tau13 = get_line_data(*Tau[lev][TauType::tau13], dir, cell);
    MultiFab my_line_tau22 = get_line_data(*Tau[lev][TauType::tau22], dir, cell);
    MultiFab my_line_tau23 = get_line_data(*Tau[lev][TauType::tau23], dir, cell);
    MultiFab my_line_tau33 = get_line_data(*Tau[lev][TauType::tau33], dir, cell);
 
    for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
    {
        // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
 
        std::ostream& sample_log = SampleLineLog(ifile);
        if (sample_log.good()) {
          sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
          const auto& my_line_arr = my_line[0].const_array();
          const auto& my_line_vels_arr = my_line_vels[0].const_array();
          const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
          const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
          const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
          const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
          const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
          const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
          const Box&  my_box = my_line[0].box();
          const int klo = my_box.smallEnd(2);
          const int khi = my_box.bigEnd(2);
          int i = cell[0];
          int j = cell[1];
          for (int n = 0; n < ncomp; n++) {
              for (int k = klo; k <= khi; k++) {
                  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
              }
          }
          for (int n = 0; n < AMREX_SPACEDIM; n++) {
              for (int k = klo; k <= khi; k++) {
                  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
              }
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
          }
          sample_log << std::endl;
        } // if good
    } // mfi
}

sample_points()

void ERF::sample_points	(	int	lev,
		double	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling MultiFab data at a specified cell index.

Parameters

lev	Level for the associated MultiFab data
time	Current time
cell	IntVect containing the indexes for the cell where we want to sample
mf	MultiFab from which we wish to sample data

{
    int ifile = 0;
 
    //
    // Sample the data at a single point in space
    //
    int ncomp = mf.nComp();
    Vector<Real> my_point = get_cell_data(mf, cell);
 
    if (!my_point.empty()) {
 
        // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
 
        std::ostream& sample_log = SamplePointLog(ifile);
        if (sample_log.good()) {
          sample_log << std::setw(datwidth) << time;
          for (int i = 0; i < ncomp; ++i)
          {
              sample_log << std::setw(datwidth) << my_point[i];
          }
          sample_log << std::endl;
        } // if good
    } // only write from processor that holds the cell
}

sample_stations()

void ERF::sample_stations

(

amrex::Real

time

)

{
    if (!station_sampler || station_sampler->empty()) { return; }
 
    BL_PROFILE("ERF::sample_stations()");
 
    auto& ss       = *station_sampler;
    auto& stations = ss.stations();
 
    resolve_station_stencils();
 
    const int n3d = static_cast<int>(station_vars_3d.size());
    const int n2d = static_cast<int>(station_vars_2d.size());
 
    // ---- fill the plot variables, on the levels that host a station ----
    Vector<int> lev_has_station(finest_level+1, 0);
    int top_station_lev = 0;
    for (const auto& station : stations) {
        for (const auto& loc : station.locs) {
            lev_has_station[loc.lev] = 1;
            top_station_lev = std::max(top_station_lev, loc.lev);
        }
    }
 
    Vector<MultiFab> mf3d(finest_level+1);
    Vector<MultiFab> mf2d(finest_level+1);
 
    if (n3d > 0) {
        // The scratch has to be built for every level up to the finest one a
        // station is on, not just the levels that host a station: the vorticity
        // path fills level l from level l-1.  Levels above that are not read, so
        // they are not built.  The sampler runs at every output step, so it also
        // asks BuildPlot3DScratch not to average the microphysics state down:
        // turning station output on must not change the answer.
        Plot3DScratch scratch;
        BuildPlot3DScratch(station_vars_3d, scratch, top_station_lev, false);
        for (int lev = 0; lev <= finest_level; ++lev) {
            if (!lev_has_station[lev]) { continue; }
            mf3d[lev].define(grids[lev], dmap[lev], n3d, 0);
            FillPlot3DVars(lev, station_vars_3d, scratch, mf3d[lev], n3d, time);
        }
    }
 
    if (n2d > 0) {
        const auto descriptors =
            plotfile2d::build_sampled_level_output_descriptors_from_definitions({},
                                                                               station_vars_2d,
                                                                               solverChoice);
        for (int lev = 0; lev <= finest_level; ++lev) {
            if (!lev_has_station[lev]) { continue; }
            mf2d[lev].define(ba2d[lev], dmap[lev], n2d, 0);
            FillPlot2DVars(lev, station_vars_2d, station_vars_2d, descriptors, mf2d[lev], n2d);
        }
    }
 
    // ---- gather the stencils onto the IO rank ----
    const int nstation = static_cast<int>(stations.size());
 
    // Flat list of (station, location) pairs, so a stencil box and the location
    // it belongs to share an index.
    Vector<std::pair<int,int>> all_locs;
    for (int is = 0; is < nstation; ++is) {
        for (int il = 0; il < static_cast<int>(stations[is].locs.size()); ++il) {
            all_locs.emplace_back(is, il);
        }
    }
 
    const int io_rank = ParallelDescriptor::IOProcessorNumber();
 
    // Per level: the stencil boxes, in all_locs order, of the locations on it
    Vector<Vector<int>> lev_locs(finest_level+1);
    for (int ip = 0; ip < static_cast<int>(all_locs.size()); ++ip) {
        const auto& loc = stations[all_locs[ip].first].locs[all_locs[ip].second];
        lev_locs[loc.lev].push_back(ip);
    }
 
    // Stencil data brought to the IO rank, indexed by the position within lev_locs
    Vector<MultiFab> st3d(finest_level+1), st2d(finest_level+1);
    Vector<MultiFab> stzc(finest_level+1), stznd(finest_level+1);
 
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        if (lev_locs[lev].empty()) { continue; }
 
        const Box& dom = geom[lev].Domain();
        const int  klo = dom.smallEnd(2);
 
        // Periodicity restricted to the horizontal: the stencils start at the
        // bottom of the domain, so there is nothing to wrap in z.
        const Periodicity period(IntVect(geom[lev].isPeriodic(0) ? dom.length(0) : 0,
                                         geom[lev].isPeriodic(1) ? dom.length(1) : 0,
                                         0));
 
        // Each stencil reaches only as far up as resolve_station_stencils found
        // the vertical interpolation needs, which is as far up as this level is
        // guaranteed to cover it.
        // One box per location, in lev_locs order, so a gathered box and the
        // location it belongs to share an index.  Two locations inside the same
        // cell give identical boxes, which is allowed: the BoxArray is only ever
        // a list of what to copy and what to index, never a cover of a region,
        // and ParallelCopy fills each of a pair of identical boxes from the same
        // source.  The 2D boxes sit at k = 0 rather than at klo because ba2d is
        // built by compressing the 3D BoxArray with setRange(2,0), so 0 is where
        // its data is whatever the domain's smallEnd is (ERF_MakeNewArrays.cpp).
        BoxList bl_cc, bl_2d, bl_nd;
        for (const int ip : lev_locs[lev]) {
            const auto& loc = stations[all_locs[ip].first].locs[all_locs[ip].second];
            Box cc(IntVect(loc.i0,loc.j0,klo), IntVect(loc.i1,loc.j1,loc.ktop));
            bl_cc.push_back(cc);
            Box b2(IntVect(loc.i0,loc.j0,0), IntVect(loc.i1,loc.j1,0));
            bl_2d.push_back(b2);
            bl_nd.push_back(Box(IntVect(loc.i0,loc.j0,klo), IntVect(loc.i1+1,loc.j1+1,klo),
                                IntVect::TheNodeVector()));
        }
 
        const int nbox = static_cast<int>(bl_cc.size());
        Vector<int> pmap(nbox, io_rank);
        DistributionMapping dm_io(pmap);
        MFInfo pinned = MFInfo().SetArena(The_Pinned_Arena());
 
        if (n3d > 0) {
            BoxArray ba(bl_cc);
            st3d[lev].define(ba, dm_io, n3d, 0, pinned);
            st3d[lev].setVal(Real(0.0));
            st3d[lev].ParallelCopy(mf3d[lev], 0, 0, n3d, IntVect(0), IntVect(0), period);
        }
        if (n2d > 0) {
            BoxArray ba(bl_2d);
            st2d[lev].define(ba, dm_io, n2d, 0, pinned);
            st2d[lev].setVal(Real(0.0));
            st2d[lev].ParallelCopy(mf2d[lev], 0, 0, n2d, IntVect(0), IntVect(0), period);
        }
        if (z_phys_cc[lev]) {
            BoxArray ba(bl_cc);
            stzc[lev].define(ba, dm_io, 1, 0, pinned);
            stzc[lev].setVal(Real(0.0));
            stzc[lev].ParallelCopy(*z_phys_cc[lev], 0, 0, 1, IntVect(0), IntVect(0), period);
        }
        if (z_phys_nd[lev]) {
            BoxArray ba(bl_nd);
            stznd[lev].define(ba, dm_io, 1, 0, pinned);
            stznd[lev].setVal(Real(0.0));
            stznd[lev].ParallelCopy(*z_phys_nd[lev], 0, 0, 1, IntVect(0), IntVect(0), period);
        }
    }
 
    // The gathers above are device-side in a GPU build, and what follows reads
    // the pinned destinations on the host.
    Gpu::streamSynchronize();
 
    // ---- interpolate, on the IO rank ----
    Vector<Real> row(ss.numColumns(), Real(0.0));
 
    if (ParallelDescriptor::IOProcessor())
    {
        for (int lev = 0; lev <= finest_level; ++lev)
        {
            if (lev_locs[lev].empty()) { continue; }
 
            const auto  problo = geom[lev].ProbLoArray();
            const auto  dx     = geom[lev].CellSizeArray();
            const Box&  dom    = geom[lev].Domain();
            const int   klo    = dom.smallEnd(2);
 
            for (int ib = 0; ib < static_cast<int>(lev_locs[lev].size()); ++ib)
            {
                const int ip = lev_locs[lev][ib];
                const Station&    station = stations[all_locs[ip].first];
                const StationLoc& loc     = station.locs[all_locs[ip].second];
 
                const int i0 = loc.i0, j0 = loc.j0, i1 = loc.i1, j1 = loc.j1;
                const Real wx = loc.wx, wy = loc.wy;
 
                auto bilinear = [&](const Array4<const Real>& a, int comp, int k)
                {
                    return (Real(1.0)-wy) * ( (Real(1.0)-wx)*a(i0,j0,k,comp) + wx*a(i1,j0,k,comp) )
                         +            wy  * ( (Real(1.0)-wx)*a(i0,j1,k,comp) + wx*a(i1,j1,k,comp) );
                };
 
                // Height of the cell centres in the station's column
                Array4<const Real> zc_arr;
                if (z_phys_cc[lev]) { zc_arr = stzc[lev].const_array(ib); }
                auto z_of_k = [&](int k)
                {
                    if (zc_arr) { return bilinear(zc_arr, 0, k); }
                    return problo[2] + (Real(k-klo) + Real(0.5)) * dx[2];
                };
 
                // Terrain elevation under the station
                Real z_surf = problo[2];
                if (z_phys_nd[lev]) {
                    const Array4<const Real>& nd = stznd[lev].const_array(ib);
                    auto corner_avg = [&](int i, int j) {
                        return Real(0.25) * (nd(i,j,klo) + nd(i+1,j,klo) + nd(i,j+1,klo) + nd(i+1,j+1,klo));
                    };
                    z_surf = (Real(1.0)-wy) * ( (Real(1.0)-wx)*corner_avg(i0,j0) + wx*corner_avg(i1,j0) )
                           +            wy  * ( (Real(1.0)-wx)*corner_avg(i0,j1) + wx*corner_avg(i1,j1) );
                }
 
                Array4<const Real> a3, a2;
                if (n3d > 0) { a3 = st3d[lev].const_array(ib); }
                if (n2d > 0) { a2 = st2d[lev].const_array(ib); }
 
                // The columns of this location, in the order buildColumns assigned them
                int icol = loc.col_begin;
 
                for (const auto& var : station.vars) {
                    if (!var.is_2d) { continue; }
                    const int gcol = station.columns[icol++];
                    row[gcol] = var.is_missing ? var.missing_value : bilinear(a2, var.comp, 0);
                }
 
                const Real z_zero = station.heights_are_agl ? z_surf : Real(0.0);
 
                for (const Real height : station.heights)
                {
                    const Real z_target = z_zero + height;
 
                    // Bracket the target height in the part of the column this
                    // level was chosen to cover
                    int kk = klo;
                    while (kk < loc.ktop && z_of_k(kk+1) < z_target) { ++kk; }
                    const Real zk  = z_of_k(kk);
                    const Real zk1 = z_of_k(std::min(kk+1, loc.ktop));
                    Real wz = (zk1 > zk) ? (z_target - zk) / (zk1 - zk) : Real(0.0);
                    wz = std::min(std::max(wz, Real(0.0)), Real(1.0));   // clamp below the first
                    const int k1 = std::min(kk+1, loc.ktop);             // cell centre and above the top
 
                    for (const auto& var : station.vars) {
                        if (var.is_2d) { continue; }
                        const int gcol = station.columns[icol++];
                        row[gcol] = var.is_missing
                                  ? var.missing_value
                                  : (Real(1.0)-wz)*bilinear(a3, var.comp, kk) + wz*bilinear(a3, var.comp, k1);
                    }
                }
            }
        }
    }
 
    ss.appendRow(time, static_cast<Real>(start_time) + time, row);
}

Here is the call graph for this function:

SampleLine()

amrex::IntVect& ERF::SampleLine ( int  i )

inlineprivate

    {
        return sampleline[i];
    }

SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int  i )

inlineprivate

    {
        return *samplelinelog[i];
    }

SampleLineLogName()

std::string ERF::SampleLineLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith samplelinelog file.

{ return samplelinelogname[i]; }

SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int  i )

inlineprivate

    {
        return samplepoint[i];
    }

SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int  i )

inlineprivate

    {
        return *sampleptlog[i];
    }

SamplePointLogName()

std::string ERF::SamplePointLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith sampleptlog file.

{ return sampleptlogname[i]; }

SetDirsForPlotfilesAndCheckpointsForDA()

void ERF::SetDirsForPlotfilesAndCheckpointsForDA	(	const int	da_iter,
		const int	ens_no
	)

SetDriverAtmosToOceanStateContract()

void ERF::SetDriverAtmosToOceanStateContract

(

bool

use_state_contract

)

{
    m_driver_atm2ocn_uses_state_contract = use_state_contract;
}

setPlotVariables()

void ERF::setPlotVariables ( const std::string &  pp_plot_var_names, amrex::Vector< std::string > &  plot_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    if (pp.contains(pp_plot_var_names.c_str()))
    {
        std::string nm;
 
        int nPltVars = pp.countval(pp_plot_var_names.c_str());
 
        for (int i = 0; i < nPltVars; i++)
        {
            pp.get(pp_plot_var_names.c_str(), nm, i);
 
            // Add the named variable to our list of plot variables
            // if it is not already in the list
            if (!containerHasElement(plot_var_names, nm)) {
                plot_var_names.push_back(nm);
            }
        }
    } else {
        //
        // The default is to add none of the variables to the list
        //
        plot_var_names.clear();
    }
 
    canonicalizePlot3DVariables(plot_var_names);
}

Referenced by ERF_shared().

Here is the call graph for this function:

Here is the caller graph for this function:

setPlotVariables2D()

void ERF::setPlotVariables2D ( const std::string &  pp_plot_var_names, amrex::Vector< std::string > &  plot_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    if (!pp.contains(pp_plot_var_names.c_str())) {
        //
        // The default is to add none of the variables to the list
        //
        plot_var_names.clear();
        return;
    }
 
    Vector<std::string> requested_plot_names;
    std::string nm;
    const int nPltVars = pp.countval(pp_plot_var_names.c_str());
    for (int i = 0; i < nPltVars; ++i) {
        pp.get(pp_plot_var_names.c_str(), nm, i);
        requested_plot_names.push_back(nm);
    }
 
    const bool has_surface_layer =
        phys_bc_type[Orientation(Direction::z, Orientation::low)] == ERF_BC::surface_layer;
    const auto active_lsm_names = lsm.Get_DataNames();
    const auto available_names = plotfile2d::available_diagnostic_names(solverChoice,
                                                                         has_surface_layer,
                                                                         active_lsm_names);
 
    // Keep the canonical built-in 2D ordering so the plotfile component layout
    // stays stable even if the input request order changes.
    const auto selection = plotfile2d::select_requested_plot_variables(requested_plot_names,
                                                                        available_names);
    plot_var_names = selection.accepted;
 
    // Unknown 2D names are skipped rather than aborting because the 2D plot
    // list is intentionally user-configurable and may include names that are not
    // compiled into a given build. The warning is still explicit so the user
    // can correct the input deck.
    warn_for_unavailable_2d_plot_vars(plotfile2d::format_plot2d_parameter_name(pp_prefix, pp_plot_var_names),
                                      selection.unavailable, available_names);
}

Here is the call graph for this function:

setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool  restarting )

private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters

[in]

restarting

Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

{
    // If we are restarting then we haven't read the input_sounding file yet
    //    so we need to read it here
    // TODO: should we store this information in the checkpoint file instead?
    bool is_moist = (solverChoice.moisture_type != MoistureType::None);
    if (restarting) {
        input_sounding_data.resize_arrays();
        for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
            input_sounding_data.read_from_file(geom[0], zlevels_stag[0], n, is_moist);
        }
    }
 
    const Real* z_inp_sound     = input_sounding_data.z_inp_sound[0].dataPtr();
    const Real* U_inp_sound     = input_sounding_data.U_inp_sound[0].dataPtr();
    const Real* V_inp_sound     = input_sounding_data.V_inp_sound[0].dataPtr();
    const Real* theta_inp_sound = input_sounding_data.theta_inp_sound[0].dataPtr();
    const int   inp_sound_size  = input_sounding_data.size(0);
 
    int refine_fac{1};
    for (int lev = 0; lev <= finest_level; lev++)
    {
        const int klo = geom[lev].Domain().smallEnd(2);
        const int khi = geom[lev].Domain().bigEnd(2);
        const int Nz = khi - klo + 1;
 
        Vector<Real> zcc(Nz);
        Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
                                 zlevels_stag[0].begin()+khi/refine_fac+2);
        expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
#if 0
        amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
        for (int k = 0; k < zlevels_sub.size(); k++) { amrex::AllPrint() << zlevels_sub[k] << " "; }
        amrex::AllPrint() << " --> ";
        for (int k = 0; k < Nz; k++) { amrex::AllPrint() << zcc[k] << " "; }
        amrex::AllPrint() << std::endl;
#endif
 
        for (int k = 0; k < Nz; k++)
        {
            h_rayleigh_ptrs[lev][Rayleigh::ubar][k]     = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
            h_rayleigh_ptrs[lev][Rayleigh::vbar][k]     = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
            h_rayleigh_ptrs[lev][Rayleigh::wbar][k]     = zero;
            h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
        }
 
        // Copy from host version to device version
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
 
        if (lev < finest_level) {
            refine_fac *= ref_ratio[lev][2];
        }
    }
}

Here is the call graph for this function:

setRecordDataInfo()

void ERF::setRecordDataInfo ( int  i, const std::string &  filename  )

inlineprivate

    {
        if (amrex::ParallelDescriptor::IOProcessor())
        {
            datalog[i] = std::make_unique<std::fstream>();
            datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
            if (!datalog[i]->good()) {
                amrex::FileOpenFailed(filename);
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
    }

setRecordDerDataInfo()

void ERF::setRecordDerDataInfo ( int  i, const std::string &  filename  )

inlineprivate

    {
        if (amrex::ParallelDescriptor::IOProcessor())
        {
            der_datalog[i] = std::make_unique<std::fstream>();
            der_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
            if (!der_datalog[i]->good()) {
                amrex::FileOpenFailed(filename);
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordDerDataInfo");
    }

setRecordEnergyDataInfo()

void ERF::setRecordEnergyDataInfo ( int  i, const std::string &  filename  )

inlineprivate

    {
        if (amrex::ParallelDescriptor::IOProcessor())
        {
            tot_e_datalog[i] = std::make_unique<std::fstream>();
            tot_e_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
            if (!tot_e_datalog[i]->good()) {
                amrex::FileOpenFailed(filename);
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordEnergyDataInfo");
    }

setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo ( int  i, int  lev, amrex::IntVect &  cell, const std::string &  filename  )

inlineprivate

    {
        amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
        for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
        {
            const amrex::Box& bx = mfi.validbox();
            if (bx.contains(cell)) {
                samplelinelog[i] = std::make_unique<std::fstream>();
                samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                if (!samplelinelog[i]->good()) {
                    amrex::FileOpenFailed(filename);
                }
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
    }

setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo ( int  i, int  lev, amrex::IntVect &  cell, const std::string &  filename  )

inlineprivate

    {
        amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
        for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
        {
            const amrex::Box& bx = mfi.validbox();
            if (bx.contains(cell)) {
                sampleptlog[i] = std::make_unique<std::fstream>();
                sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                if (!sampleptlog[i]->good()) {
                    amrex::FileOpenFailed(filename);
                }
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
    }

setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool  restarting )

private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters

[in]

restarting

Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

{
    // If we are restarting then we haven't read the input_sponge file yet
    //    so we need to read it here
    // TODO: should we store this information in the checkpoint file instead?
    if (restarting) {
        input_sponge_data.read_from_file(geom[0], zlevels_stag[0]);
    }
 
    const Real* z_inp_sponge     = input_sponge_data.z_inp_sponge.dataPtr();
    const Real* U_inp_sponge     = input_sponge_data.U_inp_sponge.dataPtr();
    const Real* V_inp_sponge     = input_sponge_data.V_inp_sponge.dataPtr();
    const int   inp_sponge_size  = input_sponge_data.size();
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        const int khi = geom[lev].Domain().bigEnd()[2];
        Vector<Real> zcc(khi+1);
 
        if (z_phys_cc[lev]) {
            // use_terrain=1
            // calculate the damping strength based on the max height at each k
            reduce_to_max_per_height(zcc, z_phys_cc[lev]);
        } else {
            const auto *const prob_lo = geom[lev].ProbLo();
            const auto *const dx = geom[lev].CellSize();
            for (int k = 0; k <= khi; k++)
            {
                zcc[k] = prob_lo[2] + (k+myhalf) * dx[2];
            }
        }
 
        for (int k = 0; k <= khi; k++)
        {
            h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
            h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
        }
 
        // Copy from host version to device version
        Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
                         d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
        Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
                         d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
    }
}

Here is the call graph for this function:

setSubVolVariables()

void ERF::setSubVolVariables ( const std::string &  pp_subvol_var_names, amrex::Vector< std::string > &  subvol_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    std::string nm;
 
    int nSubVolVars = pp.countval(pp_subvol_var_names.c_str());
 
    // We pre-populate the list with velocities, but allow these to be over-written
    //    by user input
    if (nSubVolVars == 0)
    {
        subvol_var_names.push_back("x_velocity");
        subvol_var_names.push_back("y_velocity");
        subvol_var_names.push_back("z_velocity");
 
    } else {
        for (int i = 0; i < nSubVolVars; i++)
        {
            pp.get(pp_subvol_var_names.c_str(), nm, i);
 
            // Add the named variable to our list of subvol variables
            // if it is not already in the list
            if (!containerHasElement(subvol_var_names, nm)) {
                subvol_var_names.push_back(nm);
            }
        }
    }
 
    // Get state variables in the same order as we define them,
    // since they may be in any order in the input list
    Vector<std::string> tmp_plot_names;
 
    // NOTE: raw state names are selected with exactly the same predicate the 3D
    //       plotfile uses, so the two output paths cannot drift.  It answers from
    //       the allocated width for the dry components and for the non-water
    //       species some schemes carry above the moist window, and from the
    //       moisture map inside that window -- a "rhoQn" the model does not carry
    //       would otherwise be read out of bounds when we ParallelCopy it below.
    const MoistureComponentIndices& mi = solverChoice.moisture_indices;
 
    erf_plotfile::Plot3DSelectionCapabilities capabilities;
    erf_plotfile::plot3d_set_state_capabilities(capabilities,
                                                mi,
                                                micro->Get_Qstate_Moist_Size(),
                                                micro->Get_Qstate_Size());
 
    for (int i = 0; i < cons_names.size(); ++i) {
        if ( containerHasElement(subvol_var_names, cons_names[i]) ) {
            if ( erf_plotfile::plot3d_fixed_variable_available(cons_names[i], capabilities) ) {
                tmp_plot_names.push_back(cons_names[i]);
            }
        }
    }
 
    // Check for velocity since it's not in cons_names
    if (containerHasElement(subvol_var_names, "x_velocity")) {
        tmp_plot_names.push_back("x_velocity");
    }
    if (containerHasElement(subvol_var_names, "y_velocity")) {
        tmp_plot_names.push_back("y_velocity");
    }
    if (containerHasElement(subvol_var_names, "z_velocity")) {
        tmp_plot_names.push_back("z_velocity");
    }
 
    //
    // If the model we are running doesn't have the variable listed in the inputs file,
    //     just ignore it rather than aborting
    //
    // NOTE: we walk "derived_names" (not "derived_subvol_names") because the names must be
    //       pushed in the order WriteSubvolume computes them, and that order is the
    //       "derived_names" order.  "derived_subvol_names" is the unordered allow-list of
    //       the derived quantities WriteSubvolume knows how to fill.
    //
    // NOTE: there are no terrain-dependent names to gate here -- "z_phys", "detJ" and
    //       "terrain_IB_mask" are not in "derived_subvol_names", so WriteSubvolume
    //       cannot fill them and they never reach this loop.
    //
    for (int i = 0; i < derived_names.size(); ++i) {
        if ( containerHasElement(subvol_var_names,     derived_names[i]) &&
             containerHasElement(derived_subvol_names, derived_names[i]) ) {
            // NOTE: WriteSubvolume only computes "mucape" for a moist run, so it must not be
            //       named for a dry one even though the kernel itself is dry-safe.  The
            //       moisture map does not govern that name -- the 3D plotfile writes it for
            //       dry runs too -- so it is gated here rather than in has_derived_var().
            const bool ok_to_add = ( mi.has_moisture() || (derived_names[i] != "mucape") );
            if (ok_to_add && mi.has_derived_var(derived_names[i]))
            {
                tmp_plot_names.push_back(derived_names[i]);
            }
        } // hasElement
    }
 
    subvol_var_names = tmp_plot_names;
}

Referenced by ERF_shared().

Here is the call graph for this function:

Here is the caller graph for this function:

solve_with_gmres()

void ERF::solve_with_gmres	(	int	lev,
		const amrex::Box &	subdomain,
		amrex::MultiFab &	rhs,
		amrex::MultiFab &	p,
		amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &	fluxes,
		amrex::MultiFab &	ax_sub,
		amrex::MultiFab &	ay_sub,
		amrex::MultiFab &	az_sub,
		amrex::MultiFab &	,
		amrex::MultiFab &	znd_sub
	)

Solve the Poisson equation using FFT-preconditioned GMRES

Parameters

lev	Level index for the solve
subdomain	Box over which the solve is performed
rhs	Right-hand side field for the Poisson solve
p	Solution field to fill
fluxes	Face-centered gradient fluxes to fill
ax_sub	Terrain metric coefficient on x-faces
ay_sub	Terrain metric coefficient on y-faces
az_sub	Terrain metric coefficient on z-faces
znd_sub	Node-centered physical height field

The unnamed metric argument in the class declaration is the cell-centered Jacobian determinant.

{
#ifdef ERF_USE_FFT
    BL_PROFILE("ERF::solve_with_gmres()");
 
    Real reltol = solverChoice.poisson_reltol;
    Real abstol = solverChoice.poisson_abstol;
 
    auto const dom_lo = lbound(Geom(lev).Domain());
    auto const dom_hi = ubound(Geom(lev).Domain());
 
    auto const sub_lo = lbound(subdomain);
    auto const sub_hi = ubound(subdomain);
 
    auto dx    = Geom(lev).CellSizeArray();
 
    Geometry my_geom;
 
    Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
    if (Geom(lev).isPeriodic(0) && sub_lo.x == dom_lo.x && sub_hi.x == dom_hi.x) { is_per[0] = 1;}
    if (Geom(lev).isPeriodic(1) && sub_lo.y == dom_lo.y && sub_hi.y == dom_hi.y) { is_per[1] = 1;}
 
    int coord_sys = 0;
 
    // If subdomain == domain then we pass Geom(lev) to the FFT solver
    if (subdomain == Geom(lev).Domain()) {
        my_geom.define(Geom(lev).Domain(), Geom(lev).ProbDomain(), coord_sys, is_per);
    } else {
        // else we create a new geometry based only on the subdomain
        // The information in my_geom used by the FFT routines is:
        //   1) my_geom.Domain()
        //   2) my_geom.CellSize()
        //   3) my_geom.isAllPeriodic() / my_geom.periodicity()
        RealBox rb( sub_lo.x   *dx[0],  sub_lo.y   *dx[1],  sub_lo.z   *dx[2],
                   (sub_hi.x+1)*dx[0], (sub_hi.y+1)*dx[1], (sub_hi.z+1)*dx[2]);
        my_geom.define(subdomain, rb, coord_sys, is_per);
    }
 
    amrex::GMRES<MultiFab, TerrainPoisson> gmsolver;
 
    TerrainPoisson tp(my_geom, Geom(lev), rhs.boxArray(), rhs.DistributionMap(), domain_bc_type,
                      stretched_dz_d[lev], ax_sub, ay_sub, az_sub, dJ_sub, &znd_sub,
                      solverChoice.use_real_bcs);
 
    gmsolver.define(tp);
 
    gmsolver.setVerbose(mg_verbose);
 
    gmsolver.setRestartLength(50);
 
    tp.usePrecond(true);
 
    gmsolver.solve(phi, rhs, reltol, abstol);
 
    tp.getFluxes(phi, fluxes);
 
    for (MFIter mfi(phi); mfi.isValid(); ++mfi)
    {
        Box xbx = mfi.nodaltilebox(0);
        Box ybx = mfi.nodaltilebox(1);
        const Array4<Real      >& fx_ar = fluxes[0].array(mfi);
        const Array4<Real      >& fy_ar = fluxes[1].array(mfi);
        const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
        const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
        ParallelFor(xbx,ybx,
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            fx_ar(i,j,k) *= mf_ux(i,j,0);
        },
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            fy_ar(i,j,k) *= mf_vy(i,j,0);
        });
    } // mfi
#else
    amrex::ignore_unused(lev, rhs, phi, fluxes, ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
#endif
 
    // ****************************************************************************
    // Impose bc's on pprime
    // ****************************************************************************
    ImposeBCsOnPhi(lev, phi, subdomain);
}

Here is the call graph for this function:

sum_derived_quantities()

void ERF::sum_derived_quantities

(

double

time

)

{
    if (verbose <= 0 || NumDerDataLogs() <= 0) return;
 
    int lev = 0;
 
    AMREX_ALWAYS_ASSERT(lev == 0);
 
    auto& mfx0 = *mapfac[0][MapFacType::m_x];
    auto& mfy0 = *mapfac[0][MapFacType::m_x];
    auto&  dJ0 = *detJ_cc[0];
 
    // ************************************************************************
    // WARNING: we are not filling ghost cells other than periodic outside the domain
    // ************************************************************************
 
    MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
    mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
 
    // Average all three components of velocity (on faces) to the cell center
    average_face_to_cellcenter(mf_cc_vel,0,
                               Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                        &vars_new[lev][Vars::yvel],
                                                        &vars_new[lev][Vars::zvel]});
    mf_cc_vel.FillBoundary(geom[lev].periodicity());
 
    if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
        amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
    }
 
    MultiFab r_wted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
    MultiFab unwted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
    MultiFab     enstrophysq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
    MultiFab        theta_mf(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(unwted_magvelsq, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        auto& src_fab = mf_cc_vel[mfi];
 
        auto& dest1_fab = unwted_magvelsq[mfi];
        // NOTE: we send in src_fab where we should
        derived::erf_dermagvelsq(bx, dest1_fab, 0, 1, src_fab, (*z_phys_cc[lev])[mfi], Geom(lev),
                                 static_cast<Real>(t_new[0]), nullptr, lev);
 
        auto& dest2_fab = enstrophysq[mfi];
        derived::erf_derenstrophysq(bx, dest2_fab, 0, 1, src_fab, (*z_phys_cc[lev])[mfi], Geom(lev),
                                    static_cast<Real>(t_new[0]), nullptr, lev);
    }
 
    // Copy the MF holding 1/2(u^2 + v^2 + w^2) into the MF that will hold 1/2 rho (u^2 + v^2 + w^2)d
    MultiFab::Copy(r_wted_magvelsq, unwted_magvelsq, 0, 0, 1, 0);
 
    // Multiply the MF holding 1/2(u^2 + v^2 + w^2) by rho to get  1/2 rho (u^2 + v^2 + w^2)
    MultiFab::Multiply(r_wted_magvelsq, vars_new[lev][Vars::cons], 0, 0, 1, 0);
 
    // Copy the MF holding (rho theta) into "theta_mf"
    MultiFab::Copy(theta_mf, vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, 0);
 
    // Divide (rho theta) by rho to get theta in the MF "theta_mf"
    MultiFab::Divide(theta_mf, vars_new[lev][Vars::cons], Rho_comp, 0, 1, 0);
 
    Real  unwted_avg = volWgtSumMF(lev, unwted_magvelsq, 0, dJ0, mfx0, mfy0, false);
    Real  r_wted_avg = volWgtSumMF(lev, r_wted_magvelsq, 0, dJ0, mfx0, mfy0, false);
    Real enstrsq_avg = volWgtSumMF(lev, enstrophysq,     0, dJ0, mfx0, mfy0, false);
    Real   theta_avg = volWgtSumMF(lev, theta_mf,        0, dJ0, mfx0, mfy0, false);
 
    // Get volume including terrain (consistent with volWgtSumMF routine)
    MultiFab volume(grids[lev], dmap[lev], 1, 0);
    auto const& dx = geom[lev].CellSizeArray();
    Real cell_vol  = dx[0]*dx[1]*dx[2];
    volume.setVal(cell_vol);
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
    }
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& tbx  = mfi.tilebox();
        auto dst        = volume.array(mfi);
        const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
        ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
        });
    }
    Real vol = volume.sum();
 
     unwted_avg /= vol;
     r_wted_avg /= vol;
    enstrsq_avg /= vol;
      theta_avg /= vol;
 
    const int nfoo = 4;
    Real foo[nfoo] = {unwted_avg,r_wted_avg,enstrsq_avg,theta_avg};
    Real zero_d = zero;
#ifdef AMREX_LAZY
    Lazy::QueueReduction([=]() mutable {
#endif
    ParallelDescriptor::ReduceRealSum(
        foo, nfoo, ParallelDescriptor::IOProcessorNumber());
 
      if (ParallelDescriptor::IOProcessor()) {
        int i = 0;
        unwted_avg  = foo[i++];
        r_wted_avg  = foo[i++];
        enstrsq_avg = foo[i++];
          theta_avg = foo[i++];
 
        std::ostream& data_log_der = DerDataLog(0);
 
        if (time == zero_d) {
            data_log_der << std::setw(datwidth) << "          time";
            data_log_der << std::setw(datwidth) << "        ke_den";
            data_log_der << std::setw(datwidth) << "         velsq";
            data_log_der << std::setw(datwidth) << "     enstrophy";
            data_log_der << std::setw(datwidth) << "    int_energy";
            data_log_der << std::endl;
        }
        data_log_der << std::setw(datwidth) << std::setprecision(timeprecision) << time;
        data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  <<  unwted_avg;
        data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  <<  r_wted_avg;
        data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  << enstrsq_avg;
        data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  <<   theta_avg;
        data_log_der << std::endl;
 
      } // if IOProcessor
#ifdef AMREX_LAZY
    });
#endif
}

Here is the call graph for this function:

sum_energy_quantities()

void ERF::sum_energy_quantities

(

double

time

)

{
    if ( (verbose <= 0) || (tot_e_datalog.size() < 1) ) { return; }
 
    int lev = 0;
 
    auto& mfx0 = *mapfac[0][MapFacType::m_x];
    auto& mfy0 = *mapfac[0][MapFacType::m_x];
    auto&  dJ0 = *detJ_cc[0];
 
    AMREX_ALWAYS_ASSERT(lev == 0);
 
    bool local = true;
 
    // ************************************************************************
    // WARNING: we are not filling ghost cells other than periodic outside the domain
    // ************************************************************************
 
    MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
    mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
 
    // Average all three components of velocity (on faces) to the cell center
    average_face_to_cellcenter(mf_cc_vel,0,
                               Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                        &vars_new[lev][Vars::yvel],
                                                        &vars_new[lev][Vars::zvel]});
    mf_cc_vel.FillBoundary(geom[lev].periodicity());
 
    if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
        amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
    }
 
    MultiFab tot_mass  (grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
    MultiFab tot_energy(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
 
    auto const& dx = geom[lev].CellSizeArray();
    bool is_moist = (solverChoice.moisture_type != MoistureType::None);
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(tot_mass, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
 
        const Array4<Real>& cc_vel_arr     = mf_cc_vel.array(mfi);
        const Array4<Real>& tot_mass_arr   = tot_mass.array(mfi);
        const Array4<Real>& tot_energy_arr = tot_energy.array(mfi);
        const Array4<const Real>& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
        const Array4<const Real>& z_arr    = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) :
                                                                Array4<const Real>{};
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            Real Qv   = (is_moist) ? cons_arr(i,j,k,RhoQ1_comp) : zero;
            Real Qc   = (is_moist) ? cons_arr(i,j,k,RhoQ2_comp) : zero;
            Real Qt   = Qv + Qc;
            Real Rhod = cons_arr(i,j,k,Rho_comp);
            Real Rhot = Rhod * (one + Qt);
            Real Temp = getTgivenRandRTh(Rhod, cons_arr(i,j,k,RhoTheta_comp), Qv);
            Real TKE  = myhalf * ( cc_vel_arr(i,j,k,0)*cc_vel_arr(i,j,k,0)
                              + cc_vel_arr(i,j,k,1)*cc_vel_arr(i,j,k,1)
                              + cc_vel_arr(i,j,k,2)*cc_vel_arr(i,j,k,2) );
            Real zval = (z_arr) ? z_arr(i,j,k) : Real(k)*dx[2];
 
            Real Cv   = Cp_d - R_d;
            Real Cvv  = Cp_v - R_v;
            Real Cpv  = Cp_v;
 
            tot_mass_arr(i,j,k)   = Rhot;
            tot_energy_arr(i,j,k) = Rhod * ( (Cv + Cvv*Qv + Cpv*Qc)*Temp - L_v*Qc
                                           + (one + Qt)*TKE + (one + Qt)*CONST_GRAV*zval );
 
        });
 
    }
 
    Real  tot_mass_avg   = volWgtSumMF(lev, tot_mass  , 0, dJ0, mfx0, mfy0, false, local);
    Real  tot_energy_avg = volWgtSumMF(lev, tot_energy, 0, dJ0, mfx0, mfy0, false, local);
 
    // Get volume including terrain (consistent with volWgtSumMF routine)
    MultiFab volume(grids[lev], dmap[lev], 1, 0);
    Real cell_vol  = dx[0]*dx[1]*dx[2];
    volume.setVal(cell_vol);
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
    }
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& tbx  = mfi.tilebox();
        auto dst        = volume.array(mfi);
        const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
        ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
        });
    }
    Real vol = volume.sum();
 
    // Divide by the volume
    tot_mass_avg   /= vol;
    tot_energy_avg /= vol;
 
    const int nfoo = 2;
    Real foo[nfoo] = {tot_mass_avg,tot_energy_avg};
    Real zero_d = zero;
#ifdef AMREX_LAZY
    Lazy::QueueReduction([=]() mutable {
#endif
    ParallelDescriptor::ReduceRealSum(
        foo, nfoo, ParallelDescriptor::IOProcessorNumber());
 
      if (ParallelDescriptor::IOProcessor()) {
        int i = 0;
        tot_mass_avg   = foo[i++];
        tot_energy_avg = foo[i++];
 
        std::ostream& data_log_energy = *tot_e_datalog[0];
 
        if (time == zero_d) {
            data_log_energy << std::setw(datwidth) << "          time";
            data_log_energy << std::setw(datwidth) << "      tot_mass";
            data_log_energy << std::setw(datwidth) << "    tot_energy";
            data_log_energy << std::endl;
        }
        data_log_energy << std::setw(datwidth) << std::setprecision(timeprecision) << time;
        data_log_energy << std::setw(datwidth) << std::setprecision(datprecision)  << tot_mass_avg;
        data_log_energy << std::setw(datwidth) << std::setprecision(datprecision)  << tot_energy_avg;
        data_log_energy << std::endl;
 
      } // if IOProcessor
#ifdef AMREX_LAZY
    });
#endif
}

Here is the call graph for this function:

sum_integrated_quantities()

void ERF::sum_integrated_quantities

(

double

time

)

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters

time	Current time

{
    BL_PROFILE("ERF::sum_integrated_quantities()");
 
    if (verbose <= 0)
      return;
 
    // Single level sum
    Real mass_sl;
 
    // Multilevel sums
    Real mass_ml = zero;
    Real rhth_ml = zero;
    Real scal_ml = zero;
    Real mois_ml = zero;
 
    bool local = true;
 
    auto& mfx0 = *mapfac[0][MapFacType::m_x];
    auto& mfy0 = *mapfac[0][MapFacType::m_x];
    auto&  dJ0 = *detJ_cc[0];
 
    mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,dJ0,mfx0,mfy0,false,local);
 
    for (int lev = 0; lev <= finest_level; lev++) {
        auto& mfx = *mapfac[lev][MapFacType::m_x];
        auto& mfy = *mapfac[lev][MapFacType::m_x];
        auto&  dJ = *detJ_cc[lev];
        mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,dJ,mfx,mfy,true);
    }
 
    Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,dJ0,mfx0,mfy0,false);
    Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,dJ0,mfx0,mfy0,false);
    Real mois_sl = zero;
    if (solverChoice.moisture_type != MoistureType::None) {
        int n_qstate_into_total = micro->Get_Qstate_Moist_Size() - micro->Get_Qstate_Moist_NumConc_Size();
        for (int qoff(0); qoff<n_qstate_into_total; ++qoff) {
            mois_sl += volWgtSumMF(0,vars_new[0][Vars::cons],RhoQ1_comp+qoff,dJ0,mfx0,mfy0,false);
        }
    }
 
    for (int lev = 0; lev <= finest_level; lev++) {
        auto& mfx = *mapfac[lev][MapFacType::m_x];
        auto& mfy = *mapfac[lev][MapFacType::m_x];
        auto&  dJ = *detJ_cc[lev];
        rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,dJ,mfx,mfy,true);
        scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,dJ,mfx,mfy,true);
        if (solverChoice.moisture_type != MoistureType::None) {
            int n_qstate_into_total = micro->Get_Qstate_Moist_Size() - micro->Get_Qstate_Moist_NumConc_Size();
            for (int qoff(0); qoff<n_qstate_into_total; ++qoff) {
                mois_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoQ1_comp+qoff,dJ,mfx,mfy,true);
            }
        }
    }
 
    Real h_avg_ustar = zero;
    Real h_avg_tstar = zero;
    Real h_avg_olen  = zero;
    Orientation zlo(Direction::z, Orientation::low);
    if ((m_SurfaceLayer[zlo] != nullptr) && (NumDataLogs() > 0)) {
        // Sum each surface cell once: on grids split in z the planar arrays hold one
        // duplicate box per stacked 3D box
        h_avg_ustar = m_SurfaceLayer[zlo]->surface_sum(0, *m_SurfaceLayer[zlo]->get_u_star(0));
        h_avg_tstar = m_SurfaceLayer[zlo]->surface_sum(0, *m_SurfaceLayer[zlo]->get_t_star(0));
        h_avg_olen  = m_SurfaceLayer[zlo]->surface_sum(0, *m_SurfaceLayer[zlo]->get_olen(0));
 
        // Divide by the total number of cells we are averaging over
        const Box& domain = geom[0].Domain();
        const Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
        h_avg_ustar /= area_z;
        h_avg_tstar /= area_z;
        h_avg_olen  /= area_z;
    }
 
    const int nfoo = 8;
    Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mois_sl,mass_ml,rhth_ml,scal_ml,mois_ml};
    Real zero_d = zero;
#ifdef AMREX_LAZY
    Lazy::QueueReduction([=]() mutable {
#endif
    ParallelDescriptor::ReduceRealSum(
        foo, nfoo, ParallelDescriptor::IOProcessorNumber());
 
      if (ParallelDescriptor::IOProcessor()) {
        int i = 0;
        mass_sl = foo[i++];
        rhth_sl = foo[i++];
        scal_sl = foo[i++];
        mois_sl = foo[i++];
        mass_ml = foo[i++];
        rhth_ml = foo[i++];
        scal_ml = foo[i++];
        mois_ml = foo[i++];
 
        Print() << '\n';
        Print() << "TIME= " << std::setw(datwidth) << std::setprecision(timeprecision) << std::left << time << '\n';
        if (finest_level ==  0) {
#if 1
           Print() << " MASS       = " << mass_sl << '\n';
#else
           Print() << " PERT MASS  = " << mass_sl << '\n';
#endif
           Print() << " RHO THETA  = " << rhth_sl << '\n';
           if (solverChoice.transport_scalar) { Print() << " RHO SCALAR = " << scal_sl << '\n'; }
           if (solverChoice.moisture_type != MoistureType::None) { Print() << " RHO QTOTAL = " << mois_sl << '\n'; }
        } else {
#if 1
           Print() << " MASS       SL/ML = " << mass_sl << " " << mass_ml << '\n';
#else
           Print() << " PERT MASS  SL/ML = " << mass_sl << " " << mass_ml << '\n';
#endif
           Print() << " RHO THETA  SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
           if (solverChoice.transport_scalar) { Print() << " RHO SCALAR SL/ML = " << scal_sl << " " << scal_ml << '\n'; }
           if (solverChoice.moisture_type != MoistureType::None) { Print() << " RHO QTOTAL SL/ML = " << mois_sl << " " << mois_ml << '\n'; }
        }
 
        // The first data log only holds scalars
        if (NumDataLogs() > 0)
        {
            int n_d = 0;
            std::ostream& data_log1 = DataLog(n_d);
            if (data_log1.good()) {
                if (time == zero_d) {
                    data_log1 << std::setw(datwidth) << "          time";
                    data_log1 << std::setw(datwidth) << "          u_star";
                    data_log1 << std::setw(datwidth) << "          t_star";
                    data_log1 << std::setw(datwidth) << "          olen";
                    data_log1 << std::endl;
                } // time = 0
 
              // Write the quantities at this time
              data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time;
              data_log1 << std::setw(datwidth) << std::setprecision(datprecision)  << h_avg_ustar;
              data_log1 << std::setw(datwidth) << std::setprecision(datprecision)  << h_avg_tstar;
              data_log1 << std::setw(datwidth) << std::setprecision(datprecision)  << h_avg_olen;
              data_log1 << std::endl;
            } // if good
        } // loop over i
      } // if IOProcessor
#ifdef AMREX_LAZY
    });
#endif
 
    // This is just an alias for convenience
    int lev = 0;
    if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
        for (int i = 0; i < NumSamplePoints(); ++i)
        {
            sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
        }
    }
    if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
        for (int i = 0; i < NumSampleLines(); ++i)
        {
            sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
        }
    }
}

SurfaceDataInterpolation()

void ERF::SurfaceDataInterpolation	(	const int	lev,
		const double	time,
		amrex::Vector< std::unique_ptr< amrex::MultiFab >> &	a_z_phys_nd,
		bool	regrid_forces_file_read
	)

Interpolate weather forecast surface data in time and space onto the simulation grid.

Parameters

[in]	lev	Current level.
[in]	time	Current simulation time.
[in]	a_z_phys_nd	Physical height MultiFabs.
[in]	regrid_forces_file_read	Flag indicating if a regrid requires a file read.

{
 
    static amrex::Vector<double> next_read_forecast_time;
    static amrex::Vector<double> last_read_forecast_time;
 
    const int nlevs = a_z_phys_nd.size();
 
    Real hindcast_data_interval = solverChoice.hindcast_data_interval_in_hrs*Real(3600.0);
 
    // Initialize static vectors once
    if (next_read_forecast_time.empty()) {
        next_read_forecast_time.resize(nlevs, -one);
        last_read_forecast_time.resize(nlevs, -one);
        Print() << "Initializing the time vector values here by " << lev << std::endl;
    }
 
    if (next_read_forecast_time[lev] < zero) {
        int next_multiple = static_cast<int>(time / hindcast_data_interval);
        next_read_forecast_time[lev] = next_multiple * hindcast_data_interval;
        last_read_forecast_time[lev] = next_read_forecast_time[lev];
    }
 
    if (time >= next_read_forecast_time[lev] or regrid_forces_file_read) {
 
        Print() << "Data reading happening at level " << lev << std::endl;
 
        std::string folder = solverChoice.hindcast_surface_data_dir;
 
        // Check if folder exists and is a directory
        if (!fs::exists(folder) || !fs::is_directory(folder)) {
            throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
        }
 
        std::vector<std::string> bin_files;
 
        for (const auto& entry : fs::directory_iterator(folder)) {
            if (!entry.is_regular_file()) continue;
 
            std::string fname = entry.path().filename().string();
            if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
                bin_files.push_back(entry.path().string());
            }
        }
        std::sort(bin_files.begin(), bin_files.end());
 
    // Check if no .bin files were found
        if (bin_files.empty()) {
            throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
        }
 
        std::string filename1, filename2;
 
        int idx1 = static_cast<int>(time / hindcast_data_interval);
        int idx2 = static_cast<int>(time / hindcast_data_interval)+1;
        Print() << "Reading surface data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
 
        if (idx2 >= static_cast<int>(bin_files.size())) {
            throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
        }
 
        filename1 = bin_files[idx1];
        filename2 = bin_files[idx2];
 
        FillSurfaceStateMultiFabs(lev, filename1, surface_state_1);
        FillSurfaceStateMultiFabs(lev, filename2, surface_state_2);
 
         // Create the time-interpolated forecast state
        //CreateForecastStateMultiFabs(forecast_state_interp);
        if(!regrid_forces_file_read){
            last_read_forecast_time[lev] = next_read_forecast_time[lev];
            next_read_forecast_time[lev] += hindcast_data_interval;
            Print() << "Next forecast time getting updated here " << std::endl;
        }
    }
 
    double prev_read_time = last_read_forecast_time[lev];
    Real alpha1 = static_cast<Real>(1.0 - (time - prev_read_time)/hindcast_data_interval);
    Real alpha2 = one - alpha1;
 
    amrex::Print()<< "The values of alpha1 and alpha2 are " << alpha1 << " "<< alpha2 <<std::endl;
 
    if (alpha1 < zero || alpha1 > one ||
    alpha2 < zero || alpha2 > one)
    {
        std::stringstream ss;
        ss << "Interpolation weights for hindcast files are incorrect: "
        << "alpha1 = " << alpha1 << ", alpha2 = " << alpha2;
       Abort(ss.str());
    }
 
    MultiFab& mf_surf_interp   = surface_state_interp[lev];
 
    // Fill the time-interpolated forecast states
    MultiFab::LinComb(surface_state_interp[lev],
                      alpha1, surface_state_1[lev], 0,
                      alpha2, surface_state_2[lev], 0,
                      0, mf_surf_interp.nComp(), mf_surf_interp.nGrow());
 
    /* debug plotfile dump follows — leave commented out
    std::string pltname = "plt_interp_surface";
    Vector<std::string> varnames_plot_mf = {"ls_mask", "SST"};
 
    const MultiFab& src = vars_new[0][0];
 
    MultiFab plot_mf(src.boxArray(),
                     src.DistributionMap(),
                     2, 0);
 
    plot_mf.setVal(0.0);
 
    for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
        const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
        const Array4<Real> &surf_mf_arr = surface_state_1[0].array(mfi);
 
        const Box& bx = mfi.validbox();
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
            plot_mf_arr(i,j,k,0) = surf_mf_arr(i,j,0);
            plot_mf_arr(i,j,k,1) = surf_mf_arr(i,j,1);
        });
    }
 
    WriteSingleLevelPlotfile(
            pltname,
            plot_mf,
            varnames_plot_mf,
            geom[0],
            time,
            0 // level
    );*/
}

timeStep()

void ERF::timeStep ( int  lev, double  time, int  iteration  )

private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	iteration	time step counter

{
    //
    // We need to FillPatch the coarse level before assessing whether to regrid
    // We have not done the swap yet so we fill the "new" which will become the "old"
    //
    MultiFab& S_new = vars_new[lev][Vars::cons];
    MultiFab& U_new = vars_new[lev][Vars::xvel];
    MultiFab& V_new = vars_new[lev][Vars::yvel];
    MultiFab& W_new = vars_new[lev][Vars::zvel];
 
#ifdef ERF_USE_NETCDF
    //
    // Since we now only read in a subset of the time slices in wrfbdy and
    //     wrflowinp, we need to check whether it's time to read in more.
    //
    bool use_moist = (solverChoice.moisture_type != MoistureType::None);
    if (solverChoice.use_real_bcs && (lev==0))
    {
        MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
        Array<MultiFab*, AMREX_SPACEDIM> area_vec = {ax[lev].get(), ay[lev].get(), az[lev].get()};
 
        int ntimes = bdy_data_xlo.size();
        double time_since_start_bdy = time + start_time - start_bdy_time;
        int n_time_old = std::min(static_cast<int>( (time_since_start_bdy        ) /  bdy_time_interval), ntimes-1);
        int n_time_new = std::min(static_cast<int>( (time_since_start_bdy+dt[lev]) /  bdy_time_interval), ntimes-1);
        auto repack_runtime_bdy = [&] (const int itime) {
            const bool separate_hydrometeors = solverChoice.use_wrf_bdy_qc_qi &&
                wrf_bdy_has_separate_hydrometeors(solverChoice.moisture_indices);
            repack_wrfbdy_to_realbdy(bdy_data_xlo[itime], solverChoice.use_wrf_bdy_qc_qi, separate_hydrometeors);
            repack_wrfbdy_to_realbdy(bdy_data_xhi[itime], solverChoice.use_wrf_bdy_qc_qi, separate_hydrometeors);
            repack_wrfbdy_to_realbdy(bdy_data_ylo[itime], solverChoice.use_wrf_bdy_qc_qi, separate_hydrometeors);
            repack_wrfbdy_to_realbdy(bdy_data_yhi[itime], solverChoice.use_wrf_bdy_qc_qi, separate_hydrometeors);
        };
 
        for (int itime = 0; itime < ntimes; itime++)
        {
            /*
            if (bdy_data_xlo[itime].size() > 0) {
                amrex::Print() << "HAVE  BDY DATA AT TIME " << itime << std::endl;
            } else {
                amrex::Print() << " NO   BDY DATA AT TIME " << itime << std::endl;
            }
            */
 
            // Note that we never release itime == 0 because it is used for the spatial interpolation at later times
            bool clear_itime = (itime > 0 && itime < n_time_old);
 
            if (clear_itime && bdy_data_xlo[itime].size() > 0) {
                bdy_data_xlo[itime].clear();
                bdy_data_xhi[itime].clear();
                bdy_data_ylo[itime].clear();
                bdy_data_yhi[itime].clear();
                //amrex::Print() << "CLEAR BDY DATA AT TIME " << itime << std::endl;
            }
 
            bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
            //if (need_itime) { amrex::Print()  << "NEED  BDY DATA AT TIME " << itime << std::endl; }
 
            // Handle erfbdy files (AMReX native format).
            if (use_erfbdy) {
                if (bdy_data_xlo[itime].size() == 0 && need_itime) {
                    read_from_erfbdy(itime, erfbdy_file,
                                     bdy_data_xlo, bdy_data_xhi,
                                     bdy_data_ylo, bdy_data_yhi,
                                     nvars_erfbdy, real_width);
                    repack_runtime_bdy(itime);
                }
            // Handle wrfbdy files (NetCDF format).
            } else {
                if (bdy_data_xlo[itime].size() == 0 && need_itime) {
                    bool is_anelastic = (solverChoice.anelastic[0] == 1);
                    read_and_convert_from_wrfbdy(itime,nc_bdy_file,bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                                 wrf_MUB, wrf_C1H, wrf_C2H, wrf_RDNW, wrf_PHB, z_phys_cc[lev], z_phys_nd[lev],
                                                 vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel], vars_new[lev][Vars::cons],
                                                 r_hse, area_vec, geom[lev], use_moist,
                                                 solverChoice.use_wrf_bdy_qc_qi,
                                                 solverChoice.moisture_indices.qi >= 0,
                                                 solverChoice.use_wrf_bdy_qc_qi &&
                                                 wrf_bdy_has_separate_hydrometeors(solverChoice.moisture_indices),
                                                 solverChoice.rebalance_wrf_input, domain_bcs_type,
                                                 real_width, bdy_time_interval, is_anelastic);
                    if (itime == ntimes-1 && itime > 0) {
                        repack_runtime_bdy(itime-1);
                    }
                    repack_runtime_bdy(itime);
                }
            } // use_erfbdy
        } // itime
    } // use_real_bcs && lev == 0
 
    if (!nc_low_file.empty() && (lev==0))
    {
        int ntimes = low_data_zlo.size();
        double time_since_start_low = time + start_time - start_low_time;
        int n_time_old = std::min(static_cast<int>( (time_since_start_low        ) /  low_time_interval), ntimes-1);
        int n_time_new = std::min(static_cast<int>( (time_since_start_low+dt[lev]) /  low_time_interval), ntimes-1);
 
        for (int itime = 0; itime < ntimes; itime++)
        {
            /*
            if (low_data_zlo[itime].size() > 0) {
                amrex::Print() << "HAVE  LOW DATA AT TIME " << itime << std::endl;
            } else {
                amrex::Print() << " NO   LOW DATA AT TIME " << itime << std::endl;
            }
            */
 
            bool clear_itime = (itime < n_time_old);
 
            if (clear_itime && low_data_zlo[itime].size() > 0) {
                low_data_zlo[itime].clear();
                //amrex::Print() << "CLEAR LOW DATA AT TIME " << itime << std::endl;
            }
 
            bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
            //if (need_itime) { amrex::Print()  << "NEED  LOW DATA AT TIME " << itime << std::endl; }
 
            if (low_data_zlo[itime].size() == 0 && need_itime) {
                read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
 
                update_sst_tsk(itime, geom[lev], ba2d[lev],
                               sst_lev[lev], tsk_lev[lev],
                               m_SurfaceLayer[Orientation(Direction::z, Orientation::low)], low_data_zlo,
                               S_new, *mf_PSFC[lev],
                               solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
            }
        } // itime
    } // have nc_low_file && lev == 0
#endif
 
    //
    // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
    //
    if (lev == 0) {
        FillPatchCrseLevel(lev, time, {&S_new, &U_new, &V_new, &W_new});
    } else if (lev < finest_level) {
        FillPatchFineLevel(lev, time, {&S_new, &U_new, &V_new, &W_new},
                           {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                           base_state[lev], base_state[lev]);
    }
 
    if (regrid_int > 0)  // We may need to regrid
    {
        // help keep track of whether a level was already regridded
        // from a coarser level call to regrid
        static Vector<int> last_regrid_step(max_level+1, 0);
 
        // regrid changes level "lev+1" so we don't regrid on max_level
        // also make sure we don't regrid fine levels again if
        // it was taken care of during a coarser regrid
        if (lev < max_level)
        {
            if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
            {
                // regrid could add newly refine levels (if finest_level < max_level)
                // so we save the previous finest level index
                int old_finest = finest_level;
 
                if (solverChoice.coupling_type == CouplingType::TwoWay &&
                    solverChoice.moisture_type != MoistureType::None &&
                    Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian &&
                    finest_level >= 1) {
                    micro->AverageDownMicroVars(finest_level);
                    for (int flev = finest_level-1; flev >= lev; --flev) {
                        AverageDownMoistStateTo(flev);
                    }
                }
 
#ifdef ERF_USE_PARTICLES
                // Snapshot the per-level particle BoxArrays before regrid so
                // we can identify cells that lost fine coverage and merge
                // them down to the coarse-level SD density.
                Vector<BoxArray> old_pc_ba;
                if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
                    auto* pc = dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer();
                    AMREX_ALWAYS_ASSERT(pc != nullptr);
                    old_pc_ba.resize(old_finest + 1);
                    for (int k = 0; k <= old_finest; k++) {
                        old_pc_ba[k] = pc->ParticleBoxArray(k);
                    }
                }
#endif
 
                regrid(lev, static_cast<Real>(time));
 
#ifdef ERF_USE_PARTICLES
                if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
                    auto* pc = dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer();
                    AMREX_ALWAYS_ASSERT(pc != nullptr);
                    // Sync the particle container's per-level storage with
                    // the post-regrid BoxArrays/DistributionMaps before any
                    // iMultiFab-based work touches it.  Note: this is the
                    // bare ParticleContainer::Redistribute (no SplitMerge);
                    // tag-based splitting and merging still run below.
                    pc->Redistribute();
                    // Split super-droplets that ended up on a level deeper
                    // than their tag indicates (cumulative cascading split
                    // for L0-natives that landed directly on L1 or L2 after
                    // the regrid created multiple new levels in one shot).
                    pc->SplitParticlesForRefinement(finest_level);
                }
                // Redistribute moves split daughters to their destination
                // sub-cells and runs each species' SplitMergeAtLevelBoundary,
                // whose per-level tag-normalizing merge sweep cleans up any
                // leftover super-droplets from levels that have just vanished.
                particleData.Redistribute(z_phys_nd);
 
                if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
                    auto* pc = dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer();
                    AMREX_ALWAYS_ASSERT(pc != nullptr);
                    // Reduce SD count in cells that lost fine-level coverage.
                    // Walks old_finest..1 so each level's masked merge runs
                    // while clev = k-1 still holds the just-moved fines.
                    for (int k = old_finest; k >= 1; k--) {
                        pc->MergeParticlesAtDerefinement(k, old_pc_ba[k], refRatio(k-1));
                    }
                }
#endif
 
                // mark that we have regridded this level already
                for (int k = lev; k <= finest_level; ++k) {
                    last_regrid_step[k] = istep[k];
                }
 
                // if there are newly created levels, set the time step
                for (int k = old_finest+1; k <= finest_level; ++k) {
                    dt[k] = dt[k-1] / static_cast<double>(nsubsteps[k]);
                }
            } // if
        } // lev
    }
 
    // Update what we call "old" and "new" time
    t_old[lev] = t_new[lev];
    t_new[lev] += dt[lev];
 
    if (Verbose()) {
        amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
        amrex::Print() << std::setprecision(timeprecision)
                       << "ADVANCE from elapsed time = " << t_old[lev] << " to " << t_new[lev]
                       << " with dt = " << dt[lev] << std::endl;
    }
 
#ifdef ERF_USE_WW3_COUPLING
    amrex::Print() <<  " About to call send_to_ww3 from ERF_Timestep" << std::endl;
    send_to_ww3(lev);
    amrex::Print() <<  " About to call read_waves from ERF_Timestep"  << std::endl;
    read_waves(lev);
    //send_to_ww3(lev);
    //read_waves(lev);
    //send_to_ww3(lev);
#endif
 
    // Advance a single level for a single time step
    Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
 
    ++istep[lev];
 
    if (Verbose()) {
        amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
        amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
    }
 
    if (lev < finest_level)
    {
        // recursive call for next-finer level
        for (int i = 1; i <= nsubsteps[lev+1]; ++i)
        {
            double strt_time_for_fine = time + (i-1)*dt[lev+1];
            timeStep(lev+1, strt_time_for_fine, i);
        }
    }
 
    if ( verbose && lev == 0 && solverChoice.moisture_type != MoistureType::None) {
        amrex::Print() << "Cloud fraction " << time << "  " << cloud_fraction(time) << std::endl;
    }
}

Referenced by EvolveOneStep().

Here is the call graph for this function:

Here is the caller graph for this function:

turbPert_amplitude()

void ERF::turbPert_amplitude ( const int  lev )

private

Calculate the cell-centered turbulent perturbation amplitude.

Parameters

lev	Integer specifying the current level

{
    // Accessing data
    auto& lev_new = vars_new[lev];
 
    // Creating local data
    int ncons = lev_new[Vars::cons].nComp();
    MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
 
    // Defining BoxArray type
    auto m_ixtype = cons_data.boxArray().ixType();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
        const Box &bx  = mfi.validbox();
        const auto &cons_pert_arr = cons_data.array(mfi); // Address of perturbation array
        const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
 
        turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cons_pert_arr, pert_cell);
    } // mfi
}

Here is the call graph for this function:

turbPert_amplitude_w()

void ERF::turbPert_amplitude_w ( const int  lev )

private

Calculate the turbulent perturbation amplitude for vertical velocity.

Parameters

lev	Integer specifying the current level

{
    // Accessing data
    auto& lev_new = vars_new[lev];
 
    // Creating local data
    int ncomps = lev_new[Vars::zvel].nComp();
    MultiFab wvel_data(lev_new[Vars::zvel], make_alias, 0, ncomps);
 
    // Defining BoxArray type
    auto m_ixtype = wvel_data.boxArray().ixType();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::zvel], TileNoZ()); mfi.isValid(); ++mfi) {
        const Box &bx  = mfi.validbox();
        const auto &wvel_pert_arr = wvel_data.array(mfi); // Address of perturbation array
        const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
 
        turbPert.apply_tpi(lev, bx, -1, m_ixtype, wvel_pert_arr, pert_cell);
    } // mfi
}

Here is the call graph for this function:

turbPert_update()

void ERF::turbPert_update ( const int  lev, const double  local_dt  )

private

Update turbulent perturbation timing and amplitudes for one AMR level.

Parameters

lev	Integer specifying the current level
local_dt	Local timestep used to update the perturbation state

{
    // Accessing data
    auto& lev_new = vars_new[lev];
 
    // Create aliases to state data to pass to calc_tpi_update
    int ncons = lev_new[Vars::cons].nComp();
    MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
    MultiFab xvel_data(lev_new[Vars::xvel], make_alias, 0, 1);
    MultiFab yvel_data(lev_new[Vars::yvel], make_alias, 0, 1);
 
    // Computing perturbation update time
    turbPert.calc_tpi_update(lev, local_dt, xvel_data, yvel_data, cons_data);
 
    Print() << "Successfully initialized turbulent perturbation update time and amplitude at level "
            << lev << " with type: " << turbPert.pt_type[lev] << "\n";
}

update_box_for_refinement()

void ERF::update_box_for_refinement	(	std::string &	ref_prefix,
		int &	lev_for_box,
		amrex::RealBox &	real_box,
		const double	time
	)

Update the refinement box position for moving grids.

Parameters

[in]	ref_prefix	Prefix for parsing input parameters.
[out]	lev_for_box	Level associated with the refinement box.
[out]	real_box	Updated real-space coordinates of the refinement box.
[in]	time	Current simulation time.

{
    ParmParse ppr(ref_prefix);
 
    Vector<double> move_start_time, move_stop_time;
    int ni = ppr.queryarr("move_start_time", move_start_time);
    int nj = ppr.queryarr("move_stop_time" ,  move_stop_time);
    if (ni != nj) {
        amrex::Print() << "Must be same number of start times as stop times for moving grids" << std::endl;
        amrex::Abort();
    }
    for (int i = 0; i < ni; i++) {
        if (move_stop_time[i] <= move_start_time[i]) {
            amrex::Print() << "start time for interval " << i << " is " << move_start_time[i] << std::endl;
            amrex::Print() << "stop  time for interval " << i << " is " << move_stop_time[i]  << std::endl;
            amrex::Abort("moving grid: stop time must be greater than start time");
        }
    }
    for (int i = 1; i < ni; i++) {
        if (move_start_time[i] < move_stop_time[i-1]) {
            amrex::Print() << "start time for interval " << i   << " is " << move_start_time[i] << std::endl;
            amrex::Print() << "stop  time for interval " << i-1 << " is " << move_stop_time[i-1]  << std::endl;
            amrex::Abort("moving grid: stop time must be less than start time of the next interval");
        }
    }
 
    Vector<Real> move_speed_x, move_speed_y;
    int ni2 = ppr.queryarr("move_speed_x", move_speed_x);
    int nj2 = ppr.queryarr("move_speed_y", move_speed_y);
    if (ni2 != nj2 ) {
        amrex::Print() << "Must be same number of speeds in x- and y-directions" << std::endl;
        amrex::Abort();
    }
    if (ni != ni2 ) {
        amrex::Print() << "Must be same number of speeds as time intervals" << std::endl;
        amrex::Abort();
    }
 
    Real offset_x = zero;
    Real offset_y = zero;
 
    for (int i = 0; i < ni; i++) {
        if (time > move_start_time[i]) {
            offset_x += move_speed_x[i] * static_cast<Real>(std::min(time,move_stop_time[i]) - move_start_time[i]);
            offset_y += move_speed_y[i] * static_cast<Real>(std::min(time,move_stop_time[i]) - move_start_time[i]);
        }
    }
 
    RealBox orig_real_box;
    read_box_for_refinement (ref_prefix, lev_for_box, orig_real_box);
 
    Real xlo = orig_real_box.lo(0) + offset_x; Real ylo = orig_real_box.lo(1) + offset_y;
    Real xhi = orig_real_box.hi(0) + offset_x; Real yhi = orig_real_box.hi(1) + offset_y;
    Real zlo = orig_real_box.lo(2); Real zhi = orig_real_box.hi(2);
 
    real_box.setLo(RealVect(xlo,ylo,zlo));
    real_box.setHi(RealVect(xhi,yhi,zhi));
}

update_diffusive_arrays()

void ERF::update_diffusive_arrays ( int  lev, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

private

{
    // ********************************************************************************************
    // Diffusive terms
    // ********************************************************************************************
    bool l_use_eb = (SolverChoice::terrain_type == TerrainType::EB);
    bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None && !l_use_eb);
    bool l_use_kturb   = solverChoice.turbChoice[lev].use_kturb;
    bool l_use_diff    = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
                           l_use_kturb );
    bool l_need_SmnSmn = solverChoice.turbChoice[lev].use_keqn;
    bool l_use_moist   = (  solverChoice.moisture_type != MoistureType::None  );
    bool l_rotate      = (  solverChoice.use_rotate_surface_flux  );
    bool l_Surf_X      = phys_bc_type[Orientation::xlo()] == ERF_BC::surface_layer || phys_bc_type[Orientation::xhi()] == ERF_BC::surface_layer;
    bool l_Surf_Y      = phys_bc_type[Orientation::ylo()] == ERF_BC::surface_layer || phys_bc_type[Orientation::yhi()] == ERF_BC::surface_layer;
 
 
    bool l_implicit_diff = (solverChoice.vert_implicit_fac[lev][0] > 0 ||
                            solverChoice.vert_implicit_fac[lev][1] > 0 ||
                            solverChoice.vert_implicit_fac[lev][2] > 0);
 
    bool l_eb_surface_layer = (l_use_eb && solverChoice.ebChoice.eb_boundary_type == EBBoundaryType::SurfaceLayer);
 
    BoxArray ba12 = convert(ba, IntVect(1,1,0));
    BoxArray ba13 = convert(ba, IntVect(1,0,1));
    BoxArray ba23 = convert(ba, IntVect(0,1,1));
 
    Tau[lev].resize(9);
    Tau_corr[lev].resize(3);
 
    // Always resize Tau_EB structure, even if not used, because other code checks nullptr
    Tau_EB[lev].resize(2);  // tau_eb13 and tau_eb23
    for (int comp = 0; comp < 2; ++comp) {
        Tau_EB[lev][comp].resize(3);  // xface, yface, zface
    }
 
    if (l_use_diff) {
        //
        // NOTE: We require ghost cells in the vertical when allowing grids that don't
        //       cover the entire vertical extent of the domain at this level
        //
        Tau[lev][TauType::tau11] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau11]->setVal(zero);
        Tau[lev][TauType::tau22] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau22]->setVal(zero);
        Tau[lev][TauType::tau33] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau33]->setVal(zero);
 
        Tau[lev][TauType::tau12] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau12]->setVal(zero);
        Tau[lev][TauType::tau13] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau13]->setVal(zero);
        Tau[lev][TauType::tau23] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau23]->setVal(zero);
        if (l_use_terrain || (l_Surf_X || l_Surf_Y)) {
            Tau[lev][TauType::tau21] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau21]->setVal(zero);
            Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau31]->setVal(zero);
            Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau32]->setVal(zero);
        } else if (l_implicit_diff) {
            Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau31]->setVal(zero);
            Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau32]->setVal(zero);
        } else {
            Tau[lev][TauType::tau21] = nullptr;
            Tau[lev][TauType::tau31] = nullptr;
            Tau[lev][TauType::tau32] = nullptr;
        }
 
        // EB diffusive stresses - allocate for all three staggered grids
        if (l_eb_surface_layer) {
            Tau_EB[lev][EBTauType::tau_eb13][EBGridType::xface] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
            Tau_EB[lev][EBTauType::tau_eb13][EBGridType::yface] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
            Tau_EB[lev][EBTauType::tau_eb13][EBGridType::zface] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
            Tau_EB[lev][EBTauType::tau_eb13][EBGridType::xface]->setVal(0.);
            Tau_EB[lev][EBTauType::tau_eb13][EBGridType::yface]->setVal(0.);
            Tau_EB[lev][EBTauType::tau_eb13][EBGridType::zface]->setVal(0.);
 
            Tau_EB[lev][EBTauType::tau_eb23][EBGridType::xface] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
            Tau_EB[lev][EBTauType::tau_eb23][EBGridType::yface] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
            Tau_EB[lev][EBTauType::tau_eb23][EBGridType::zface] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
            Tau_EB[lev][EBTauType::tau_eb23][EBGridType::xface]->setVal(0.);
            Tau_EB[lev][EBTauType::tau_eb23][EBGridType::yface]->setVal(0.);
            Tau_EB[lev][EBTauType::tau_eb23][EBGridType::zface]->setVal(0.);
        } else {
            for (int comp = 0; comp < 2; ++comp) {
                for (int grid = 0; grid < 3; ++grid) {
                    Tau_EB[lev][comp][grid] = nullptr;
                }
            }
        }
 
        if (l_implicit_diff && solverChoice.implicit_momentum_diffusion)
        {
            Tau_corr[lev][0] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); // Tau31
            Tau_corr[lev][1] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); // Tau32
            Tau_corr[lev][0]->setVal(zero);
            Tau_corr[lev][1]->setVal(zero);
#ifdef ERF_IMPLICIT_W
            Tau_corr[lev][2] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) ); // Tau33
            Tau_corr[lev][2]->setVal(zero);
#else
            Tau_corr[lev][2] = nullptr;
#endif
        } else {
            Tau_corr[lev][0] = nullptr;
            Tau_corr[lev][1] = nullptr;
            Tau_corr[lev][2] = nullptr;
        }
 
        SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
        SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
        SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
        SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
        SFS_hfx1_lev[lev]->setVal(zero);
        SFS_hfx2_lev[lev]->setVal(zero);
        SFS_hfx3_lev[lev]->setVal(zero);
        SFS_diss_lev[lev]->setVal(zero);
 
        // EB heat fluxes
        if (l_use_eb) {
            hfx3_EB[lev] = std::make_unique<MultiFab>( ba, dm, 1, IntVect(1,1,1) );
            hfx3_EB[lev]->setVal(zero);
        } else {
            hfx3_EB[lev] = nullptr;
        }
 
        if (l_use_moist) {
            SFS_q1fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
            SFS_q2fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
            SFS_q1fx3_lev[lev]->setVal(zero);
            SFS_q2fx3_lev[lev]->setVal(zero);
            if (l_rotate || (l_Surf_X || l_Surf_Y)) {
                SFS_q1fx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
                SFS_q1fx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
                SFS_q1fx1_lev[lev]->setVal(zero);
                SFS_q1fx2_lev[lev]->setVal(zero);
            } else {
                SFS_q1fx1_lev[lev] = nullptr;
                SFS_q1fx2_lev[lev] = nullptr;
            }
        } else {
            SFS_q1fx1_lev[lev] = nullptr;
            SFS_q1fx2_lev[lev] = nullptr;
            SFS_q1fx3_lev[lev] = nullptr;
            SFS_q2fx3_lev[lev] = nullptr;
        }
    } else {
        for (int i = 0; i < 9; i++) {
            Tau[lev][i] = nullptr;
        }
        SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
        SFS_q1fx1_lev[lev] = nullptr; SFS_q1fx2_lev[lev] = nullptr; SFS_q1fx3_lev[lev] = nullptr;
        SFS_q2fx3_lev[lev] = nullptr;
        SFS_diss_lev[lev] = nullptr;
    }
 
    if (l_use_kturb) {
        eddyDiffs_lev[lev] = std::make_unique<MultiFab>(ba, dm, EddyDiff::NumDiffs, 2);
        eddyDiffs_lev[lev]->setVal(zero);
        if(l_need_SmnSmn) {
            SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
            SmnSmn_lev[lev]->setVal(zero);
        } else {
            SmnSmn_lev[lev] = nullptr;
        }
    } else {
        eddyDiffs_lev[lev] = nullptr;
        SmnSmn_lev[lev]    = nullptr;
    }
}

update_terrain_arrays()

void ERF::update_terrain_arrays

(

int

lev

)

{
    if (SolverChoice::mesh_type == MeshType::StretchedDz ||
        SolverChoice::mesh_type == MeshType::VariableDz) {
        make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
        make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
        make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
    } else { // MeshType::ConstantDz
        if (SolverChoice::terrain_type == TerrainType::EB) {
            const auto& ebfact = *eb[lev]->get_const_factory();
            const MultiFab& volfrac = ebfact.getVolFrac();
            detJ_cc[lev] = std::make_unique<MultiFab>(volfrac, amrex::make_alias, 0, volfrac.nComp());
        }
    }
}

Here is the call graph for this function:

volWgtColumnSum()

void ERF::volWgtColumnSum	(	int	lev,
		const amrex::MultiFab &	mf,
		int	comp,
		amrex::MultiFab &	mf_2d,
		const amrex::MultiFab &	dJ
	)

Compute a volume-weighted column sum of MultiFab data.

Parameters

	lev	Current level.
[in]	mf_to_be_summed	MultiFab to be summed.
[in]	comp	Component index to be summed.
[out]	mf_2d	MultiFab to store the resulting column sums.
[in]	dJ	Volume weighting due to metric terms.

{
    BL_PROFILE("ERF::volWgtSumColumnMF()");
 
    mf_2d.setVal(0.);
 
    // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
    // m is the map scale factor at cell centers
    //
    // NOTE: TileNoZ, not TilingIfNotGPU: the atomic adds accumulate a whole column into
    //       dst_arr(i,j,0), so if the grid were tiled in z the tiles covering one column
    //       would add into the same entry concurrently.  On the host HostDevice::Atomic::Add
    //       is "#pragma omp atomic" -- atomic but unordered -- so the column sum would then
    //       depend on thread scheduling in its last bits.  Keeping each column inside a
    //       single tile makes the k accumulation serial and the result independent of both
    //       the tile size and the thread count.
    //
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(mf_to_be_summed, TileNoZ()); mfi.isValid(); ++mfi) {
        const Box& bx   = mfi.tilebox();
        const auto  dst_arr = mf_2d.array(mfi);
        const auto  src_arr = mf_to_be_summed.array(mfi);
        if (SolverChoice::mesh_type == MeshType::ConstantDz) {
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                amrex::HostDevice::Atomic::Add(&dst_arr(i,j,0),src_arr(i,j,k,comp));
            });
        } else {
            const auto& dJ_arr = dJ.const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                amrex::HostDevice::Atomic::Add(&dst_arr(i,j,0),src_arr(i,j,k,comp)*dJ_arr(i,j,k));
            });
        }
    } // mfi
 
    auto const& dx = geom[lev].CellSizeArray();
 
    mf_2d.mult(dx[2]);
}

Here is the call graph for this function:

volWgtSumMF()

Real ERF::volWgtSumMF	(	int	lev,
		const amrex::MultiFab &	mf,
		int	comp,
		const amrex::MultiFab &	dJ,
		const amrex::MultiFab &	mfx,
		const amrex::MultiFab &	mfy,
		bool	finemask,
		bool	local = true
	)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters

lev	Current level
mf_to_be_summed	: MultiFab on which we do the volume weighted sum
dJ	: volume weighting due to metric terms
mfmx	: map factor in x-direction at cell centers
mfmy	: map factor in y-direction at cell centers
comp	: Index of the component we want to sum
finemask	: If a finer level is available, determines whether we mask fine data
local	: Boolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)

{
    BL_PROFILE("ERF::volWgtSumMF()");
 
    Real sum = zero;
    MultiFab tmp(mf_to_be_summed.boxArray(), mf_to_be_summed.DistributionMap(), 1, 0);
 
    // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
    // m is the map scale factor at cell centers
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx   = mfi.tilebox();
        const auto  dst_arr = tmp.array(mfi);
        const auto  src_arr = mf_to_be_summed.array(mfi);
        const auto& mfx_arr = mfmx.const_array(mfi);
        const auto& mfy_arr = mfmy.const_array(mfi);
 
        if (SolverChoice::terrain_type != TerrainType::EB) {
            if (SolverChoice::mesh_type == MeshType::ConstantDz) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    dst_arr(i,j,k,0) = src_arr(i,j,k,comp) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                });
            } else {
                const auto&  dJ_arr = dJ.const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    dst_arr(i,j,k,0) = src_arr(i,j,k,comp) * dJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                });
            }
        } else {
            const auto&  dJ_arr = dJ.const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                dst_arr(i,j,k,0) = src_arr(i,j,k,comp) * dJ_arr(i,j,k);
            });
        }
 
    } // mfi
 
    if (lev < finest_level && finemask) {
        MultiFab::Multiply(tmp, *fine_mask[lev+1].get(), 0, 0, 1, 0);
    }
 
    // If local = true  then "sum" will be the sum only over the FABs on each rank
    // If local = false then "sum" will be the sum over the whole MultiFab, and will be broadcast to all ranks
    sum = tmp.sum(0,local);
 
    auto const& dx = geom[lev].CellSizeArray();
 
    sum *= dx[0]*dx[1]*dx[2];
 
    return sum;
}

Here is the call graph for this function:

WeatherDataInterpolation()

void ERF::WeatherDataInterpolation	(	const int	lev,
		const double	time,
		amrex::Vector< std::unique_ptr< amrex::MultiFab >> &	a_z_phys_nd,
		bool	regrid_forces_file_read
	)

Interpolate weather forecast data onto the simulation mesh.

Parameters

[in]	lev	Level index.
[in]	time	Current simulation time.
[in]	a_z_phys_nd	Nodal physical height fields across all levels.
[in]	regrid_forces_file_read	Flag to force reading of forecast files during regridding.

{
 
    static amrex::Vector<double> next_read_forecast_time;
    static amrex::Vector<double> last_read_forecast_time;
 
    const int nlevs = static_cast<int>(a_z_phys_nd.size());
 
    Real hindcast_data_interval = solverChoice.hindcast_data_interval_in_hrs*Real(3600.0);
 
    // Initialize static vectors once
    if (next_read_forecast_time.empty()) {
        next_read_forecast_time.resize(nlevs, -one);
        last_read_forecast_time.resize(nlevs, -one);
        Print() << "Initializing the time vector values here by " << lev << std::endl;
    }
 
    if (next_read_forecast_time[lev] < zero) {
        int next_multiple = static_cast<int>(time / hindcast_data_interval);
        next_read_forecast_time[lev] = next_multiple * hindcast_data_interval;
        last_read_forecast_time[lev] = next_read_forecast_time[lev];
    }
 
    if (time >= next_read_forecast_time[lev] or regrid_forces_file_read) {
 
        Print() << "Data reading happening at level " << lev << std::endl;
 
        std::string folder = solverChoice.hindcast_boundary_data_dir;
 
        // Check if folder exists and is a directory
        if (!fs::exists(folder) || !fs::is_directory(folder)) {
            throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
        }
 
        std::vector<std::string> bin_files;
 
        for (const auto& entry : fs::directory_iterator(folder)) {
            if (!entry.is_regular_file()) continue;
 
            std::string fname = entry.path().filename().string();
            if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
                bin_files.push_back(entry.path().string());
            }
        }
        std::sort(bin_files.begin(), bin_files.end());
 
    // Check if no .bin files were found
        if (bin_files.empty()) {
            throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
        }
 
        std::string filename1, filename2;
 
        int idx1 = static_cast<int>(time / hindcast_data_interval);
        int idx2 = static_cast<int>(time / hindcast_data_interval)+1;
        Print() << "Reading weather data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
 
        if (idx2 >= static_cast<int>(bin_files.size())) {
            throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
        }
 
        filename1 = bin_files[idx1];
        filename2 = bin_files[idx2];
 
        FillForecastStateMultiFabs(lev, filename1, a_z_phys_nd[lev], forecast_state_1);
        FillForecastStateMultiFabs(lev, filename2, a_z_phys_nd[lev], forecast_state_2);
 
         // Create the time-interpolated forecast state
        //CreateForecastStateMultiFabs(forecast_state_interp);
        if(!regrid_forces_file_read){
            last_read_forecast_time[lev] = next_read_forecast_time[lev];
            next_read_forecast_time[lev] += hindcast_data_interval;
            Print() << "Next forecast time getting updated here " << std::endl;
        }
    }
 
    double prev_read_time = last_read_forecast_time[lev];
    Real alpha1 = static_cast<Real>(1.0 - (time - prev_read_time)/hindcast_data_interval);
    Real alpha2 = one - alpha1;
 
    amrex::Print()<< "The values of alpha1 and alpha2 are " << alpha1 << " "<< alpha2 <<std::endl;
 
    if (alpha1 < zero || alpha1 > one ||
    alpha2 < zero || alpha2 > one)
    {
        std::stringstream ss;
        ss << "Interpolation weights for hindcast files are incorrect: "
        << "alpha1 = " << alpha1 << ", alpha2 = " << alpha2;
       Abort(ss.str());
    }
 
    MultiFab& erf_mf_cons   = forecast_state_interp[lev][Vars::cons];
    MultiFab& erf_mf_xvel   = forecast_state_interp[lev][Vars::xvel];
    MultiFab& erf_mf_yvel   = forecast_state_interp[lev][Vars::yvel];
    //MultiFab& erf_mf_zvel   = forecast_state_interp[0][Vars::zvel];
    MultiFab& erf_mf_latlon = forecast_state_interp[lev][4];
 
    // Fill the time-interpolated forecast states
    MultiFab::LinComb(forecast_state_interp[lev][Vars::cons],
                      alpha1, forecast_state_1[lev][Vars::cons], 0,
                      alpha2, forecast_state_2[lev][Vars::cons], 0,
                      0, erf_mf_cons.nComp(), forecast_state_interp[lev][Vars::cons].nGrow());
    MultiFab::LinComb(forecast_state_interp[lev][Vars::xvel],
                      alpha1, forecast_state_1[lev][Vars::xvel], 0,
                      alpha2, forecast_state_2[lev][Vars::xvel], 0,
                      0, erf_mf_xvel.nComp(), forecast_state_interp[lev][Vars::xvel].nGrow());
    MultiFab::LinComb(forecast_state_interp[lev][Vars::yvel],
                      alpha1, forecast_state_1[lev][Vars::yvel], 0,
                      alpha2, forecast_state_2[lev][Vars::yvel], 0,
                      0, erf_mf_yvel.nComp(), forecast_state_interp[lev][Vars::yvel].nGrow());
    MultiFab::LinComb(forecast_state_interp[lev][4],
                      alpha1, forecast_state_1[lev][4], 0,
                      alpha2, forecast_state_2[lev][4], 0,
                      0, erf_mf_latlon.nComp(), forecast_state_interp[lev][4].nGrow());
 
    /*Vector<std::string> varnames_plot_mf = {
    "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
    }; // Customize variable names
 
    std::string pltname = "plt_interp";
 
    MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
                     10, 0);
 
    plot_mf.setVal(0.0);
 
    for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
        const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
        const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
        const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
        const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
        const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
        const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
 
        const Box& bx = mfi.validbox();
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
            plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
            plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
            plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
            plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
            plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
 
            plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/two;
            plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/two;
            plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/two;
 
            plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
            plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
        });
    }
 
 
    WriteSingleLevelPlotfile(
            pltname,
            plot_mf,
            varnames_plot_mf,
            geom[0],
            time,
            0 // level
        );*/
}

Write2DPlotFile()

void ERF::Write2DPlotFile	(	int	which,
		PlotFileType	plotfile_type,
		amrex::Vector< std::string >	plot_var_names
	)

{
    const int plot_step = istep[0];
    const std::string plotfilename = make_2d_plotfile_name(which, plot_step,
                                                           plot2d_file_1, plot2d_file_2,
                                                           file_name_digits);
 
    const auto output_descriptors =
        plotfile2d::build_sampled_level_output_descriptors(pp_prefix, which,
                                                           plot_var_names, solverChoice);
 
    Vector<std::string> varnames;
    varnames.reserve(output_descriptors.size());
    for (const auto& descriptor : output_descriptors) {
        varnames.push_back(descriptor.name);
    }
    const int ncomp_mf = static_cast<int>(varnames.size());
 
    if (ncomp_mf == 0) return;
 
    // Vector of MultiFabs for cell-centered data
    Vector<MultiFab> mf(finest_level+1);
    for (int lev = 0; lev <= finest_level; ++lev) {
        mf[lev].define(ba2d[lev], dmap[lev], ncomp_mf, 0);
    }
 
 
    // **********************************************************************************************
    // (Effectively) 2D arrays
    // **********************************************************************************************
    for (int lev = 0; lev <= finest_level; ++lev) {
        FillPlot2DVars(lev, plot_var_names, varnames, output_descriptors, mf[lev], ncomp_mf);
    }
 
    Vector<Geometry> my_geom = make_2d_plot_geometries(geom, finest_level);
 
    if (plotfile_type == PlotFileType::Amrex)
    {
        Print() << "Writing 2D native plotfile " << plotfilename << "\n";
        WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                GetVecOfConstPtrs(mf),
                                varnames, my_geom, static_cast<Real>(t_new[0]), istep, refRatio());
        // Native AMReX 2D plotfiles write a JSON sidecar with catalog
        // metadata for the selected output variables only.
        plotfile2d::write_2d_metadata_json(plotfilename, output_descriptors);
        writeJobInfo(plotfilename, erf_provenance::ArtifactType::Plotfile2D,
                     istep[0], t_new[0]);
 
#ifdef ERF_USE_NETCDF
    } else if (plotfile_type == PlotFileType::Netcdf) {
         int lev   = 0;
         int l_which = 0;
         const Real* p_lo = my_geom[lev].ProbLo();
         const Real* p_hi = my_geom[lev].ProbHi();
         const auto dx    = my_geom[lev].CellSize();
         writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
                         {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],dx[2]}, {dx[0],dx[1],dx[2]},
                         my_geom[lev].Domain(), static_cast<Real>(t_new[0]),
                         static_cast<Real>(start_bdy_time), solverChoice, zlevels_stag[lev]);
#endif
    } else {
        // Here we assume the plotfile_type is PlotFileType::None
        Print() << "Writing no 2D plotfile since plotfile_type is none" << std::endl;
    }
}

Here is the call graph for this function:

Write3DPlotFile()

bool ERF::Write3DPlotFile	(	int	which,
		PlotFileType	plotfile_type,
		amrex::Vector< std::string >	plot_var_names
	)

{
    auto dPlotTime0 = amrex::second();
 
    Real tnew = static_cast<Real>(t_new[0]);
 
    const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
    const int ncomp_mf = static_cast<int>(varnames.size());
 
    if (ncomp_mf == 0) return false;
 
    Plot3DScratch scratch;
    BuildPlot3DScratch(plot_var_names, scratch);
 
    // Vector of MultiFabs for cell-centered data
    Vector<MultiFab> mf(finest_level+1);
    for (int lev = 0; lev <= finest_level; ++lev) {
        mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
    }
 
    // Vector of MultiFabs for nodal data
    Vector<MultiFab> mf_nd(finest_level+1);
    if ( SolverChoice::mesh_type != MeshType::ConstantDz) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
            mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
            mf_nd[lev].setVal(0.);
        }
    }
 
    // Vector of MultiFabs for face-centered velocity
    Vector<MultiFab> mf_u(finest_level+1);
    Vector<MultiFab> mf_v(finest_level+1);
    Vector<MultiFab> mf_w(finest_level+1);
    if (m_plot_face_vels) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            BoxArray grid_stag_u(grids[lev]); grid_stag_u.surroundingNodes(0);
            BoxArray grid_stag_v(grids[lev]); grid_stag_v.surroundingNodes(1);
            BoxArray grid_stag_w(grids[lev]); grid_stag_w.surroundingNodes(2);
            mf_u[lev].define(grid_stag_u, dmap[lev], 1, 0);
            mf_v[lev].define(grid_stag_v, dmap[lev], 1, 0);
            mf_w[lev].define(grid_stag_w, dmap[lev], 1, 0);
            MultiFab::Copy(mf_u[lev],vars_new[lev][Vars::xvel],0,0,1,0);
            MultiFab::Copy(mf_v[lev],vars_new[lev][Vars::yvel],0,0,1,0);
            MultiFab::Copy(mf_w[lev],vars_new[lev][Vars::zvel],0,0,1,0);
        }
    }
 
    for (int lev = 0; lev <= finest_level; ++lev) {
        FillPlot3DVars(lev, plot_var_names, scratch, mf[lev], ncomp_mf, tnew);
    }
 
    if (solverChoice.terrain_type == TerrainType::EB)
    {
        for (int lev = 0; lev <= finest_level; ++lev) {
            EB_set_covered(mf[lev], zero);
        }
    }
 
    // Fill terrain distortion MF (nu_nd)
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        for (int lev(0); lev <= finest_level; ++lev) {
            MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
            Real dz = Geom()[lev].CellSizeArray()[2];
            for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                Array4<Real> mf_arr = mf_nd[lev].array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    mf_arr(i,j,k,2) -= k * dz;
                });
            }
        }
    }
 
    std::string plotfilename;
    std::string plotfilenameU;
    std::string plotfilenameV;
    std::string plotfilenameW;
 
    if (which == 1) {
       if (use_real_time_in_pltname) {
           const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
           plotfilename = plot3d_file_1+"_"+getTimestamp(start_time+tnew, dt_format,false);
       } else {
           plotfilename  = Concatenate(plot3d_file_1, istep[0], file_name_digits);
       }
       plotfilenameU = Concatenate(plot3d_file_1+"U", istep[0], file_name_digits);
       plotfilenameV = Concatenate(plot3d_file_1+"V", istep[0], file_name_digits);
       plotfilenameW = Concatenate(plot3d_file_1+"W", istep[0], file_name_digits);
    } else if (which == 2) {
       if (use_real_time_in_pltname) {
           const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
           plotfilename = plot3d_file_2+"_"+getTimestamp(start_time+tnew, dt_format,false);
       } else {
           plotfilename  = Concatenate(plot3d_file_2, istep[0], file_name_digits);
       }
       plotfilenameU = Concatenate(plot3d_file_2+"U", istep[0], file_name_digits);
       plotfilenameV = Concatenate(plot3d_file_2+"V", istep[0], file_name_digits);
       plotfilenameW = Concatenate(plot3d_file_2+"W", istep[0], file_name_digits);
    }
 
    // LSM writes it's own data
    if (which==1 && plot_lsm) {
        lsm.Plot_Lsm_Data(tnew, finest_level, istep, refRatio());
    }
 
#ifdef ERF_USE_RRTMGP
    /*
    // write additional RRTMGP data
    // TODO: currently single level only
    if (which==1 && plot_rad) {
        rad[0]->writePlotfile(plot_file_1, tnew, istep[0]);
    }
    */
#endif
 
    // Single level
    if (finest_level == 0)
    {
        if (plotfile_type == PlotFileType::Amrex)
        {
            Print() << "Writing native 3D plotfile " << plotfilename << "\n";
            if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                   GetVecOfConstPtrs(mf),
                                                   GetVecOfConstPtrs(mf_nd),
                                                   varnames,
                                                   Geom(), tnew, istep, refRatio());
            } else {
                WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                        GetVecOfConstPtrs(mf),
                                        varnames,
                                        Geom(), tnew, istep, refRatio());
            }
            writeJobInfo(plotfilename, erf_provenance::ArtifactType::Plotfile3D,
                         istep[0], t_new[0]);
 
            if (m_plot_face_vels) {
                Print() << "Writing face velocities" << std::endl;
                WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
                                        GetVecOfConstPtrs(mf_u),
                                        {"x_velocity_stag"},
                                        Geom(), tnew, istep, refRatio());
                WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
                                        GetVecOfConstPtrs(mf_v),
                                        {"y_velocity_stag"},
                                        Geom(), tnew, istep, refRatio());
                WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
                                        GetVecOfConstPtrs(mf_w),
                                        {"z_velocity_stag"},
                                        Geom(), tnew, istep, refRatio());
            }
 
            if (m_plot_face_terrain_blanking &&
                (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                 solverChoice.buildings_type == BuildingsType::ImmersedForcing) &&
                terrain_blanking_xface[0]) {  // Check if face arrays are allocated
                Print() << "Writing face terrain blanking" << std::endl;
                std::string plotfilenameTBX = plotfilename; plotfilenameTBX += "_terrain_blank_xface";
                std::string plotfilenameTBY = plotfilename; plotfilenameTBY += "_terrain_blank_yface";
                std::string plotfilenameTBZ = plotfilename; plotfilenameTBZ += "_terrain_blank_zface";
                WriteMultiLevelPlotfile(plotfilenameTBX, finest_level+1,
                                        GetVecOfConstPtrs(terrain_blanking_xface),
                                        {"terrain_blank_xface"},
                                        Geom(), tnew, istep, refRatio());
                WriteMultiLevelPlotfile(plotfilenameTBY, finest_level+1,
                                        GetVecOfConstPtrs(terrain_blanking_yface),
                                        {"terrain_blank_yface"},
                                        Geom(), tnew, istep, refRatio());
                WriteMultiLevelPlotfile(plotfilenameTBZ, finest_level+1,
                                        GetVecOfConstPtrs(terrain_blanking_zface),
                                        {"terrain_blank_zface"},
                                        Geom(), tnew, istep, refRatio());
            }
 
#ifdef ERF_USE_PARTICLES
            particleData.writePlotFile(plotfilename, z_phys_nd);
#endif
#ifdef ERF_USE_NETCDF
        } else if (plotfile_type == PlotFileType::Netcdf) {
             AMREX_ALWAYS_ASSERT(solverChoice.mesh_type != MeshType::VariableDz);
             int lev   = 0;
             int l_which = 0;
             const Real* p_lo = geom[lev].ProbLo();
             const Real* p_hi = geom[lev].ProbHi();
             const auto dx    = geom[lev].CellSize();
             writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
                             {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
                             geom[lev].Domain(), tnew, start_bdy_time, solverChoice, zlevels_stag[lev]);
#endif
        } else {
            // Here we assume the plotfile_type is PlotFileType::None
            Print() << "Writing no 3D plotfile since plotfile_type is none" << std::endl;
        }
 
    } else { // Multilevel
 
        if (plotfile_type == PlotFileType::Amrex) {
 
            int lev0 = 0;
            int desired_ratio = std::max(std::max(ref_ratio[lev0][0],ref_ratio[lev0][1]),ref_ratio[lev0][2]);
            bool any_ratio_one = ( ( (ref_ratio[lev0][0] == 1) || (ref_ratio[lev0][1] == 1) ) ||
                                     (ref_ratio[lev0][2] == 1) );
            for (int lev = 1; lev < finest_level; lev++) {
                any_ratio_one = any_ratio_one ||
                                     ( ( (ref_ratio[lev][0] == 1) || (ref_ratio[lev][1] == 1) ) ||
                                         (ref_ratio[lev][2] == 1) );
            }
 
            if (any_ratio_one && m_expand_plotvars_to_unif_rr)
            {
                Vector<IntVect>   r2(finest_level);
                Vector<Geometry>  g2(finest_level+1);
                Vector<MultiFab> mf2(finest_level+1);
 
                mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
 
                // Copy level 0 as is
                MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
 
                // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
                Array<int,AMREX_SPACEDIM> periodicity =
                             {Geom()[lev0].isPeriodic(0),Geom()[lev0].isPeriodic(1),Geom()[lev0].isPeriodic(2)};
                g2[lev0].define(Geom()[lev0].Domain(),&(Geom()[lev0].ProbDomain()),0,periodicity.data());
 
                r2[0] = IntVect(desired_ratio/ref_ratio[lev0][0],
                                desired_ratio/ref_ratio[lev0][1],
                                desired_ratio/ref_ratio[lev0][2]);
 
                for (int lev = 1; lev <= finest_level; ++lev) {
                    if (lev > 1) {
                        r2[lev-1][0] = r2[lev-2][0] * desired_ratio / ref_ratio[lev-1][0];
                        r2[lev-1][1] = r2[lev-2][1] * desired_ratio / ref_ratio[lev-1][1];
                        r2[lev-1][2] = r2[lev-2][2] * desired_ratio / ref_ratio[lev-1][2];
                    }
 
                    mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
 
                    // Set the new problem domain
                    Box d2(Geom()[lev].Domain());
                    d2.refine(r2[lev-1]);
 
                    g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
                }
 
                //
                // We need to make a temporary that is the size of ncomp_mf
                // in order to not get an out of bounds error
                // even though the values will not be used
                //
                Vector<BCRec> temp_domain_bcs_type;
                temp_domain_bcs_type.resize(ncomp_mf);
 
                //
                // Do piecewise constant interpolation of mf into mf2
                //
                for (int lev = 1; lev <= finest_level; ++lev) {
                    Interpolater* mapper_c = &pc_interp;
                    InterpFromCoarseLevel(mf2[lev], static_cast<Real>(t_new[lev]), mf[lev],
                                          0, 0, ncomp_mf,
                                          geom[lev], g2[lev],
                                          null_bc_for_fill, 0, null_bc_for_fill, 0,
                                          r2[lev-1], mapper_c, temp_domain_bcs_type, 0);
                }
 
                // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
                Vector<IntVect> rr(finest_level);
                for (int lev = 0; lev < finest_level; ++lev) {
                    rr[lev] = IntVect(desired_ratio);
                }
 
               Print() << "Writing 3D plotfile " << plotfilename << "\n";
               if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                   WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                      GetVecOfConstPtrs(mf2),
                                                      GetVecOfConstPtrs(mf_nd),
                                                      varnames,
                                                      g2, tnew, istep, rr);
               } else {
                   WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                           GetVecOfConstPtrs(mf2), varnames,
                                           g2, tnew, istep, rr);
               }
 
            } else {
               if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                    WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                       GetVecOfConstPtrs(mf),
                                                       GetVecOfConstPtrs(mf_nd),
                                                       varnames,
                                                       geom, tnew, istep, ref_ratio);
                } else {
                    WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                            GetVecOfConstPtrs(mf), varnames,
                                            geom, tnew, istep, ref_ratio);
                }
                if (m_plot_face_vels) {
                    Print() << "Writing face velocities" << std::endl;
                    WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
                                            GetVecOfConstPtrs(mf_u),
                                            {"x_velocity_stag"},
                                            geom, tnew, istep, ref_ratio);
                    WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
                                            GetVecOfConstPtrs(mf_v),
                                            {"y_velocity_stag"},
                                            geom, tnew, istep, ref_ratio);
                    WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
                                            GetVecOfConstPtrs(mf_w),
                                            {"z_velocity_stag"},
                                            geom, tnew, istep, ref_ratio);
                }
            } // ref_ratio test
 
            writeJobInfo(plotfilename, erf_provenance::ArtifactType::Plotfile3D,
                         istep[0], t_new[0]);
 
#ifdef ERF_USE_PARTICLES
            particleData.writePlotFile(plotfilename, z_phys_nd);
#endif
 
#ifdef ERF_USE_NETCDF
        } else if (plotfile_type == PlotFileType::Netcdf) {
             AMREX_ALWAYS_ASSERT(solverChoice.mesh_type != MeshType::VariableDz);
             for (int lev = 0; lev <= finest_level; ++lev) {
                 for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
                     Box bounding_region = (lev == 0)  ? geom[lev].Domain() : boxes_at_level[lev][which_box];
                     const Real* p_lo = geom[lev].ProbLo();
                     const Real* p_hi = geom[lev].ProbHi();
                     const auto dx    = geom[lev].CellSizeArray();
                     writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
                                     {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
                                     bounding_region, tnew, start_bdy_time, solverChoice, zlevels_stag[lev]);
                 }
             }
#endif
        }
    } // end multi-level
 
    if (verbose > 0)
    {
        auto dPlotTime = amrex::second() - dPlotTime0;
        ParallelDescriptor::ReduceRealMax(dPlotTime,ParallelDescriptor::IOProcessorNumber());
        amrex::Print() << "3DPlotfile write time = " << dPlotTime << " seconds." << '\n';
    }
 
    return true;
 
}

Here is the call graph for this function:

write_1D_profiles()

void ERF::write_1D_profiles

(

double

time

)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters

time	Current time

{
    BL_PROFILE("ERF::write_1D_profiles()");
 
    if (NumDataLogs() > 1)
    {
        // Define the 1d arrays we will need
        Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
        Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
        Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
        Gpu::HostVector<Real> h_avg_wthv;
        Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
        Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
        Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
        Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
        Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
        Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
 
        Gpu::HostVector<Real> h_avg_ttend, h_avg_qtend, h_avg_wsub, h_avg_tnudge, h_avg_qnudge, h_avg_unudge, h_avg_vnudge;
        Gpu::HostVector<Real> h_avg_thtend, h_avg_qhtend, h_avg_tvtend, h_avg_qvtend, h_avg_qcvtend;
        if (NumDataLogs() > 1) {
            derive_diag_profiles(time,
                                 h_avg_u, h_avg_v, h_avg_w,
                                 h_avg_rho, h_avg_th, h_avg_ksgs,
                                 h_avg_Kmv, h_avg_Khv,
                                 h_avg_qv, h_avg_qc, h_avg_qr,
                                 h_avg_wqv, h_avg_wqc, h_avg_wqr,
                                 h_avg_qi, h_avg_qs, h_avg_qg,
                                 h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                 h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                 h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                 h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
                                 h_avg_wthv);
        }
 
        if (NumDataLogs() > 3 && time > zero) {
            derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                   h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                   h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
                                   h_avg_sgsdiss);
        }
 
        if (NumDataLogs() > 4 && time > 0.) {
            derive_forcing_profiles_stag(h_avg_ttend, h_avg_qtend, h_avg_wsub,
                                         h_avg_thtend, h_avg_qhtend, h_avg_tvtend,
                                         h_avg_qvtend, h_avg_qcvtend, h_avg_tnudge,
                                         h_avg_qnudge, h_avg_unudge, h_avg_vnudge);
        }
 
        int hu_size =  h_avg_u.size();
 
        auto const& dx = geom[0].CellSizeArray();
        if (ParallelDescriptor::IOProcessor()) {
            if (NumDataLogs() > 1) {
                std::ostream& data_log1 = DataLog(1);
                if (data_log1.good()) {
                  // Write the quantities at this time
                  for (int k = 0; k < hu_size; k++) {
                      Real z;
                      if (zlevels_stag[0].size() > 1) {
                          z = myhalf * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                      } else {
                          z = (k + myhalf)* dx[2];
                      }
                      data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_u[k]   << " " << h_avg_v[k]   << " " << h_avg_w[k]     << " "
                                << h_avg_rho[k] << " " << h_avg_th[k]  << " " << h_avg_ksgs[k] << " "
                                << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
                                << h_avg_qv[k]  << " " << h_avg_qc[k]  << " " << h_avg_qr[k]    << " "
                                << h_avg_qi[k]  << " " << h_avg_qs[k]  << " " << h_avg_qg[k]
                                << std::endl;
                  } // loop over z
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 2) {
                std::ostream& data_log2 = DataLog(2);
                if (data_log2.good()) {
                  // Write the perturbational quantities at this time
                  for (int k = 0; k < hu_size; k++) {
                      Real z;
                      if (zlevels_stag[0].size() > 1) {
                          z = myhalf * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                      } else {
                          z = (k + myhalf)* dx[2];
                      }
                      Real thv = h_avg_th[k] * (one + epsv*h_avg_qv[k] - h_avg_qc[k] - h_avg_qr[k]);
                      data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]  << " "
                                << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]  << " "
                                << h_avg_uw[k]   - h_avg_u[k]*h_avg_w[k]  << " "
                                << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]  << " "
                                << h_avg_vw[k]   - h_avg_v[k]*h_avg_w[k]  << " "
                                << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]  << " "
                                << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k] << " "
                                << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k] << " "
                                << h_avg_wth[k]  - h_avg_w[k]*h_avg_th[k] << " "
                                << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
                                // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                //                        -   <u_i u_i> * <u_j>
                                //                        - 2*<u_i> * <u_i u_j>
                                //                        + 2*<u_i>*<u_i> * <u_j>
                                << h_avg_uiuiu[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
                                 - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
                                   << " " // (u'_i u'_i)u'
                                << h_avg_uiuiv[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
                                 - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
                                   << " " // (u'_i u'_i)v'
                                << h_avg_uiuiw[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
                                 - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                   << " " // (u'_i u'_i)w'
                                << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k] << " "
                                << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k] << " "
                                << h_avg_pw[k]   - h_avg_p[k]*h_avg_w[k] << " "
                                << h_avg_wqv[k]  - h_avg_qv[k]*h_avg_w[k] << " "
                                << h_avg_wqc[k]  - h_avg_qc[k]*h_avg_w[k] << " "
                                << h_avg_wqr[k]  - h_avg_qr[k]*h_avg_w[k] << " "
                                << h_avg_wthv[k] - h_avg_w[k]*thv
                                << std::endl;
                  } // loop over z
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 3 && time > zero) {
                std::ostream& data_log3 = DataLog(3);
                if (data_log3.good()) {
                  // Write the average stresses
                  for (int k = 0; k < hu_size; k++) {
                      Real z;
                      if (zlevels_stag[0].size() > 1) {
                          z = myhalf * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                      } else {
                          z = (k + myhalf)* dx[2];
                      }
                      data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_tau11[k]  << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                << h_avg_tau22[k]  << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                << h_avg_sgshfx[k] << " "
                                << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
                                << h_avg_sgsdiss[k]
                                << std::endl;
                  } // loop over z
                } // if good
            } // if (NumDataLogs() > 3)
 
            if (NumDataLogs() > 4 && time > 0.) {
                std::ostream& data_log4 = DataLog(4);
                if (data_log4.good()) {
                    for (int k = 0; k < hu_size; k++) {
                        Real z;
                        if (zlevels_stag[0].size() > 1) {
                            z = myhalf * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                        } else {
                            z = (k + myhalf)* dx[2];
                        }
                        data_log4 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                  << h_avg_ttend[k]  << " " << h_avg_qtend[k]   << " " << h_avg_wsub[k]   << " "
                                  << h_avg_thtend[k] << " " << h_avg_qhtend[k]  << " " << h_avg_tvtend[k] << " "
                                  << h_avg_qvtend[k] << " " << h_avg_qcvtend[k] << " " << h_avg_tnudge[k] << " "
                                  << h_avg_qnudge[k] << " " << h_avg_unudge[k]  << " " << h_avg_vnudge[k]
                                  << std::endl;
                  } // loop over z
                }
            }
        } // if IOProcessor
    } // if (NumDataLogs() > 1)
}

write_1D_profiles_stag()

void ERF::write_1D_profiles_stag

(

double

time

)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Quantities are output at their native grid locations. Therefore, w and associated flux quantities <(•)'w'>, tau13, and tau23 (where '•' includes u, v, p, theta, ...) will be output at staggered heights (i.e., coincident with z faces) rather than cell-center heights to avoid performing additional averaging. Unstaggered (i.e., cell-centered) quantities are output alongside staggered quantities at the lower cell faces in the log file; these quantities will have a zero value at the big end, corresponding to k=Nz+one

The structure of file should follow ERF_Write1DProfiles.cpp

Parameters

time	Current time

{
    BL_PROFILE("ERF::write_1D_profiles_stag()");
 
    int prof_datwidth = 14;
    int prof_datprecision = 9;
    int prof_timeprecision = 13; // e.g., 1-yr LES: 31,536,000 s with dt ~ 0.01 ==> min prec = 10
 
 
    if (NumDataLogs() > 1)
    {
        // Define the 1d arrays we will need
        Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
        Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
        Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
        Gpu::HostVector<Real> h_avg_wthv;
        Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
        Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
        Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
        Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
        Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
        Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
 
 
        Gpu::HostVector<Real> h_avg_ttend, h_avg_qtend, h_avg_wsub, h_avg_tnudge, h_avg_qnudge, h_avg_unudge, h_avg_vnudge;
        Gpu::HostVector<Real> h_avg_thtend, h_avg_qhtend, h_avg_tvtend, h_avg_qvtend, h_avg_qcvtend;
        {
            derive_diag_profiles_stag(time,
                                      h_avg_u, h_avg_v, h_avg_w,
                                      h_avg_rho, h_avg_th, h_avg_ksgs,
                                      h_avg_Kmv, h_avg_Khv,
                                      h_avg_qv, h_avg_qc, h_avg_qr,
                                      h_avg_wqv, h_avg_wqc, h_avg_wqr,
                                      h_avg_qi, h_avg_qs, h_avg_qg,
                                      h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                      h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                      h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                      h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
                                      h_avg_wthv);
        }
 
        if (NumDataLogs() > 3 && time > zero) {
            derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                        h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                        h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
                                        h_avg_sgsdiss);
        }
 
        if (NumDataLogs() > 4 && time > 0.) {
            derive_forcing_profiles_stag(h_avg_ttend, h_avg_qtend, h_avg_wsub,
                                         h_avg_thtend, h_avg_qhtend, h_avg_tvtend,
                                         h_avg_qvtend, h_avg_qcvtend, h_avg_tnudge,
                                         h_avg_qnudge, h_avg_unudge, h_avg_vnudge);
        }
 
        int unstag_size =  h_avg_w.size() - 1; // _un_staggered heights
 
        auto const& dx = geom[0].CellSizeArray();
        if (ParallelDescriptor::IOProcessor()) {
            if (NumDataLogs() > 1) {
                std::ostream& data_log1 = DataLog(1);
                if (data_log1.good()) {
                  // Write the quantities at this time
                  for (int k = 0; k < unstag_size; k++) {
                      Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                      data_log1 << std::setw(prof_datwidth) << std::setprecision(prof_timeprecision) << time << " "
                                << std::setw(prof_datwidth) << std::setprecision(prof_datprecision) << z << " "
                                << h_avg_u[k]   << " " << h_avg_v[k]   << " " << h_avg_w[k]     << " "
                                << h_avg_rho[k] << " " << h_avg_th[k]  << " " << h_avg_ksgs[k] << " "
                                << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
                                << h_avg_qv[k]  << " " << h_avg_qc[k]  << " " << h_avg_qr[k]    << " "
                                << h_avg_qi[k]  << " " << h_avg_qs[k]  << " " << h_avg_qg[k]
                                << std::endl;
                  } // loop over z
                  // Write top face values
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log1 << std::setw(prof_datwidth) << std::setprecision(prof_timeprecision) << time << " "
                            << std::setw(prof_datwidth) << std::setprecision(prof_datprecision) << z << " "
                            << 0 << " " << 0 << " " << h_avg_w[unstag_size] << " "
                            << 0 << " " << 0 << " " << 0 << " " // rho, theta, ksgs
                            << 0 << " " << 0 << " "             // Kmv, Khv
                            << 0 << " " << 0 << " " << 0 << " " // qv, qc, qr
                            << 0 << " " << 0 << " " << 0        // qi, qs, qg
                            << std::endl;
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 2) {
                std::ostream& data_log2 = DataLog(2);
                if (data_log2.good()) {
                  // Write the perturbational quantities at this time
                  // For surface values (k=0), assume w = uw = vw = ww = 0
                  Real w_cc  = h_avg_w[1] / 2;  // w at first cell center
                  Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
                  Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
                  Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
                  data_log2 << std::setw(prof_datwidth) << std::setprecision(prof_timeprecision) << time << " "
                            << std::setw(prof_datwidth) << std::setprecision(prof_datprecision) << 0 << " "
                            << h_avg_uu[0]   - h_avg_u[0]*h_avg_u[0]   << " " // u'u'
                            << h_avg_uv[0]   - h_avg_u[0]*h_avg_v[0]   << " " // u'v'
                            << 0                                       << " " // u'w'
                            << h_avg_vv[0]   - h_avg_v[0]*h_avg_v[0]   << " " // v'v'
                            << 0                                       << " " // v'w'
                            << 0                                       << " " // w'w'
                            << h_avg_uth[0]  - h_avg_u[0]*h_avg_th[0]  << " " // u'th'
                            << h_avg_vth[0]  - h_avg_v[0]*h_avg_th[0]  << " " // v'th'
                            << 0                                       << " " // w'th'
                            << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
                            << h_avg_uiuiu[0]
                             - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
                             - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
                             + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
                               << " " // (u'_i u'_i)u'
                            << h_avg_uiuiv[0]
                             - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
                             - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
                             + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
                               << " " // (u'_i u'_i)v'
                            << 0 << " " // (u'_i u'_i)w'
                            << h_avg_pu[0]   - h_avg_p[0]*h_avg_u[0]   << " " // p'u'
                            << h_avg_pv[0]   - h_avg_p[0]*h_avg_v[0]   << " " // p'v'
                            << 0                                       << " " // p'w'
                            << 0                                       << " " // qv'w'
                            << 0                                       << " " // qc'w'
                            << 0                                       << " " // qr'w'
                            << 0                                              // thv'w'
                            << std::endl;
 
                  // For internal values, interpolate scalar quantities to faces
                  for (int k = 1; k < unstag_size; k++) {
                      Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                      Real uface  = myhalf*(h_avg_u[k]  + h_avg_u[k-1]);
                      Real vface  = myhalf*(h_avg_v[k]  + h_avg_v[k-1]);
                      Real thface = myhalf*(h_avg_th[k] + h_avg_th[k-1]);
                      Real pface  = myhalf*(h_avg_p[k]  + h_avg_p[k-1]);
                      Real qvface = myhalf*(h_avg_qv[k] + h_avg_qv[k-1]);
                      Real qcface = myhalf*(h_avg_qc[k] + h_avg_qc[k-1]);
                      Real qrface = myhalf*(h_avg_qr[k] + h_avg_qr[k-1]);
                      Real uuface = myhalf*(h_avg_uu[k] + h_avg_uu[k-1]);
                      Real vvface = myhalf*(h_avg_vv[k] + h_avg_vv[k-1]);
                      Real thvface = thface * (one + epsv*qvface - qcface - qrface);
                      w_cc   = myhalf*(h_avg_w[k-1]  + h_avg_w[k]);
                      uw_cc  = myhalf*(h_avg_uw[k-1] + h_avg_uw[k]);
                      vw_cc  = myhalf*(h_avg_vw[k-1] + h_avg_vw[k]);
                      ww_cc  = myhalf*(h_avg_ww[k-1] + h_avg_ww[k]);
                      data_log2 << std::setw(prof_datwidth) << std::setprecision(prof_timeprecision) << time << " "
                                << std::setw(prof_datwidth) << std::setprecision(prof_datprecision) << z << " "
                                << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]   << " " // u'u'
                                << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]   << " " // u'v'
                                << h_avg_uw[k]   -      uface*h_avg_w[k]   << " " // u'w'
                                << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]   << " " // v'v'
                                << h_avg_vw[k]   -      vface*h_avg_w[k]   << " " // v'w'
                                << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]   << " " // w'w'
                                << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k]  << " " // u'th'
                                << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k]  << " " // v'th'
                                << h_avg_wth[k]  - h_avg_w[k]*thface       << " " // w'th'
                                << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
                                // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                //                        -   <u_i u_i> * <u_j>
                                //                        - 2*<u_i> * <u_i u_j>
                                //                        + 2*<u_i>*<u_i> * <u_j>
                                << h_avg_uiuiu[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
                                 - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
                                   << " " // cell-centered (u'_i u'_i)u'
                                << h_avg_uiuiv[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
                                 - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
                                   << " " // cell-centered (u'_i u'_i)v'
                                << h_avg_uiuiw[k]
                                 - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                                 - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                 + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                   << " " // face-centered (u'_i u'_i)w'
                                << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k]   << " " // cell-centered p'u'
                                << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k]   << " " // cell-centered p'v'
                                << h_avg_pw[k]   -      pface*h_avg_w[k]   << " " // face-centered p'w'
                                << h_avg_wqv[k]  -     qvface*h_avg_w[k]   << " "
                                << h_avg_wqc[k]  -     qcface*h_avg_w[k]   << " "
                                << h_avg_wqr[k]  -     qrface*h_avg_w[k]   << " "
                                << h_avg_wthv[k] -    thvface*h_avg_w[k]
                                << std::endl;
                  } // loop over z
 
                  // Write top face values, extrapolating scalar quantities
                  const int k = unstag_size;
                  Real uface  = Real(1.5)*h_avg_u[k-1]  - myhalf*h_avg_u[k-2];
                  Real vface  = Real(1.5)*h_avg_v[k-1]  - myhalf*h_avg_v[k-2];
                  Real thface = Real(1.5)*h_avg_th[k-1] - myhalf*h_avg_th[k-2];
                  Real pface  = Real(1.5)*h_avg_p[k-1]  - myhalf*h_avg_p[k-2];
                  Real qvface = Real(1.5)*h_avg_qv[k-1] - myhalf*h_avg_qv[k-2];
                  Real qcface = Real(1.5)*h_avg_qc[k-1] - myhalf*h_avg_qc[k-2];
                  Real qrface = Real(1.5)*h_avg_qr[k-1] - myhalf*h_avg_qr[k-2];
                  Real uuface = Real(1.5)*h_avg_uu[k-1] - myhalf*h_avg_uu[k-2];
                  Real vvface = Real(1.5)*h_avg_vv[k-1] - myhalf*h_avg_vv[k-2];
                  Real thvface = thface * (one + epsv*qvface - qcface - qrface);
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log2 << std::setw(prof_datwidth) << std::setprecision(prof_timeprecision) << time << " "
                            << std::setw(prof_datwidth) << std::setprecision(prof_datprecision) << z << " "
                            << 0                                     << " " // u'u'
                            << 0                                     << " " // u'v'
                            << h_avg_uw[k]   -      uface*h_avg_w[k] << " " // u'w'
                            << 0                                     << " " // v'v'
                            << h_avg_vw[k]   -      vface*h_avg_w[k] << " " // v'w'
                            << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k] << " " // w'w'
                            << 0                                     << " " // u'th'
                            << 0                                     << " " // v'th'
                            << h_avg_wth[k]  -     thface*h_avg_w[k] << " " // w'th'
                            << 0                                     << " " // th'th'
                            << 0                                     << " " // (u'_i u'_i)u'
                            << 0                                     << " " // (u'_i u'_i)v'
                            << h_avg_uiuiw[k]
                             - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                             - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                             + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                               << " " // (u'_i u'_i)w'
                            << 0                                     << " " // pu'
                            << 0                                     << " " // pv'
                            << h_avg_pw[k]   -      pface*h_avg_w[k] << " " // pw'
                            << h_avg_wqv[k]  -     qvface*h_avg_w[k] << " "
                            << h_avg_wqc[k]  -     qcface*h_avg_w[k] << " "
                            << h_avg_wqr[k]  -     qrface*h_avg_w[k] << " "
                            << h_avg_wthv[k] -    thvface*h_avg_w[k]
                            << std::endl;
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 3 && time > zero) {
                std::ostream& data_log3 = DataLog(3);
                if (data_log3.good()) {
                  // Write the average stresses
                  for (int k = 0; k < unstag_size; k++) {
                      Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                      data_log3 << std::setw(prof_datwidth) << std::setprecision(prof_timeprecision) << time << " "
                                << std::setw(prof_datwidth) << std::setprecision(prof_datprecision) << z << " "
                                << h_avg_tau11[k]  << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                << h_avg_tau22[k]  << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                << h_avg_sgshfx[k] << " "
                                << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
                                << h_avg_sgsdiss[k]
                                << std::endl;
                  } // loop over z
                  // Write top face values
                  Real NANval = zero;
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log3 << std::setw(prof_datwidth) << std::setprecision(prof_timeprecision) << time << " "
                            << std::setw(prof_datwidth) << std::setprecision(prof_datprecision) << z << " "
                            << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
                            << NANval << " " << h_avg_tau23[unstag_size] << " " << NANval << " "
                            << h_avg_sgshfx[unstag_size] << " "
                            << h_avg_sgsq1fx[unstag_size] << " " << h_avg_sgsq2fx[unstag_size] << " "
                            << NANval
                            << std::endl;
                } // if good
            } // if (NumDataLogs() > 3)
 
            if (NumDataLogs() > 4 && time > 0.) {
                std::ostream& data_log4 = DataLog(4);
                if (data_log4.good()) {
                    for (int k = 0; k < unstag_size; k++) {
                        Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                        data_log4 << std::setw(prof_datwidth) << std::setprecision(prof_timeprecision) << time << " "
                                  << std::setw(prof_datwidth) << std::setprecision(prof_datprecision) << z << " "
                                  << h_avg_ttend[k]  << " " << h_avg_qtend[k]   << " " << h_avg_wsub[k]   << " "
                                  << h_avg_thtend[k] << " " << h_avg_qhtend[k]  << " " << h_avg_tvtend[k] << " "
                                  << h_avg_qvtend[k] << " " << h_avg_qcvtend[k] << " " << h_avg_tnudge[k] << " "
                                  << h_avg_qnudge[k] << " " << h_avg_unudge[k]  << " " << h_avg_vnudge[k]
                                  << std::endl;
                  } // loop over z
                  // Write top face values
                  Real NANval = zero;
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log4 << std::setw(prof_datwidth) << std::setprecision(prof_timeprecision) << time << " "
                            << std::setw(prof_datwidth) << std::setprecision(prof_datprecision) << z << " "
                            << NANval << " " << NANval << " " << NANval << " "
                            << NANval << " " << NANval << " " << NANval << " "
                            << NANval << " " << NANval << " " << NANval << " "
                            << NANval << " " << NANval << " " << NANval
                            << std::endl;
                }
            }
        } // if IOProcessor
    } // if (NumDataLogs() > 1)
}

WriteAtFinalTime()

void ERF::WriteAtFinalTime

(

)

{
    // Write plotfiles at final time
    int plotfiles_3d_written = 0;
    bool interval_diagnostic_consumed = false;
    if ( (m_plot3d_int_1 > 0 || m_plot3d_per_1 > zero) && istep[0] > last_plot3d_file_step_1 ) {
        const bool wrote_plotfile = Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
        if (wrote_plotfile) {
            ++plotfiles_3d_written;
            interval_diagnostic_consumed = interval_diagnostic_consumed ||
                erf_plotfile::plot3d_selection_has_interval_mean_diagnostic(plot3d_var_names_1);
        }
        if (m_plot3d_per_1 > zero) {last_plot3d_file_time_1 += m_plot3d_per_1;}
    }
    if ( (m_plot3d_int_2 > 0 || m_plot3d_per_2 > zero) && istep[0] > last_plot3d_file_step_2) {
        const bool wrote_plotfile = Write3DPlotFile(2,plotfile3d_type_2,plot3d_var_names_2);
        if (wrote_plotfile) {
            ++plotfiles_3d_written;
            interval_diagnostic_consumed = interval_diagnostic_consumed ||
                erf_plotfile::plot3d_selection_has_interval_mean_diagnostic(plot3d_var_names_2);
        }
        if (m_plot3d_per_2 > zero) {last_plot3d_file_time_2 += m_plot3d_per_2;}
    }
    ResetIntervalMeansAfter3DPlotfileBatch(plotfiles_3d_written,
                                           interval_diagnostic_consumed);
    if ( (m_plot2d_int_1 > 0 || m_plot2d_per_1 > zero) && istep[0] > last_plot2d_file_step_1 ) {
        Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
        if (m_plot2d_per_1 > zero) {last_plot2d_file_time_1 += m_plot2d_per_1;}
    }
    if ( (m_plot2d_int_2 > 0 || m_plot2d_per_2 > zero) && istep[0] > last_plot2d_file_step_2) {
        Write2DPlotFile(2,plotfile2d_type_2,plot2d_var_names_2);
        if (m_plot2d_per_2 > zero) {last_plot2d_file_time_2 += m_plot2d_per_2;}
    }
 
    for (int i = 0; i < m_subvol_int.size(); i++) {
        if ( (m_subvol_int[i] > 0 || m_subvol_per[i] > zero) && istep[0] > last_subvol_step[i]) {
            WriteSubvolume(i,subvol3d_var_names);
            if (m_subvol_per[i] > zero) {last_subvol_time[i] += m_subvol_per[i];}
        }
    }
 
    if ( (m_check_int > 0 || m_check_per > zero) && istep[0] > last_check_file_step) {
        WriteCheckpointFile();
        if (m_check_per > zero) {last_check_file_time += m_check_per;}
    }
}

Here is the call graph for this function:

WriteAtIntermediateTime()

void ERF::WriteAtIntermediateTime	(	int	nstep,
		double	time
	)

{
    int plotfiles_3d_written = 0;
    bool interval_diagnostic_consumed = false;
    if (writeNow(cur_time, step+1, m_plot3d_int_1, m_plot3d_per_1, dt[0], last_plot3d_file_time_1)) {
        last_plot3d_file_step_1 = step+1;
        const bool wrote_plotfile = Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
        if (wrote_plotfile) {
            ++plotfiles_3d_written;
            interval_diagnostic_consumed = interval_diagnostic_consumed ||
                erf_plotfile::plot3d_selection_has_interval_mean_diagnostic(plot3d_var_names_1);
        }
        for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
        if (m_plot3d_per_1 > zero) {last_plot3d_file_time_1 += m_plot3d_per_1;}
    }
    if (writeNow(cur_time, step+1, m_plot3d_int_2, m_plot3d_per_2, dt[0], last_plot3d_file_time_2)) {
        last_plot3d_file_step_2 = step+1;
        const bool wrote_plotfile = Write3DPlotFile(2,plotfile3d_type_2,plot3d_var_names_2);
        if (wrote_plotfile) {
            ++plotfiles_3d_written;
            interval_diagnostic_consumed = interval_diagnostic_consumed ||
                erf_plotfile::plot3d_selection_has_interval_mean_diagnostic(plot3d_var_names_2);
        }
        for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
        if (m_plot3d_per_2 > zero) {last_plot3d_file_time_2 += m_plot3d_per_2;}
    }
    ResetIntervalMeansAfter3DPlotfileBatch(plotfiles_3d_written,
                                           interval_diagnostic_consumed);
 
    if (writeNow(cur_time, step+1, m_plot2d_int_1, m_plot2d_per_1, dt[0], last_plot2d_file_time_1)) {
        last_plot2d_file_step_1 = step+1;
        Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
        if (m_plot2d_per_1 > zero) {last_plot2d_file_time_1 += m_plot2d_per_1;}
    }
 
    if (writeNow(cur_time, step+1, m_plot2d_int_2, m_plot2d_per_2, dt[0], last_plot2d_file_time_2)) {
        last_plot2d_file_step_2 = step+1;
        Write2DPlotFile(2,plotfile2d_type_2,plot2d_var_names_2);
        if (m_plot2d_per_2 > zero) {last_plot2d_file_time_2 += m_plot2d_per_2;}
    }
 
    for (int i = 0; i < m_subvol_int.size(); i++) {
        if (writeNow(cur_time, step+1, m_subvol_int[i], m_subvol_per[i], dt[0], last_subvol_time[i])) {
            last_subvol_step[i] = step+1;
            WriteSubvolume(i,subvol3d_var_names);
            if (m_subvol_per[i] > zero) {last_subvol_time[i] += m_subvol_per[i];}
        }
    }
 
    if (writeNow(cur_time, step+1, m_check_int, m_check_per, dt[0], last_check_file_time)) {
        last_check_file_step = step+1;
        WriteCheckpointFile();
        if (m_check_per > zero) {last_check_file_time += m_check_per;}
    }
}

Referenced by EvolveOneStep().

Here is the call graph for this function:

Here is the caller graph for this function:

writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream &  os )

static

{
    std::string PrettyLine = std::string(78, '=') + "\n";
    std::string OtherLine = std::string(78, '-') + "\n";
    std::string SkipSpace = std::string(8, ' ');
 
    // build information
    os << PrettyLine;
    os << " ERF Build Information\n";
    os << PrettyLine;
 
    os << "build date:    " << buildInfoGetBuildDate() << "\n";
    os << "build machine: " << buildInfoGetBuildMachine() << "\n";
    os << "build dir:     " << buildInfoGetBuildDir() << "\n";
    os << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
 
    os << "\n";
 
    os << "COMP:          " << buildInfoGetComp() << "\n";
    os << "COMP version:  " << buildInfoGetCompVersion() << "\n";
 
    os << "C++ compiler:  " << buildInfoGetCXXName() << "\n";
    os << "C++ flags:     " << buildInfoGetCXXFlags() << "\n";
 
    os << "\n";
 
    os << "Link flags:    " << buildInfoGetLinkFlags() << "\n";
    os << "Libraries:     " << buildInfoGetLibraries() << "\n";
 
    os << "\n";
 
    for (int n = 1; n <= buildInfoGetNumModules(); n++) {
      os << buildInfoGetModuleName(n) << ": "
         << buildInfoGetModuleVal(n) << "\n";
    }
 
    os << "\n";
    const char* githash1 = buildInfoGetGitHash(1);
    const char* githash2 = buildInfoGetGitHash(2);
    if (strlen(githash1) > 0) {
      os << "ERF       git hash: " << githash1 << "\n";
    }
    if (strlen(githash2) > 0) {
      os << "AMReX       git hash: " << githash2 << "\n";
    }
 
    const char* buildgithash = buildInfoGetBuildGitHash();
    const char* buildgitname = buildInfoGetBuildGitName();
    if (strlen(buildgithash) > 0) {
      os << buildgitname << " git hash: " << buildgithash << "\n";
    }
 
    os << "\n";
    os << " ERF Compile time variables: \n";
 
    os << "\n";
    os << " ERF Defines: \n";
#ifdef _OPENMP
    os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
       << std::endl;
#else
    os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
       << std::endl;
#endif
 
#ifdef MPI_VERSION
    os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
       << MPI_VERSION << std::endl;
#else
    os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
       << "UNDEFINED" << std::endl;
#endif
 
#ifdef MPI_SUBVERSION
    os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
       << MPI_SUBVERSION << std::endl;
#else
    os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
       << "UNDEFINED" << std::endl;
#endif
 
    os << "\n\n";
}

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

WriteCheckpointFile()

void ERF::WriteCheckpointFile

(

)

const

ERF function for writing a checkpoint file.

{
    auto dCheckTime0 = amrex::second();
 
    // Station rows are buffered, and a restart from this checkpoint appends to
    // the station files: without this the rows between the last flush and the
    // checkpoint would be missing from the series the restart continues.
    flush_stations();
 
    // chk00010            write a checkpoint file with this root directory
    // chk00010/Header     this contains information you need to save (e.g., finest_level, t_new, etc.) and also
    //                     the BoxArrays at each level
    // chk00010/Level_0/
    // chk00010/Level_1/
    // etc.                these subdirectories will hold the MultiFab data at each level of refinement
 
    // checkpoint file name, e.g., chk00010
    const std::string& checkpointname = Concatenate(check_file,istep[0],file_name_digits);
 
    Print() << "Writing native checkpoint " << checkpointname << "\n";
 
    const int nlevels = finest_level+1;
 
    // ---- prebuild a hierarchy of directories
    // ---- dirName is built first.  if dirName exists, it is renamed.  then build
    // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
    // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
    // ---- after all directories are built
    // ---- ParallelDescriptor::IOProcessor() creates the directories
    PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
 
    if (ParallelDescriptor::IOProcessor()) {
        write_surface_temperature_contract(checkpointname);
    }
 
    int ncomp_cons = vars_new[0][Vars::cons].nComp();
 
    // write Header file
    if (ParallelDescriptor::IOProcessor()) {
 
       std::string HeaderFileName(checkpointname + "/Header");
       VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
       std::ofstream HeaderFile;
       HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
       HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                               std::ofstream::trunc |
                                               std::ofstream::binary);
       if(! HeaderFile.good()) {
           FileOpenFailed(HeaderFileName);
       }
 
       HeaderFile.precision(17);
 
       // write out title line
       HeaderFile << "Checkpoint file for ERF\n";
 
       // write out finest_level
       HeaderFile << finest_level << "\n";
 
       // write the number of components
       // for each variable we store
 
       // conservative, cell-centered vars
       HeaderFile << ncomp_cons << "\n";
 
       // x-velocity on faces
       HeaderFile << 1 << "\n";
 
       // y-velocity on faces
       HeaderFile << 1 << "\n";
 
       // z-velocity on faces
       HeaderFile << 1 << "\n";
 
       // write out array of istep
       for (int i = 0; i < istep.size(); ++i) {
           HeaderFile << istep[i] << " ";
       }
       HeaderFile << "\n";
 
       // write out array of dt
       for (int i = 0; i < dt.size(); ++i) {
           HeaderFile << dt[i] << " ";
       }
       HeaderFile << "\n";
 
       // write out array of t_new
       for (int i = 0; i < t_new.size(); ++i) {
           HeaderFile << t_new[i] << " ";
       }
       HeaderFile << "\n";
 
       // write the BoxArray at each level
       for (int lev = 0; lev <= finest_level; ++lev) {
           boxArray(lev).writeOn(HeaderFile);
           HeaderFile << '\n';
       }
 
       // write out array of t_avg_cnt, the normalizer for the time-averaged velocity
       //
       // NOTE: this is written *after* the BoxArrays, i.e. last, on purpose.  Nothing
       //       else is parsed out of the header past that point, so a reader that does
       //       not know about this line simply ignores it, and a reader that does can
       //       treat its absence (a checkpoint from before issue 3654 was fixed) as
       //       "restart the average", which is the behavior those files already had.
       for (int i = 0; i < t_avg_cnt.size(); ++i) {
           HeaderFile << t_avg_cnt[i] << " ";
       }
       HeaderFile << "\n";
 
       // Write separate file that tells how many components we have of the base state
       std::string BaseStateFileName(checkpointname + "/num_base_state_comps");
       std::ofstream BaseStateFile;
       BaseStateFile.open(BaseStateFileName.c_str(), std::ofstream::out   |
                                                     std::ofstream::trunc |
                                                     std::ofstream::binary);
       if(! BaseStateFile.good()) {
           FileOpenFailed(BaseStateFileName);
       } else {
           // write out number of components in base state
           BaseStateFile << BaseState::num_comps      << "\n";
           BaseStateFile << base_state[0].nGrowVect() << "\n";
       }
 
       // Persist the LSM scalar step counter (e.g. NoahMP itimestep). Without
       // this, restart resets the substep schedule, which changes the LSM->MOST
       // flux firing times and produces a non-bitwise trajectory vs. cold start.
       if (solverChoice.lsm_type != LandSurfaceType::None) {
           std::string LsmStepFileName(checkpointname + "/lsm_step");
           std::ofstream LsmStepFile;
           LsmStepFile.open(LsmStepFileName.c_str(), std::ofstream::out   |
                                                     std::ofstream::trunc |
                                                     std::ofstream::binary);
           if(! LsmStepFile.good()) {
               FileOpenFailed(LsmStepFileName);
           } else {
               for (int lev = 0; lev <= finest_level; ++lev) {
                   LsmStepFile << lsm.Get_LSM_Step(lev) << " ";
               }
               LsmStepFile << "\n";
           }
       }
 
       // Persist the state of the exponential time filter used by the surface layer
       // when erf.most.time_average is on.  The plane and EB averaging policies hold
       // that state in the plane averages below; every policy needs the per-level
       // "is the filter initialized" flag, since restoring an average that was never
       // filled would blend bogus_large_value into the next average.  Without this
       // file the filter simply starts over on restart, which is what checkpoints
       // written before this was added already did.
       int n_faces = 0;
       for (OrientationIter oit; oit; ++oit) {
           Orientation ori = oit();
           if (phys_bc_type[ori] == ERF_BC::surface_layer) {
               n_faces += 1;
           }
       }
       for (OrientationIter oit; oit; ++oit) {
           Orientation ori = oit();
           if (m_SurfaceLayer[ori] && m_SurfaceLayer[ori]->mac_avg_is_time_averaged()) {
               std::string face = "_" + std::to_string(ori);
               if (n_faces == 1 && static_cast<int>(ori) == Orientation::zlo()) {
                   face = "";
               }
 
               std::string MostAvgFileName(checkpointname + "/most_time_average" + face);
               std::ofstream MostAvgFile;
               MostAvgFile.open(MostAvgFileName.c_str(), std::ofstream::out   |
                                                         std::ofstream::trunc |
                                                         std::ofstream::binary);
               if(! MostAvgFile.good()) {
                   FileOpenFailed(MostAvgFileName);
               } else {
                   // Number of average components, then one line per level holding the
                   // initialization flag, how many plane averages follow (zero for the
                   // region policy, which keeps no plane averages), and those averages
                   MostAvgFile << m_SurfaceLayer[ori]->get_num_mac_avg() << "\n";
                   MostAvgFile.precision(17);
                   for (int lev = 0; lev <= finest_level; ++lev) {
                       const Vector<Real> pavg = m_SurfaceLayer[ori]->get_mac_plane_avg(lev);
                       MostAvgFile << (m_SurfaceLayer[ori]->mac_avg_is_initialized(lev) ? 1 : 0)
                                   << " " << pavg.size();
                       for (int iavg(0); iavg < static_cast<int>(pavg.size()); ++iavg) {
                           MostAvgFile << " " << pavg[iavg];
                       }
                       MostAvgFile << "\n";
                   }
               }
           }
       }
   }
 
    // Interval means are stored separately from the legacy Header so older
    // readers continue to parse that file unchanged.  The metadata records
    // the level/component layout; the v1 processor map is retained only for
    // format compatibility and is not a restart constraint.
    if (solverChoice.compute_mean_vars && ParallelDescriptor::IOProcessor()) {
        const std::string metadata_name(checkpointname + "/IntervalMeansHeader");
        std::ofstream metadata(metadata_name, std::ofstream::out |
                                            std::ofstream::trunc |
                                            std::ofstream::binary);
        if (!metadata.good()) {
            FileOpenFailed(metadata_name);
        }
        metadata.precision(17);
        metadata << "ERF interval means checkpoint v1\n";
        metadata << finest_level + 1 << " " << 10 << "\n";
        for (int lev = 0; lev <= finest_level; ++lev) {
            metadata << lev << " " << t_mean_cnt[lev] << " "
                     << mean_vars_time_reset_done << "\n";
            boxArray(lev).writeOn(metadata);
            metadata << '\n';
            const auto& pmap = dmap[lev].ProcessorMap();
            metadata << pmap.size();
            for (const int proc : pmap) {
                metadata << " " << proc;
            }
            metadata << '\n';
        }
    }
 
    // write the MultiFab data to, e.g., chk00010/Level_0/
    // Here we make copies of the MultiFab with no ghost cells
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
        MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
        VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
 
        MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
        MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
        VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
 
        MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
        MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
        VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
 
        MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
        MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
        VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
 
        if (solverChoice.anelastic[lev] == 1) {
            MultiFab ppinc(grids[lev],dmap[lev],1,0);
            MultiFab::Copy(ppinc,pp_inc[lev],0,0,1,0);
            VisMF::Write(ppinc, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PP_Inc"));
 
            MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
            MultiFab::Copy(gpx,gradp[lev][GpVars::gpx],0,0,1,0);
            VisMF::Write(gpx, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpx"));
 
            MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
            MultiFab::Copy(gpy,gradp[lev][GpVars::gpy],0,0,1,0);
            VisMF::Write(gpy, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpy"));
 
            MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
            MultiFab::Copy(gpz,gradp[lev][GpVars::gpz],0,0,1,0);
            VisMF::Write(gpz, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpz"));
        }
 
        // Surface momentum and scalar fluxes handed to an ocean model in
        // flux-passing mode. These are computed inside a timestep's RHS rather
        // than carried as state, so after a restart they hold zero until the
        // first advance -- but the coupler packs them for the ocean *before*
        // that advance, which delivers zero wind stress and zero heat flux for a
        // whole ocean step with no diagnostic. They are part of the coupled
        // interface and have to survive the checkpoint.
        //
        // Written only when a coupling driver is attached, so standalone
        // checkpoints are byte-for-byte unchanged. Each field is written
        // separately because diffusion and moisture settings decide which exist;
        // the read side probes for each rather than assuming.
        if (m_driver_has_atm2ocn_coupling) {
            if (!Tau.empty() && Tau[lev][TauType::tau13]) {
                MultiFab tau13(convert(grids[lev],IntVect(1,0,1)),dmap[lev],1,0);
                MultiFab::Copy(tau13,*Tau[lev][TauType::tau13],0,0,1,0);
                VisMF::Write(tau13, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Tau13"));
            }
            if (!Tau.empty() && Tau[lev][TauType::tau23]) {
                MultiFab tau23(convert(grids[lev],IntVect(0,1,1)),dmap[lev],1,0);
                MultiFab::Copy(tau23,*Tau[lev][TauType::tau23],0,0,1,0);
                VisMF::Write(tau23, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Tau23"));
            }
            if (!SFS_hfx3_lev.empty() && SFS_hfx3_lev[lev]) {
                MultiFab hfx3(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
                MultiFab::Copy(hfx3,*SFS_hfx3_lev[lev],0,0,1,0);
                VisMF::Write(hfx3, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SFS_hfx3"));
            }
            if (!SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev]) {
                MultiFab q1fx3(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
                MultiFab::Copy(q1fx3,*SFS_q1fx3_lev[lev],0,0,1,0);
                VisMF::Write(q1fx3, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SFS_q1fx3"));
            }
        }
 
        // The running sum of the time-averaged velocity.  Its normalizer, t_avg_cnt,
        // goes in the header above; both are needed or the average silently restarts
        // from zero across a checkpoint/restart (issue 3654).
        // NOTE: no ghost cells to strip here, so this is written in place.
        if (solverChoice.time_avg_vel) {
            AMREX_ALWAYS_ASSERT(vel_t_avg[lev] != nullptr);
            VisMF::Write(*vel_t_avg[lev], MultiFabFileFullPrefix(lev, checkpointname, "Level_", "VelTimeAvg"));
        }
 
        // Face state of the immersed-boundary surface energy balance (no-op unless enabled);
        // read back in init_ibseb() once the face list has been rebuilt from the blanking.
        ibseb_write_checkpoint(checkpointname, lev);
 
        if (solverChoice.compute_mean_vars) {
            AMREX_ALWAYS_ASSERT(interval_means[lev] != nullptr);
            VisMF::Write(*interval_means[lev],
                         MultiFabFileFullPrefix(lev, checkpointname, "Level_", "IntervalMeans"));
        }
 
        // Note that we write the ghost cells of the base state (unlike above)
        IntVect ng_base = base_state[lev].nGrowVect();
        int  ncomp_base = base_state[lev].nComp();
        MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
        MultiFab::Copy(base,base_state[lev],0,0,ncomp_base,ng_base);
        VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
 
        if (SolverChoice::mesh_type != MeshType::ConstantDz)  {
            // Note that we also write the ghost cells of z_phys_nd
            IntVect ng = z_phys_nd[lev]->nGrowVect();
            MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
            MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
            VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
        }
 
        // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
        // Write the moisture model restart variables
        std::vector<int> qmoist_indices;
        std::vector<std::string> qmoist_names;
        micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
        int qmoist_nvar = static_cast<int>(qmoist_indices.size());
        for (int var = 0; var < qmoist_nvar; var++) {
            const int ncomp  = 1;
            IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
            MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
            MultiFab::Copy(moist_vars,*(qmoist[lev][qmoist_indices[var]]),0,0,ncomp,ng_moist);
            VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", qmoist_names[var]));
        }
 
#if defined(ERF_USE_WINDFARM)
        if(solverChoice.windfarm_type == WindFarmType::Fitch or
           solverChoice.windfarm_type == WindFarmType::EWP or
           solverChoice.windfarm_type == WindFarmType::SimpleAD){
            IntVect ng_turb = Nturb[lev].nGrowVect();
            MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng_turb);
            MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng_turb);
            VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
        }
#endif
 
        // Two-stream radiation: the force-restore surface state
        if (solverChoice.rad_type == RadiationType::TwoStream) {
            two_stream_rad.write_checkpoint(lev, checkpointname);
        }
 
        // Write the LSM data
        if (solverChoice.lsm_type != LandSurfaceType::None) {
            for (int ivar(0); ivar<lsm_data[lev].size(); ++ivar) {
                BoxArray ba = lsm_data[lev][ivar]->boxArray();
                DistributionMapping dm = lsm_data[lev][ivar]->DistributionMap();
                IntVect ng = lsm_data[lev][ivar]->nGrowVect();
                int nvar   = lsm_data[lev][ivar]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                MultiFab::Copy(lsm_vars,*(lsm_data[lev][ivar]),0,0,nvar,ng);
                VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmData" + std::to_string(ivar)));
            }
            for (int iflux(0); iflux<lsm_flux[lev].size(); ++iflux) {
                BoxArray ba = lsm_flux[lev][iflux]->boxArray();
                DistributionMapping dm = lsm_flux[lev][iflux]->DistributionMap();
                IntVect ng = lsm_flux[lev][iflux]->nGrowVect();
                int nvar   = lsm_flux[lev][iflux]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                MultiFab::Copy(lsm_vars,*(lsm_flux[lev][iflux]),0,0,nvar,ng);
                VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmFlux" + std::to_string(iflux)));
            }
 
            // Write the full LSM prognostic state (e.g. NoahMP soil/snow/canopy)
            // to chk*/noahmp_restart/Level_<lev>.nc so a restart reproduces a
            // cold-start trajectory bitwise (issue #3255). The LSM model writes
            // its blocks collectively; no-op for models without such state.
            // Create the subdir on rank 0 and barrier before the collective open
            // so no rank races ahead of the directory existing.
            std::string LsmRestartDir(checkpointname + "/noahmp_restart");
            if (ParallelDescriptor::IOProcessor()) {
                amrex::UtilCreateDirectory(LsmRestartDir, 0755);
            }
            ParallelDescriptor::Barrier();
            lsm.Write_Lsm_Restart(lev, LsmRestartDir);
        }
 
        // Write the radiation heating rates
        if ((solverChoice.rad_type != RadiationType::None) && (qheating_rates[lev])) {
            int nrad = qheating_rates[lev]->nComp();
            MultiFab mf_rad(grids[lev],dmap[lev],nrad,0);
            MultiFab::Copy(mf_rad,*qheating_rates[lev],0,0,nrad,0);
            VisMF::Write(mf_rad, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qrad"));
        }
 
        IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
        MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
        MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_x],0,0,1,ng);
        VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_mx"));
 
#if 0
        if (MapFacType::m_x != MapFacType::m_y) {
            MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_y],0,0,1,ng);
            VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_my"));
        }
#endif
 
        ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
        MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
        MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_x],0,0,1,ng);
        VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_ux"));
 
#if 0
        if (MapFacType::m_x != MapFacType::m_y) {
            MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_y],0,0,1,ng);
            VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_uy"));
        }
#endif
 
        ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
        MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
        MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_x],0,0,1,ng);
        VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vx"));
 
#if 0
        if (MapFacType::m_x != MapFacType::m_y) {
            MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_y],0,0,1,ng);
            VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vy"));
        }
#endif
 
        // Count number of surface layer boundaries to determine face prefix
        int n_faces = 0;
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            if (phys_bc_type[ori] == ERF_BC::surface_layer) {
                n_faces += 1;
            }
        }
 
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            if (m_SurfaceLayer[ori])  {
                amrex::Print() << "Writing SurfaceLayer variables at level " << lev << " for face " << ori << std::endl;
                std::string face = "_" + std::to_string(ori);
                // if we only use a single surface layer, don't add the face prefix
                if (n_faces == 1 && static_cast<int>(ori) == Orientation::zlo()) {
                    face = "";
                }
 
                // These MultiFabs live on a 2D BoxArray for planar terrain but on the full 3D
                // BoxArray (with a z ghost cell) for EB terrain, so we write each one on its own
                // BoxArray -- copying into a hard-wired 2D MultiFab would silently drop everything
                // above k=0 for EB (issue #3560)
                auto write_sl_var = [&] (MultiFab* src, const std::string& name) {
                    VisMF::Write(*src, MultiFabFileFullPrefix(lev, checkpointname, "Level_", name));
                };
 
                write_sl_var(m_SurfaceLayer[ori]->get_u_star(lev), "Ustar" + face);
                write_sl_var(m_SurfaceLayer[ori]->get_w_star(lev), "Wstar" + face);
                write_sl_var(m_SurfaceLayer[ori]->get_t_star(lev), "Tstar" + face);
                write_sl_var(m_SurfaceLayer[ori]->get_q_star(lev), "Qstar" + face);
                write_sl_var(m_SurfaceLayer[ori]->get_olen(lev)  , "Olen"  + face);
                write_sl_var(m_SurfaceLayer[ori]->get_q_surf(lev), "Qsurf" + face);
                write_sl_var(m_SurfaceLayer[ori]->get_t_surf(lev), "Tsurf" + face);
                write_sl_var(m_SurfaceLayer[ori]->get_pblh(lev)  , "PBLH"  + face);
                write_sl_var(m_SurfaceLayer[ori]->get_z0(lev)    , "Z0"    + face);
                // The exponentially filtered averages behind erf.most.time_average. The
                // region policy carries the filter state in these MultiFabs (the plane and
                // EB policies carry it in the plane averages written to the
                // most_time_average file above), so without them the filter history is lost
                // across a restart and the first update falls back to the instantaneous
                // average.
                //
                // NOTE: we only write these once the filter has been initialized; before
                //       that they hold bogus_large_value, which must never be blended in.
                if (m_SurfaceLayer[ori]->mac_avg_is_initialized(lev)) {
                    for (int iavg(0); iavg < m_SurfaceLayer[ori]->get_num_mac_avg(); ++iavg) {
                        write_sl_var(m_SurfaceLayer[ori]->get_mac_avg_ptr(lev,iavg),
                                     "MOSTAvg" + std::to_string(iavg) + face);
                    }
                }
            }
        }
 
        if (sst_lev[lev][0]) {
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                MultiFab::Copy(sst_at_t,*sst_lev[lev][nt],0,0,1,ng);
                VisMF::Write(sst_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
                                                             "SST_" + std::to_string(nt)));
            }
        }
 
        if (tsk_lev[lev][0]) {
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                MultiFab::Copy(tsk_at_t,*tsk_lev[lev][nt],0,0,1,ng);
                VisMF::Write(tsk_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
                                                             "TSK_" + std::to_string(nt)));
            }
        }
 
        {
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
                    const Box& bx = mfi.growntilebox();
                    Array4<int>  const& src_arr = lmask_lev[lev][nt]->array(mfi);
                    Array4<Real> const& dst_arr = lmask_at_t.array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        dst_arr(i,j,k) = Real(src_arr(i,j,k));
                    });
                }
                VisMF::Write(lmask_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
                                                              "LMASK_" + std::to_string(nt)));
            }
        }
 
        IntVect ngv = ng; ngv[2] = 0;
 
        // Write lat/lon if it exists
        if (lat_m[lev] && lon_m[lev]) {
            amrex::Print() << "Writing Lat/Lon variables at level " << lev << std::endl;
            MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
            MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
            MultiFab::Copy(lat,*lat_m[lev],0,0,1,ngv);
            MultiFab::Copy(lon,*lon_m[lev],0,0,1,ngv);
            VisMF::Write(lat, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LAT"));
            VisMF::Write(lon, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LON"));
        }
 
 
#ifdef ERF_USE_NETCDF
        // Write sinPhi and cosPhi if it exists
        if (cosPhi_m[lev] && sinPhi_m[lev] && solverChoice.variable_coriolis) {
            amrex::Print() << "Writing Coriolis factors at level " << lev << std::endl;
            MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
            MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
            MultiFab::Copy(sphi,*sinPhi_m[lev],0,0,1,ngv);
            MultiFab::Copy(cphi,*cosPhi_m[lev],0,0,1,ngv);
            VisMF::Write(sphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SinPhi"));
            VisMF::Write(cphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "CosPhi"));
        }
 
        if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
            if (lev == 0) {
                amrex::Print() << "Writing C1H/C2H/RDNW/MUB/PHB variables at level " << lev << std::endl;
                MultiFab tmp1d(ba1d[0],dmap[0],1,0);
 
                MultiFab::Copy(tmp1d,*wrf_C1H,0,0,1,0);
                VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C1H"));
 
                MultiFab::Copy(tmp1d,*wrf_C2H,0,0,1,0);
                VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C2H"));
 
                MultiFab::Copy(tmp1d,*wrf_RDNW,0,0,1,0);
                VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "RDNW"));
 
                MultiFab tmp2d(ba2d[0],dmap[0],1,wrf_MUB->nGrowVect());
 
                MultiFab::Copy(tmp2d,*wrf_MUB,0,0,1,wrf_MUB->nGrowVect());
                VisMF::Write(tmp2d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MUB"));
 
                MultiFab tmp3d(convert(grids[0],IntVect(0,0,1)),dmap[0],1,wrf_PHB->nGrowVect());
 
                MultiFab::Copy(tmp3d,*wrf_PHB,0,0,1,wrf_PHB->nGrowVect());
                VisMF::Write(tmp3d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PHB"));
            }
        }
#endif
    } // for lev
 
    // Write zlevels to its own directory and read it as well, similar to bdy data
    if (ParallelDescriptor::IOProcessor()) {
        std::string ZLevelsFileName(checkpointname + "/zlevels");
        std::ofstream ZLevelsFile;
        ZLevelsFile.open(ZLevelsFileName.c_str(), std::ofstream::out   |
                                                  std::ofstream::trunc |
                                                  std::ofstream::binary);
        if(! ZLevelsFile.good()) {
            FileOpenFailed(ZLevelsFileName);
        } else {
            ZLevelsFile.precision(17);
 
            // Write every level we hold (max_level+1) rather than finest_level+1, so
            // that levels which are not currently active are still restored.
            ZLevelsFile << zlevels_stag.size() << "\n";
 
            for (int lev = 0; lev < static_cast<int>(zlevels_stag.size()); ++lev) {
                ZLevelsFile << zlevels_stag[lev].size();
                for (int k = 0; k < static_cast<int>(zlevels_stag[lev].size()); ++k) {
                    ZLevelsFile << " " << zlevels_stag[lev][k];
                }
                ZLevelsFile << "\n";
            }
        }
    }
 
#ifdef ERF_USE_PARTICLES
   particleData.Checkpoint(checkpointname);
#endif
 
#if 0
#ifdef ERF_USE_NETCDF
   // Write bdy_data files
   if ( ParallelDescriptor::IOProcessor() &&
        ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
         solverChoice.use_real_bcs )
   {
       // Vector dimensions
       int num_time = bdy_data_xlo.size();
       int num_var  = bdy_data_xlo[0].size();
 
       // Open header file and write to it
       std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
       bdy_h_file << std::setprecision(1) << std::fixed;
       bdy_h_file << num_time << "\n";
       bdy_h_file << num_var  << "\n";
       bdy_h_file << start_bdy_time << "\n";
       bdy_h_file << bdy_time_interval << "\n";
       bdy_h_file << real_width << "\n";
       for (int ivar(0); ivar<num_var; ++ivar) {
           bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
           bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
           bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
           bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
       }
 
       // Open data file and write to it
       std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
       for (int itime(0); itime<num_time; ++itime) {
           if (bdy_data_xlo[itime].size() > 0) {
               for (int ivar(0); ivar<num_var; ++ivar) {
                   bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
                   bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
                   bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
                   bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
               }
           }
       }
   }
#endif
#endif
 
    // Write job_info after checkpoint state so the provenance record describes
    // the completed output attempt and can seed a later restart lineage.
    writeJobInfo(checkpointname, erf_provenance::ArtifactType::Checkpoint,
                 istep[0], t_new[0]);
 
    if (verbose > 0)
    {
        auto dCheckTime = amrex::second() - dCheckTime0;
        ParallelDescriptor::ReduceRealMax(dCheckTime,ParallelDescriptor::IOProcessorNumber());
        amrex::Print() << "Checkpoint write time = " << dCheckTime << " seconds." << '\n';
    }
}

Here is the call graph for this function:

WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain	(	std::ostream &	HeaderFile,
		int	nlevels,
		const amrex::Vector< amrex::BoxArray > &	bArray,
		const amrex::Vector< std::string > &	varnames,
		const amrex::Vector< amrex::Geometry > &	my_geom,
		double	time,
		const amrex::Vector< int > &	level_steps,
		const amrex::Vector< amrex::IntVect > &	my_ref_ratio,
		const std::string &	versionName,
		const std::string &	levelPrefix,
		const std::string &	mfPrefix
	)		const

{
    AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
    AMREX_ALWAYS_ASSERT(nlevels <= my_ref_ratio.size()+1);
    AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
 
    HeaderFile.precision(17);
 
    // ---- this is the generic plot file type name
    HeaderFile << versionName << '\n';
 
    HeaderFile << varnames.size() << '\n';
 
    for (int ivar = 0; ivar < varnames.size(); ++ivar) {
        HeaderFile << varnames[ivar] << "\n";
    }
    HeaderFile << AMREX_SPACEDIM << '\n';
    HeaderFile << my_time << '\n';
    HeaderFile << finest_level << '\n';
    for (int i = 0; i < AMREX_SPACEDIM; ++i) {
        HeaderFile << my_geom[0].ProbLo(i) << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i < AMREX_SPACEDIM; ++i) {
        HeaderFile << my_geom[0].ProbHi(i) << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i < finest_level; ++i) {
        HeaderFile << my_ref_ratio[i][0] << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i <= finest_level; ++i) {
        HeaderFile << my_geom[i].Domain() << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i <= finest_level; ++i) {
        HeaderFile << level_steps[i] << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i <= finest_level; ++i) {
        for (int k = 0; k < AMREX_SPACEDIM; ++k) {
            HeaderFile << my_geom[i].CellSize()[k] << ' ';
        }
        HeaderFile << '\n';
    }
    HeaderFile << (int) my_geom[0].Coord() << '\n';
    HeaderFile << "0\n";
 
    for (int level = 0; level <= finest_level; ++level) {
        HeaderFile << level << ' ' << bArray[level].size() << ' ' << my_time << '\n';
        HeaderFile << level_steps[level] << '\n';
 
        const IntVect& domain_lo = my_geom[level].Domain().smallEnd();
        for (int i = 0; i < bArray[level].size(); ++i)
        {
            // Need to shift because the RealBox ctor we call takes the
            // physical location of index (0,0,0).  This does not affect
            // the usual cases where the domain index starts with zero
            const Box& b = shift(bArray[level][i], -domain_lo);
            RealBox loc = RealBox(b, my_geom[level].CellSize(), my_geom[level].ProbLo());
            for (int n = 0; n < AMREX_SPACEDIM; ++n) {
                HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
            }
        }
 
        HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
    }
    HeaderFile << "1" << "\n";
    HeaderFile << "3" << "\n";
    HeaderFile << "amrexvec_nu_x" << "\n";
    HeaderFile << "amrexvec_nu_y" << "\n";
    HeaderFile << "amrexvec_nu_z" << "\n";
    std::string mf_nodal_prefix = "Nu_nd";
    for (int level = 0; level <= finest_level; ++level) {
        HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
    }
}

Here is the call graph for this function:

writeJobInfo()

void ERF::writeJobInfo	(	const std::string &	dir,
		erf_provenance::ArtifactType	artifact_type,
		int	artifact_step,
		double	artifact_time_seconds
	)		const

{
    if (!ParallelDescriptor::IOProcessor()) {
        return;
    }
 
    // job_info file with details about the run
    std::ofstream jobInfoFile;
    std::string FullPathJobInfoFile = dir;
    FullPathJobInfoFile += "/job_info";
    jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out | std::ios::trunc);
    if (!jobInfoFile.good()) {
        FileOpenFailed(FullPathJobInfoFile);
    }
 
    erf_provenance::ProvenanceRecord provenance;
    provenance.execution = execution_provenance;
    provenance.artifact.artifact_uuid = erf_provenance::generate_uuid_v4();
    provenance.artifact.artifact_type = artifact_type;
    provenance.artifact.artifact_step = artifact_step;
    provenance.artifact.artifact_time_seconds = artifact_time_seconds;
    provenance.artifact.artifact_created_utc = erf_provenance::current_utc();
 
    std::string PrettyLine = "==================================================="
      "============================\n";
    std::string OtherLine = "----------------------------------------------------"
      "----------------------------\n";
    std::string SkipSpace = "        ";
 
    // job information
    jobInfoFile << PrettyLine;
    jobInfoFile << " ERF Job Information\n";
    jobInfoFile << PrettyLine;
 
    jobInfoFile << erf_provenance::serialize_provenance_block(provenance);
    jobInfoFile << "\n";
 
    jobInfoFile << "inputs file: " << inputs_name << "\n\n";
 
    jobInfoFile << "number of MPI processes: "
                << ParallelDescriptor::NProcs() << "\n";
#ifdef _OPENMP
    jobInfoFile << "number of threads:       " << omp_get_max_threads() << "\n";
#endif
 
    jobInfoFile << "\n";
    jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
                << getCPUTime() / Real(3600.0);
 
    jobInfoFile << "\n\n";
 
    if (use_datetime) {
        const std::string dt_format = "%Y-%m-%d %H:%M:%S"; // ISO 8601 standard
        jobInfoFile << "Simulation time: " << getTimestamp(start_time+t_new[0], dt_format) << "\n";
        jobInfoFile << "\n\n";
    }
 
    // Output information
    jobInfoFile << PrettyLine;
    jobInfoFile << " Output Information\n";
    jobInfoFile << PrettyLine;
 
    jobInfoFile << "artifact type:    "
                << erf_provenance::artifact_type_token(artifact_type) << "\n";
    jobInfoFile << "artifact step:    " << artifact_step << "\n";
    const auto existing_precision = jobInfoFile.precision();
    jobInfoFile << std::setprecision(17)
                << "simulation time:  " << artifact_time_seconds << "\n";
    jobInfoFile.precision(existing_precision);
 
    time_t now = time(nullptr);
 
    // Convert now to tm struct for local timezone
    tm* localtm = localtime(&now);
    jobInfoFile << "output data / time: " << asctime(localtm);
 
    std::string currentDir = FileSystem::CurrentPath();
    jobInfoFile << "output dir:         " << currentDir << "\n";
 
    jobInfoFile << "\n\n";
 
    // build information
    jobInfoFile << PrettyLine;
    jobInfoFile << " Build Information\n";
    jobInfoFile << PrettyLine;
 
    jobInfoFile << "build date:    " << buildInfoGetBuildDate() << "\n";
    jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
    jobInfoFile << "build dir:     " << buildInfoGetBuildDir() << "\n";
    jobInfoFile << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
 
    jobInfoFile << "\n";
 
    jobInfoFile << "COMP:          " << buildInfoGetComp() << "\n";
    jobInfoFile << "COMP version:  " << buildInfoGetCompVersion() << "\n";
 
    jobInfoFile << "\n";
 
    for (int n = 1; n <= buildInfoGetNumModules(); n++) {
      jobInfoFile << buildInfoGetModuleName(n) << ": "
                  << buildInfoGetModuleVal(n) << "\n";
    }
 
    jobInfoFile << "\n";
 
    const char* githash1 = buildInfoGetGitHash(1);
    const char* githash2 = buildInfoGetGitHash(2);
    if (strlen(githash1) > 0) {
      jobInfoFile << "ERF       git hash: " << githash1 << "\n";
    }
    if (strlen(githash2) > 0) {
      jobInfoFile << "AMReX       git hash: " << githash2 << "\n";
    }
 
    const char* buildgithash = buildInfoGetBuildGitHash();
    const char* buildgitname = buildInfoGetBuildGitName();
    if (strlen(buildgithash) > 0) {
      jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
    }
 
    jobInfoFile << "\n\n";
 
    // grid information
    jobInfoFile << PrettyLine;
    jobInfoFile << " Grid Information\n";
    jobInfoFile << PrettyLine;
 
    int f_lev = finest_level;
 
    for (int i = 0; i <= f_lev; i++) {
      jobInfoFile << " level: " << i << "\n";
      jobInfoFile << "   number of boxes = " << grids[i].size() << "\n";
      jobInfoFile << "   maximum zones   = ";
      for (int n = 0; n < AMREX_SPACEDIM; n++) {
        jobInfoFile << geom[i].Domain().length(n) << " ";
      }
      jobInfoFile << "\n\n";
    }
 
    jobInfoFile << " Boundary conditions\n";
 
    jobInfoFile << "   -x: " << domain_bc_type[0] << "\n";
    jobInfoFile << "   +x: " << domain_bc_type[3] << "\n";
    jobInfoFile << "   -y: " << domain_bc_type[1] << "\n";
    jobInfoFile << "   +y: " << domain_bc_type[4] << "\n";
    jobInfoFile << "   -z: " << domain_bc_type[2] << "\n";
    jobInfoFile << "   +z: " << domain_bc_type[5] << "\n";
 
    jobInfoFile << "\n\n";
 
    // runtime parameters
    jobInfoFile << PrettyLine;
    jobInfoFile << " Inputs File Parameters\n";
    jobInfoFile << PrettyLine;
 
    ParmParse::dumpTable(jobInfoFile, true);
    jobInfoFile.close();
}

Here is the call graph for this function:

WriteLinePlot()

void ERF::WriteLinePlot	(	const std::string &	filename,
		amrex::Vector< std::array< amrex::Real, 2 >> &	points_xy
	)

{
    std::ofstream ofs(filename);
    if (!ofs.is_open()) {
        amrex::Print() << "Error: Could not open file " << filename << " for writing.\n";
        return;
    }
 
    ofs << std::setprecision(10) << std::scientific;
    ofs << "# x y\n";
 
    for (const auto& p : points_xy) {
        ofs << p[0] << " " << p[1] << "\n";
    }
 
    ofs.close();
 
    amrex::Print() << "Line plot data written to " << filename << "\n";
}

WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain	(	const std::string &	plotfilename,
		int	nlevels,
		const amrex::Vector< const amrex::MultiFab * > &	mf,
		const amrex::Vector< const amrex::MultiFab * > &	mf_nd,
		const amrex::Vector< std::string > &	varnames,
		const amrex::Vector< amrex::Geometry > &	my_geom,
		double	time,
		const amrex::Vector< int > &	level_steps,
		const amrex::Vector< amrex::IntVect > &	my_ref_ratio,
		const std::string &	versionName = "HyperCLaw-V1.1",
		const std::string &	levelPrefix = "Level_",
		const std::string &	mfPrefix = "Cell",
		const amrex::Vector< std::string > &	extra_dirs = amrex::Vector<std::string>()
	)		const

{
    BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
 
    AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
    AMREX_ALWAYS_ASSERT(nlevels <= rr.size()+1);
    AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
    AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
 
    bool callBarrier(false);
    PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
    if (!extra_dirs.empty()) {
        for (const auto& d : extra_dirs) {
            const std::string ed = plotfilename+"/"+d;
            PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
        }
    }
    ParallelDescriptor::Barrier();
 
    if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
        Vector<BoxArray> boxArrays(nlevels);
        for(int level(0); level < boxArrays.size(); ++level) {
            boxArrays[level] = mf[level]->boxArray();
        }
 
        auto f = [=,this]() {
            VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
            std::string HeaderFileName(plotfilename + "/Header");
            std::ofstream HeaderFile;
            HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
            HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                                    std::ofstream::trunc |
                                                    std::ofstream::binary);
            if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
            WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
                                                  my_geom, time, level_steps, rr, versionName,
                                                  levelPrefix, mfPrefix);
        };
 
        if (AsyncOut::UseAsyncOut()) {
            AsyncOut::Submit(std::move(f));
        } else {
            f();
        }
    }
 
    std::string mf_nodal_prefix = "Nu_nd";
    for (int level = 0; level <= finest_level; ++level)
    {
        if (AsyncOut::UseAsyncOut()) {
            VisMF::AsyncWrite(*mf[level],
                              MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
                              true);
            VisMF::AsyncWrite(*mf_nd[level],
                              MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
                              true);
        } else {
            const MultiFab* data;
            std::unique_ptr<MultiFab> mf_tmp;
            if (mf[level]->nGrowVect() != 0) {
                mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
                                                    mf[level]->DistributionMap(),
                                                    mf[level]->nComp(), 0, MFInfo(),
                                                    mf[level]->Factory());
                MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
                data = mf_tmp.get();
            } else {
                data = mf[level];
            }
            VisMF::Write(*data        , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
            VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
        }
    }
}

Here is the call graph for this function:

WriteMyEBSurface()

void ERF::WriteMyEBSurface

(

)

writeNow()

bool ERF::writeNow	(	double	cur_time,
		const int	nstep,
		const int	plot_int,
		const double	plot_per,
		const double	dt_0,
		double &	last_file_time
	)

{
    bool write_now = false;
 
    if ( plot_int > 0) {
 
        write_now = (nstep % plot_int == 0);
 
    } else if (plot_per > zero) {
 
        amrex::Print() << "CUR NEXT PER " << cur_time << " " << next_file_time << " " << plot_per << std::endl;
 
        // Only write now if nstep newly matches the number of elapsed periods
        write_now = (cur_time > (next_file_time - 0.1*dt_0));
    }
 
    return write_now;
}

WriteSubvolume()

void ERF::WriteSubvolume	(	int	isub,
		amrex::Vector< std::string >	subvol_var_names
	)

{
    ParmParse pp("erf.subvol");
 
    Vector<Real> origin;
    Vector< int> ncell;
    Vector<Real> delta;
 
    // **************************************************************
    // Read in the origin, number of cells in each dir, and resolution
    // **************************************************************
 
    int lev_for_sub = 0;
    int offset = isub * AMREX_SPACEDIM;
 
    pp.getarr("origin",origin,offset,AMREX_SPACEDIM);
    pp.getarr("nxnynz", ncell,offset,AMREX_SPACEDIM);
    pp.getarr("dxdydz", delta,offset,AMREX_SPACEDIM);
 
    bool found = false;
    for (int i = 0; i <= finest_level; i++) {
        if (!found) {
            if (almostEqual(delta[offset+0],geom[i].CellSize(0)) &&
                almostEqual(delta[offset+1],geom[i].CellSize(1)) &&
                almostEqual(delta[offset+2],geom[i].CellSize(2)) ) {
 
                amrex::Print() << "WriteSubvolume:Resolution specified matches that of level " << i << std::endl;
                found = true;
                lev_for_sub = i;
            }
        }
    }
 
    if (!found) {
        amrex::Abort("Resolution specified for subvol does not match the resolution of any of the levels.");
    }
 
 
    // **************************************************************
    // Now that we know which level we're at, we can figure out which (i,j,k) the origin corresponds to
    // Note we use Real(1.0001) as a fudge factor since the division of two reals --> integer will do a floor
    // **************************************************************
    int i0 = static_cast<int>((origin[offset+0] - geom[lev_for_sub].ProbLo(0)) * Real(1.0001) / delta[offset+0]);
    int j0 = static_cast<int>((origin[offset+1] - geom[lev_for_sub].ProbLo(1)) * Real(1.0001) / delta[offset+1]);
    int k0 = static_cast<int>((origin[offset+2] - geom[lev_for_sub].ProbLo(2)) * Real(1.0001) / delta[offset+2]);
 
    found = false;
    if (almostEqual(geom[lev_for_sub].ProbLo(0)+i0*delta[offset+0],origin[offset+0]) &&
        almostEqual(geom[lev_for_sub].ProbLo(1)+j0*delta[offset+1],origin[offset+1]) &&
        almostEqual(geom[lev_for_sub].ProbLo(2)+k0*delta[offset+2],origin[offset+2]) )
    {
        amrex::Print() << "WriteSubvolume:Specified origin is the lower left corner of cell " << IntVect(i0,j0,k0) << std::endl;
        found = true;
    }
 
    if (!found) {
        amrex::Abort("Origin specified does not correspond to a node at this level.");
    }
 
    Box domain(geom[lev_for_sub].Domain());
 
    Box bx(IntVect(i0,j0,k0),IntVect(i0+ncell[offset+0]-1,j0+ncell[offset+1]-1,k0+ncell[offset+2]-1));
    amrex::Print() << "WriteSubvolume:Box requested is " << bx << std::endl;
 
    if (!domain.contains(bx))
    {
        amrex::Abort("WriteSubvolume:Box requested is larger than the existing domain");
    }
 
    Vector<int> cs(AMREX_SPACEDIM);
    int count = pp.countval("chunk_size");
    if (count > 0) {
        pp.queryarr("chunk_size",cs,0,AMREX_SPACEDIM);
    } else {
        cs[0] = max_grid_size[0][0];
        cs[1] = max_grid_size[0][1];
        cs[2] = max_grid_size[0][2];
    }
    IntVect chunk_size(cs[0],cs[1],cs[2]);
 
    BoxArray ba(bx);
    ba.maxSize(chunk_size);
 
    amrex::Print() << "WriteSubvolume:BoxArray is " << ba << std::endl;
 
    Vector<std::string> varnames;
    varnames.insert(varnames.end(), subvol_var_names.begin(), subvol_var_names.end());
 
    int ncomp_mf = subvol_var_names.size();
 
    DistributionMapping dm(ba);
 
    MultiFab mf(ba, dm, ncomp_mf, 0);
 
    int mf_comp = 0;
 
    // *****************************************************************************************
 
    // First, copy any of the conserved state variables into the output plotfile
    for (int i = 0; i < cons_names.size(); ++i) {
        if (containerHasElement(subvol_var_names, cons_names[i])) {
            mf.ParallelCopy(vars_new[lev_for_sub][Vars::cons],i,mf_comp,1,1,0);
            mf_comp++;
        }
    }
 
    // *****************************************************************************************
 
    if (containerHasElement(subvol_var_names, "x_velocity") ||
        containerHasElement(subvol_var_names, "y_velocity") ||
        containerHasElement(subvol_var_names, "z_velocity"))
    {
        MultiFab mf_cc_vel(grids[lev_for_sub], dmap[lev_for_sub], AMREX_SPACEDIM, 0);
        average_face_to_cellcenter(mf_cc_vel,0,
                                   Array<const MultiFab*,3>{&vars_new[lev_for_sub][Vars::xvel],
                                                            &vars_new[lev_for_sub][Vars::yvel],
                                                            &vars_new[lev_for_sub][Vars::zvel]});
        if (containerHasElement(subvol_var_names, "x_velocity")) {
            mf.ParallelCopy(mf_cc_vel,0,mf_comp,1,0,0);
            mf_comp++;
        }
        if (containerHasElement(subvol_var_names, "y_velocity")) {
            mf.ParallelCopy(mf_cc_vel,1,mf_comp,1,0,0);
            mf_comp++;
        }
        if (containerHasElement(subvol_var_names, "z_velocity")) {
            mf.ParallelCopy(mf_cc_vel,2,mf_comp,1,0,0);
            mf_comp++;
        }
    }
 
    // *****************************************************************************************
 
    // Finally, check for any derived quantities and compute them, inserting
    // them into our output multifab
    auto calculate_derived = [&](const std::string& der_name,
                                 MultiFab& src_mf,
                                 decltype(derived::erf_dernull)& der_function)
    {
        if (containerHasElement(subvol_var_names, der_name)) {
            MultiFab dmf(src_mf.boxArray(), src_mf.DistributionMap(), 1, 0);
            //
            // NOTE: we must not tile in z here because some of the derived quantities
            //       ("precipitable", "mucape") are whole-column operations and require
            //       the full column in each box
            //
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(dmf, TileNoZ()); mfi.isValid(); ++mfi)
            {
                const Box& tbx = mfi.tilebox();
                auto& dfab = dmf[mfi];
                auto& sfab = src_mf[mfi];
                auto& zfab = (*z_phys_cc[lev_for_sub])[mfi];
                der_function(tbx, dfab, 0, 1, sfab, zfab, Geom(lev_for_sub),
                             static_cast<Real>(t_new[0]), nullptr, lev_for_sub);
            }
            mf.ParallelCopy(dmf,0,mf_comp,1,0,0);
            mf_comp++;
        }
    };
 
    // *****************************************************************************************
    // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
    //       defined in ERF.H
    // *****************************************************************************************
 
    // NOTE: the moisture map is the authority on what moist data exists, the same
    //       test setSubVolVariables used when it selected these names.
    const MoistureComponentIndices& mi = solverChoice.moisture_indices;
 
    if (mi.has_moisture()) {
        calculate_derived("temp",    vars_new[lev_for_sub][Vars::cons], derived::erf_dermoisttemp);
    } else {
        calculate_derived("temp",    vars_new[lev_for_sub][Vars::cons], derived::erf_dertemp);
    }
    calculate_derived("theta",       vars_new[lev_for_sub][Vars::cons], derived::erf_dertheta);
    calculate_derived("KE",          vars_new[lev_for_sub][Vars::cons], derived::erf_derKE);
    calculate_derived("scalar",      vars_new[lev_for_sub][Vars::cons], derived::erf_derscalar);
    calculate_derived("soundspeed",  vars_new[lev_for_sub][Vars::cons], derived::erf_dersoundspeed);
    if (mi.has_moisture()) {
        calculate_derived("precipitable", vars_new[lev_for_sub][Vars::cons], derived::erf_derprecipitable);
        calculate_derived("mucape",      vars_new[lev_for_sub][Vars::cons], derived::erf_dermucape);
    }
 
    // *****************************************************************************************
 
    double time = t_new[lev_for_sub];
 
    std::string sf = subvol_file + "_" + std::to_string(isub);
    std::string subvol_filename;
 
    if (use_real_time_in_pltname) {
        const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
        subvol_filename = sf + getTimestamp(start_time+time, dt_format);
    } else {
       subvol_filename = Concatenate(sf + "_", istep[0], file_name_digits);
    }
 
    amrex::Print() <<"Writing subvolume into " << subvol_filename << std::endl;
    WriteSingleLevelPlotfile(subvol_filename,mf,varnames,geom[lev_for_sub],static_cast<amrex::Real>(time),istep[0]);
 
}

Here is the call graph for this function:

WriteVTKPolyline()

void ERF::WriteVTKPolyline	(	const std::string &	filename,
		amrex::Vector< std::array< amrex::Real, 2 >> &	points_xy
	)

{
    std::ofstream vtkfile(filename);
    if (!vtkfile.is_open()) {
        std::cerr << "Error: Cannot open file " << filename << std::endl;
        return;
    }
 
    int num_points = points_xy.size();
    if (num_points == 0) {
        vtkfile << "# vtk DataFile Version three\n";
        vtkfile << "Storm Track\n";
        vtkfile << "ASCII\n";
        vtkfile << "DATASET POLYDATA\n";
        vtkfile << "POINTS " << num_points << " float\n";
        vtkfile.close();
        return;
    }
    if (num_points < 2) {
        points_xy.push_back(points_xy[0]);
    }
    num_points = points_xy.size();
 
    vtkfile << "# vtk DataFile Version three\n";
    vtkfile << "Storm Track\n";
    vtkfile << "ASCII\n";
    vtkfile << "DATASET POLYDATA\n";
 
    // Write points (Z=0 assumed)
    vtkfile << "POINTS " << num_points << " float\n";
    for (const auto& pt : points_xy) {
        vtkfile << pt[0] << " " << pt[1] << " " << 10000.0 << std::endl;
    }
 
    // Write polyline connectivity
    vtkfile << "LINES 1 " << num_points + 1 << "\n";
    vtkfile << num_points << " ";
    for (int i = 0; i < num_points; ++i) {
        vtkfile << i << " ";
    }
    vtkfile << "\n";
 
    vtkfile.close();
}

Member Data Documentation

advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg

private

Referenced by ERF_shared(), and getAdvFluxReg().

avg_xmom

amrex::Vector<amrex::MultiFab> ERF::avg_xmom

private

Referenced by ERF_shared().

avg_ymom

amrex::Vector<amrex::MultiFab> ERF::avg_ymom

private

Referenced by ERF_shared().

avg_zmom

amrex::Vector<amrex::MultiFab> ERF::avg_zmom

private

Referenced by ERF_shared().

ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax

private

Referenced by ERF_shared().

ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src

private

Referenced by ERF_shared().

ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay

private

Referenced by ERF_shared().

ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src

private

Referenced by ERF_shared().

az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az

private

Referenced by ERF_shared().

az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src

private

Referenced by ERF_shared().

ba1d

amrex::Vector<amrex::BoxArray> ERF::ba1d

private

Referenced by ERF_shared().

ba2d

amrex::Vector<amrex::BoxArray> ERF::ba2d

private

Referenced by ERF_shared(), GetOceanToAtmosSurfaceLayout(), and PackAtmosphericStates().

base_state

amrex::Vector<amrex::MultiFab> ERF::base_state

private

Referenced by ERF_shared().

base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new

private

Referenced by ERF_shared().

bckgnd_state

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::bckgnd_state

bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1

staticprivate

bndry_output_planes_per

double ERF::bndry_output_planes_per = -1.0

staticprivate

bndry_output_planes_start_time

double ERF::bndry_output_planes_start_time = 0.0

staticprivate

boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level

private

cf_set_width

int ERF::cf_set_width {0}

private

cf_width

int ERF::cf_width {0}

private

cfl

Real ERF::cfl = Real(0.8)

staticprivate

change_max

Real ERF::change_max = Real(1.1)

staticprivate

check_file

std::string ERF::check_file {"chk"}

private

check_for_nans

int ERF::check_for_nans = 0

staticprivate

cloud_chamber_budget

std::unique_ptr<CloudChamberBudget> ERF::cloud_chamber_budget

private

Referenced by ERF_shared().

cloud_chamber_config

erf_cloud_chamber::Config ERF::cloud_chamber_config

private

Referenced by ERF_shared().

column_file_name

std::string ERF::column_file_name = "column_data.nc"

staticprivate

column_interval

int ERF::column_interval = -1

staticprivate

column_kextent

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::column_kextent

private

Referenced by ERF_shared().

column_kextent_hi_sentinel

constexpr int ERF::column_kextent_hi_sentinel = std::numeric_limits<int>::max()/2

staticconstexprprivate

column_kextent_lo_sentinel

constexpr int ERF::column_kextent_lo_sentinel = std::numeric_limits<int>::min()/2

staticconstexprprivate

column_loc_x

Real ERF::column_loc_x = zero

staticprivate

column_loc_y

Real ERF::column_loc_y = zero

staticprivate

column_per

double ERF::column_per = -1.0

staticprivate

cons_names

const amrex::Vector<std::string> ERF::cons_names

private

Initial value:

{"density", "rhotheta", "rhoKE", "rhoadv_0",
                                                     "rhoQ1", "rhoQ2", "rhoQ3",
                                                     "rhoQ4", "rhoQ5", "rhoQ6",
                                                     "rhoQ7", "rhoQ8", "rhoQ9",
                                                     "rhoQ10", "rhoQ11"}

cosPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::cosPhi_m

private

Referenced by ERF_shared().

d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density

private

d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure

private

d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc

private

d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv

private

d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature

private

d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs

private

Referenced by ERF_shared().

d_sinesq_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_ptrs

private

Referenced by ERF_shared().

d_sinesq_stag_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_stag_ptrs

private

Referenced by ERF_shared().

d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs

private

d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos

private

d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos

private

d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid

private

datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog

private

Referenced by DataLog(), NumDataLogs(), and setRecordDataInfo().

datalogname

amrex::Vector<std::string> ERF::datalogname

private

Referenced by DataLogName().

datetime_format

const std::string ERF::datetime_format = "%Y-%m-%d %H:%M:%S"

private

datprecision

const int ERF::datprecision = 6

private

datwidth

const int ERF::datwidth = 14

private

der_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::der_datalog

private

Referenced by DerDataLog(), NumDerDataLogs(), and setRecordDerDataInfo().

der_datalogname

amrex::Vector<std::string> ERF::der_datalogname

private

Referenced by DerDataLogName().

derived_names

const amrex::Vector<std::string> ERF::derived_names

private

derived_subvol_names

const amrex::Vector<std::string> ERF::derived_subvol_names

private

Initial value:

{"soundspeed", "temp", "theta", "KE", "scalar",
                                                           "precipitable", "mucape"}

destag_profiles

bool ERF::destag_profiles = true

private

detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc

private

Referenced by ERF_shared().

detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new

private

Referenced by ERF_shared().

detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src

private

Referenced by ERF_shared().

domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type

private

domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type

private

domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d

private

dt

amrex::Vector<double> ERF::dt

private

Referenced by ERF_shared(), and EvolveOneStep().

dt_max

double ERF::dt_max = 1.0e9

staticprivate

dt_max_initial

double ERF::dt_max_initial = static_cast<double>(bogus_large_value)

staticprivate

Referenced by ERF_shared().

dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio

private

Referenced by ERF_shared().

dz_min

amrex::Vector<amrex::Real> ERF::dz_min

private

eb

amrex::Vector<std::unique_ptr<eb_> > ERF::eb

private

Referenced by EBFactory(), ERF_shared(), and get_eb().

eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev

private

Referenced by ERF_shared().

erfbdy_file

std::string ERF::erfbdy_file {"erfbdy"}

private

execution_provenance

erf_provenance::ExecutionProvenance ERF::execution_provenance

private

Referenced by ERF_shared().

file_name_digits

int ERF::file_name_digits = 5

private

fine_mask

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::fine_mask

private

Referenced by ERF_shared().

finished_wave

bool ERF::finished_wave = false

private

fixed_dt

amrex::Vector<double> ERF::fixed_dt

private

fixed_fast_dt

amrex::Vector<double> ERF::fixed_fast_dt

private

fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0

staticprivate

forecast_state_1

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_1

Referenced by ERF_shared().

forecast_state_2

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_2

Referenced by ERF_shared().

forecast_state_interp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_interp

Referenced by ERF_shared().

FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c

private

FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u

private

FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v

private

FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w

private

gradp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::gradp

private

Referenced by ERF_shared().

h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density

private

h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure

private

h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc

private

h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv

private

h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature

private

h_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs

private

Referenced by ERF_shared().

h_sinesq_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_ptrs

private

Referenced by ERF_shared().

h_sinesq_stag_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_stag_ptrs

private

Referenced by ERF_shared().

h_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs

private

h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos

private

h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos

private

h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid

private

have_read_nc_init_file

Vector< Vector< int > > ERF::have_read_nc_init_file = {{0}}

staticprivate

hfx3_EB

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::hfx3_EB

private

Referenced by ERF_shared().

hurricane_eye_track_latlon

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_latlon

hurricane_eye_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_xy

hurricane_maxvel_vs_time

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_maxvel_vs_time

hurricane_minpressure_vs_time

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_minpressure_vs_time

hurricane_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_track_xy

hurricane_tracker_circle

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_tracker_circle

Hwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave

private

Referenced by ERF_shared().

Hwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave_onegrid

private

Referenced by ERF_shared().

ibseb_params

IBSEBParams ERF::ibseb_params

private

init_shrink

Real ERF::init_shrink = one

staticprivate

input_bndry_planes

int ERF::input_bndry_planes = 0

staticprivate

input_sounding_data

InputSoundingData ERF::input_sounding_data

private

input_sponge_data

InputSpongeData ERF::input_sponge_data

private

interpolation_type

StateInterpType ERF::interpolation_type

staticprivate

interval_means

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::interval_means

private

Referenced by ERF_shared().

istep

amrex::Vector<int> ERF::istep

private

Referenced by EvolveOneStep().

lagged_delta_rt

amrex::Vector<amrex::MultiFab> ERF::lagged_delta_rt

private

Referenced by ERF_shared().

land_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::land_type_lev

private

Referenced by ERF_shared().

last_check_file_step

int ERF::last_check_file_step = -1

staticprivate

last_check_file_time

double ERF::last_check_file_time = 0.0

staticprivate

last_plot2d_file_step_1

int ERF::last_plot2d_file_step_1 = -1

staticprivate

last_plot2d_file_step_2

int ERF::last_plot2d_file_step_2 = -1

staticprivate

last_plot2d_file_time_1

double ERF::last_plot2d_file_time_1 = 0.0

staticprivate

last_plot2d_file_time_2

double ERF::last_plot2d_file_time_2 = 0.0

staticprivate

last_plot3d_file_step_1

int ERF::last_plot3d_file_step_1 = -1

staticprivate

last_plot3d_file_step_2

int ERF::last_plot3d_file_step_2 = -1

staticprivate

last_plot3d_file_time_1

double ERF::last_plot3d_file_time_1 = 0.0

staticprivate

last_plot3d_file_time_2

double ERF::last_plot3d_file_time_2 = 0.0

staticprivate

last_subvol_step

amrex::Vector<int> ERF::last_subvol_step

private

last_subvol_time

amrex::Vector<double> ERF::last_subvol_time

private

lat_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lat_m

private

Referenced by ERF_shared(), and GetOceanToAtmosCellCenterLonLat().

line_sampler

std::unique_ptr<LineSampler> ERF::line_sampler = nullptr

private

line_sampling_interval

int ERF::line_sampling_interval = -1

private

line_sampling_per

amrex::Real ERF::line_sampling_per = -one

private

lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev

private

Referenced by ERF_shared(), and GetLandMask().

lon_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lon_m

private

Referenced by ERF_shared(), and GetOceanToAtmosCellCenterLonLat().

lsf

LargeScaleForcingData ERF::lsf

private

lsf_data

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lsf_data

private

Referenced by ERF_shared().

lsm

LandSurface ERF::lsm

private

Referenced by ERF_shared().

lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data

private

Referenced by ERF_shared().

lsm_data_name

amrex::Vector<std::string> ERF::lsm_data_name

private

lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux

private

Referenced by ERF_shared().

lsm_flux_name

amrex::Vector<std::string> ERF::lsm_flux_name

private

Lwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave

private

Referenced by ERF_shared().

Lwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave_onegrid

private

Referenced by ERF_shared().

m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_extdir_vals

private

m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_neumann_vals

private

m_bc_nonreflecting

amrex::Array<bool, AMREX_SPACEDIM*2> ERF::m_bc_nonreflecting = {{false}}

private

m_check_int

int ERF::m_check_int = -1

private

m_check_per

double ERF::m_check_per = -1.0

private

m_coupled_sst

std::unique_ptr<amrex::MultiFab> ERF::m_coupled_sst

Referenced by ApplyOceanSurfaceState().

m_coupled_sst_valid

std::unique_ptr<amrex::iMultiFab> ERF::m_coupled_sst_valid

Referenced by ApplyOceanSurfaceState().

m_driver_atm2ocn_uses_state_contract

bool ERF::m_driver_atm2ocn_uses_state_contract = true

Referenced by ConfigureDriverAtmosToOceanCoupling(), and SetDriverAtmosToOceanStateContract().

m_driver_has_atm2ocn_coupling

bool ERF::m_driver_has_atm2ocn_coupling = false

Referenced by ConfigureDriverAtmosToOceanCoupling().

m_driver_uses_two_way_coupling

bool ERF::m_driver_uses_two_way_coupling = false

Referenced by ConfigureDriverAtmosToOceanCoupling().

m_expand_plotvars_to_unif_rr

bool ERF::m_expand_plotvars_to_unif_rr = false

private

m_forest_drag

amrex::Vector<std::unique_ptr<ForestDrag> > ERF::m_forest_drag

private

Referenced by ERF_shared().

m_ibseb

amrex::Vector<std::unique_ptr<IBFaceSet> > ERF::m_ibseb

private

m_plot2d_int_1

int ERF::m_plot2d_int_1 = -1

private

m_plot2d_int_2

int ERF::m_plot2d_int_2 = -1

private

m_plot2d_per_1

double ERF::m_plot2d_per_1 = -1.0

private

m_plot2d_per_2

double ERF::m_plot2d_per_2 = -1.0

private

m_plot3d_int_1

int ERF::m_plot3d_int_1 = -1

private

m_plot3d_int_2

int ERF::m_plot3d_int_2 = -1

private

m_plot3d_per_1

double ERF::m_plot3d_per_1 = -1.0

private

m_plot3d_per_2

double ERF::m_plot3d_per_2 = -1.0

private

m_plot_face_terrain_blanking

bool ERF::m_plot_face_terrain_blanking = false

private

m_plot_face_vels

bool ERF::m_plot_face_vels = false

private

m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr

private

m_subvol_int

amrex::Vector<int> ERF::m_subvol_int

private

m_subvol_per

amrex::Vector<double> ERF::m_subvol_per

private

m_SurfaceLayer

amrex::Vector<std::unique_ptr<SurfaceLayer> > ERF::m_SurfaceLayer

private

Referenced by ApplyOceanSurfaceState(), and ERF_shared().

m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr

private

m_wall_scalar_bc_intent

amrex::Array<amrex::Array<erf_wall_scalar_bc::WallScalarBCIntent, AMREX_SPACEDIM*2>, NBCVAR_max> ERF::m_wall_scalar_bc_intent {}

private

m_warned_coupled_sst_declined

bool ERF::m_warned_coupled_sst_declined = false

Referenced by ApplyOceanSurfaceState().

mapfac

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::mapfac

private

Referenced by ERF_shared().

max_step

int ERF::max_step = -1

private

mean_vars_time_reset_done

int ERF::mean_vars_time_reset_done {0}

private

Referenced by ERF_shared().

metgrid_basic_linear

bool ERF::metgrid_basic_linear {false}

private

metgrid_debug_dry

bool ERF::metgrid_debug_dry {false}

private

metgrid_debug_isothermal

bool ERF::metgrid_debug_isothermal {false}

private

metgrid_debug_msf

bool ERF::metgrid_debug_msf {false}

private

metgrid_debug_psfc

bool ERF::metgrid_debug_psfc {false}

private

metgrid_debug_quiescent

bool ERF::metgrid_debug_quiescent {false}

private

metgrid_force_sfc_k

int ERF::metgrid_force_sfc_k {6}

private

metgrid_interp_theta

bool ERF::metgrid_interp_theta {false}

private

metgrid_order

int ERF::metgrid_order {2}

private

metgrid_proximity

amrex::Real ERF::metgrid_proximity {amrex::Real(500.0)}

private

metgrid_retain_sfc

bool ERF::metgrid_retain_sfc {false}

private

metgrid_use_below_sfc

bool ERF::metgrid_use_below_sfc {true}

private

metgrid_use_sfc

bool ERF::metgrid_use_sfc {true}

private

mf_PSFC

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_PSFC

private

Referenced by ERF_shared().

mg_verbose

int ERF::mg_verbose = 0

staticprivate

micro

std::unique_ptr<Microphysics> ERF::micro

private

mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem

private

Referenced by ERF_shared().

native_shoc_driver

amrex::Vector<std::unique_ptr<ShocDriver> > ERF::native_shoc_driver

private

Referenced by ERF_shared().

nc_bdy_file

std::string ERF::nc_bdy_file

staticprivate

nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}

staticprivate

nc_low_file

std::string ERF::nc_low_file

staticprivate

ng_dens_hse

int ERF::ng_dens_hse

staticprivate

ng_pres_hse

int ERF::ng_pres_hse

staticprivate

nsubsteps

amrex::Vector<int> ERF::nsubsteps

private

nudge_data

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::nudge_data

private

Referenced by ERF_shared().

num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level

private

num_files_at_level

amrex::Vector<int> ERF::num_files_at_level

private

nvars_erfbdy

int ERF::nvars_erfbdy {0}

private

output_1d_column

int ERF::output_1d_column = 0

staticprivate

output_bndry_planes

int ERF::output_bndry_planes = 0

staticprivate

pert_interval

int ERF::pert_interval = -1

staticprivate

phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type

private

physbcs_base

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_base> > ERF::physbcs_base

private

Referenced by ERF_shared().

physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons

private

Referenced by ERF_shared().

physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u

private

Referenced by ERF_shared().

physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v

private

Referenced by ERF_shared().

physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w

private

Referenced by ERF_shared().

plane_sampler

std::unique_ptr<PlaneSampler> ERF::plane_sampler = nullptr

private

plane_sampling_interval

int ERF::plane_sampling_interval = -1

private

plane_sampling_per

amrex::Real ERF::plane_sampling_per = -one

private

plot2d_file_1

std::string ERF::plot2d_file_1 {"plt2d_1_"}

private

plot2d_file_2

std::string ERF::plot2d_file_2 {"plt2d_2_"}

private

plot2d_var_names_1

amrex::Vector<std::string> ERF::plot2d_var_names_1

private

plot2d_var_names_2

amrex::Vector<std::string> ERF::plot2d_var_names_2

private

plot3d_file_1

std::string ERF::plot3d_file_1 {"plt_1_"}

private

plot3d_file_2

std::string ERF::plot3d_file_2 {"plt_2_"}

private

plot3d_var_names_1

amrex::Vector<std::string> ERF::plot3d_var_names_1

private

Referenced by ERF_shared().

plot3d_var_names_2

amrex::Vector<std::string> ERF::plot3d_var_names_2

private

Referenced by ERF_shared().

plot_file_on_restart

bool ERF::plot_file_on_restart = true

staticprivate

plot_lsm

bool ERF::plot_lsm = false

private

plot_rad

bool ERF::plot_rad = false

private

plotfile2d_type_1

PlotFileType ERF::plotfile2d_type_1 = PlotFileType::None

staticprivate

plotfile2d_type_2

PlotFileType ERF::plotfile2d_type_2 = PlotFileType::None

staticprivate

plotfile3d_type_1

PlotFileType ERF::plotfile3d_type_1 = PlotFileType::None

staticprivate

plotfile3d_type_2

PlotFileType ERF::plotfile3d_type_2 = PlotFileType::None

staticprivate

pp_inc

amrex::Vector<amrex::MultiFab> ERF::pp_inc

private

Referenced by ERF_shared().

pp_prefix

std::string ERF::pp_prefix {"erf"}

previousCPUTimeUsed

double ERF::previousCPUTimeUsed = 0.0

staticprivate

Referenced by getCPUTime().

prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr

private

Referenced by ERF_shared().

profile_int

int ERF::profile_int = -1

private

qfx3_EB

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::qfx3_EB

private

qheating_rates

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::qheating_rates

private

Referenced by ERF_shared().

qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist

private

Qr_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qr_prim

private

Referenced by ERF_shared().

Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim

private

Referenced by ERF_shared().

r_plane_avg

amrex::Vector<amrex::TableData<amrex::Real, 1> > ERF::r_plane_avg

private

Referenced by ERF_shared().

rad

amrex::Vector<std::unique_ptr<IRadiation> > ERF::rad

private

Referenced by ERF_shared().

rad_datalog_int

int ERF::rad_datalog_int = -1

private

Referenced by ERF_shared().

rad_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rad_fluxes

private

Referenced by ERF_shared(), HasRadiation(), and PackAtmosphericStates().

rad_interp_from_coarse_pending

amrex::Vector<int> ERF::rad_interp_from_coarse_pending

private

real_extrap_w

bool ERF::real_extrap_w {true}

private

real_width

int ERF::real_width {0}

private

ref_tag_indicator_names

Vector< std::string > ERF::ref_tag_indicator_names

staticprivate

ref_tags

Vector< AMRErrorTag > ERF::ref_tags

staticprivate

regrid_int

int ERF::regrid_int = -1

private

regrid_level_0_on_restart

bool ERF::regrid_level_0_on_restart = false

private

restart_chkfile

std::string ERF::restart_chkfile = ""

private

rhoqt_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rhoqt_src

private

Referenced by ERF_shared().

rhotheta_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rhotheta_src

private

Referenced by ERF_shared().

rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new

private

Referenced by ERF_shared().

rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old

private

Referenced by ERF_shared().

rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new

private

Referenced by ERF_shared().

rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old

private

Referenced by ERF_shared().

rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new

private

Referenced by ERF_shared().

rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old

private

Referenced by ERF_shared().

sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline

private

Referenced by NumSampleLines(), and SampleLine().

samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog

private

Referenced by NumSampleLineLogs(), SampleLineLog(), and setRecordSampleLineInfo().

samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname

private

Referenced by SampleLineLogName().

samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint

private

Referenced by NumSamplePoints(), and SamplePoint().

sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog

private

Referenced by NumSamplePointLogs(), SamplePointLog(), and setRecordSamplePointInfo().

sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname

private

Referenced by SamplePointLogName().

SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev

private

Referenced by ERF_shared().

SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev

private

Referenced by ERF_shared().

SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev

private

Referenced by ERF_shared().

SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev

private

Referenced by ERF_shared(), GetFillableAtmosToOceanLanes(), and PackAtmosphericStates().

SFS_q1fx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx1_lev

private

Referenced by ERF_shared().

SFS_q1fx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx2_lev

private

Referenced by ERF_shared().

SFS_q1fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx3_lev

private

Referenced by ERF_shared(), GetFillableAtmosToOceanLanes(), and PackAtmosphericStates().

SFS_q2fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q2fx3_lev

private

Referenced by ERF_shared().

sinPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sinPhi_m

private

Referenced by ERF_shared().

SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev

private

Referenced by ERF_shared().

soil_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::soil_type_lev

private

Referenced by ERF_shared().

solverChoice

SolverChoice ERF::solverChoice

staticprivate

Referenced by ApplyOceanSurfaceState(), ERF_shared(), GetFillableAtmosToOceanLanes(), HasCloudWater(), and PackAtmosphericStates().

sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev

private

Referenced by ERF_shared().

start_time

double ERF::start_time = 0.0

staticprivate

Referenced by EvolveOneStep().

startCPUTime

double ERF::startCPUTime = 0.0

staticprivate

Referenced by getCPUTime().

station_sampler

std::unique_ptr<StationSampler> ERF::station_sampler = nullptr

private

station_sampling_interval

int ERF::station_sampling_interval = -1

private

station_sampling_per

amrex::Real ERF::station_sampling_per = -one

private

station_vars_2d

amrex::Vector<std::string> ERF::station_vars_2d

private

station_vars_3d

amrex::Vector<std::string> ERF::station_vars_3d

private

stop_time

double ERF::stop_time = std::numeric_limits<double>::max()

staticprivate

Referenced by EvolveOneStep().

stretched_dz_d

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::stretched_dz_d

private

Referenced by ERF_shared().

stretched_dz_h

amrex::Vector<amrex::Vector<amrex::Real> > ERF::stretched_dz_h

private

Referenced by ERF_shared().

sub_cfl

Real ERF::sub_cfl = one

staticprivate

subdomains

amrex::Vector<amrex::Vector<amrex::BoxArray> > ERF::subdomains

private

subvol3d_var_names

amrex::Vector<std::string> ERF::subvol3d_var_names

private

Referenced by ERF_shared().

subvol_file

std::string ERF::subvol_file {"subvol"}

private

sum_interval

int ERF::sum_interval = -1

staticprivate

sum_per

double ERF::sum_per = -1.0

staticprivate

surface_state_1

amrex::Vector<amrex::MultiFab> ERF::surface_state_1

Referenced by ERF_shared().

surface_state_2

amrex::Vector<amrex::MultiFab> ERF::surface_state_2

Referenced by ERF_shared().

surface_state_interp

amrex::Vector<amrex::MultiFab> ERF::surface_state_interp

Referenced by ERF_shared().

t_avg_cnt

amrex::Vector<double> ERF::t_avg_cnt

private

Referenced by ERF_shared().

t_mean_cnt

amrex::Vector<double> ERF::t_mean_cnt

private

Referenced by ERF_shared().

t_new

amrex::Vector<double> ERF::t_new

private

Referenced by ERF_shared(), and EvolveOneStep().

t_old

amrex::Vector<double> ERF::t_old

private

Referenced by ERF_shared().

t_plane_avg

amrex::Vector<amrex::TableData<amrex::Real, 1> > ERF::t_plane_avg

private

Referenced by ERF_shared().

Tau

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau

private

Referenced by ERF_shared(), GetFillableAtmosToOceanLanes(), and PackAtmosphericStates().

Tau_corr

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau_corr

private

Referenced by ERF_shared().

Tau_EB

amrex::Vector<amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > > ERF::Tau_EB

private

Referenced by ERF_shared().

terrain_blanking

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking

private

Referenced by ERF_shared().

terrain_blanking_xface

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking_xface

private

Referenced by ERF_shared().

terrain_blanking_yface

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking_yface

private

Referenced by ERF_shared().

terrain_blanking_zface

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking_zface

private

Referenced by ERF_shared().

th_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::th_bc_data

private

Referenced by ERF_shared().

Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim

private

Referenced by ERF_shared().

thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce

private

Referenced by ERF_shared().

thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce

private

Referenced by ERF_shared().

thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce

private

Referenced by ERF_shared().

timeprecision

const int ERF::timeprecision = 13

private

tot_e_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::tot_e_datalog

private

Referenced by setRecordEnergyDataInfo().

tot_e_datalogname

amrex::Vector<std::string> ERF::tot_e_datalogname

private

tsk_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::tsk_lev

private

Referenced by ERF_shared().

turbPert

TurbulentPerturbation ERF::turbPert

private

two_stream_rad

TwoStreamRadiation ERF::two_stream_rad

private

Referenced by ERF_shared().

urb_frac_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::urb_frac_lev

private

Referenced by ERF_shared().

use_datetime

bool ERF::use_datetime = false

private

use_erfbdy

bool ERF::use_erfbdy {false}

private

use_fft

bool ERF::use_fft = false

staticprivate

use_real_time_in_pltname

bool ERF::use_real_time_in_pltname = false

private

user_specified_inputs

std::set<std::string> ERF::user_specified_inputs

vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new

private

Referenced by ERF_shared(), and PackAtmosphericStates().

vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old

private

Referenced by ERF_shared().

vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg

private

Referenced by ERF_shared().

verbose

int ERF::verbose = 0

staticprivate

Referenced by PackAtmosphericStates().

walldist

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::walldist

private

Referenced by ERF_shared().

weather_forecast_data_1

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_1

weather_forecast_data_2

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_2

wrf_bsp

BaseStateParams ERF::wrf_bsp

private

wrf_C1H

std::unique_ptr<amrex::MultiFab> ERF::wrf_C1H

private

wrf_C2H

std::unique_ptr<amrex::MultiFab> ERF::wrf_C2H

private

wrf_MUB

std::unique_ptr<amrex::MultiFab> ERF::wrf_MUB

private

wrf_PHB

std::unique_ptr<amrex::MultiFab> ERF::wrf_PHB

private

wrf_RDNW

std::unique_ptr<amrex::MultiFab> ERF::wrf_RDNW

private

write_erfbdy

bool ERF::write_erfbdy {false}

private

xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask

private

Referenced by ERF_shared().

xvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::xvel_bc_data

private

Referenced by ERF_shared().

yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask

private

Referenced by ERF_shared().

yvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::yvel_bc_data

private

Referenced by ERF_shared().

z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc

private

Referenced by ERF_shared().

z_phys_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc_src

private

Referenced by ERF_shared().

z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd

private

Referenced by ERF_shared().

z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new

private

Referenced by ERF_shared().

z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src

private

Referenced by ERF_shared().

z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk

private

Referenced by ERF_shared().

zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask

private

Referenced by ERF_shared().

zlevels_stag

amrex::Vector<amrex::Vector<amrex::Real> > ERF::zlevels_stag

private

Referenced by ERF_shared().

zmom_crse_rhs

amrex::Vector<amrex::MultiFab> ERF::zmom_crse_rhs

private

Referenced by ERF_shared().

zvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::zvel_bc_data

private

Referenced by ERF_shared().

---

The documentation for this class was generated from the following files:

• Source/ERF.H
• Source/BoundaryConditions/ERF_BoundaryConditionsBndryReg.cpp
• Source/BoundaryConditions/ERF_FillBdyCCVels.cpp
• Source/BoundaryConditions/ERF_FillCoarsePatch.cpp
• Source/BoundaryConditions/ERF_FillIntermediatePatch.cpp
• Source/BoundaryConditions/ERF_FillPatch.cpp
• Source/Coupling/ERF_to_REMORA.cpp
• Source/EB/ERF_EBInit.cpp
• Source/ERF.cpp
• Source/ERF_Constructors.cpp
• Source/ERF_Diagnostics.cpp
• Source/ERF_MakeNewArrays.cpp
• Source/ERF_MakeNewLevel.cpp
• Source/ImmersedBoundarySEB/ERF_IBSEB.cpp
• Source/Initialization/ERF_Init1D.cpp
• Source/Initialization/ERF_InitBCs.cpp
• Source/Initialization/ERF_InitCustomPertState.cpp
• Source/Initialization/ERF_InitForEnsemble.cpp
• Source/Initialization/ERF_InitFromHSE.cpp
• Source/Initialization/ERF_InitFromInputSounding.cpp
• Source/Initialization/ERF_InitGeowind.cpp
• Source/Initialization/ERF_InitImmersedForcing.cpp
• Source/Initialization/ERF_InitRayleigh.cpp
• Source/Initialization/ERF_InitSponge.cpp
• Source/Initialization/ERF_InitTurbPert.cpp
• Source/IO/ERF_Checkpoint.cpp
• Source/IO/ERF_ConsoleIO.cpp
• Source/IO/ERF_Plotfile.cpp
• Source/IO/ERF_Plotfile2D.cpp
• Source/IO/ERF_StationSampler.cpp
• Source/IO/ERF_TrackerOutput.cpp
• Source/IO/ERF_Write1DProfiles.cpp
• Source/IO/ERF_Write1DProfiles_stag.cpp
• Source/IO/ERF_WriteJobInfo.cpp
• Source/IO/ERF_WriteScalarProfiles.cpp
• Source/IO/ERF_WriteSubvolume.cpp
• Source/LinearSolvers/ERF_ComputeDivergence.cpp
• Source/LinearSolvers/ERF_ImposeBCsOnPhi.cpp
• Source/LinearSolvers/ERF_PoissonSolve.cpp
• Source/LinearSolvers/ERF_PoissonSolve_tb.cpp
• Source/LinearSolvers/ERF_PoissonWallDist.cpp
• Source/LinearSolvers/ERF_SolveWithGMRES.cpp
• Source/PBL/Shoc/ERF_ShocCoupling.cpp
• Source/Refinement/ERF_RefineBox.cpp
• Source/Refinement/ERF_RefineHurricane.cpp
• Source/Refinement/ERF_Tagging.cpp
• Source/TimeIntegration/ERF_Advance.cpp
• Source/TimeIntegration/ERF_AdvanceDycore.cpp
• Source/TimeIntegration/ERF_AdvanceLSM.cpp
• Source/TimeIntegration/ERF_AdvanceMicrophysics.cpp
• Source/TimeIntegration/ERF_AdvanceRadiation.cpp
• Source/TimeIntegration/ERF_ComputeTimestep.cpp
• Source/TimeIntegration/ERF_TimeStep.cpp
• Source/Utils/ERF_AverageDown.cpp
• Source/Utils/ERF_HurricaneDiagnostics.cpp
• Source/Utils/ERF_MakeSubdomains.cpp
• Source/Utils/ERF_SurfaceDataInterpolation.cpp
• Source/Utils/ERF_VolWgtSum.cpp
• Source/Utils/ERF_WeatherDataInterpolation.cpp