ERF Class Reference

#include <ERF.H>

Inheritance diagram for ERF:

Collaboration diagram for ERF:

Public Member Functions
	ERF ()
	~ERF () override
void	ERF_shared ()
	ERF (ERF &&) noexcept=delete
ERF &	operator= (ERF &&other) noexcept=delete
	ERF (const ERF &other)=delete
ERF &	operator= (const ERF &other)=delete
void	Evolve ()
void	ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override
void	InitData ()
void	InitData_pre ()
void	InitData_post ()
void	compute_divergence (int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
void	project_velocities (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars, amrex::MultiFab &p)
void	project_velocities_tb (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars, amrex::MultiFab &p)
void	poisson_wall_dist (int lev)
void	solve_with_gmres (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
void	solve_with_mlmg (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM >	get_projection_bc (amrex::Orientation::Side side) const noexcept
bool	projection_has_dirichlet (amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > bcs) const
void	init_only (int lev, amrex::Real time)
void	restart ()
bool	writeNow (const amrex::Real cur_time, const amrex::Real dt, const int nstep, const int plot_int, const amrex::Real plot_per)
void	post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev)
void	sum_integrated_quantities (amrex::Real time)
void	write_1D_profiles (amrex::Real time)
void	write_1D_profiles_stag (amrex::Real time)
amrex::Real	cloud_fraction (amrex::Real time)
void	FillBdyCCVels (amrex::Vector< amrex::MultiFab > &mf_cc_vel)
void	sample_points (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
void	sample_lines (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
void	derive_diag_profiles (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
void	derive_diag_profiles_stag (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
void	derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
void	derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
amrex::Real	volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, const amrex::MultiFab &mapfac, bool local, bool finemask)
void	MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
void	RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
void	ClearLevel (int lev) override
void	MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
amrex::Real	estTimeStep (int lev, long &dt_fast_ratio) const
void	advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
void	advance_microphysics (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
void	advance_lsm (int lev, amrex::MultiFab &, const amrex::Real &dt_advance)
amrex::MultiFab &	build_fine_mask (int lev)
void	MakeHorizontalAverages ()
void	MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)
void	derive_upwp (amrex::Vector< amrex::Real > &h_havg)
void	WritePlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
void	WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
void	WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
void	erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, std::unique_ptr< amrex::MultiFab > &z_cc)
void	init_from_input_sounding (int lev)
void	input_sponge (int lev)
void	init_from_hse (int lev)
void	init_thin_body (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
void	fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
void	AverageDownTo (int crse_lev, int scomp, int ncomp)
void	WriteCheckpointFile () const
void	ReadCheckpointFile ()
void	ReadCheckpointFileMOST ()
void	writeJobInfo (const std::string &dir) const

Static Public Member Functions
static bool	is_it_time_for_action (int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
static void	writeBuildInfo (std::ostream &os)
static void	print_banner (MPI_Comm, std::ostream &)
static void	print_usage (MPI_Comm, std::ostream &)
static void	print_error (MPI_Comm, const std::string &msg)
static void	print_summary (std::ostream &)
static void	print_tpls (std::ostream &)

Public Attributes
std::string	pp_prefix {"erf"}

Private Member Functions
void	ReadParameters ()
void	ParameterSanityChecks ()
void	AverageDown ()
void	update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
void	init_zphys (int lev, amrex::Real time)
void	remake_zphys (int lev, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
void	update_terrain_arrays (int lev)
void	Construct_ERFFillPatchers (int lev)
void	Define_ERFFillPatchers (int lev)
void	init1DArrays ()
void	init_bcs ()
void	init_custom (int lev)
void	init_uniform (int lev)
void	init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
void	turbPert_update (const int lev, const amrex::Real dt)
void	turbPert_amplitude (const int lev)
void	initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
void	make_physbcs (int lev)
void	initializeMicrophysics (const int &)
void	FillPatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
void	FillPatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
void	FillIntermediatePatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons, bool allow_most_bcs=true)
void	FillCoarsePatch (int lev, amrex::Real time)
void	timeStep (int lev, amrex::Real time, int iteration)
void	Advance (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
void	initHSE ()
	Initialize HSE. More...
void	initHSE (int lev)
void	initRayleigh ()
	Initialize Rayleigh damping profiles. More...
void	initSponge ()
	Initialize sponge profiles. More...
void	setRayleighRefFromSounding (bool restarting)
	Set Rayleigh mean profiles from input sounding. More...
void	setSpongeRefFromSounding (bool restarting)
	Set sponge mean profiles from input sounding. More...
void	ComputeDt (int step=-1)
std::string	PlotFileName (int lev) const
void	setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
void	appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
void	init_Dirichlet_bc_data (const std::string input_file)
void	InitializeFromFile ()
void	InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)
void	post_update (amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
void	fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
void	init_geo_wind_profile (const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
void	refinement_criteria_setup ()
AMREX_FORCE_INLINE amrex::YAFluxRegister *	getAdvFluxReg (int lev)
AMREX_FORCE_INLINE std::ostream &	DataLog (int i)
AMREX_FORCE_INLINE int	NumDataLogs () noexcept
AMREX_FORCE_INLINE std::ostream &	SamplePointLog (int i)
AMREX_FORCE_INLINE int	NumSamplePointLogs () noexcept
AMREX_FORCE_INLINE std::ostream &	SampleLineLog (int i)
AMREX_FORCE_INLINE int	NumSampleLineLogs () noexcept
amrex::IntVect &	SamplePoint (int i)
AMREX_FORCE_INLINE int	NumSamplePoints () noexcept
amrex::IntVect &	SampleLine (int i)
AMREX_FORCE_INLINE int	NumSampleLines () noexcept
void	setRecordDataInfo (int i, const std::string &filename)
void	setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
void	setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
std::string	DataLogName (int i) const noexcept
	The filename of the ith datalog file. More...
std::string	SamplePointLogName (int i) const noexcept
	The filename of the ith sampleptlog file. More...
std::string	SampleLineLogName (int i) const noexcept
	The filename of the ith samplelinelog file. More...

Static Private Member Functions
static amrex::Vector< std::string >	PlotFileVarNames (amrex::Vector< std::string > plot_var_names)
static void	GotoNextLine (std::istream &is)
static AMREX_FORCE_INLINE int	ComputeGhostCells (const AdvChoice &advChoice, bool use_num_diff)
static amrex::Real	getCPUTime ()

Private Attributes
InputSoundingData	input_sounding_data
InputSpongeData	input_sponge_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	xvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	yvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	zvel_bc_data
std::unique_ptr< ProblemBase >	prob = nullptr
amrex::Vector< int >	num_boxes_at_level
amrex::Vector< int >	num_files_at_level
amrex::Vector< amrex::Vector< amrex::Box > >	boxes_at_level
amrex::Vector< int >	istep
amrex::Vector< int >	nsubsteps
amrex::Vector< amrex::Real >	t_new
amrex::Vector< amrex::Real >	t_old
amrex::Vector< amrex::Real >	dt
amrex::Vector< long >	dt_mri_ratio
amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_new
amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_old
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vel_t_avg
amrex::Vector< amrex::Real >	t_avg_cnt
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > >	mri_integrator_mem
amrex::Vector< amrex::MultiFab >	pp_inc
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > >	physbcs_cons
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > >	physbcs_u
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > >	physbcs_v
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > >	physbcs_w
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > >	physbcs_base
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Theta_prim
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Qv_prim
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Qr_prim
amrex::Vector< amrex::MultiFab >	rU_old
amrex::Vector< amrex::MultiFab >	rU_new
amrex::Vector< amrex::MultiFab >	rV_old
amrex::Vector< amrex::MultiFab >	rV_new
amrex::Vector< amrex::MultiFab >	rW_old
amrex::Vector< amrex::MultiFab >	rW_new
amrex::Vector< amrex::MultiFab >	zmom_crse_rhs
std::unique_ptr< Microphysics >	micro
amrex::Vector< amrex::Vector< amrex::MultiFab * > >	qmoist
LandSurface	lsm
amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_data
amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_flux
int	cf_width {0}
int	cf_set_width {0}
amrex::Vector< ERFFillPatcher >	FPr_c
amrex::Vector< ERFFillPatcher >	FPr_u
amrex::Vector< ERFFillPatcher >	FPr_v
amrex::Vector< ERFFillPatcher >	FPr_w
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau11_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau22_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau33_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau12_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau21_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau13_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau31_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau23_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau32_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	eddyDiffs_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SmnSmn_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	sst_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	lmask_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx1_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx2_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx3_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_diss_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx1_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx2_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx3_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q2fx3_lev
amrex::Vector< amrex::Vector< amrex::Real > >	zlevels_stag
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_cc
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_t_rk
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	walldist
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mapfac_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mapfac_u
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mapfac_v
amrex::Vector< amrex::Vector< amrex::Real > >	stretched_dz_h
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	stretched_dz_d
amrex::Vector< amrex::MultiFab >	base_state
amrex::Vector< amrex::MultiFab >	base_state_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Hwave
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Lwave
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Hwave_onegrid
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Lwave_onegrid
bool	finished_wave = false
amrex::Vector< amrex::YAFluxRegister * >	advflux_reg
amrex::Vector< amrex::BCRec >	domain_bcs_type
amrex::Gpu::DeviceVector< amrex::BCRec >	domain_bcs_type_d
amrex::Array< std::string, 2 *AMREX_SPACEDIM >	domain_bc_type
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max >	m_bc_extdir_vals
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max >	m_bc_neumann_vals
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 >	phys_bc_type
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	xflux_imask
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	yflux_imask
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	zflux_imask
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_xforce
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_yforce
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_zforce
int	last_plot_file_step_1
int	last_plot_file_step_2
int	last_check_file_step
int	plot_file_on_restart = 1
int	max_step = std::numeric_limits<int>::max()
amrex::Real	start_time = 0.0
amrex::Real	stop_time = std::numeric_limits<amrex::Real>::max()
std::string	restart_chkfile = ""
amrex::Vector< amrex::Real >	fixed_dt
amrex::Vector< amrex::Real >	fixed_fast_dt
int	regrid_int = -1
std::string	plot_file_1 {"plt_1_"}
std::string	plot_file_2 {"plt_2_"}
bool	m_expand_plotvars_to_unif_rr = false
int	m_plot_int_1 = -1
int	m_plot_int_2 = -1
amrex::Real	m_plot_per_1 = -1.0
amrex::Real	m_plot_per_2 = -1.0
bool	m_plot_face_vels = false
bool	plot_lsm = false
int	profile_int = -1
bool	destag_profiles = true
std::string	check_file {"chk"}
std::string	check_type {"native"}
std::string	restart_type {"native"}
int	m_check_int = -1
amrex::Real	m_check_per = -1.0
amrex::Vector< std::string >	plot_var_names_1
amrex::Vector< std::string >	plot_var_names_2
const amrex::Vector< std::string >	cons_names
const amrex::Vector< std::string >	derived_names
TurbulentPerturbation	turbPert
int	real_width {0}
int	real_set_width {0}
bool	metgrid_debug_quiescent {false}
bool	metgrid_debug_isothermal {false}
bool	metgrid_debug_dry {false}
bool	metgrid_debug_psfc {false}
bool	metgrid_debug_msf {false}
bool	metgrid_interp_theta {false}
bool	metgrid_basic_linear {false}
bool	metgrid_use_below_sfc {true}
bool	metgrid_use_sfc {true}
bool	metgrid_retain_sfc {false}
amrex::Real	metgrid_proximity {1000.0}
int	metgrid_order {2}
int	metgrid_force_sfc_k {0}
amrex::Vector< amrex::Vector< amrex::Real > >	h_rhotheta_src
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_rhotheta_src
amrex::Vector< amrex::Vector< amrex::Real > >	h_rhoqt_src
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_rhoqt_src
amrex::Vector< amrex::Vector< amrex::Real > >	h_w_subsid
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_w_subsid
amrex::Vector< amrex::Vector< amrex::Real > >	h_u_geos
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_u_geos
amrex::Vector< amrex::Vector< amrex::Real > >	h_v_geos
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_v_geos
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_rayleigh_ptrs
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_sponge_ptrs
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_rayleigh_ptrs
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_sponge_ptrs
amrex::Vector< amrex::Real >	h_havg_density
amrex::Vector< amrex::Real >	h_havg_temperature
amrex::Vector< amrex::Real >	h_havg_pressure
amrex::Vector< amrex::Real >	h_havg_qv
amrex::Vector< amrex::Real >	h_havg_qc
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_density
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_temperature
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_pressure
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qv
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qc
std::unique_ptr< WriteBndryPlanes >	m_w2d = nullptr
std::unique_ptr< ReadBndryPlanes >	m_r2d = nullptr
std::unique_ptr< ABLMost >	m_most = nullptr
amrex::Vector< std::unique_ptr< ForestDrag > >	m_forest_drag
amrex::Vector< std::unique_ptr< TerrainDrag > >	m_terrain_drag
amrex::MultiFab	fine_mask
amrex::Vector< amrex::Real >	dz_min
int	sampler_interval = -1
amrex::Real	sampler_per = -1.0
std::unique_ptr< SampleData >	data_sampler = nullptr
amrex::Vector< std::unique_ptr< std::fstream > >	datalog
amrex::Vector< std::string >	datalogname
amrex::Vector< std::unique_ptr< std::fstream > >	sampleptlog
amrex::Vector< std::string >	sampleptlogname
amrex::Vector< amrex::IntVect >	samplepoint
amrex::Vector< std::unique_ptr< std::fstream > >	samplelinelog
amrex::Vector< std::string >	samplelinelogname
amrex::Vector< amrex::IntVect >	sampleline
amrex::Vector< std::unique_ptr< amrex::FabFactory< amrex::FArrayBox > > >	m_factory

Static Private Attributes
static amrex::Real	cfl = 0.8
static amrex::Real	sub_cfl = 1.0
static amrex::Real	init_shrink = 1.0
static amrex::Real	change_max = 1.1
static int	fixed_mri_dt_ratio = 0
static SolverChoice	solverChoice
static int	verbose = 0
static int	mg_verbose = 0
static bool	use_fft = false
static int	sum_interval = -1
static int	pert_interval = -1
static amrex::Real	sum_per = -1.0
static PlotFileType	plotfile_type_1 = PlotFileType::None
static PlotFileType	plotfile_type_2 = PlotFileType::None
static InitType	init_type
static StateInterpType	interpolation_type
static std::string	sponge_type
static bool	use_real_bcs
static amrex::Vector< amrex::Vector< std::string > >	nc_init_file = {{""}}
static std::string	nc_bdy_file
static bool	init_sounding_ideal = false
static int	output_1d_column = 0
static int	column_interval = -1
static amrex::Real	column_per = -1.0
static amrex::Real	column_loc_x = 0.0
static amrex::Real	column_loc_y = 0.0
static std::string	column_file_name = "column_data.nc"
static int	output_bndry_planes = 0
static int	bndry_output_planes_interval = -1
static amrex::Real	bndry_output_planes_per = -1.0
static amrex::Real	bndry_output_planes_start_time = 0.0
static int	input_bndry_planes = 0
static int	ng_dens_hse
static int	ng_pres_hse
static amrex::Vector< amrex::AMRErrorTag >	ref_tags
static amrex::Real	startCPUTime = 0.0
static amrex::Real	previousCPUTimeUsed = 0.0

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

ERF() [1/3]

ERF::ERF

(

)

{
    int fix_random_seed = 0;
    ParmParse pp("erf"); pp.query("fix_random_seed", fix_random_seed);
    // Note that the value of 1024UL is not significant -- the point here is just to set the
    // same seed for all MPI processes for the purpose of regression testing
    if (fix_random_seed) {
        Print() << "Fixing the random seed" << std::endl;
        InitRandom(1024UL);
    }
 
    ERF_shared();
}

Here is the call graph for this function:

~ERF()

ERF::~ERF ( )

overridedefault

ERF() [2/3]

ERF::ERF ( ERF &&  )

deletenoexcept

ERF() [3/3]

ERF::ERF ( const ERF &  other )

delete

Member Function Documentation

Advance()

void ERF::Advance ( int  lev, amrex::Real  time, amrex::Real  dt_lev, int  iteration, int  ncycle  )

private

Function that advances the solution at one level for a single time step – this does some preliminaries then calls erf_advance

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	dt_lev	time step for this time advance

{
    BL_PROFILE("ERF::Advance()");
 
    // We must swap the pointers so the previous step's "new" is now this step's "old"
    std::swap(vars_old[lev], vars_new[lev]);
 
    MultiFab& S_old = vars_old[lev][Vars::cons];
    MultiFab& S_new = vars_new[lev][Vars::cons];
 
    MultiFab& U_old = vars_old[lev][Vars::xvel];
    MultiFab& V_old = vars_old[lev][Vars::yvel];
    MultiFab& W_old = vars_old[lev][Vars::zvel];
 
    MultiFab& U_new = vars_new[lev][Vars::xvel];
    MultiFab& V_new = vars_new[lev][Vars::yvel];
    MultiFab& W_new = vars_new[lev][Vars::zvel];
 
    // We need to set these because otherwise in the first call to erf_advance we may
    //    read uninitialized data on ghost values in setting the bc's on the velocities
    U_new.setVal(1.e34,U_new.nGrowVect());
    V_new.setVal(1.e34,V_new.nGrowVect());
    W_new.setVal(1.e34,W_new.nGrowVect());
 
    //
    // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
    // If lev == 0 we have already FillPatched this in ERF::TimeStep
    //
//  if (lev == 0) {
//      FillPatch(lev, time, {&S_old, &U_old, &V_old, &W_old});
//  } else {
    if (lev > 0) {
        FillPatch(lev, time, {&S_old, &U_old, &V_old, &W_old},
                             {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]},
                             base_state[lev], base_state[lev]);
    }
 
    //
    // So we must convert the fillpatched to momenta, including the ghost values
    //
    VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
                       V_old, rV_old[lev].nGrowVect(),
                       W_old, rW_old[lev].nGrowVect(),
                       S_old, rU_old[lev], rV_old[lev], rW_old[lev],
                       Geom(lev).Domain(),
                       domain_bcs_type);
 
    // TODO: Can test on multiple levels later
    // Update the inflow perturbation update time and amplitude
    if (lev == 0) {
        if (solverChoice.pert_type == PerturbationType::Source ||
            solverChoice.pert_type == PerturbationType::Direct)
        {
            turbPert.calc_tpi_update(lev, dt_lev, U_old, V_old, S_old);
        }
 
        // If PerturbationType::Direct is selected, directly add the computed perturbation
        // on the conserved field
        if (solverChoice.pert_type == PerturbationType::Direct)
        {
            auto m_ixtype = S_old.boxArray().ixType(); // Conserved term
            for (MFIter mfi(S_old,TileNoZ()); mfi.isValid(); ++mfi) {
                Box bx  = mfi.tilebox();
                const Array4<Real> &cell_data  = S_old.array(mfi);
                const Array4<const Real> &pert_cell = turbPert.pb_cell.array(mfi);
                turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cell_data, pert_cell);
            }
        }
    }
 
    // configure ABLMost params if used MostWall boundary condition
    if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::MOST) {
        if (m_most) {
            IntVect ng = Theta_prim[lev]->nGrowVect();
            MultiFab::Copy(  *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
            MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp     , 0, 1, ng);
            if (solverChoice.moisture_type != MoistureType::None) {
                ng = Qv_prim[lev]->nGrowVect();
 
                MultiFab::Copy(  *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
                MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp  , 0, 1, ng);
 
                if (solverChoice.RhoQr_comp > -1) {
                    MultiFab::Copy(  *Qr_prim[lev], S_old, solverChoice.RhoQr_comp, 0, 1, ng);
                    MultiFab::Divide(*Qr_prim[lev], S_old, Rho_comp  , 0, 1, ng);
                } else {
                    Qr_prim[lev]->setVal(0.0);
                }
            }
            // NOTE: std::swap above causes the field ptrs to be out of date.
            //       Reassign the field ptrs for MAC avg computation.
            m_most->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim, Qr_prim);
            m_most->update_pblh(lev, vars_old, z_phys_cc[lev].get(),
                                solverChoice.RhoQv_comp,
                                solverChoice.RhoQc_comp,
                                solverChoice.RhoQr_comp);
            m_most->update_fluxes(lev, time);
        }
    }
 
#if defined(ERF_USE_WINDFARM)
    if (solverChoice.windfarm_type != WindFarmType::None) {
        advance_windfarm(Geom(lev), dt_lev, S_old,
                         U_old, V_old, W_old, vars_windfarm[lev], Nturb[lev], SMark[lev], time);
    }
 
#endif
 
    const BoxArray&            ba = S_old.boxArray();
    const DistributionMapping& dm = S_old.DistributionMap();
 
    int nvars = S_old.nComp();
 
    // Source array for conserved cell-centered quantities -- this will be filled
    //     in the call to make_sources in ERF_TI_slow_rhs_fun.H
    MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0.0);
 
    // Source arrays for momenta -- these will be filled
    //     in the call to make_mom_sources in ERF_TI_slow_rhs_fun.H
    BoxArray ba_x(ba); ba_x.surroundingNodes(0);
    MultiFab xmom_source(ba_x,dm,nvars,1); xmom_source.setVal(0.0);
 
    BoxArray ba_y(ba); ba_y.surroundingNodes(1);
    MultiFab ymom_source(ba_y,dm,nvars,1); ymom_source.setVal(0.0);
 
    BoxArray ba_z(ba); ba_z.surroundingNodes(2);
    MultiFab zmom_source(ba_z,dm,nvars,1); zmom_source.setVal(0.0);
 
    // We don't need to call FillPatch on cons_mf because we have fillpatch'ed S_old above
    MultiFab cons_mf(ba,dm,nvars,S_old.nGrowVect());
    MultiFab::Copy(cons_mf,S_old,0,0,S_old.nComp(),S_old.nGrowVect());
 
    amrex::Vector<MultiFab> state_old;
    amrex::Vector<MultiFab> state_new;
 
    // **************************************************************************************
    // Here we define state_old and state_new which are to be advanced
    // **************************************************************************************
    // Initial solution
    // Note that "old" and "new" here are relative to each RK stage.
    state_old.push_back(MultiFab(cons_mf    , amrex::make_alias, 0, nvars)); // cons
    state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0,     1)); // xmom
    state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0,     1)); // ymom
    state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0,     1)); // zmom
 
    // Final solution
    // state_new at the end of the last RK stage holds the t^{n+1} data
    state_new.push_back(MultiFab(S_new      , amrex::make_alias, 0, nvars)); // cons
    state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0,     1)); // xmom
    state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0,     1)); // ymom
    state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0,     1)); // zmom
 
    // **************************************************************************************
    // Update the dycore
    // **************************************************************************************
    advance_dycore(lev, state_old, state_new,
                   U_old, V_old, W_old,
                   U_new, V_new, W_new,
                   cc_source, xmom_source, ymom_source, zmom_source,
                   Geom(lev), dt_lev, time);
 
    // **************************************************************************************
    // Update the microphysics (moisture)
    // **************************************************************************************
    advance_microphysics(lev, S_new, dt_lev, iteration, time);
 
    // **************************************************************************************
    // Update the land surface model
    // **************************************************************************************
    advance_lsm(lev, S_new, dt_lev);
 
#if defined(ERF_USE_RRTMGP)
    // **************************************************************************************
    // Update the radiation
    // **************************************************************************************
    advance_radiation(lev, S_new, dt_lev);
#endif
 
#ifdef ERF_USE_PARTICLES
    // **************************************************************************************
    // Update the particle positions
    // **************************************************************************************
   evolveTracers( lev, dt_lev, vars_new, z_phys_nd );
#endif
 
    // ***********************************************************************************************
    // Impose domain boundary conditions here so that in FillPatching the fine data we won't
    // need to re-fill these
    // ***********************************************************************************************
    if (lev < finest_level) {
         IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
         (*physbcs_cons[lev])(vars_new[lev][Vars::cons],0,vars_new[lev][Vars::cons].nComp(),
                              vars_new[lev][Vars::cons].nGrowVect(),time,BCVars::cons_bc,true);
            (*physbcs_u[lev])(vars_new[lev][Vars::xvel],0,1,ngvect_vels,time,BCVars::xvel_bc,true);
            (*physbcs_v[lev])(vars_new[lev][Vars::yvel],0,1,ngvect_vels,time,BCVars::yvel_bc,true);
            (*physbcs_w[lev])(vars_new[lev][Vars::zvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                              ngvect_vels,time,BCVars::zvel_bc,true);
    }
 
    // **************************************************************************************
    // Register old and new coarse data if we are at a level less than the finest level
    // **************************************************************************************
    if (lev < finest_level)
    {
        if (cf_width > 0) {
            // We must fill the ghost cells of these so that the parallel copy works correctly
            state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
            FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
                                          {time, time + dt_lev});
        }
 
        if (cf_width >= 0) {
            // We must fill the ghost cells of these so that the parallel copy works correctly
            state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
            FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
                                          {time, time + dt_lev});
 
            state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
            FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
                                          {time, time + dt_lev});
 
            state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
            FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
                                          {time, time + dt_lev});
        }
 
            //
            // Now create a MultiFab that holds (S_new - S_old) / dt from the coarse level interpolated
            //     on to the coarse/fine boundary at the fine resolution
            //
            Interpolater* mapper_f = &face_cons_linear_interp;
 
            // PhysBCFunctNoOp null_bc;
            // MultiFab tempx(vars_new[lev+1][Vars::xvel].boxArray(),vars_new[lev+1][Vars::xvel].DistributionMap(),1,0);
            // tempx.setVal(0.0);
            // xmom_crse_rhs[lev+1].setVal(0.0);
            // FPr_u[lev].FillSet(tempx               , time       , null_bc, domain_bcs_type);
            // FPr_u[lev].FillSet(xmom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
            // MultiFab::Subtract(xmom_crse_rhs[lev+1],tempx,0,0,1,IntVect{0});
            // xmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
 
            // MultiFab tempy(vars_new[lev+1][Vars::yvel].boxArray(),vars_new[lev+1][Vars::yvel].DistributionMap(),1,0);
            // tempy.setVal(0.0);
            // ymom_crse_rhs[lev+1].setVal(0.0);
            // FPr_v[lev].FillSet(tempy               , time       , null_bc, domain_bcs_type);
            // FPr_v[lev].FillSet(ymom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
            // MultiFab::Subtract(ymom_crse_rhs[lev+1],tempy,0,0,1,IntVect{0});
            // ymom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
 
            MultiFab temp_state(zmom_crse_rhs[lev+1].boxArray(),zmom_crse_rhs[lev+1].DistributionMap(),1,0);
            InterpFromCoarseLevel(temp_state,            IntVect{0}, IntVect{0}, state_old[IntVars::zmom], 0, 0, 1,
                                  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
            InterpFromCoarseLevel(zmom_crse_rhs[lev+1],  IntVect{0}, IntVect{0}, state_new[IntVars::zmom], 0, 0, 1,
                                  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
            MultiFab::Subtract(zmom_crse_rhs[lev+1],temp_state,0,0,1,IntVect{0});
            zmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
    }
 
    // ***********************************************************************************************
    // Update the time averaged velocities if they are requested
    // ***********************************************************************************************
    if (solverChoice.time_avg_vel) {
        Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
    }
}

Here is the call graph for this function:

advance_dycore()

void ERF::advance_dycore	(	int	level,
		amrex::Vector< amrex::MultiFab > &	state_old,
		amrex::Vector< amrex::MultiFab > &	state_new,
		amrex::MultiFab &	xvel_old,
		amrex::MultiFab &	yvel_old,
		amrex::MultiFab &	zvel_old,
		amrex::MultiFab &	xvel_new,
		amrex::MultiFab &	yvel_new,
		amrex::MultiFab &	zvel_new,
		amrex::MultiFab &	source,
		amrex::MultiFab &	xmom_src,
		amrex::MultiFab &	ymom_src,
		amrex::MultiFab &	zmom_src,
		amrex::Geometry	fine_geom,
		amrex::Real	dt,
		amrex::Real	time
	)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters

[in]	level	level of refinement (coarsest level is 0)
[in]	state_old	old-time conserved variables
[in]	state_new	new-time conserved variables
[in]	xvel_old	old-time x-component of velocity
[in]	yvel_old	old-time y-component of velocity
[in]	zvel_old	old-time z-component of velocity
[in]	xvel_new	new-time x-component of velocity
[in]	yvel_new	new-time y-component of velocity
[in]	zvel_new	new-time z-component of velocity
[in]	cc_src	source term for conserved variables
[in]	xmom_src	source term for x-momenta
[in]	ymom_src	source term for y-momenta
[in]	zmom_src	source term for z-momenta
[in]	fine_geom	container for geometry information at current level
[in]	dt_advance	time step for this time advance
[in]	old_time	old time for this time advance

{
    BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
 
    const Box& domain = fine_geom.Domain();
 
    DiffChoice dc    = solverChoice.diffChoice;
    TurbChoice tc    = solverChoice.turbChoice[level];
    SpongeChoice sc  = solverChoice.spongeChoice;
 
    MultiFab r_hse (base_state[level], make_alias, BaseState::r0_comp , 1);
    MultiFab p_hse (base_state[level], make_alias, BaseState::p0_comp , 1);
    MultiFab pi_hse(base_state[level], make_alias, BaseState::pi0_comp, 1);
 
    // These pointers are used in the MRI utility functions
    MultiFab* r0  = &r_hse;
    MultiFab* p0  = &p_hse;
    MultiFab* pi0 = &pi_hse;
 
    Real* dptr_rhotheta_src = solverChoice.custom_rhotheta_forcing ? d_rhotheta_src[level].data() : nullptr;
    Real* dptr_rhoqt_src    = solverChoice.custom_moisture_forcing ? d_rhoqt_src[level].data()    : nullptr;
    Real* dptr_wbar_sub     = solverChoice.custom_w_subsidence     ? d_w_subsid[level].data()     : nullptr;
 
    // Turbulent Perturbation Pointer
    //Real* dptr_rhotheta_src = solverChoice.pert_type ? d_rhotheta_src[level].data() : nullptr;
 
    Vector<Real*> d_rayleigh_ptrs_at_lev;
    d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
    d_rayleigh_ptrs_at_lev[Rayleigh::ubar]     = solverChoice.rayleigh_damp_U   ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
    d_rayleigh_ptrs_at_lev[Rayleigh::vbar]     = solverChoice.rayleigh_damp_V   ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
    d_rayleigh_ptrs_at_lev[Rayleigh::wbar]     = solverChoice.rayleigh_damp_W   ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
    d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.rayleigh_damp_T   ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
 
    Vector<Real*> d_sponge_ptrs_at_lev;
    if(sc.sponge_type=="input_sponge")
    {
        d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
        d_sponge_ptrs_at_lev[Sponge::ubar_sponge]  =  d_sponge_ptrs[level][Sponge::ubar_sponge].data();
        d_sponge_ptrs_at_lev[Sponge::vbar_sponge]  =  d_sponge_ptrs[level][Sponge::vbar_sponge].data();
    }
 
    bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None);
    bool l_use_diff    = ( (dc.molec_diff_type != MolecDiffType::None) ||
                           (tc.les_type        !=       LESType::None) ||
                           (tc.pbl_type        !=       PBLType::None) );
    bool l_use_kturb   = ( (tc.les_type != LESType::None)   ||
                           (tc.pbl_type != PBLType::None) );
    bool l_use_moisture = ( solverChoice.moisture_type != MoistureType::None );
    bool l_implicit_substepping = ( solverChoice.substepping_type[level] == SubsteppingType::Implicit );
 
    const bool use_most = (m_most != nullptr);
    const bool exp_most = (solverChoice.use_explicit_most);
    amrex::ignore_unused(use_most);
 
    const BoxArray& ba            = state_old[IntVars::cons].boxArray();
    const BoxArray& ba_z          = zvel_old.boxArray();
    const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
 
    int num_prim = state_old[IntVars::cons].nComp() - 1;
 
    MultiFab    S_prim  (ba  , dm, num_prim,          state_old[IntVars::cons].nGrowVect());
    MultiFab  pi_stage  (ba  , dm,        1,          state_old[IntVars::cons].nGrowVect());
    MultiFab fast_coeffs(ba_z, dm,        5,          0);
    MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
    MultiFab* SmnSmn    = SmnSmn_lev[level].get();
 
    // **************************************************************************************
    // Compute strain for use in slow RHS, Smagorinsky model, and MOST
    // **************************************************************************************
    {
    BL_PROFILE("erf_advance_strain");
    if (l_use_diff) {
 
        const BCRec* bc_ptr_h = domain_bcs_type.data();
        const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
 
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
        {
            Box bxcc  = mfi.growntilebox(IntVect(1,1,0));
            Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
            Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
            Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
 
            if (bxcc.smallEnd(2) != domain.smallEnd(2)) {
                 bxcc.growLo(2,1);
                tbxxy.growLo(2,1);
                tbxxz.growLo(2,1);
                tbxyz.growLo(2,1);
            }
 
            if (bxcc.bigEnd(2) != domain.bigEnd(2)) {
                 bxcc.growHi(2,1);
                tbxxy.growHi(2,1);
                tbxxz.growHi(2,1);
                tbxyz.growHi(2,1);
            }
 
            const Array4<const Real> & u = xvel_old.array(mfi);
            const Array4<const Real> & v = yvel_old.array(mfi);
            const Array4<const Real> & w = zvel_old.array(mfi);
 
            Array4<Real> tau11 = Tau11_lev[level].get()->array(mfi);
            Array4<Real> tau22 = Tau22_lev[level].get()->array(mfi);
            Array4<Real> tau33 = Tau33_lev[level].get()->array(mfi);
            Array4<Real> tau12 = Tau12_lev[level].get()->array(mfi);
            Array4<Real> tau13 = Tau13_lev[level].get()->array(mfi);
            Array4<Real> tau23 = Tau23_lev[level].get()->array(mfi);
 
            Array4<Real> tau21  = l_use_terrain ? Tau21_lev[level].get()->array(mfi) : Array4<Real>{};
            Array4<Real> tau31  = l_use_terrain ? Tau31_lev[level].get()->array(mfi) : Array4<Real>{};
            Array4<Real> tau32  = l_use_terrain ? Tau32_lev[level].get()->array(mfi) : Array4<Real>{};
            const Array4<const Real>& z_nd = l_use_terrain ? z_phys_nd[level]->const_array(mfi) : Array4<const Real>{};
 
            const Array4<const Real> mf_m = mapfac_m[level]->array(mfi);
            const Array4<const Real> mf_u = mapfac_u[level]->array(mfi);
            const Array4<const Real> mf_v = mapfac_v[level]->array(mfi);
 
            if (l_use_terrain) {
                ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                u, v, w,
                                tau11, tau22, tau33,
                                tau12, tau13,
                                tau21, tau23,
                                tau31, tau32,
                                z_nd, detJ_cc[level]->const_array(mfi), bc_ptr_h, dxInv,
                                mf_m, mf_u, mf_v);
            } else {
                ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                u, v, w,
                                tau11, tau22, tau33,
                                tau12, tau13, tau23,
                                bc_ptr_h, dxInv,
                                mf_m, mf_u, mf_v);
            }
        } // mfi
    } // l_use_diff
    } // profile
 
#include "ERF_TI_utils.H"
 
    // Additional SFS quantities, calculated once per timestep
    MultiFab* Hfx1 = SFS_hfx1_lev[level].get();
    MultiFab* Hfx2 = SFS_hfx2_lev[level].get();
    MultiFab* Hfx3 = SFS_hfx3_lev[level].get();
    MultiFab* Q1fx1 = SFS_q1fx1_lev[level].get();
    MultiFab* Q1fx2 = SFS_q1fx2_lev[level].get();
    MultiFab* Q1fx3 = SFS_q1fx3_lev[level].get();
    MultiFab* Q2fx3 = SFS_q2fx3_lev[level].get();
    MultiFab* Diss = SFS_diss_lev[level].get();
 
    // *************************************************************************
    // Calculate cell-centered eddy viscosity & diffusivities
    //
    // Notes -- we fill all the data in ghost cells before calling this so
    //    that we can fill the eddy viscosity in the ghost regions and
    //    not have to call a boundary filler on this data itself
    //
    // LES - updates both horizontal and vertical eddy viscosity components
    // PBL - only updates vertical eddy viscosity components so horizontal
    //       components come from the LES model or are left as zero.
    // *************************************************************************
    if (l_use_kturb)
    {
        // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
        const BCRec* bc_ptr_h = domain_bcs_type.data();
        ComputeTurbulentViscosity(xvel_old, yvel_old,
                                  *Tau11_lev[level].get(), *Tau22_lev[level].get(), *Tau33_lev[level].get(),
                                  *Tau12_lev[level].get(), *Tau13_lev[level].get(), *Tau23_lev[level].get(),
                                  state_old[IntVars::cons],
                                  *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
                                  fine_geom, *mapfac_u[level], *mapfac_v[level],
                                  z_phys_nd[level], solverChoice,
                                  m_most, exp_most, l_use_moisture, level, bc_ptr_h);
    }
 
    // ***********************************************************************************************
    // Update user-defined source terms -- these are defined once per time step (not per RK stage)
    // ***********************************************************************************************
    if (solverChoice.custom_rhotheta_forcing) {
        prob->update_rhotheta_sources(old_time,
                                      h_rhotheta_src[level], d_rhotheta_src[level],
                                      fine_geom, z_phys_cc[level]);
    }
 
    if (solverChoice.custom_moisture_forcing) {
        prob->update_rhoqt_sources(old_time,
                                   h_rhoqt_src[level], d_rhoqt_src[level],
                                   fine_geom, z_phys_cc[level]);
    }
 
    if (solverChoice.custom_geostrophic_profile) {
        prob->update_geostrophic_profile(old_time,
                                   h_u_geos[level], d_u_geos[level],
                                   h_v_geos[level], d_v_geos[level],
                                   fine_geom, z_phys_cc[level]);
    }
 
    // ***********************************************************************************************
    // Convert old velocity available on faces to old momentum on faces to be used in time integration
    // ***********************************************************************************************
    MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
 
    //
    // This is an optimization since we won't need more than one ghost
    // cell of momentum in the integrator if not using numerical diffusion
    //
    IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
    IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
    IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
 
    VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
                       state_old[IntVars::xmom],
                       state_old[IntVars::ymom],
                       state_old[IntVars::zmom],
                       domain, domain_bcs_type);
 
    MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
    MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
    MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
 
    bool fast_only = false;
    bool vel_and_mom_synced = true;
 
    apply_bcs(state_old, old_time,
              state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
              fast_only, vel_and_mom_synced);
    cons_to_prim(state_old[IntVars::cons], state_old[IntVars::cons].nGrow());
 
#include "ERF_TI_no_substep_fun.H"
#include "ERF_TI_substep_fun.H"
#include "ERF_TI_slow_rhs_fun.H"
 
    // ***************************************************************************************
    // Setup the integrator and integrate for a single timestep
    // **************************************************************************************
    MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
 
    // Define rhs and 'post update' utility function that is called after calculating
    // any state data (e.g. at RK stages or at the end of a timestep)
    mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
    mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
 
    if (solverChoice.anelastic[level]) {
        mri_integrator.set_slow_rhs_inc(slow_rhs_fun_inc);
    }
 
    mri_integrator.set_fast_rhs(fast_rhs_fun);
    mri_integrator.set_slow_fast_timestep_ratio(fixed_mri_dt_ratio > 0 ? fixed_mri_dt_ratio : dt_mri_ratio[level]);
    mri_integrator.set_no_substep(no_substep_fun);
 
    mri_integrator.advance(state_old, state_new, old_time, dt_advance);
 
    if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
}

Here is the call graph for this function:

advance_lsm()

void ERF::advance_lsm	(	int	lev,
		amrex::MultiFab &	,
		const amrex::Real &	dt_advance
	)

{
    if (solverChoice.lsm_type != LandSurfaceType::None) {
        lsm.Advance(lev, dt_advance);
    }
}

advance_microphysics()

void ERF::advance_microphysics	(	int	lev,
		amrex::MultiFab &	cons_in,
		const amrex::Real &	dt_advance,
		const int &	iteration,
		const amrex::Real &	time
	)

{
    if (solverChoice.moisture_type != MoistureType::None) {
        micro->Update_Micro_Vars_Lev(lev, cons);
        micro->Advance(lev, dt_advance, iteration, time, solverChoice, vars_new, z_phys_nd, phys_bc_type);
        micro->Update_State_Vars_Lev(lev, cons);
    }
}

appendPlotVariables()

void ERF::appendPlotVariables ( const std::string &  pp_plot_var_names, amrex::Vector< std::string > &  plot_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    Vector<std::string> plot_var_names(0);
    if (pp.contains(pp_plot_var_names.c_str())) {
        std::string nm;
        int nPltVars = pp.countval(pp_plot_var_names.c_str());
        for (int i = 0; i < nPltVars; i++) {
            pp.get(pp_plot_var_names.c_str(), nm, i);
            // Add the named variable to our list of plot variables
            // if it is not already in the list
            if (!containerHasElement(plot_var_names, nm)) {
                plot_var_names.push_back(nm);
            }
        }
    }
 
    Vector<std::string> tmp_plot_names(0);
#ifdef ERF_USE_PARTICLES
    Vector<std::string> particle_mesh_plot_names;
    particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
    for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
        std::string tmp(particle_mesh_plot_names[i]);
        if (containerHasElement(plot_var_names, tmp) ) {
            tmp_plot_names.push_back(tmp);
        }
    }
#endif
 
    for (int i = 0; i < tmp_plot_names.size(); i++) {
        a_plot_var_names.push_back( tmp_plot_names[i] );
    }
 
    // Finally, check to see if we found all the requested variables
    for (const auto& plot_name : plot_var_names) {
        if (!containerHasElement(a_plot_var_names, plot_name)) {
             if (amrex::ParallelDescriptor::IOProcessor()) {
                 Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
             }
        }
    }
}

Here is the call graph for this function:

AverageDown()

void ERF::AverageDown ( )

private

{
    AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
 
    int src_comp, num_comp;
    for (int lev = finest_level-1; lev >= 0; --lev)
    {
        // If anelastic we don't average down rho because rho == rho0.
        if (solverChoice.anelastic[lev]) {
            src_comp = 1;
        } else {
            src_comp = 0;
        }
        num_comp = vars_new[0][Vars::cons].nComp() - src_comp;
        AverageDownTo(lev,src_comp,num_comp);
    }
}

AverageDownTo()

void ERF::AverageDownTo	(	int	crse_lev,
		int	scomp,
		int	ncomp
	)

{
    if (solverChoice.anelastic[crse_lev]) {
        AMREX_ALWAYS_ASSERT(scomp == 1);
    } else {
        AMREX_ALWAYS_ASSERT(scomp == 0);
    }
 
    AMREX_ALWAYS_ASSERT(ncomp == vars_new[crse_lev][Vars::cons].nComp() - scomp);
    AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
 
    // ******************************************************************************************
    // First do cell-centered quantities
    // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
    // m is the map scale factor at cell centers
    // Here we pre-divide (rho S) by m^2 before average down
    // ******************************************************************************************
    for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
      for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
        const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
            });
        } else {
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
            });
        }
      } // mfi
    } // lev
 
    int fine_lev = crse_lev+1;
 
    if (interpolation_type == StateInterpType::Perturbational) {
        // Make the fine rho and (rho theta) be perturbational
        MultiFab::Divide(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
                         Rho_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                           BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                           BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
 
        // Make the crse rho and (rho theta) be perturbational
        MultiFab::Divide(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                         Rho_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                           BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                           BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
    }
 
    average_down(vars_new[crse_lev+1][Vars::cons],vars_new[crse_lev  ][Vars::cons],
                 scomp, ncomp, refRatio(crse_lev));
 
    if (interpolation_type == StateInterpType::Perturbational) {
        // Restore the fine data to what it was
        MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                      BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                      BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Multiply(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
                           Rho_comp,RhoTheta_comp,1,IntVect{0});
 
        // Make the crse data be full values not perturbational
        MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                      BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                      BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Multiply(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                           Rho_comp,RhoTheta_comp,1,IntVect{0});
    }
 
    vars_new[crse_lev][Vars::cons].FillBoundary(geom[crse_lev].periodicity());
 
    // Here we multiply (rho S) by m^2 after average down
    for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
      for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
        const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)) / detJ_arr(i,j,k);
            });
        } else {
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
            });
        }
      } // mfi
    } // lev
 
    // ******************************************************************************************
    // Now average down momenta.
    // Note that vars_new holds velocities not momenta, but we want to do conservative
    //    averaging so we first convert to momentum, then average down, then convert
    //    back to velocities -- only on the valid region
    // ******************************************************************************************
    for (int lev = crse_lev; lev <= crse_lev+1; lev++)
    {
        // FillBoundary for density so we can go back and forth between velocity and momentum
        vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
 
        VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
                           vars_new[lev][Vars::yvel], IntVect(0,0,0),
                           vars_new[lev][Vars::zvel], IntVect(0,0,0),
                           vars_new[lev][Vars::cons],
                             rU_new[lev],
                             rV_new[lev],
                             rW_new[lev],
                           Geom(lev).Domain(),
                           domain_bcs_type);
    }
 
    average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
    average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
    average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
 
    for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
        MomentumToVelocity(vars_new[lev][Vars::xvel],
                           vars_new[lev][Vars::yvel],
                           vars_new[lev][Vars::zvel],
                           vars_new[lev][Vars::cons],
                             rU_new[lev],
                             rV_new[lev],
                             rW_new[lev],
                           Geom(lev).Domain(),
                           domain_bcs_type);
    }
}

Here is the call graph for this function:

build_fine_mask()

MultiFab & ERF::build_fine_mask

(

int

level

)

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters

level

Fine level index which masks underlying coarser data

{
    // Mask for zeroing covered cells
    AMREX_ASSERT(level > 0);
 
    const BoxArray& cba = grids[level-1];
    const DistributionMapping& cdm = dmap[level-1];
 
    // TODO -- we should make a vector of these a member of ERF class
    fine_mask.define(cba, cdm, 1, 0, MFInfo());
    fine_mask.setVal(1.0);
 
    BoxArray fba = grids[level];
    iMultiFab ifine_mask = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
 
    const auto  fma =  fine_mask.arrays();
    const auto ifma = ifine_mask.arrays();
    ParallelFor(fine_mask, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
    {
       fma[bno](i,j,k) = ifma[bno](i,j,k);
    });
 
    return fine_mask;
}

ClearLevel()

void ERF::ClearLevel ( int  lev )

override

{
    for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
        vars_new[lev][var_idx].clear();
        vars_old[lev][var_idx].clear();
    }
 
    base_state[lev].clear();
 
    rU_new[lev].clear();
    rU_old[lev].clear();
    rV_new[lev].clear();
    rV_old[lev].clear();
    rW_new[lev].clear();
    rW_old[lev].clear();
 
    if (lev > 0) {
        zmom_crse_rhs[lev].clear();
    }
 
    if (solverChoice.anelastic[lev] == 1) {
        pp_inc[lev].clear();
    }
 
    // Clears the integrator memory
    mri_integrator_mem[lev].reset();
 
    // Clears the physical boundary condition routines
    physbcs_cons[lev].reset();
    physbcs_u[lev].reset();
    physbcs_v[lev].reset();
    physbcs_w[lev].reset();
    physbcs_base[lev].reset();
 
    // Clears the flux register array
    advflux_reg[lev]->reset();
}

cloud_fraction()

Real ERF::cloud_fraction

(

amrex::Real

time

)

{
    BL_PROFILE("ERF::cloud_fraction()");
 
    int lev = 0;
    // This holds all of qc
    MultiFab qc(vars_new[lev][Vars::cons],make_alias,RhoQ2_comp,1);
 
    int direction = 2; // z-direction
    Box const& domain = geom[lev].Domain();
 
    auto const& qc_arr = qc.const_arrays();
 
    // qc_2d is an BaseFab<int> holding the max value over the column
    auto qc_2d = ReduceToPlane<ReduceOpMax,int>(direction, domain, qc,
         [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) -> int
         {
             if (qc_arr[box_no](i,j,k) > 0) {
                 return 1;
             } else {
                 return 0;
             }
         });
 
    auto* p = qc_2d.dataPtr();
 
    Long numpts = qc_2d.numPts();
 
    AMREX_ASSERT(numpts < Long(std::numeric_limits<int>::max));
 
#if 1
    if (ParallelDescriptor::UseGpuAwareMpi()) {
        ParallelDescriptor::ReduceIntMax(p,static_cast<int>(numpts));
    } else {
        Gpu::PinnedVector<int> hv(numpts);
        Gpu::copyAsync(Gpu::deviceToHost, p, p+numpts, hv.data());
        Gpu::streamSynchronize();
        ParallelDescriptor::ReduceIntMax(hv.data(),static_cast<int>(numpts));
        Gpu::copyAsync(Gpu::hostToDevice, hv.data(), hv.data()+numpts, p);
    }
 
    // Sum over component 0
    Long num_cloudy = qc_2d.template sum<RunOn::Device>(0);
 
#else
    //
    // We need this if we allow domain decomposition in the vertical
    //    but for now we leave it commented out
    //
    Long num_cloudy = Reduce::Sum<Long>(numpts,
         [=] AMREX_GPU_DEVICE (Long i) -> Long {
             if (p[i] == 1) {
                 return 1;
             } else {
                 return 0;
             }
         });
    ParallelDescriptor::ReduceLongSum(num_cloudy);
#endif
 
    Real num_total = qc_2d.box().d_numPts();
 
    Real cloud_frac = num_cloudy / num_total;
 
    return cloud_frac;
}

compute_divergence()

void ERF::compute_divergence	(	int	lev,
		amrex::MultiFab &	rhs,
		amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM >	rho0_u_const,
		amrex::Geometry const &	geom_at_lev
	)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level 0.

{
    BL_PROFILE("ERF::compute_divergence()");
 
    auto dxInv = geom_at_lev.InvCellSizeArray();
 
    // ****************************************************************************
    // Compute divergence which will form RHS
    // Note that we replace "rho0w" with the contravariant momentum, Omega
    // ****************************************************************************
#ifdef ERF_USE_EB
    bool already_on_centroids = true;
    EB_computeDivergence(rhs, rho0_u_const, geom_at_lev, already_on_centroids);
#else
    if (SolverChoice::mesh_type == MeshType::ConstantDz)
    {
        computeDivergence(rhs, rho0_u_const, geom_at_lev);
    }
    else
    {
        for ( MFIter mfi(rhs,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            Box bx = mfi.tilebox();
            const Array4<Real const>& rho0u_arr = rho0_u_const[0]->const_array(mfi);
            const Array4<Real const>& rho0v_arr = rho0_u_const[1]->const_array(mfi);
            const Array4<Real const>& rho0w_arr = rho0_u_const[2]->const_array(mfi);
            const Array4<Real      >&  rhs_arr = rhs.array(mfi);
 
            if (SolverChoice::mesh_type == MeshType::StretchedDz) {
                Real* stretched_dz_d_ptr = stretched_dz_d[lev].data();
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                    Real dz = stretched_dz_d_ptr[k];
                    rhs_arr(i,j,k) =  (rho0u_arr(i+1,j,k) - rho0u_arr(i,j,k)) * dxInv[0]
                                     +(rho0v_arr(i,j+1,k) - rho0v_arr(i,j,k)) * dxInv[1]
                                     +(rho0w_arr(i,j,k+1) - rho0w_arr(i,j,k)) / dz;
                });
            } else {
 
                //
                // Note we compute the divergence using "rho0w" == Omega
                //
                const Array4<Real const>& ax_arr = ax[lev]->const_array(mfi);
                const Array4<Real const>& ay_arr = ay[lev]->const_array(mfi);
                const Array4<Real const>& dJ_arr = detJ_cc[lev]->const_array(mfi);
                //
                // az == 1 for terrain-fitted coordinates
                //
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    rhs_arr(i,j,k) =   ((ax_arr(i+1,j,k)*rho0u_arr(i+1,j,k) - ax_arr(i,j,k)*rho0u_arr(i,j,k)) * dxInv[0]
                                       +(ay_arr(i,j+1,k)*rho0v_arr(i,j+1,k) - ay_arr(i,j,k)*rho0v_arr(i,j,k)) * dxInv[1]
                                       +(                rho0w_arr(i,j,k+1) -               rho0w_arr(i,j,k)) * dxInv[2]) / dJ_arr(i,j,k);
                });
            }
        } // mfi
    }
#endif
}

ComputeDt()

void ERF::ComputeDt ( int  step = -1 )

private

Function that calls estTimeStep for each level

{
    Vector<Real> dt_tmp(finest_level+1);
 
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
    }
 
    ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
 
    Real dt_0 = dt_tmp[0];
    int n_factor = 1;
    for (int lev = 0; lev <= finest_level; ++lev) {
        dt_tmp[lev] = amrex::min(dt_tmp[lev], change_max*dt[lev]);
        n_factor *= nsubsteps[lev];
        dt_0 = amrex::min(dt_0, n_factor*dt_tmp[lev]);
        if (step == 0){
            dt_0 *= init_shrink;
            if (verbose && init_shrink != 1.0) {
                Print() << "Timestep 0: shrink initial dt at level " << lev << " by " << init_shrink << std::endl;
            }
        }
    }
 
    // Limit dt's by the value of stop_time.
    const Real eps = 1.e-3*dt_0;
    if (t_new[0] + dt_0 > stop_time - eps) {
        dt_0 = stop_time - t_new[0];
    }
 
    dt[0] = dt_0;
    for (int lev = 1; lev <= finest_level; ++lev) {
        dt[lev] = dt[lev-1] / nsubsteps[lev];
    }
}

ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells ( const AdvChoice &  advChoice, bool  use_num_diff  )

inlinestaticprivate

    {
        if (use_num_diff)
        {
            return 3;
        } else {
            if (
                 (advChoice.dycore_horiz_adv_type    == AdvType::Centered_6th)
              || (advChoice.dycore_vert_adv_type     == AdvType::Centered_6th)
              || (advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
              || (advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
              || (advChoice.dryscal_horiz_adv_type   == AdvType::Centered_6th)
              || (advChoice.dryscal_vert_adv_type    == AdvType::Centered_6th) )
            { return 3; }
            else if (
                 (advChoice.dycore_horiz_adv_type    == AdvType::Upwind_5th)
              || (advChoice.dycore_vert_adv_type     == AdvType::Upwind_5th)
              || (advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
              || (advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
              || (advChoice.dryscal_horiz_adv_type   == AdvType::Upwind_5th)
              || (advChoice.dryscal_vert_adv_type    == AdvType::Upwind_5th) )
            { return 3; }
            else if (
                  (advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5)
               || (advChoice.dryscal_vert_adv_type    == AdvType::Weno_5)
               || (advChoice.moistscal_horiz_adv_type == AdvType::Weno_5)
               || (advChoice.moistscal_vert_adv_type  == AdvType::Weno_5)
               || (advChoice.moistscal_horiz_adv_type == AdvType::Weno_5Z)
               || (advChoice.moistscal_vert_adv_type  == AdvType::Weno_5Z)
               || (advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5Z)
               || (advChoice.dryscal_vert_adv_type    == AdvType::Weno_5Z) )
            { return 3; }
            else if (
                  (advChoice.dryscal_horiz_adv_type   == AdvType::Weno_7)
               || (advChoice.dryscal_vert_adv_type    == AdvType::Weno_7)
               || (advChoice.moistscal_horiz_adv_type == AdvType::Weno_7)
               || (advChoice.moistscal_vert_adv_type  == AdvType::Weno_7)
               || (advChoice.moistscal_horiz_adv_type == AdvType::Weno_7Z)
               || (advChoice.moistscal_vert_adv_type  == AdvType::Weno_7Z)
               || (advChoice.dryscal_horiz_adv_type   == AdvType::Weno_7Z)
               || (advChoice.dryscal_vert_adv_type    == AdvType::Weno_7Z) )
            { return 4; }
            else
            { return 2; }
        }
    }

Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int  lev )

private

{
    auto& fine_new = vars_new[lev];
    auto& crse_new = vars_new[lev-1];
    auto& ba_fine  = fine_new[Vars::cons].boxArray();
    auto& ba_crse  = crse_new[Vars::cons].boxArray();
    auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
    auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
 
    int ncomp = vars_new[lev][Vars::cons].nComp();
 
    FPr_c.emplace_back(ba_fine, dm_fine, geom[lev]  ,
                       ba_crse, dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
    FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                       convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                       convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                       convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
}

DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int  i )

inlineprivate

    {
        return *datalog[i];
    }

DataLogName()

std::string ERF::DataLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith datalog file.

{ return datalogname[i]; }

Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int  lev )

private

{
    auto& fine_new = vars_new[lev];
    auto& crse_new = vars_new[lev-1];
    auto& ba_fine  = fine_new[Vars::cons].boxArray();
    auto& ba_crse  = crse_new[Vars::cons].boxArray();
    auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
    auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
 
    int ncomp = fine_new[Vars::cons].nComp();
 
    FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev]  ,
                        ba_crse, dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
    FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
}

derive_diag_profiles()

void ERF::derive_diag_profiles	(	amrex::Real	time,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Kmv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Khv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qi,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qg,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wthv
	)

Computes the profiles for diagnostic quantities.

Parameters

h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_uiuiu	Profile for u_i*u_i*u triple product on Host
h_avg_uiuiv	Profile for u_i*u_i*v triple product on Host
h_avg_uiuiw	Profile for u_i*u_i*w triple product on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host

{
    // We assume that this is always called at level 0
    int lev = 0;
 
    bool l_use_kturb = ((solverChoice.turbChoice[lev].les_type != LESType::None) ||
                        (solverChoice.turbChoice[lev].pbl_type != PBLType::None));
    bool l_use_KE   = ( (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) ||
                        (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) );
 
    // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, uw, vv, vw, ww, uth, vth, wth,
    //                  0      1     2    3    4   5   6   7   8   9  10   11   12   13
    //                thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr,
    //                  14     15     16     17 18  19  20  21  22  23  24   25   26   27
    //                  qi, qs, qg, wthv
    //                  28  29  30    31
    MultiFab mf_out(grids[lev], dmap[lev], 32, 0);
 
    MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
 
    MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
    MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
    MultiFab  w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
 
    average_face_to_cellcenter(mf_vels,0,
        Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
 
    int zdir = 2;
    auto domain = geom[0].Domain();
 
    // Sum in the horizontal plane
    h_avg_u  = sumToLine(mf_vels ,0,1,domain,zdir);
    h_avg_v  = sumToLine(mf_vels ,1,1,domain,zdir);
    h_avg_w  = sumToLine(mf_vels ,2,1,domain,zdir);
 
    int hu_size =  h_avg_u.size();
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < hu_size; ++k) {
        h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
    }
 
    Gpu::DeviceVector<Real> d_avg_u(hu_size, Real(0.0));
    Gpu::DeviceVector<Real> d_avg_v(hu_size, Real(0.0));
    Gpu::DeviceVector<Real> d_avg_w(hu_size, Real(0.0));
 
#if 0
    auto* avg_u_ptr = d_avg_u.data();
    auto* avg_v_ptr = d_avg_v.data();
    auto* avg_w_ptr = d_avg_w.data();
#endif
 
    Gpu::copy(Gpu::hostToDevice, h_avg_u.begin(), h_avg_u.end(), d_avg_u.begin());
    Gpu::copy(Gpu::hostToDevice, h_avg_v.begin(), h_avg_v.end(), d_avg_v.begin());
    Gpu::copy(Gpu::hostToDevice, h_avg_w.begin(), h_avg_w.end(), d_avg_w.begin());
 
    int nvars = vars_new[lev][Vars::cons].nComp();
    MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
 
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
 
    for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
        const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
        const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
        const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
        const Array4<Real>& cons_arr = mf_cons.array(mfi);
        const Array4<Real>&   p0_arr = p_hse.array(mfi);
        const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
                                                            Array4<const Real>{};
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 1) = theta;
            Real ksgs = 0.0;
            if (l_use_KE) {
                ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
            }
            fab_arr(i, j, k, 2) = ksgs;
#if 1
            if (l_use_kturb) {
                fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
                fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
            } else {
                fab_arr(i, j, k, 3) = 0.0;
                fab_arr(i, j, k, 4) = 0.0;
            }
#else
            // Here we hijack the "Kturb" variable name to print out the resolved kinetic energy
            Real upert = u_cc_arr(i,j,k) - avg_u_ptr[k];
            Real vpert = v_cc_arr(i,j,k) - avg_v_ptr[k];
            Real wpert = w_cc_arr(i,j,k) - avg_w_ptr[k];
            fab_arr(i, j, k, 3) = 0.5 * (upert*upert + vpert*vpert + wpert*wpert);
#endif
            fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
            fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
            fab_arr(i, j, k, 7) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // u*w
            fab_arr(i, j, k, 8) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
            fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // v*w
            fab_arr(i, j, k,10) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // w*w
            fab_arr(i, j, k,11) = u_cc_arr(i,j,k) * theta; // u*th
            fab_arr(i, j, k,12) = v_cc_arr(i,j,k) * theta; // v*th
            fab_arr(i, j, k,13) = w_cc_arr(i,j,k) * theta; // w*th
            fab_arr(i, j, k,14) = theta * theta;           // th*th
 
            // if the number of fields is changed above, then be sure to update
            // the following def!
            Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,8) + fab_arr(i,j,k,10);
            fab_arr(i, j, k,15) = uiui * u_cc_arr(i,j,k);   // (ui*ui)*u
            fab_arr(i, j, k,16) = uiui * v_cc_arr(i,j,k);   // (ui*ui)*v
            fab_arr(i, j, k,17) = uiui * w_cc_arr(i,j,k);   // (ui*ui)*w
 
            if (!use_moisture) {
                Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,18) = p;                       // p
                fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k);     // p*w
                fab_arr(i, j, k,22) = 0.;  // qv
                fab_arr(i, j, k,23) = 0.;  // qc
                fab_arr(i, j, k,24) = 0.;  // qr
                fab_arr(i, j, k,25) = 0.;  // w*qv
                fab_arr(i, j, k,26) = 0.;  // w*qc
                fab_arr(i, j, k,27) = 0.;  // w*qr
                fab_arr(i, j, k,28) = 0.;  // qi
                fab_arr(i, j, k,29) = 0.;  // qs
                fab_arr(i, j, k,30) = 0.;  // qg
                fab_arr(i, j, k,31) = 0.;  // w*thv
            }
        });
    } // mfi
 
    if (use_moisture)
    {
        int n_qstate = micro->Get_Qstate_Size();
 
        for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& fab_arr  = mf_out.array(mfi);
            const Array4<Real>& cons_arr = mf_cons.array(mfi);
            const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
            const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
            const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
            const Array4<Real>&   p0_arr = p_hse.array(mfi);
 
            int rhoqr_comp = solverChoice.RhoQr_comp;
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
                Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
                Real qr = (rhoqr_comp > -1) ?  cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
                                               Real(0.0);
                Real p  = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
 
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,18) = p;                       // p
                fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k);     // p*w
                fab_arr(i, j, k,22) = qv;  // qv
                fab_arr(i, j, k,23) = qc;  // qc
                fab_arr(i, j, k,24) = qr;  // qr
                fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * qv;  // w*qv
                fab_arr(i, j, k,26) = w_cc_arr(i,j,k) * qc;  // w*qc
                fab_arr(i, j, k,27) = w_cc_arr(i,j,k) * qr;  // w*qr
                if (n_qstate > 3) {
                    fab_arr(i, j, k,28) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp);  // qi
                    fab_arr(i, j, k,29) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp);  // qs
                    fab_arr(i, j, k,30) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp);  // qg
                } else {
                    fab_arr(i, j, k,28) = 0.0;  // qi
                    fab_arr(i, j, k,29) = 0.0;  // qs
                    fab_arr(i, j, k,30) = 0.0;  // qg
                }
                Real ql    = qc + qr;
                Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
                Real thv   = theta * (1 + 0.61*qv - ql);
                fab_arr(i, j, k,31) = w_cc_arr(i,j,k) * thv; // w*thv
            });
        } // mfi
    } // use_moisture
 
    h_avg_rho   = sumToLine(mf_out, 0,1,domain,zdir);
    h_avg_th    = sumToLine(mf_out, 1,1,domain,zdir);
    h_avg_ksgs  = sumToLine(mf_out, 2,1,domain,zdir);
    h_avg_Kmv   = sumToLine(mf_out, 3,1,domain,zdir);
    h_avg_Khv   = sumToLine(mf_out, 4,1,domain,zdir);
    h_avg_uu    = sumToLine(mf_out, 5,1,domain,zdir);
    h_avg_uv    = sumToLine(mf_out, 6,1,domain,zdir);
    h_avg_uw    = sumToLine(mf_out, 7,1,domain,zdir);
    h_avg_vv    = sumToLine(mf_out, 8,1,domain,zdir);
    h_avg_vw    = sumToLine(mf_out, 9,1,domain,zdir);
    h_avg_ww    = sumToLine(mf_out,10,1,domain,zdir);
    h_avg_uth   = sumToLine(mf_out,11,1,domain,zdir);
    h_avg_vth   = sumToLine(mf_out,12,1,domain,zdir);
    h_avg_wth   = sumToLine(mf_out,13,1,domain,zdir);
    h_avg_thth  = sumToLine(mf_out,14,1,domain,zdir);
    h_avg_uiuiu = sumToLine(mf_out,15,1,domain,zdir);
    h_avg_uiuiv = sumToLine(mf_out,16,1,domain,zdir);
    h_avg_uiuiw = sumToLine(mf_out,17,1,domain,zdir);
    h_avg_p     = sumToLine(mf_out,18,1,domain,zdir);
    h_avg_pu    = sumToLine(mf_out,19,1,domain,zdir);
    h_avg_pv    = sumToLine(mf_out,20,1,domain,zdir);
    h_avg_pw    = sumToLine(mf_out,21,1,domain,zdir);
    h_avg_qv    = sumToLine(mf_out,22,1,domain,zdir);
    h_avg_qc    = sumToLine(mf_out,23,1,domain,zdir);
    h_avg_qr    = sumToLine(mf_out,24,1,domain,zdir);
    h_avg_wqv   = sumToLine(mf_out,25,1,domain,zdir);
    h_avg_wqc   = sumToLine(mf_out,26,1,domain,zdir);
    h_avg_wqr   = sumToLine(mf_out,27,1,domain,zdir);
    h_avg_qi    = sumToLine(mf_out,28,1,domain,zdir);
    h_avg_qs    = sumToLine(mf_out,29,1,domain,zdir);
    h_avg_qg    = sumToLine(mf_out,30,1,domain,zdir);
    h_avg_wthv  = sumToLine(mf_out,31,1,domain,zdir);
 
    // Divide by the total number of cells we are averaging over
    int h_avg_u_size = static_cast<int>(h_avg_u.size());
    for (int k = 0; k < h_avg_u_size; ++k) {
        h_avg_rho[k] /= area_z;
        h_avg_ksgs[k]  /= area_z;
        h_avg_Kmv[k]   /= area_z;
        h_avg_Khv[k]   /= area_z;
        h_avg_th[k]    /= area_z;
        h_avg_thth[k]  /= area_z;
        h_avg_uu[k]    /= area_z;
        h_avg_uv[k]    /= area_z;
        h_avg_uw[k]    /= area_z;
        h_avg_vv[k]    /= area_z;
        h_avg_vw[k]    /= area_z;
        h_avg_ww[k]    /= area_z;
        h_avg_uth[k]   /= area_z;
        h_avg_vth[k]   /= area_z;
        h_avg_wth[k]   /= area_z;
        h_avg_uiuiu[k] /= area_z;
        h_avg_uiuiv[k] /= area_z;
        h_avg_uiuiw[k] /= area_z;
        h_avg_p[k]     /= area_z;
        h_avg_pu[k]    /= area_z;
        h_avg_pv[k]    /= area_z;
        h_avg_pw[k]    /= area_z;
        h_avg_qv[k]    /= area_z;
        h_avg_qc[k]    /= area_z;
        h_avg_qr[k]    /= area_z;
        h_avg_wqv[k]   /= area_z;
        h_avg_wqc[k]   /= area_z;
        h_avg_wqr[k]   /= area_z;
        h_avg_qi[k]    /= area_z;
        h_avg_qs[k]    /= area_z;
        h_avg_qg[k]    /= area_z;
        h_avg_wthv[k]  /= area_z;
    }
 
#if 0
    // Here we print the integrated total kinetic energy as computed in the 1D profile above
    Real sum = 0.;
    Real dz = geom[0].ProbHi(2) / static_cast<Real>(h_avg_u_size);
    for (int k = 0; k < h_avg_u_size; ++k) {
        sum += h_avg_kturb[k] * h_avg_rho[k] * dz;
    }
    amrex::Print() << "ITKE " << time << " " << sum << " using " << h_avg_u_size << " " << dz << std::endl;
#endif
}

Here is the call graph for this function:

derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag	(	amrex::Real	time,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Kmv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Khv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qi,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qg,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wthv
	)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters

h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_uiuiu	Profile for u_i*u_i*u triple product on Host
h_avg_uiuiv	Profile for u_i*u_i*v triple product on Host
h_avg_uiuiw	Profile for u_i*u_i*w triple product on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host

{
    // We assume that this is always called at level 0
    int lev = 0;
 
    bool l_use_kturb = ((solverChoice.turbChoice[lev].les_type != LESType::None) ||
                        (solverChoice.turbChoice[lev].pbl_type != PBLType::None));
    bool l_use_KE   = ( (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) ||
                        (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) );
 
    // Note: "uiui" == u_i*u_i = u*u + v*v + w*w
    // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, vv, uth, vth,
    //       indices:   0      1     2    3    4   5   6   7    8    9
    //                thth, uiuiu, uiuiv, p, pu, pv, qv, qc, qr, qi, qs, qg
    //                  10     11     12 13  14  15  16  17  18  19  20  21
    MultiFab mf_out(grids[lev], dmap[lev], 22, 0);
 
    // This will hold uw, vw, ww, wth, uiuiw, pw, wqv, wqc, wqr, wthv
    //       indices:  0   1   2    3      4   5    6    7    8     9
    MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 10, 0);
 
    // This is only used to average u and v; w is not averaged to cell centers
    MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
 
    MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
    MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
    MultiFab  w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers (staggered)
 
    int zdir = 2;
    auto domain = geom[0].Domain();
    Box stag_domain = domain;
    stag_domain.convert(IntVect(0,0,1));
 
    int nvars = vars_new[lev][Vars::cons].nComp();
    MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
 
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
 
    for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
        const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
        const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
        const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
        const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
        const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
        const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
        const Array4<Real>& cons_arr = mf_cons.array(mfi);
        const Array4<Real>&   p0_arr = p_hse.array(mfi);
        const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
                                                            Array4<const Real>{};
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            u_cc_arr(i,j,k) = 0.5 * (u_arr(i,j,k) + u_arr(i+1,j  ,k));
            v_cc_arr(i,j,k) = 0.5 * (v_arr(i,j,k) + v_arr(i  ,j+1,k));
 
            Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 1) = theta;
            Real ksgs = 0.0;
            if (l_use_KE) {
                ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
            }
            fab_arr(i, j, k, 2) = ksgs;
            if (l_use_kturb) {
                fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
                fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
            } else {
                fab_arr(i, j, k, 3) = 0.0;
                fab_arr(i, j, k, 4) = 0.0;
            }
            fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
            fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
            fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
            fab_arr(i, j, k, 8) = u_cc_arr(i,j,k) * theta;             // u*th
            fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * theta;             // v*th
            fab_arr(i, j, k,10) = theta * theta;                       // th*th
 
            Real wcc = 0.5 * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
 
            // if the number of fields is changed above, then be sure to update
            // the following def!
            Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,7) + wcc*wcc;
            fab_arr(i, j, k,11) = uiui * u_cc_arr(i,j,k);           // (ui*ui)*u
            fab_arr(i, j, k,12) = uiui * v_cc_arr(i,j,k);           // (ui*ui)*v
 
            if (!use_moisture) {
                Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,13) = p;                       // p
                fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,16) = 0.;  // qv
                fab_arr(i, j, k,17) = 0.;  // qc
                fab_arr(i, j, k,18) = 0.;  // qr
                fab_arr(i, j, k,19) = 0.;  // qi
                fab_arr(i, j, k,20) = 0.;  // qs
                fab_arr(i, j, k,21) = 0.;  // qg
            }
        });
 
        const Box& zbx = mfi.tilebox(IntVect(0,0,1));
        ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            // average to z faces (first to cell centers, then in z)
            Real uface = 0.25 * ( u_arr(i  ,j,k) + u_arr(i  ,j,k-1)
                                + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
            Real vface = 0.25 * ( v_arr(i,j  ,k) + v_arr(i,j  ,k-1)
                                + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
            Real theta0 = cons_arr(i,j,k  ,RhoTheta_comp) / cons_arr(i,j,k  ,Rho_comp);
            Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
            Real thface = 0.5*(theta0 + theta1);
            fab_arr_stag(i,j,k,0) =           uface * w_fc_arr(i,j,k); // u*w
            fab_arr_stag(i,j,k,1) =           vface * w_fc_arr(i,j,k); // v*w
            fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
            fab_arr_stag(i,j,k,3) =          thface * w_fc_arr(i,j,k); // th*w
            Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
            fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
            if (!use_moisture) {
                Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp)) - p0_arr(i,j,k  );
                Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
                Real pface = 0.5 * (p0 + p1);
                fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k);       // p*w
                fab_arr_stag(i,j,k,6) = 0.;  // w*qv
                fab_arr_stag(i,j,k,7) = 0.;  // w*qc
                fab_arr_stag(i,j,k,8) = 0.;  // w*qr
                fab_arr_stag(i,j,k,9) = 0.;  // w*thv
            }
        });
 
    } // mfi
 
    if (use_moisture)
    {
        int n_qstate = micro->Get_Qstate_Size();
 
        for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& fab_arr  = mf_out.array(mfi);
            const Array4<Real>& fab_arr_stag  = mf_out_stag.array(mfi);
            const Array4<Real>& cons_arr = mf_cons.array(mfi);
            const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
            const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
            const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
            const Array4<Real>&   p0_arr = p_hse.array(mfi);
 
            int rhoqr_comp = solverChoice.RhoQr_comp;
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
                Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
                Real qr = (rhoqr_comp > -1) ?  cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
                                               Real(0.0);
                Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
 
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,13) = p;                       // p
                fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,16) = qv;  // qv
                fab_arr(i, j, k,17) = qc;  // qc
                fab_arr(i, j, k,18) = qr;  // qr
                if (n_qstate > 3) { // SAM model
                    fab_arr(i, j, k,19) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp);  // qi
                    fab_arr(i, j, k,20) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp);  // qs
                    fab_arr(i, j, k,21) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp);  // qg
                } else {
                    fab_arr(i, j, k,19) = 0.0;  // qi
                    fab_arr(i, j, k,20) = 0.0;  // qs
                    fab_arr(i, j, k,21) = 0.0;  // qg
                }
            });
 
            const Box& zbx = mfi.tilebox(IntVect(0,0,1));
            ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                Real qv0 = cons_arr(i,j,k  ,RhoQ1_comp) / cons_arr(i,j,k  ,Rho_comp);
                Real qv1 = cons_arr(i,j,k-1,RhoQ1_comp) / cons_arr(i,j,k-1,Rho_comp);
                Real qc0 = cons_arr(i,j,k  ,RhoQ2_comp) / cons_arr(i,j,k  ,Rho_comp);
                Real qc1 = cons_arr(i,j,k-1,RhoQ2_comp) / cons_arr(i,j,k-1,Rho_comp);
                Real qr0 = (rhoqr_comp > -1) ? cons_arr(i,j,k  ,RhoQ3_comp) / cons_arr(i,j,k  ,Rho_comp) :
                                               Real(0.0);
                Real qr1 = (rhoqr_comp > -1) ? cons_arr(i,j,k-1,RhoQ3_comp) / cons_arr(i,j,k-1,Rho_comp) :
                                               Real(0.0);
                Real qvface = 0.5 * (qv0 + qv1);
                Real qcface = 0.5 * (qc0 + qc1);
                Real qrface = 0.5 * (qr0 + qr1);
 
                Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp), qv0) - p0_arr(i,j,k  );
                Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv1) - p0_arr(i,j,k-1);
                Real pface = 0.5 * (p0 + p1);
 
                Real theta0 = cons_arr(i,j,k  ,RhoTheta_comp) / cons_arr(i,j,k  ,Rho_comp);
                Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
                Real thface = 0.5*(theta0 + theta1);
                Real ql = qcface + qrface;
                Real thv = thface * (1 + 0.61*qvface - ql);
 
                fab_arr_stag(i,j,k,5) = pface  * w_fc_arr(i,j,k); // p*w
                fab_arr_stag(i,j,k,6) = qvface * w_fc_arr(i,j,k); // w*qv
                fab_arr_stag(i,j,k,7) = qcface * w_fc_arr(i,j,k); // w*qc
                fab_arr_stag(i,j,k,8) = qrface * w_fc_arr(i,j,k); // w*qr
                fab_arr_stag(i,j,k,9) = thv    * w_fc_arr(i,j,k); // w*thv
            });
        } // mfi
    } // use_moisture
 
    // Sum in the horizontal plane
    h_avg_u = sumToLine(u_cc,0,1,     domain,zdir);
    h_avg_v = sumToLine(v_cc,0,1,     domain,zdir);
    h_avg_w = sumToLine(w_fc,0,1,stag_domain,zdir);
 
    h_avg_rho   = sumToLine(mf_out, 0,1,domain,zdir);
    h_avg_th    = sumToLine(mf_out, 1,1,domain,zdir);
    h_avg_ksgs  = sumToLine(mf_out, 2,1,domain,zdir);
    h_avg_Kmv   = sumToLine(mf_out, 3,1,domain,zdir);
    h_avg_Khv   = sumToLine(mf_out, 4,1,domain,zdir);
    h_avg_uu    = sumToLine(mf_out, 5,1,domain,zdir);
    h_avg_uv    = sumToLine(mf_out, 6,1,domain,zdir);
    h_avg_vv    = sumToLine(mf_out, 7,1,domain,zdir);
    h_avg_uth   = sumToLine(mf_out, 8,1,domain,zdir);
    h_avg_vth   = sumToLine(mf_out, 9,1,domain,zdir);
    h_avg_thth  = sumToLine(mf_out,10,1,domain,zdir);
    h_avg_uiuiu = sumToLine(mf_out,11,1,domain,zdir);
    h_avg_uiuiv = sumToLine(mf_out,12,1,domain,zdir);
    h_avg_p     = sumToLine(mf_out,13,1,domain,zdir);
    h_avg_pu    = sumToLine(mf_out,14,1,domain,zdir);
    h_avg_pv    = sumToLine(mf_out,15,1,domain,zdir);
    h_avg_qv    = sumToLine(mf_out,16,1,domain,zdir);
    h_avg_qc    = sumToLine(mf_out,17,1,domain,zdir);
    h_avg_qr    = sumToLine(mf_out,18,1,domain,zdir);
    h_avg_qi    = sumToLine(mf_out,19,1,domain,zdir);
    h_avg_qs    = sumToLine(mf_out,20,1,domain,zdir);
    h_avg_qg    = sumToLine(mf_out,21,1,domain,zdir);
 
    h_avg_uw    = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
    h_avg_vw    = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
    h_avg_ww    = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
    h_avg_wth   = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
    h_avg_uiuiw = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
    h_avg_pw    = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
    h_avg_wqv   = sumToLine(mf_out_stag,6,1,stag_domain,zdir);
    h_avg_wqc   = sumToLine(mf_out_stag,7,1,stag_domain,zdir);
    h_avg_wqr   = sumToLine(mf_out_stag,8,1,stag_domain,zdir);
    h_avg_wthv  = sumToLine(mf_out_stag,9,1,stag_domain,zdir);
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
    for (int k = 0; k < unstag_size; ++k) {
        h_avg_u[k]     /= area_z;
        h_avg_v[k]     /= area_z;
        h_avg_rho[k]   /= area_z;
        h_avg_ksgs[k]  /= area_z;
        h_avg_Kmv[k]   /= area_z;
        h_avg_Khv[k]   /= area_z;
        h_avg_th[k]    /= area_z;
        h_avg_thth[k]  /= area_z;
        h_avg_uu[k]    /= area_z;
        h_avg_uv[k]    /= area_z;
        h_avg_vv[k]    /= area_z;
        h_avg_uth[k]   /= area_z;
        h_avg_vth[k]   /= area_z;
        h_avg_uiuiu[k] /= area_z;
        h_avg_uiuiv[k] /= area_z;
        h_avg_p[k]     /= area_z;
        h_avg_pu[k]    /= area_z;
        h_avg_pv[k]    /= area_z;
        h_avg_qv[k]    /= area_z;
        h_avg_qc[k]    /= area_z;
        h_avg_qr[k]    /= area_z;
        h_avg_qi[k]    /= area_z;
        h_avg_qs[k]    /= area_z;
        h_avg_qg[k]    /= area_z;
    }
 
    for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
        h_avg_w[k]     /= area_z;
        h_avg_uw[k]    /= area_z;
        h_avg_vw[k]    /= area_z;
        h_avg_ww[k]    /= area_z;
        h_avg_wth[k]   /= area_z;
        h_avg_uiuiw[k] /= area_z;
        h_avg_pw[k]    /= area_z;
        h_avg_wqv[k]   /= area_z;
        h_avg_wqc[k]   /= area_z;
        h_avg_wqr[k]   /= area_z;
        h_avg_wthv[k]  /= area_z;
    }
}

Here is the call graph for this function:

derive_stress_profiles()

void ERF::derive_stress_profiles	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q1fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q2fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

{
    int lev = 0;
 
    // This will hold the stress tensor components
    MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
 
    bool l_use_moist   = ( solverChoice.moisture_type != MoistureType::None );
 
    for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
 
        // NOTE: These are from the last RK stage...
        const Array4<const Real>& tau11_arr = Tau11_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau12_arr = Tau12_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau13_arr = Tau13_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau22_arr = Tau22_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau23_arr = Tau23_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau33_arr = Tau33_lev[lev]->const_array(mfi);
 
        // These should be re-calculated during ERF_slow_rhs_post
        // -- just vertical SFS kinematic heat flux for now
        //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
        //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
        const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
        const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            fab_arr(i, j, k, 0) = tau11_arr(i,j,k);
            fab_arr(i, j, k, 1) = 0.25 * ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                         + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) );
            fab_arr(i, j, k, 2) = 0.25 * ( tau13_arr(i,j,k  ) + tau13_arr(i+1,j,k)
                                         + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) );
            fab_arr(i, j, k, 3) = tau22_arr(i,j,k);
            fab_arr(i, j, k, 4) = 0.25 * ( tau23_arr(i,j,k  ) + tau23_arr(i,j+1,k)
                                         + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) );
            fab_arr(i, j, k, 5) = tau33_arr(i,j,k);
            fab_arr(i, j, k, 6) = 0.5 * ( hfx3_arr(i,j,k) + hfx3_arr(i,j,k+1) );
            fab_arr(i, j, k, 7) = (l_use_moist) ? 0.5 * ( q1fx3_arr(i,j,k) + q1fx3_arr(i,j,k+1) ) : 0.0;
            fab_arr(i, j, k, 8) = (l_use_moist) ? 0.5 * ( q2fx3_arr(i,j,k) + q2fx3_arr(i,j,k+1) ) : 0.0;
            fab_arr(i, j, k, 9) =  diss_arr(i,j,k);
        });
    }
 
    int zdir = 2;
    auto domain = geom[0].Domain();
 
    h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
    h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
    h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
    h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
    h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
    h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
    h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
    h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
    h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
    h_avg_diss  = sumToLine(mf_out,9,1,domain,zdir);
 
    int ht_size =  h_avg_tau11.size();
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < ht_size; ++k) {
        h_avg_tau11[k] /= area_z; h_avg_tau12[k] /= area_z; h_avg_tau13[k] /= area_z;
        h_avg_tau22[k] /= area_z; h_avg_tau23[k] /= area_z;
        h_avg_tau33[k] /= area_z;
        h_avg_hfx3[k] /= area_z;
        h_avg_q1fx3[k] /= area_z; h_avg_q2fx3[k] /= area_z;
        h_avg_diss[k] /= area_z;
    }
}

derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q1fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q2fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

{
    int lev = 0;
 
    // This will hold the stress tensor components
    MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
 
    // This will hold Tau13 and Tau23
    MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 5, 0);
 
    bool l_use_moist   = ( solverChoice.moisture_type != MoistureType::None );
 
    for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
        const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
 
        // NOTE: These are from the last RK stage...
        const Array4<const Real>& tau11_arr = Tau11_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau12_arr = Tau12_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau13_arr = Tau13_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau22_arr = Tau22_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau23_arr = Tau23_lev[lev]->const_array(mfi);
        const Array4<const Real>& tau33_arr = Tau33_lev[lev]->const_array(mfi);
 
        // These should be re-calculated during ERF_slow_rhs_post
        // -- just vertical SFS kinematic heat flux for now
        //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
        //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
        const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
        const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            fab_arr(i, j, k, 0) = tau11_arr(i,j,k);
            fab_arr(i, j, k, 1) = 0.25 * ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                         + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) );
//          fab_arr(i, j, k, 2) = 0.25 * ( tau13_arr(i,j,k  ) + tau13_arr(i+1,j,k)
//                                       + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) );
            fab_arr(i, j, k, 3) = tau22_arr(i,j,k);
//          fab_arr(i, j, k, 4) = 0.25 * ( tau23_arr(i,j,k  ) + tau23_arr(i,j+1,k)
//                                       + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) );
            fab_arr(i, j, k, 5) = tau33_arr(i,j,k);
//          fab_arr(i, j, k, 6) =  hfx3_arr(i,j,k);
//          fab_arr(i, j, k, 7) =  (l_use_moist) ? q1fx3_arr(i,j,k) : 0.0;
//          fab_arr(i, j, k, 8) =  (l_use_moist) ? q2fx3_arr(i,j,k) : 0.0;
            fab_arr(i, j, k, 9) =  diss_arr(i,j,k);
        });
 
        const Box& zbx = mfi.tilebox(IntVect(0,0,1));
        ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            // average from edge to face center
            fab_arr_stag(i,j,k,0) = 0.5*(tau13_arr(i,j,k) + tau13_arr(i+1,j  ,k));
            fab_arr_stag(i,j,k,1) = 0.5*(tau23_arr(i,j,k) + tau23_arr(i  ,j+1,k));
 
            fab_arr_stag(i,j,k,2) =  hfx3_arr(i,j,k);
            fab_arr_stag(i,j,k,3) =  (l_use_moist) ? q1fx3_arr(i,j,k) : 0.0;
            fab_arr_stag(i,j,k,4) =  (l_use_moist) ? q2fx3_arr(i,j,k) : 0.0;
        });
    }
 
    int zdir = 2;
    auto domain = geom[0].Domain();
    Box stag_domain = domain;
    stag_domain.convert(IntVect(0,0,1));
 
    h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
    h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
//  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
    h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
//  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
    h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
//  h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
//  h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
//  h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
    h_avg_diss  = sumToLine(mf_out,9,1,domain,zdir);
 
    h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
    h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
    h_avg_hfx3  = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
    h_avg_q1fx3 = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
    h_avg_q2fx3 = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
 
    int ht_size =  h_avg_tau11.size(); // _un_staggered
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < ht_size; ++k) {
        h_avg_tau11[k] /= area_z; h_avg_tau12[k] /= area_z;
        h_avg_tau22[k] /= area_z;
        h_avg_tau33[k] /= area_z;
        h_avg_diss[k] /= area_z;
    }
    for (int k = 0; k < ht_size+1; ++k) { // staggered heights
        h_avg_tau13[k] /= area_z;
        h_avg_tau23[k] /= area_z;
        h_avg_hfx3[k] /= area_z;
        h_avg_q1fx3[k] /= area_z; h_avg_q2fx3[k] /= area_z;
    }
}

derive_upwp()

void ERF::derive_upwp

(

amrex::Vector< amrex::Real > &

h_havg

)

erf_enforce_hse()

void ERF::erf_enforce_hse	(	int	lev,
		amrex::MultiFab &	dens,
		amrex::MultiFab &	pres,
		amrex::MultiFab &	pi,
		amrex::MultiFab &	th,
		std::unique_ptr< amrex::MultiFab > &	z_cc
	)

Enforces hydrostatic equilibrium when using terrain.

Parameters

[in]	lev	Integer specifying the current level
[out]	dens	MultiFab storing base state density
[out]	pres	MultiFab storing base state pressure
[out]	pi	MultiFab storing base state Exner function
[in]	z_cc	Pointer to MultiFab storing cell centered z-coordinates

{
    Real l_gravity = solverChoice.gravity;
    bool l_use_terrain = (solverChoice.mesh_type != MeshType::ConstantDz);
 
    const auto geomdata = geom[lev].data();
    const Real dz = geomdata.CellSize(2);
 
    for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
    {
        // Create a flat box with same horizontal extent but only one cell in vertical
        const Box& tbz = mfi.nodaltilebox(2);
        int klo = tbz.smallEnd(2);
        int khi = tbz.bigEnd(2);
 
        // Note we only grow by 1 because that is how big z_cc is.
        Box b2d = tbz; // Copy constructor
        b2d.grow(0,1);
        b2d.grow(1,1);
        b2d.setRange(2,0);
 
        // We integrate to the first cell (and below) by using rho in this cell
        // If gravity == 0 this is constant pressure
        // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
        // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
        // Note ng_pres_hse = 1
 
       // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
       // Note that gravity is positive
 
        Array4<Real>  rho_arr = dens.array(mfi);
        Array4<Real> pres_arr = pres.array(mfi);
        Array4<Real>   pi_arr =   pi.array(mfi);
        Array4<Real>   th_arr = theta.array(mfi);
        Array4<Real> zcc_arr;
        if (l_use_terrain) {
           zcc_arr = z_cc->array(mfi);
        }
 
        const Real rdOcp = solverChoice.rdOcp;
 
        ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
            // Set value at surface from Newton iteration for rho
            if (klo == 0)
            {
                // Physical height of the terrain at cell center
                Real hz;
                if (l_use_terrain) {
                    hz = zcc_arr(i,j,klo);
                } else {
                    hz = 0.5*dz;
                }
 
                pres_arr(i,j,klo) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
                  pi_arr(i,j,klo) = getExnergivenP(pres_arr(i,j,klo), rdOcp);
                  th_arr(i,j,klo) =getRhoThetagivenP(pres_arr(i,j,klo)) / rho_arr(i,j,klo);
 
                //
                // Set ghost cell with dz and rho at boundary
                // (We will set the rest of the ghost cells in the boundary condition routine)
                //
                pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
                  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
                  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
 
            } else {
 
                // If level > 0 and klo > 0, we need to use the value of pres_arr(i,j,klo-1) which was
                //    filled from FillPatch-ing it.
                Real dz_loc;
                if (l_use_terrain) {
                    dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
                } else {
                    dz_loc = dz;
                }
 
                Real dens_interp = 0.5*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
                pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
 
                pi_arr(i,j,klo  ) = getExnergivenP(pres_arr(i,j,klo  ), rdOcp);
                th_arr(i,j,klo  ) = getRhoThetagivenP(pres_arr(i,j,klo  )) / rho_arr(i,j,klo  );
 
                pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
                th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
            }
 
            Real dens_interp;
            if (l_use_terrain) {
                for (int k = klo+1; k <= khi; k++) {
                    Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
                    dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                    pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
                    pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                    th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
                }
            } else {
                for (int k = klo+1; k <= khi; k++) {
                    dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                    pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
                    pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                    th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
                }
            }
        });
 
    } // mfi
 
     dens.FillBoundary(geom[lev].periodicity());
     pres.FillBoundary(geom[lev].periodicity());
       pi.FillBoundary(geom[lev].periodicity());
    theta.FillBoundary(geom[lev].periodicity());
}

Here is the call graph for this function:

ERF_shared()

void ERF::ERF_shared

(

)

{
    if (ParallelDescriptor::IOProcessor()) {
        const char* erf_hash = buildInfoGetGitHash(1);
        const char* amrex_hash = buildInfoGetGitHash(2);
        const char* buildgithash = buildInfoGetBuildGitHash();
        const char* buildgitname = buildInfoGetBuildGitName();
 
        if (strlen(erf_hash) > 0) {
          Print() << "\n"
                         << "ERF git hash: " << erf_hash << "\n";
        }
        if (strlen(amrex_hash) > 0) {
          Print() << "AMReX git hash: " << amrex_hash << "\n";
        }
        if (strlen(buildgithash) > 0) {
          Print() << buildgitname << " git hash: " << buildgithash << "\n";
        }
 
        Print() << "\n";
    }
 
    int nlevs_max = max_level + 1;
 
#ifdef ERF_USE_WINDFARM
    Nturb.resize(nlevs_max);
    vars_windfarm.resize(nlevs_max);
    SMark.resize(nlevs_max);
#endif
 
#if defined(ERF_USE_RRTMGP)
    qheating_rates.resize(nlevs_max);
    sw_lw_fluxes.resize(nlevs_max);
    solar_zenith.resize(nlevs_max);
#endif
 
    // NOTE: size lsm before readparams (chooses the model at all levels)
    lsm.ReSize(nlevs_max);
    lsm_data.resize(nlevs_max);
    lsm_flux.resize(nlevs_max);
 
    // NOTE: size canopy model before readparams (if file exists, we construct)
    m_forest_drag.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev) { m_forest_drag[lev] = nullptr;}
 
    // Immersed Forcing Representation of Terrain
    m_terrain_drag.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev) { m_terrain_drag[lev] = nullptr;}
 
 
    ReadParameters();
    initializeMicrophysics(nlevs_max);
 
#ifdef ERF_USE_WINDFARM
    initializeWindFarm(nlevs_max);
#endif
 
    const std::string& pv1 = "plot_vars_1"; setPlotVariables(pv1,plot_var_names_1);
    const std::string& pv2 = "plot_vars_2"; setPlotVariables(pv2,plot_var_names_2);
 
    // This is only used when we have mesh_type == MeshType::StretchedDz
    stretched_dz_h.resize(nlevs_max);
    stretched_dz_d.resize(nlevs_max);
 
    // Initialize staggered vertical levels for grid stretching or terrain, and
    // to simplify Rayleigh damping layer calculations.
    zlevels_stag.resize(max_level+1);
    init_zlevels(zlevels_stag,
                 stretched_dz_h,
                 stretched_dz_d,
                 geom,
                 refRatio(),
                 solverChoice.grid_stretching_ratio,
                 solverChoice.zsurf,
                 solverChoice.dz0);
 
    if (SolverChoice::mesh_type == MeshType::StretchedDz ||
        SolverChoice::mesh_type == MeshType::VariableDz) {
        int nz = geom[0].Domain().length(2) + 1; // staggered
        if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
            Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
                << " does not match highest requested z level " << zlevels_stag[0][nz-1]
                << std::endl;
        }
        if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > 1.0e-4) {
            Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
                << " does not match lowest requested level " << zlevels_stag[0][0]
                << std::endl;
        }
 
        // Redefine the problem domain here?
    }
 
    prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
 
    // Geometry on all levels has been defined already.
 
    // No valid BoxArray and DistributionMapping have been defined.
    // But the arrays for them have been resized.
 
    istep.resize(nlevs_max, 0);
    nsubsteps.resize(nlevs_max, 1);
    for (int lev = 1; lev <= max_level; ++lev) {
        nsubsteps[lev] = MaxRefRatio(lev-1);
    }
 
    t_new.resize(nlevs_max, 0.0);
    t_old.resize(nlevs_max, -1.e100);
    dt.resize(nlevs_max, 1.e100);
    dt_mri_ratio.resize(nlevs_max, 1);
 
    vars_new.resize(nlevs_max);
    vars_old.resize(nlevs_max);
 
    // We resize this regardless in order to pass it without error
    pp_inc.resize(nlevs_max);
 
    rU_new.resize(nlevs_max);
    rV_new.resize(nlevs_max);
    rW_new.resize(nlevs_max);
 
    rU_old.resize(nlevs_max);
    rV_old.resize(nlevs_max);
    rW_old.resize(nlevs_max);
 
    // xmom_crse_rhs.resize(nlevs_max);
    // ymom_crse_rhs.resize(nlevs_max);
    zmom_crse_rhs.resize(nlevs_max);
 
    for (int lev = 0; lev < nlevs_max; ++lev) {
        vars_new[lev].resize(Vars::NumTypes);
        vars_old[lev].resize(Vars::NumTypes);
    }
 
    // Time integrator
    mri_integrator_mem.resize(nlevs_max);
 
    // Physical boundary conditions
    physbcs_cons.resize(nlevs_max);
    physbcs_u.resize(nlevs_max);
    physbcs_v.resize(nlevs_max);
    physbcs_w.resize(nlevs_max);
    physbcs_base.resize(nlevs_max);
 
    // Planes to hold Dirichlet values at boundaries
    xvel_bc_data.resize(nlevs_max);
    yvel_bc_data.resize(nlevs_max);
    zvel_bc_data.resize(nlevs_max);
 
    advflux_reg.resize(nlevs_max);
 
    // Stresses
    Tau11_lev.resize(nlevs_max); Tau22_lev.resize(nlevs_max); Tau33_lev.resize(nlevs_max);
    Tau12_lev.resize(nlevs_max); Tau21_lev.resize(nlevs_max);
    Tau13_lev.resize(nlevs_max); Tau31_lev.resize(nlevs_max);
    Tau23_lev.resize(nlevs_max); Tau32_lev.resize(nlevs_max);
    SFS_hfx1_lev.resize(nlevs_max); SFS_hfx2_lev.resize(nlevs_max); SFS_hfx3_lev.resize(nlevs_max);
    SFS_diss_lev.resize(nlevs_max);
    SFS_q1fx1_lev.resize(nlevs_max); SFS_q1fx2_lev.resize(nlevs_max); SFS_q1fx3_lev.resize(nlevs_max);
    SFS_q2fx3_lev.resize(nlevs_max);
    eddyDiffs_lev.resize(nlevs_max);
    SmnSmn_lev.resize(nlevs_max);
 
    // Sea surface temps
    sst_lev.resize(nlevs_max);
    lmask_lev.resize(nlevs_max);
 
    // Metric terms
    z_phys_nd.resize(nlevs_max);
    z_phys_cc.resize(nlevs_max);
    detJ_cc.resize(nlevs_max);
    ax.resize(nlevs_max);
    ay.resize(nlevs_max);
    az.resize(nlevs_max);
 
    z_phys_nd_new.resize(nlevs_max);
    detJ_cc_new.resize(nlevs_max);
    ax_new.resize(nlevs_max);
    ay_new.resize(nlevs_max);
    az_new.resize(nlevs_max);
 
    z_phys_nd_src.resize(nlevs_max);
    detJ_cc_src.resize(nlevs_max);
    ax_src.resize(nlevs_max);
    ay_src.resize(nlevs_max);
    az_src.resize(nlevs_max);
 
    z_t_rk.resize(nlevs_max);
 
    // Wall distance
    walldist.resize(nlevs_max);
 
    // Mapfactors
    mapfac_m.resize(nlevs_max);
    mapfac_u.resize(nlevs_max);
    mapfac_v.resize(nlevs_max);
 
    // Thin immersed body
    xflux_imask.resize(nlevs_max);
    yflux_imask.resize(nlevs_max);
    zflux_imask.resize(nlevs_max);
    //overset_imask.resize(nlevs_max);
    thin_xforce.resize(nlevs_max);
    thin_yforce.resize(nlevs_max);
    thin_zforce.resize(nlevs_max);
 
    // Base state
    base_state.resize(nlevs_max);
    base_state_new.resize(nlevs_max);
 
    // Wave coupling data
    Hwave.resize(nlevs_max);
    Lwave.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev)
    {
        Hwave[lev] = nullptr;
        Lwave[lev] = nullptr;
    }
    Hwave_onegrid.resize(nlevs_max);
    Lwave_onegrid.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev)
    {
        Hwave_onegrid[lev] = nullptr;
        Lwave_onegrid[lev] = nullptr;
    }
 
    // Theta prim for MOST
    Theta_prim.resize(nlevs_max);
 
    // Qv prim for MOST
    Qv_prim.resize(nlevs_max);
 
    // Qr prim for MOST
    Qr_prim.resize(nlevs_max);
 
    // Time averaged velocity field
    vel_t_avg.resize(nlevs_max);
    t_avg_cnt.resize(nlevs_max);
 
#ifdef ERF_USE_NETCDF
    // Size lat long arrays if using netcdf
    lat_m.resize(nlevs_max);
    lon_m.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev)
    {
        lat_m[lev] = nullptr;
        lon_m[lev] = nullptr;
    }
#endif
 
    // Initialize tagging criteria for mesh refinement
    refinement_criteria_setup();
 
    for (int lev = 0; lev < max_level; ++lev)
    {
       Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
          ref_ratio[lev][0]  << " " << ref_ratio[lev][1]  <<  " " << ref_ratio[lev][2] << std::endl;
    }
 
    // We will create each of these in MakeNewLevel.../RemakeLevel
    m_factory.resize(max_level+1);
 
#ifdef ERF_USE_EB
    // This is needed before initializing level MultiFabs
    MakeEBGeometry();
#endif
}

Here is the call graph for this function:

ErrorEst()

void ERF::ErrorEst ( int  lev, amrex::TagBoxArray &  tags, amrex::Real  time, int  ngrow  )

override

Function to tag cells for refinement – this overrides the pure virtual function in AmrCore

Parameters

[in]	levc	level of refinement at which we tag cells (0 is coarsest level)
[out]	tags	array of tagged cells
[in]	time	current time

{
    const int clearval = TagBox::CLEAR;
    const int   tagval = TagBox::SET;
 
    for (int j=0; j < ref_tags.size(); ++j)
    {
        std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 0);
 
        // This allows dynamic refinement based on the value of the density
        if (ref_tags[j].Field() == "density")
        {
            MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,0);
 
        // This allows dynamic refinement based on the value of qv
        } else if ( ref_tags[j].Field() == "qv" ) {
            MultiFab::Copy(  *mf, vars_new[levc][Vars::cons], RhoQ1_comp, 0, 1, 0);
            MultiFab::Divide(*mf, vars_new[levc][Vars::cons],   Rho_comp, 0, 1, 0);
 
 
        // This allows dynamic refinement based on the value of qc
        } else if (ref_tags[j].Field() == "qc" ) {
            MultiFab::Copy(  *mf, vars_new[levc][Vars::cons], RhoQ2_comp, 0, 1, 0);
            MultiFab::Divide(*mf, vars_new[levc][Vars::cons],   Rho_comp, 0, 1, 0);
 
 
        // This allows dynamic refinement based on the value of the scalar/pressure/theta
        } else if ( (ref_tags[j].Field() == "scalar"  ) ||
                    (ref_tags[j].Field() == "pressure") ||
                    (ref_tags[j].Field() == "theta"   ) )
        {
            for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                auto& dfab = (*mf)[mfi];
                auto& sfab = vars_new[levc][Vars::cons][mfi];
                if (ref_tags[j].Field() == "scalar") {
                    derived::erf_derscalar(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                } else if (ref_tags[j].Field() == "theta") {
                    derived::erf_dertheta(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                }
            } // mfi
#ifdef ERF_USE_PARTICLES
        } else {
            //
            // This allows dynamic refinement based on the number of particles per cell
            //
            // Note that we must count all the particles in levels both at and above the current,
            //      since otherwise, e.g., if the particles are all at level 1, counting particles at
            //      level 0 will not trigger refinement when regridding so level 1 will disappear,
            //      then come back at the next regridding
            //
            const auto& particles_namelist( particleData.getNames() );
            mf->setVal(0.0);
            for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
            {
                std::string tmp_string(particles_namelist[i]+"_count");
                IntVect rr = IntVect::TheUnitVector();
                if (ref_tags[j].Field() == tmp_string) {
                    for (int lev = levc; lev <= finest_level; lev++)
                    {
                        MultiFab temp_dat(grids[lev], dmap[lev], 1, 0); temp_dat.setVal(0);
                        particleData[particles_namelist[i]]->IncrementWithTotal(temp_dat, lev);
 
                        MultiFab temp_dat_crse(grids[levc], dmap[levc], 1, 0); temp_dat_crse.setVal(0);
 
                        if (lev == levc) {
                            MultiFab::Copy(*mf, temp_dat, 0, 0, 1, 0);
                        } else {
                            for (int d = 0; d < AMREX_SPACEDIM; d++) {
                                rr[d] *= ref_ratio[levc][d];
                            }
                            average_down(temp_dat, temp_dat_crse, 0, 1, rr);
                            MultiFab::Add(*mf, temp_dat_crse, 0, 0, 1, 0);
                        }
                    }
                }
            }
#endif
        }
 
        ref_tags[j](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
    } // loop over j
}

Here is the call graph for this function:

estTimeStep()

Real ERF::estTimeStep	(	int	level,
		long &	dt_fast_ratio
	)		const

Function that calls estTimeStep for each level

Parameters

[in]	level	level of refinement (coarsest level i 0)
[out]	dt_fast_ratio	ratio of slow to fast time step

{
    BL_PROFILE("ERF::estTimeStep()");
 
    Real estdt_comp = 1.e20;
    Real estdt_lowM = 1.e20;
 
    // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation
    const int nxc = geom[0].Domain().length(0);
    const int nyc = geom[0].Domain().length(1);
 
    auto const dxinv = geom[level].InvCellSizeArray();
    auto const dzinv = 1.0 / dz_min[level];
 
    MultiFab const& S_new = vars_new[level][Vars::cons];
 
    MultiFab ccvel(grids[level],dmap[level],3,0);
 
    average_face_to_cellcenter(ccvel,0,
                               Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
                                                        &vars_new[level][Vars::yvel],
                                                        &vars_new[level][Vars::zvel]});
 
    int l_implicit_substepping = (solverChoice.substepping_type[level] == SubsteppingType::Implicit);
    int l_anelastic      = solverChoice.anelastic[level];
 
#ifdef ERF_USE_EB
    EBFArrayBoxFactory ebfact = EBFactory(level);
    const MultiFab& detJ = ebfact.getVolFrac();
#endif
 
#ifdef ERF_USE_EB
    Real estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
       [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                  Array4<Real const> const& s,
                                  Array4<Real const> const& u,
                                  Array4<Real const> const& vf) -> Real
#else
    Real estdt_comp_inv = ReduceMax(S_new, ccvel, 0,
       [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                  Array4<Real const> const& s,
                                  Array4<Real const> const& u) -> Real
#endif
       {
           Real new_comp_dt = -1.e100;
           amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
           {
#ifdef ERF_USE_EB
               if (vf(i,j,k) > 0.)
#endif
               {
                   const Real rho      = s(i, j, k, Rho_comp);
                   const Real rhotheta = s(i, j, k, RhoTheta_comp);
 
                   // NOTE: even when moisture is present,
                   //       we only use the partial pressure of the dry air
                   //       to compute the soundspeed
                   Real pressure = getPgivenRTh(rhotheta);
                   Real c = std::sqrt(Gamma * pressure / rho);
 
                   // If we are doing implicit acoustic substepping, then the z-direction does not contribute
                   //    to the computation of the time step
                   if (l_implicit_substepping) {
                       if ((nxc > 1) && (nyc==1)) {
                           // 2-D in x-z
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
                       } else if ((nyc > 1) && (nxc==1)) {
                           // 2-D in y-z
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                       } else {
                           // 3-D or SCM
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                    ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                       }
 
                   // If we are not doing implicit acoustic substepping, then the z-direction contributes
                   //    to the computation of the time step
                   } else {
                       if (nxc > 1 && nyc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                    ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       } else if (nxc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       } else if (nyc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       } else {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       }
 
                   }
               }
           });
           return new_comp_dt;
       });
 
    ParallelDescriptor::ReduceRealMax(estdt_comp_inv);
    estdt_comp = cfl / estdt_comp_inv;
 
     Real estdt_lowM_inv = ReduceMax(ccvel, 0,
       [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                  Array4<Real const> const& u) -> Real
       {
           Real new_lm_dt = -1.e100;
           Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
           {
               new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
                                      ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
                                      ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
           });
           return new_lm_dt;
       });
 
     ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
     if (estdt_lowM_inv > 0.0_rt)
         estdt_lowM = cfl / estdt_lowM_inv;
 
     if (verbose) {
         if (fixed_dt[level] <= 0.0) {
             Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " <<
               1.0/dxinv[0] << " " << 1.0/dxinv[1] << " " << dz_min[level] << std::endl;
             Print() << "Compressible dt at level " << level << ":  " << estdt_comp << std::endl;
             if (estdt_lowM_inv > 0.0_rt) {
                 Print() << "Anelastic dt at level " << level << ":  " << estdt_lowM << std::endl;
             } else {
                 Print() << "Anelastic dt at level " << level << ": undefined " << std::endl;
             }
         }
 
         if (fixed_dt[level] > 0.0) {
             Print() << "Based on cfl of 1.0 " << std::endl;
             Print() << "Compressible dt at level " << level << " would be:  " << estdt_comp/cfl << std::endl;
             if (estdt_lowM_inv > 0.0_rt) {
                 Print() << "Anelastic dt at level " << level << " would be:  " << estdt_lowM/cfl << std::endl;
             } else {
                 Print() << "Anelastic dt at level " << level << " would be undefined " << std::endl;
             }
             Print() << "Fixed dt at level " << level << "       is:  " << fixed_dt[level] << std::endl;
             if (fixed_fast_dt[level] > 0.0) {
                 Print() << "Fixed fast dt at level " << level << "       is:  " << fixed_fast_dt[level] << std::endl;
             }
         }
     }
 
     if (solverChoice.substepping_type[level] != SubsteppingType::None) {
         if (fixed_dt[level] > 0. && fixed_fast_dt[level] > 0.) {
             dt_fast_ratio = static_cast<long>( fixed_dt[level] / fixed_fast_dt[level] );
         } else if (fixed_dt[level] > 0.) {
             // Max CFL_c = 1.0 for substeps by default, but we enforce a min of 4 substeps
             auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
             dt_fast_ratio = static_cast<long>( std::max(fixed_dt[level]/dt_sub_max,4.) );
         }
 
         // Force time step ratio to be an even value
         if (solverChoice.force_stage1_single_substep) {
             if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
         } else {
             if ( dt_fast_ratio%6 != 0) {
                 Print() << "mri_dt_ratio = " << dt_fast_ratio
                         << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
                 dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/6.0) * 6);
             }
         }
 
         if (verbose) {
             Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
         }
     } // if substepping, either explicit or implicit
 
     if (fixed_dt[level] > 0.0) {
         return fixed_dt[level];
     } else {
         // Anelastic (substepping is not allowed)
         if (l_anelastic) {
             return estdt_lowM;
 
         // Compressible with or without substepping
         } else {
             return estdt_comp;
         }
     }
}

Here is the call graph for this function:

Evolve()

void ERF::Evolve

(

)

{
    BL_PROFILE_VAR("ERF::Evolve()", evolve);
 
    Real cur_time = t_new[0];
 
    // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
    //      for finer levels (with or without subcycling)
    for (int step = istep[0]; step < max_step && cur_time < stop_time; ++step)
    {
        Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
 
        ComputeDt(step);
 
        // Make sure we have read enough of the boundary plane data to make it through this timestep
        if (input_bndry_planes)
        {
            m_r2d->read_input_files(cur_time,dt[0],m_bc_extdir_vals);
        }
 
        int lev = 0;
        int iteration = 1;
        timeStep(lev, cur_time, iteration);
 
        cur_time  += dt[0];
 
        Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
                << " DT = " << dt[0]  << std::endl;
 
        post_timestep(step, cur_time, dt[0]);
 
        if (writeNow(cur_time, dt[0], step+1, m_plot_int_1, m_plot_per_1)) {
            last_plot_file_step_1 = step+1;
            WritePlotFile(1,plotfile_type_1,plot_var_names_1);
        }
        if (writeNow(cur_time, dt[0], step+1, m_plot_int_2, m_plot_per_2)) {
            last_plot_file_step_2 = step+1;
            WritePlotFile(2,plotfile_type_2,plot_var_names_2);
        }
 
        if (writeNow(cur_time, dt[0], step+1, m_check_int, m_check_per)) {
            last_check_file_step = step+1;
#ifdef ERF_USE_NETCDF
            if (check_type == "netcdf") {
               WriteNCCheckpointFile();
            }
#endif
            if (check_type == "native") {
               WriteCheckpointFile();
            }
        }
 
#ifdef AMREX_MEM_PROFILING
        {
            std::ostringstream ss;
            ss << "[STEP " << step+1 << "]";
            MemProfiler::report(ss.str());
        }
#endif
 
        if (cur_time >= stop_time - 1.e-6*dt[0]) break;
    }
 
    // Write plotfiles at final time
    if ( (m_plot_int_1 > 0 || m_plot_per_1 > 0.) && istep[0] > last_plot_file_step_1 ) {
        WritePlotFile(1,plotfile_type_1,plot_var_names_1);
    }
    if ( (m_plot_int_2 > 0 || m_plot_per_2 > 0.) && istep[0] > last_plot_file_step_2) {
        WritePlotFile(2,plotfile_type_1,plot_var_names_2);
    }
 
    if ( (m_check_int > 0 || m_check_per > 0.) && istep[0] > last_check_file_step) {
#ifdef ERF_USE_NETCDF
        if (check_type == "netcdf") {
           WriteNCCheckpointFile();
        }
#endif
        if (check_type == "native") {
           WriteCheckpointFile();
        }
    }
 
    BL_PROFILE_VAR_STOP(evolve);
}

Referenced by main().

Here is the caller graph for this function:

fill_from_bndryregs()

void ERF::fill_from_bndryregs	(	const amrex::Vector< amrex::MultiFab * > &	mfs,
		amrex::Real	time
	)

{
    //
    // We now assume that if we read in on one face, we read in on all faces
    //
    AMREX_ALWAYS_ASSERT(m_r2d);
 
    int lev = 0;
    const Box& domain = geom[lev].Domain();
 
    const auto& dom_lo = lbound(domain);
    const auto& dom_hi = ubound(domain);
 
    Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time);
 
    const BCRec* bc_ptr = domain_bcs_type_d.data();
 
    // xlo: ori = 0
    // ylo: ori = 1
    // zlo: ori = 2
    // xhi: ori = 3
    // yhi: ori = 4
    // zhi: ori = 5
    const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
    const auto& bdatylo = (*bndry_data[1])[lev].const_array();
    const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
    const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
 
    int bccomp;
 
    for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
    {
        MultiFab& mf = *mfs[var_idx];
        const int icomp = 0;
        const int ncomp = mf.nComp();
 
        if (var_idx == Vars::xvel) {
           bccomp = BCVars::xvel_bc;
        } else if (var_idx == Vars::yvel) {
           bccomp = BCVars::yvel_bc;
        } else if (var_idx == Vars::zvel) {
           bccomp = BCVars::zvel_bc;
        } else  if (var_idx == Vars::cons) {
           bccomp = BCVars::cons_bc;
        }
 
#ifdef AMREX_USE_OMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        for (MFIter mfi(mf); mfi.isValid(); ++mfi)
        {
            const Array4<Real>& dest_arr = mf.array(mfi);
            Box bx = mfi.growntilebox();
 
            // x-faces
            {
            Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
            if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
 
            Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
            if (var_idx == Vars::xvel) bx_xhi.setSmall(0,dom_hi.x);
 
            ParallelFor(
                bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].lo(0) == ERFBCType::ext_dir_ingested) {
                        int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
                    }
                },
                bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].hi(0) == ERFBCType::ext_dir_ingested) {
                        int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
                    }
                }
            );
            } // x-faces
 
            // y-faces
            {
            Box bx_ylo(bx); bx_ylo.setBig  (1,dom_lo.y-1);
            if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
 
            Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
            if (var_idx == Vars::yvel) bx_yhi.setSmall(1,dom_hi.y);
 
            ParallelFor(
               bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].lo(1) == ERFBCType::ext_dir_ingested) {
                        int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
                    }
                },
                bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].hi(1) == ERFBCType::ext_dir_ingested) {
                        int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
                    }
                }
            );
            } // y-faces
        } // mf
    } // var_idx
}

fill_rhs()

void ERF::fill_rhs ( amrex::MultiFab &  rhs_mf, const amrex::MultiFab &  state_mf, amrex::Real  time, const amrex::Geometry &  geom  )

private

FillBdyCCVels()

void ERF::FillBdyCCVels

(

amrex::Vector< amrex::MultiFab > &

mf_cc_vel

)

{
    // Impose bc's at domain boundaries
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        Box domain(Geom(lev).Domain());
 
        int ihi = domain.bigEnd(0);
        int jhi = domain.bigEnd(1);
        int khi = domain.bigEnd(2);
 
        // Impose periodicity first
        mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
 
        for (MFIter mfi(mf_cc_vel[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            // Note that we don't fill corners here -- only the cells that share a face
            //      with interior cells -- this is all that is needed to calculate vorticity
            const Box& bx = mfi.tilebox();
            const Array4<Real>& vel_arr = mf_cc_vel[lev].array(mfi);
 
            if (!Geom(lev).isPeriodic(0)) {
                // Low-x side
                if (bx.smallEnd(0) <= domain.smallEnd(0)) {
                    Real mult = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    ParallelFor(makeSlab(bx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                    {
                        vel_arr(-1,j,k,1) = mult*vel_arr(0,j,k,1); // v
                        vel_arr(-1,j,k,2) = mult*vel_arr(0,j,k,2); // w
                    });
                }
 
                // High-x side
                if (bx.bigEnd(0) >= domain.bigEnd(0)) {
                    Real mult = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    ParallelFor(makeSlab(bx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                    {
                        vel_arr(ihi+1,j,k,1) = mult*vel_arr(ihi,j,k,1); // v
                        vel_arr(ihi+1,j,k,2) = mult*vel_arr(ihi,j,k,2); // w
                    });
                }
            } // !periodic
 
            if (!Geom(lev).isPeriodic(1)) {
                // Low-y side
                if (bx.smallEnd(1) <= domain.smallEnd(1)) {
                    Real mult = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    ParallelFor(makeSlab(bx,1,0), [=] AMREX_GPU_DEVICE(int i, int  , int k) noexcept
                    {
                        vel_arr(i,-1,k,0) = mult*vel_arr(i,0,k,0); // u
                        vel_arr(i,-1,k,2) = mult*vel_arr(i,0,k,2); // w
                    });
                }
 
                // High-y side
                if (bx.bigEnd(1) >= domain.bigEnd(1)) {
                    Real mult = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    ParallelFor(makeSlab(bx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
                    {
                        vel_arr(i,jhi+1,k,0) = mult*vel_arr(i,jhi,k,0); // u
                        vel_arr(i,jhi+1,k,2) = mult*-vel_arr(i,jhi,k,2); // w
                    });
                }
            } // !periodic
 
            if (!Geom(lev).isPeriodic(2)) {
                // Low-z side
                if (bx.smallEnd(2) <= domain.smallEnd(2)) {
                    Real mult = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    ParallelFor(makeSlab(bx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                    {
                        vel_arr(i,j,-1,0) = mult*vel_arr(i,j,0,0); // u
                        vel_arr(i,j,-1,1) = mult*vel_arr(i,j,0,1); // v
                    });
                }
 
                // High-z side
                if (bx.bigEnd(2) >= domain.bigEnd(2)) {
                    Real mult = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    ParallelFor(makeSlab(bx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                    {
                        vel_arr(i,j,khi+1,0) = mult*vel_arr(i,j,khi,0); // u
                        vel_arr(i,j,khi+1,1) = mult*vel_arr(i,j,khi,1); // v
                    });
                }
            } // !periodic
        } // MFIter
 
    } // lev
}

FillCoarsePatch()

void ERF::FillCoarsePatch ( int  lev, amrex::Real  time  )

private

{
    BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
    AMREX_ASSERT(lev > 0);
 
    //
    //****************************************************************************************************************
    // First fill velocities and density at the COARSE level so we can convert velocity to momenta at the COARSE level
    //****************************************************************************************************************
    //
    bool cons_only = false;
    if (lev == 1) {
        FillPatch(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
                                &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
                                cons_only);
    } else {
        FillPatch(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
                                &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
                               {&vars_new[lev-1][Vars::cons],
                                &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
                                base_state[lev-1], base_state[lev-1],
                                false, cons_only);
    }
 
    //
    // ************************************************
    // Convert velocity to momentum at the COARSE level
    // ************************************************
    //
    VelocityToMomentum(vars_new[lev-1][Vars::xvel], IntVect{0},
                       vars_new[lev-1][Vars::yvel], IntVect{0},
                       vars_new[lev-1][Vars::zvel], IntVect{0},
                       vars_new[lev-1][Vars::cons],
                         rU_new[lev-1],
                         rV_new[lev-1],
                         rW_new[lev-1],
                       Geom(lev).Domain(),
                       domain_bcs_type);
    //
    // *****************************************************************
    // Interpolate all cell-centered variables from coarse to fine level
    // *****************************************************************
    //
    Interpolater* mapper_c = &cell_cons_interp;
    Interpolater* mapper_f = &face_cons_linear_interp;
 
    //
    //************************************************************************************************
    // Interpolate cell-centered data from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells have already been filled
    // ************************************************************************************************
    IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
    int      ncomp_cons = vars_new[lev][Vars::cons].nComp();
 
    InterpFromCoarseLevel(vars_new[lev  ][Vars::cons], ngvect_cons, IntVect(0,0,0),
                          vars_new[lev-1][Vars::cons], 0, 0, ncomp_cons,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_c, domain_bcs_type, BCVars::cons_bc);
 
    //
    //************************************************************************************************
    // Interpolate x-momentum from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells have already been filled
    // ************************************************************************************************
    //
    InterpFromCoarseLevel(rU_new[lev], IntVect{0}, IntVect{0}, rU_new[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::xvel_bc);
 
    //
    //************************************************************************************************
    // Interpolate y-momentum from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells have already been filled
    // ************************************************************************************************
    //
    InterpFromCoarseLevel(rV_new[lev], IntVect{0}, IntVect{0}, rV_new[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::yvel_bc);
 
    //************************************************************************************************
    // Interpolate z-momentum from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells have already been filled
    // ************************************************************************************************
    InterpFromCoarseLevel(rW_new[lev],  IntVect{0}, IntVect{0}, rW_new[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::zvel_bc);
    //
    // *********************************************************
    // After interpolation of momentum, convert back to velocity
    // *********************************************************
    //
    for (int which_lev = lev-1; which_lev <= lev; which_lev++)
    {
        MomentumToVelocity(vars_new[which_lev][Vars::xvel],
                           vars_new[which_lev][Vars::yvel],
                           vars_new[which_lev][Vars::zvel],
                           vars_new[which_lev][Vars::cons],
                             rU_new[which_lev],
                             rV_new[which_lev],
                             rW_new[which_lev],
                           Geom(lev).Domain(),
                           domain_bcs_type);
    }
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
 
    (*physbcs_cons[lev])(vars_new[lev][Vars::cons],0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc,true);
    (   *physbcs_u[lev])(vars_new[lev][Vars::xvel],0,1         ,ngvect_vels,time,BCVars::xvel_bc,true);
    (   *physbcs_v[lev])(vars_new[lev][Vars::yvel],0,1         ,ngvect_vels,time,BCVars::yvel_bc,true);
    (   *physbcs_w[lev])(vars_new[lev][Vars::zvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                         ngvect_vels,time,BCVars::zvel_bc,true);
 
    // ***************************************************************************
    // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
    // ***************************************************************************
}

Here is the call graph for this function:

FillIntermediatePatch()

void ERF::FillIntermediatePatch ( int  lev, amrex::Real  time, const amrex::Vector< amrex::MultiFab * > &  mfs_vel, const amrex::Vector< amrex::MultiFab * > &  mfs_mom, int  ng_cons, int  ng_vel, bool  cons_only, int  icomp_cons, int  ncomp_cons, bool  allow_most_bcs = true  )

private

{
    BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
    Interpolater* mapper;
 
    PhysBCFunctNoOp null_bc;
 
    //
    // ***************************************************************************
    // The first thing we do is interpolate the momenta on the "valid" faces of
    // the fine grids (where the interface is coarse/fine not fine/fine) -- this
    // will not be over-written by interpolation below because the FillPatch
    // operators see these as valid faces.  But we must have these interpolated
    // values in the fine data before we call FillPatchTwoLevels.
    //
    // Also -- note that we might be filling values by interpolation at physical boundaries
    //         here but that's ok because we will overwrite those values when we impose
    //         the physical bc's below
    // ***************************************************************************
    if (lev>0) {
        if (cf_set_width > 0) {
            // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
            FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
        }
        if ( !cons_only && (cf_set_width >= 0) ) {
            FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
            FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
            FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
        }
    }
 
    // amrex::Print() << "LEVEL " << lev << " CONS ONLY   " << cons_only <<
    //                   " ICOMP NCOMP " << icomp_cons << " " << ncomp_cons << " NGHOST " << ng_cons << std::endl;
 
    if (!cons_only) {
        AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
        AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
    }
 
    // Enforce no penetration for thin immersed body
    if (!cons_only) {
        // Enforce no penetration for thin immersed body
        if (xflux_imask[lev]) {
            ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
        }
        if (yflux_imask[lev]) {
            ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
        }
        if (zflux_imask[lev]) {
            ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
        }
    }
 
    //
    // We now start working on conserved quantities + VELOCITY
    //
    if (lev == 0)
    {
        // We don't do anything here because we will call the physbcs routines below,
        // which calls FillBoundary and fills other domain boundary conditions
        // Physical boundaries will be filled below
 
        if (!cons_only)
        {
            // ***************************************************************************
            // We always come in to this call with updated momenta but we need to create updated velocity
            //    in order to impose the rest of the bc's
            // ***************************************************************************
            // This only fills VALID region of velocity
            MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(), domain_bcs_type);
        }
    }
    else
    {
        //
        // We must fill a temporary then copy it back so we don't double add/subtract
        //
        MultiFab mf(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
                    mfs_vel[Vars::cons]->nComp()   ,mfs_vel[Vars::cons]->nGrowVect());
        //
        // Set all components to 1.789e19, then copy just the density from *mfs_vel[Vars::cons]
        //
        mf.setVal(1.789e19);
        MultiFab::Copy(mf,*mfs_vel[Vars::cons],Rho_comp,Rho_comp,1,mf.nGrowVect());
 
        Vector<MultiFab*> fmf = {mfs_vel[Vars::cons],mfs_vel[Vars::cons]};
        Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
        Vector<Real> ctime    = {t_old[lev-1], t_new[lev-1]};
        Vector<Real> ftime    = {time,time};
 
        // Impose physical bc's on fine data (note time and 0 are not used)
        bool do_fb = true; bool do_terrain_adjustment = false;
        (*physbcs_cons[lev])(*mfs_vel[Vars::cons],icomp_cons,ncomp_cons,IntVect{ng_cons},time,BCVars::cons_bc,
                             do_fb, do_terrain_adjustment);
 
        if ( (icomp_cons+ncomp_cons > 1) && (interpolation_type == StateInterpType::Perturbational) )
        {
            // Divide (rho theta) by rho to get theta
            MultiFab::Divide(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
 
            // Subtract theta_0 from theta
            MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
 
            if (!amrex::almostEqual(time,ctime[1])) {
                MultiFab::Divide(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                                 Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                   BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
            }
            if (!amrex::almostEqual(time,ctime[0])) {
                MultiFab::Divide(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                                 Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                   BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
            }
        }
 
        // Subtract rho_0 from rho before we interpolate -- note we only subtract
        //    on valid region of mf since the ghost cells will be filled below
        if (icomp_cons == 0 && (interpolation_type == StateInterpType::Perturbational))
        {
            MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
 
            if (!amrex::almostEqual(time,ctime[1])) {
                MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                   BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
            }
            if (!amrex::almostEqual(time,ctime[0])) {
                MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                   BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
            }
        }
 
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells have already been filled
        mapper = &cell_cons_interp;
        FillPatchTwoLevels(mf, IntVect{ng_cons}, IntVect(0,0,0),
                           time, cmf, ctime, fmf, ftime,
                           icomp_cons, icomp_cons, ncomp_cons, geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper, domain_bcs_type,
                           icomp_cons);
 
        if (icomp_cons == 0 && (interpolation_type == StateInterpType::Perturbational))
        {
            // Restore the coarse values to what they were
            if (!amrex::almostEqual(time,ctime[1])) {
                MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                              BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
            }
            if (!amrex::almostEqual(time,ctime[0])) {
                MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                              BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
            }
 
            // Set values in the cells outside the domain boundary so that we can do the Add
            //     without worrying about uninitialized values outside the domain -- these
            //     will be filled in the physbcs call
            mf.setDomainBndry(1.234e20,Rho_comp,1,geom[lev]);
 
            // Add rho_0 back to rho after we interpolate -- on all the valid + ghost region
            MultiFab::Add(mf, base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{ng_cons});
        }
 
        if ( (icomp_cons+ncomp_cons > 1) && (interpolation_type == StateInterpType::Perturbational) )
        {
            // Add theta_0 to theta
            if (!amrex::almostEqual(time,ctime[1])) {
                MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                              BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                                   Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
            }
            if (!amrex::almostEqual(time,ctime[0])) {
                MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                              BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                                   Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
            }
 
            // Multiply theta by rho to get (rho theta)
            MultiFab::Multiply(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
 
            // Add theta_0 to theta
            MultiFab::Add(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
 
            // Add theta_0 back to theta
            MultiFab::Add(mf,base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{ng_cons});
 
            // Multiply (theta) by rho to get (rho theta)
            MultiFab::Multiply(mf,mf,Rho_comp,RhoTheta_comp,1,IntVect{ng_cons});
        }
 
        // Make sure to only copy back the components we worked on
        MultiFab::Copy(*mfs_vel[Vars::cons],mf,icomp_cons,icomp_cons,ncomp_cons,IntVect{ng_cons});
 
        // *****************************************************************************************
 
        if (!cons_only)
        {
            // ***************************************************************************
            // We always come in to this call with updated momenta but we need to create updated velocity
            //    in order to impose the rest of the bc's
            // ***************************************************************************
            // This only fills VALID region of velocity
            MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(), domain_bcs_type);
 
            mapper = &face_cons_linear_interp;
 
            //
            // NOTE: All interpolation here happens on velocities not momenta;
            //       note we only do the interpolation and FillBoundary here,
            //       physical bc's are imposed later
            //
            // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
            //       unlike the FillSet calls above which filled momenta on the coarse/fine bdy
            //
 
            MultiFab& mfu = *mfs_vel[Vars::xvel];
 
            fmf = {&mfu,&mfu};
            cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells have already been filled
            FillPatchTwoLevels(mfu, IntVect{ng_vel}, IntVect(0,0,0),
                               time, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::xvel_bc);
 
            // *****************************************************************************************
 
            MultiFab& mfv = *mfs_vel[Vars::yvel];
 
            fmf = {&mfv,&mfv};
            cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells have already been filled
            FillPatchTwoLevels(mfv, IntVect{ng_vel}, IntVect(0,0,0),
                               time, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::yvel_bc);
 
            // *****************************************************************************************
 
            MultiFab& mfw = *mfs_vel[Vars::zvel];
 
            fmf = {&mfw,&mfw};
            cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells have already been filled
            FillPatchTwoLevels(mfw, IntVect{ng_vel}, IntVect(0,0,0),
                               time, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::zvel_bc);
        } // !cons_only
    } // lev > 0
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
    IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
 
    bool do_fb = true;
 
#ifdef ERF_USE_NETCDF
    // We call this here because it is an ERF routine
    if (use_real_bcs && (lev==0)) {
        fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
        do_fb = false;
    }
#endif
 
    if (m_r2d) fill_from_bndryregs(mfs_vel,time);
 
    // We call this even if init_type == InitType::Real because this routine will fill the vertical bcs
    (*physbcs_cons[lev])(*mfs_vel[Vars::cons],icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
    if (!cons_only) {
        (*physbcs_u[lev])(*mfs_vel[Vars::xvel],0,1,ngvect_vels,time,BCVars::xvel_bc, do_fb);
        (*physbcs_v[lev])(*mfs_vel[Vars::yvel],0,1,ngvect_vels,time,BCVars::yvel_bc, do_fb);
        (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::zvel_bc, do_fb);
    }
    // ***************************************************************************
 
    // MOST boundary conditions
    if (!(cons_only && ncomp_cons == 1) && m_most && allow_most_bcs) {
        m_most->impose_most_bcs(lev,mfs_vel,
                                Tau11_lev[lev].get(),
                                Tau22_lev[lev].get(),
                                Tau33_lev[lev].get(),
                                Tau12_lev[lev].get(), Tau21_lev[lev].get(),
                                Tau13_lev[lev].get(), Tau31_lev[lev].get(),
                                Tau23_lev[lev].get(), Tau32_lev[lev].get(),
                                SFS_hfx1_lev[lev].get(),
                                SFS_hfx2_lev[lev].get(),
                                SFS_hfx3_lev[lev].get(),
                                SFS_q1fx1_lev[lev].get(),
                                SFS_q1fx2_lev[lev].get(),
                                SFS_q1fx3_lev[lev].get(),
                                z_phys_nd[lev].get());
    }
 
    // We always come in to this call with momenta so we need to leave with momenta!
    // We need to make sure we convert back on all ghost cells/faces because this is
    // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
    if (!cons_only)
    {
        IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
        IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
        IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
 
        VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
                           *mfs_vel[Vars::yvel], ngv,
                           *mfs_vel[Vars::zvel], ngw,
                           *mfs_vel[Vars::cons],
                           *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                           Geom(lev).Domain(),
                           domain_bcs_type);
    }
 
    mfs_mom[IntVars::cons]->FillBoundary(geom[lev].periodicity());
    mfs_mom[IntVars::xmom]->FillBoundary(geom[lev].periodicity());
    mfs_mom[IntVars::ymom]->FillBoundary(geom[lev].periodicity());
    mfs_mom[IntVars::zmom]->FillBoundary(geom[lev].periodicity());
}

Here is the call graph for this function:

FillPatch() [1/2]

void ERF::FillPatch ( int  lev, amrex::Real  time, const amrex::Vector< amrex::MultiFab * > &  mfs_vel, bool  cons_only = false  )

private

FillPatch() [2/2]

void ERF::FillPatch ( int  lev, amrex::Real  time, const amrex::Vector< amrex::MultiFab * > &  mfs_vel, const amrex::Vector< amrex::MultiFab * > &  mfs_mom, const amrex::MultiFab &  old_base_state, const amrex::MultiFab &  new_base_state, bool  fillset = true, bool  cons_only = false  )

private

get_projection_bc()

Array< LinOpBCType, AMREX_SPACEDIM > ERF::get_projection_bc ( amrex::Orientation::Side  side ) const

noexcept

{
    amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> r;
    for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
        if (geom[0].isPeriodic(dir)) {
            r[dir] = LinOpBCType::Periodic;
        } else {
            auto bc_type = domain_bc_type[Orientation(dir,side)];
            if (bc_type == "Outflow") {
                r[dir] = LinOpBCType::Dirichlet;
            } else
            {
                r[dir] = LinOpBCType::Neumann;
            }
        }
    }
    return r;
}

getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int  lev )

inlineprivate

    {
        return advflux_reg[lev];
    }

getCPUTime()

static amrex::Real ERF::getCPUTime ( )

inlinestaticprivate

    {
      int numCores = amrex::ParallelDescriptor::NProcs();
#ifdef _OPENMP
      numCores = numCores * omp_get_max_threads();
#endif
 
      amrex::Real T =
        numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
        previousCPUTimeUsed;
 
      return T;
    }

GotoNextLine()

void ERF::GotoNextLine ( std::istream &  is )

staticprivate

Utility to skip to next line in Header file input stream.

{
    constexpr std::streamsize bl_ignore_max { 100000 };
    is.ignore(bl_ignore_max, '\n');
}

init1DArrays()

void ERF::init1DArrays ( )

private

init_bcs()

void ERF::init_bcs ( )

private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

{
    bool rho_read = false;
    Vector<Real> cons_dir_init(NBCVAR_max,0.0);
    cons_dir_init[BCVars::Rho_bc_comp] = 1.0;
    cons_dir_init[BCVars::RhoTheta_bc_comp] = -1.0;
    auto f = [this,&rho_read] (std::string const& bcid, Orientation ori)
    {
        // These are simply defaults for Dirichlet faces -- they should be over-written below
        m_bc_extdir_vals[BCVars::Rho_bc_comp][ori]       =  1.0;
        m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori]  = -1.0; // It is important to set this negative
                                                                 // because the sign is tested on below
        m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori]     = 0.0;
        m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = 0.0;
        m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori]     = 0.0;
        m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori]     = 0.0;
        m_bc_extdir_vals[BCVars::RhoQ3_bc_comp][ori]     = 0.0;
        m_bc_extdir_vals[BCVars::RhoQ4_bc_comp][ori]     = 0.0;
        m_bc_extdir_vals[BCVars::RhoQ5_bc_comp][ori]     = 0.0;
        m_bc_extdir_vals[BCVars::RhoQ6_bc_comp][ori]     = 0.0;
 
        m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.0; // default
        m_bc_extdir_vals[BCVars::yvel_bc][ori] = 0.0;
        m_bc_extdir_vals[BCVars::zvel_bc][ori] = 0.0;
 
        // These are simply defaults for Neumann gradients -- they should be over-written below
        m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]       = 0.0;
        m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]  = 0.0;
 
        m_bc_neumann_vals[BCVars::RhoKE_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoScalar_bc_comp][ori] = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ1_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ2_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ3_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ4_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ5_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ6_bc_comp][ori]     = 0.0;
 
        m_bc_neumann_vals[BCVars::xvel_bc][ori] = 0.0;
        m_bc_neumann_vals[BCVars::yvel_bc][ori] = 0.0;
        m_bc_neumann_vals[BCVars::zvel_bc][ori] = 0.0;
 
        std::string pp_text;
        if (pp_prefix == "erf") {
          pp_text = bcid;
        } else {
          pp_text = pp_prefix + "." + bcid;
        }
        ParmParse pp(pp_text);
        std::string bc_type_in = "null";
        pp.query("type", bc_type_in);
        std::string bc_type = amrex::toLower(bc_type_in);
 
        if (bc_type == "symmetry")
        {
            // Print() << bcid << " set to symmetry.\n";
              phys_bc_type[ori] = ERF_BC::symmetry;
            domain_bc_type[ori] = "Symmetry";
        }
        else if (bc_type == "outflow")
        {
            // Print() << bcid << " set to outflow.\n";
              phys_bc_type[ori] = ERF_BC::outflow;
            domain_bc_type[ori] = "Outflow";
        }
        else if (bc_type == "open")
        {
            // Print() << bcid << " set to open.\n";
            AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
              phys_bc_type[ori] = ERF_BC::open;
            domain_bc_type[ori] = "Open";
        }
        else if (bc_type == "ho_outflow")
        {
            phys_bc_type[ori] = ERF_BC::ho_outflow;
            domain_bc_type[ori] = "HO_Outflow";
        }
 
        else if (bc_type == "inflow")
        {
            // Print() << bcid << " set to inflow.\n";
              phys_bc_type[ori] = ERF_BC::inflow;
            domain_bc_type[ori] = "Inflow";
 
            std::vector<Real> v;
            if (input_bndry_planes && m_r2d->ingested_velocity()) {
                m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.;
                m_bc_extdir_vals[BCVars::yvel_bc][ori] = 0.;
                m_bc_extdir_vals[BCVars::zvel_bc][ori] = 0.;
            } else {
                // Test for input data file if at xlo face
                std::string dirichlet_file;
                auto file_exists = pp.query("dirichlet_file", dirichlet_file);
                if (file_exists) {
                    init_Dirichlet_bc_data(dirichlet_file);
                } else {
                    pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
                    m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                    m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                    m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
                }
            }
 
            Real rho_in = 0.;
            if (input_bndry_planes && m_r2d->ingested_density()) {
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = 0.;
            } else {
                if (!pp.query("density", rho_in)) {
                    amrex::Print() << "Using interior values to set conserved vars" << std::endl;
                }
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
            }
            Real theta_in = 0.;
            if (input_bndry_planes && m_r2d->ingested_theta()) {
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = 0.;
            } else {
                if (rho_in > 0) {
                    pp.get("theta", theta_in);
                }
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
            }
            Real scalar_in = 0.;
            if (input_bndry_planes && m_r2d->ingested_scalar()) {
                m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = 0.;
            } else {
                if (pp.query("scalar", scalar_in))
                m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
            }
 
            if (solverChoice.moisture_type != MoistureType::None) {
                Real qv_in = 0.;
                if (input_bndry_planes && m_r2d->ingested_q1()) {
                    m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = 0.;
                } else {
                    if (pp.query("qv", qv_in))
                    m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
                }
                Real qc_in = 0.;
                if (input_bndry_planes && m_r2d->ingested_q2()) {
                    m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = 0.;
                } else {
                    if (pp.query("qc", qc_in))
                    m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
                }
            }
 
            Real KE_in = 0.;
            if (input_bndry_planes && m_r2d->ingested_KE()) {
                m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = 0.;
            } else {
                if (pp.query("KE", KE_in))
                m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
            }
        }
        else if (bc_type == "noslipwall")
        {
            // Print() << bcid <<" set to no-slip wall.\n";
              phys_bc_type[ori] = ERF_BC::no_slip_wall;
            domain_bc_type[ori] = "NoSlipWall";
 
            std::vector<Real> v;
 
            // The values of m_bc_extdir_vals default to 0.
            // But if we find "velocity" in the inputs file, use those values instead.
            if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
            {
                v[ori.coordDir()] = 0.0;
                m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
            }
 
            Real rho_in;
            rho_read = pp.query("density", rho_in);
            if (rho_read)
            {
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
            }
 
            Real theta_in;
            if (pp.query("theta", theta_in))
            {
               m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
            }
 
            Real theta_grad_in;
            if (pp.query("theta_grad", theta_grad_in))
            {
                m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
            }
 
            Real qv_in;
            if (pp.query("qv", qv_in))
            {
                m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = qv_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
            }
        }
        else if (bc_type == "slipwall")
        {
            // Print() << bcid <<" set to slip wall.\n";
 
              phys_bc_type[ori] = ERF_BC::slip_wall;
            domain_bc_type[ori] = "SlipWall";
 
            Real rho_in;
            if (pp.query("density", rho_in))
            {
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
            }
 
            Real theta_in;
            if (pp.query("theta", theta_in))
            {
               m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
            }
 
            Real rho_grad_in;
            if (pp.query("density_grad", rho_grad_in))
            {
               m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = rho_grad_in;
            }
 
            Real theta_grad_in;
            if (pp.query("theta_grad", theta_grad_in))
            {
               m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
            }
        }
        else if (bc_type == "most")
        {
              phys_bc_type[ori] = ERF_BC::MOST;
            domain_bc_type[ori] = "MOST";
        }
        else
        {
            phys_bc_type[ori] = ERF_BC::undefined;
        }
 
        if (geom[0].isPeriodic(ori.coordDir())) {
            domain_bc_type[ori] = "Periodic";
            if (phys_bc_type[ori] == ERF_BC::undefined)
            {
                phys_bc_type[ori] = ERF_BC::periodic;
            } else {
                Abort("Wrong BC type for periodic boundary");
            }
        }
 
        if (phys_bc_type[ori] == ERF_BC::undefined)
        {
             Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
             Abort("This BC type is unknown");
        }
    };
 
    f("xlo", Orientation(Direction::x,Orientation::low));
    f("xhi", Orientation(Direction::x,Orientation::high));
    f("ylo", Orientation(Direction::y,Orientation::low));
    f("yhi", Orientation(Direction::y,Orientation::high));
    f("zlo", Orientation(Direction::z,Orientation::low));
    f("zhi", Orientation(Direction::z,Orientation::high));
 
    // *****************************************************************************
    //
    // Here we translate the physical boundary conditions -- one type per face --
    //     into logical boundary conditions for each velocity component
    //
    // *****************************************************************************
    {
        domain_bcs_type.resize(AMREX_SPACEDIM+NBCVAR_max);
        domain_bcs_type_d.resize(AMREX_SPACEDIM+NBCVAR_max);
 
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            int dir = ori.coordDir();
            Orientation::Side side = ori.faceDir();
            auto const bct = phys_bc_type[ori];
            if ( bct == ERF_BC::symmetry )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::reflect_even);
                    }
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::reflect_odd);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::reflect_even);
                    }
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::reflect_odd);
                }
            }
            else if (bct == ERF_BC::outflow or bct == ERF_BC::ho_outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::neumann_int);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::neumann_int);
                }
            }
            else if (bct == ERF_BC::open)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::open);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::open);
                }
            }
            else if (bct == ERF_BC::inflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                        if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                            domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                        }
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                        if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                            domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                        }
                    }
                }
            }
            else if (bct == ERF_BC::no_slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                    }
                }
            }
            else if (bct == ERF_BC::slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::ext_dir);
 
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::ext_dir);
                }
            }
            else if (bct == ERF_BC::periodic)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::int_dir);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::int_dir);
                    }
                }
            }
            else if ( bct == ERF_BC::MOST )
            {
                AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
                domain_bcs_type[BCVars::xvel_bc+0].setLo(dir, ERFBCType::reflect_odd);
                domain_bcs_type[BCVars::xvel_bc+1].setLo(dir, ERFBCType::reflect_odd);
                domain_bcs_type[BCVars::xvel_bc+2].setLo(dir, ERFBCType::ext_dir);
            }
        }
    }
 
    // *****************************************************************************
    //
    // Here we translate the physical boundary conditions -- one type per face --
    //     into logical boundary conditions for each cell-centered variable
    //     (including the base state variables)
    // NOTE: all "scalars" share the same type of boundary condition
    //
    // *****************************************************************************
    {
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            int dir = ori.coordDir();
            Orientation::Side side = ori.faceDir();
            auto const bct = phys_bc_type[ori];
            if ( bct == ERF_BC::symmetry )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::reflect_even);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::reflect_even);
                    }
                }
            }
            else if ( bct == ERF_BC::outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                }
            }
            else if ( bct == ERF_BC::ho_outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::hoextrapcc);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::hoextrapcc);
                    }
                }
            }
            else if ( bct == ERF_BC::open )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::open);
                } else {
                    for (int i = 0; i < NBCVAR_max; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::open);
                }
            }
            else if ( bct == ERF_BC::no_slip_wall )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                            if (rho_read) {
                                domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                            } else {
                                domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
                            }
                        }
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                            if (rho_read) {
                                domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                            } else {
                                domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
                            }
                        }
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
                    }
                }
            }
            else if (bct == ERF_BC::slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                            if (rho_read) {
                                domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                            } else {
                                domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
                            }
                        }
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::Rho_bc_comp].setLo(dir, ERFBCType::neumann);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                            if (rho_read) {
                                domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                            } else {
                                domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
                            }
                        }
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::Rho_bc_comp].setHi(dir, ERFBCType::neumann);
                    }
                }
            }
            else if (bct == ERF_BC::inflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                        if (input_bndry_planes && dir < 2 && (
                           ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )) )
                        {
                            domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                        }
                        else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        }
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                        if (input_bndry_planes && dir < 2 && (
                           ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )
                           ) )
                        {
                            domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                        }
                        else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        }
                    }
                }
            }
            else if (bct == ERF_BC::periodic)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::int_dir);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                       domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::int_dir);
                    }
                }
            }
            else if ( bct == ERF_BC::MOST )
            {
                AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
                for (int i = 0; i < NBCVAR_max; i++) {
                    domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                }
            }
        }
    }
 
    // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
    Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
}

Here is the call graph for this function:

init_custom()

void ERF::init_custom ( int  lev )

private

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
 
    MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
                       lev_new[Vars::cons].nComp()   , lev_new[Vars::cons].nGrow());
    MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
    MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
    MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
 
    // Default all perturbations to zero
    cons_pert.setVal(0.);
    xvel_pert.setVal(0.);
    yvel_pert.setVal(0.);
    zvel_pert.setVal(0.);
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
    {
        const Box &bx  = mfi.tilebox();
        const Box &xbx = mfi.tilebox(IntVect(1,0,0));
        const Box &ybx = mfi.tilebox(IntVect(0,1,0));
        const Box &zbx = mfi.tilebox(IntVect(0,0,1));
 
        const auto &cons_pert_arr = cons_pert.array(mfi);
        const auto &xvel_pert_arr = xvel_pert.array(mfi);
        const auto &yvel_pert_arr = yvel_pert.array(mfi);
        const auto &zvel_pert_arr = zvel_pert.array(mfi);
 
        Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
        Array4<Real const> z_nd_arr = (solverChoice.terrain_type != TerrainType::None) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
        Array4<Real const> z_cc_arr = (solverChoice.terrain_type != TerrainType::None) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
 
        Array4<Real const> mf_m     = mapfac_m[lev]->array(mfi);
        Array4<Real const> mf_u     = mapfac_m[lev]->array(mfi);
        Array4<Real const> mf_v     = mapfac_m[lev]->array(mfi);
 
        Array4<Real> r_hse_arr = r_hse.array(mfi);
        Array4<Real> p_hse_arr = p_hse.array(mfi);
 
        prob->init_custom_pert(bx, xbx, ybx, zbx, cons_arr, cons_pert_arr,
                               xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
                               r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
                               geom[lev].data(), mf_m, mf_u, mf_v,
                               solverChoice);
    } //mfi
 
    // Add problem-specific perturbation to background flow
    MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp,      Rho_comp,             1, cons_pert.nGrow());
    MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp,        1, cons_pert.nGrow());
    MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,NSCALARS, cons_pert.nGrow());
 
    // RhoKE is only relevant if using Deardorff with LES or MYNN with PBL
    if ((solverChoice.turbChoice[lev].les_type == LESType::Deardorff) ||
        (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25)) {
        MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp,    RhoKE_comp,    1, cons_pert.nGrow());
    }
 
    if (solverChoice.moisture_type != MoistureType::None) {
        int qstate_size = micro->Get_Qstate_Size();
        MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoQ1_comp,    RhoQ1_comp,    1, cons_pert.nGrow());
        MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoQ2_comp,    RhoQ2_comp,    1, cons_pert.nGrow());
        for (int q_offset(2); q_offset<qstate_size; ++q_offset) {
            int q_idx = RhoQ1_comp+q_offset;
            MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
        }
    }
 
    MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0,             0,             1, xvel_pert.nGrowVect());
    MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0,             0,             1, yvel_pert.nGrowVect());
    MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0,             0,             1, zvel_pert.nGrowVect());
}

Here is the call graph for this function:

init_Dirichlet_bc_data()

void ERF::init_Dirichlet_bc_data ( const std::string  input_file )

private

{
    const bool use_terrain = (solverChoice.terrain_type != TerrainType::None);
 
    // Read the dirichlet_input file
    Print() << "dirichlet_input file location : " << input_file << std::endl;
    std::ifstream input_reader(input_file);
    if (!input_reader.is_open()) {
        amrex::Abort("Error opening the dirichlet_input file.\n");
    }
 
    Print() << "Successfully opened the dirichlet_input file. Now reading... " << std::endl;
    std::string line;
 
    // Size of Ninp (number of z points in input file)
    Vector<Real> z_inp_tmp, u_inp_tmp, v_inp_tmp, w_inp_tmp;
 
    const int klo = geom[0].Domain().smallEnd()[2];
    const int khi = geom[0].Domain().bigEnd()[2];
 
    const Real zbot = (use_terrain) ? zlevels_stag[0][klo]   : geom[0].ProbLo(2);
    const Real ztop = (use_terrain) ? zlevels_stag[0][khi+1] : geom[0].ProbHi(2);
 
    // Add surface
    z_inp_tmp.push_back(zbot); // height above sea level [m]
    u_inp_tmp.push_back(0.);
    v_inp_tmp.push_back(0.);
    w_inp_tmp.push_back(0.);
 
    // Read the vertical profile at each given height
    Real z, u, v, w;
    while(std::getline(input_reader, line)) {
        std::istringstream iss_z(line);
        iss_z >> z >> u >> v >> w;
        if (z == zbot) {
            u_inp_tmp[0] = u;
            v_inp_tmp[0] = v;
            w_inp_tmp[0] = w;
        } else {
            AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
            z_inp_tmp.push_back(z);
            u_inp_tmp.push_back(u);
            v_inp_tmp.push_back(v);
            w_inp_tmp.push_back(w);
            if (z >= ztop) break;
        }
    }
 
    amrex::Print() << "Successfully read and interpolated the dirichlet_input file..." << std::endl;
    input_reader.close();
 
    for (int lev = 0; lev <= max_level; lev++) {
 
        const int Nz  = geom[lev].Domain().size()[2];
        const Real dz = geom[lev].CellSize()[2];
 
        // Size of Nz (domain grid)
        Vector<Real> zcc_inp(Nz  );
        Vector<Real> znd_inp(Nz+1);
        Vector<Real> u_inp(Nz  ); xvel_bc_data[lev].resize(Nz  ,0.0);
        Vector<Real> v_inp(Nz  ); yvel_bc_data[lev].resize(Nz  ,0.0);
        Vector<Real> w_inp(Nz+1); zvel_bc_data[lev].resize(Nz+1,0.0);
 
        // At this point, we have an input from zbot up to
        // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights
        const int Ninp = z_inp_tmp.size();
        for (int k(0); k<Nz; ++k) {
            zcc_inp[k] = (use_terrain) ? 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1])
                                         : zbot + (k + 0.5) * dz;
            znd_inp[k] = (use_terrain) ? zlevels_stag[0][k+1] : zbot + (k) * dz;
            u_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), u_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
            v_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), v_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
            w_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), znd_inp[k], Ninp);
        }
        znd_inp[Nz] = ztop;
        w_inp[Nz]   = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), ztop, Ninp);
 
        // Copy host data to the device
        Gpu::copy(Gpu::hostToDevice, u_inp.begin(), u_inp.end(), xvel_bc_data[lev].begin());
        Gpu::copy(Gpu::hostToDevice, v_inp.begin(), v_inp.end(), yvel_bc_data[lev].begin());
        Gpu::copy(Gpu::hostToDevice, w_inp.begin(), w_inp.end(), zvel_bc_data[lev].begin());
 
        // NOTE: These device vectors are passed to the PhysBC constructors when that
        //       class is instantiated in ERF_MakeNewArrays.cpp.
    } // lev
}

Here is the call graph for this function:

init_from_hse()

void ERF::init_from_hse

(

int

lev

)

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

• call prob->erf_init_dens_hse(...)
  • call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
  • save r_hse
• call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse and th_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
 
    MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
    {
        const Box &gbx = mfi.growntilebox(1);
        const Array4<Real      >& cons_arr  = lev_new[Vars::cons].array(mfi);
        const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
        const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
 
        ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            cons_arr(i,j,k,Rho_comp)      = r_hse_arr(i,j,k);
            cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
        });
    } //mfi
}

Here is the call graph for this function:

init_from_input_sounding()

void ERF::init_from_input_sounding

(

int

lev

)

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters

lev	Integer specifying the current level

{
    // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
    if (lev == 0) {
        if (input_sounding_data.input_sounding_file.empty()) {
            Error("input_sounding file name must be provided via input");
        }
 
        input_sounding_data.resize_arrays();
 
        // this will interpolate the input profiles to the nominal height levels
        // (ranging from 0 to the domain top)
        for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
            input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n);
        }
 
        // this will calculate the hydrostatically balanced density and pressure
        // profiles following WRF ideal.exe
        if (init_sounding_ideal) input_sounding_data.calc_rho_p(0);
 
    } else {
        //
        // We need to do this interp from coarse level in order to set the values of
        // the base state inside the domain but outside of the fine region
        //
        base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
        //
        // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
        //       have been pre-filled - this includes ghost cells both inside and outside
        //       the domain
        //
        InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                              IntVect(0,0,0), // do not fill ghost cells outside the domain
                              base_state[lev-1], 0, 0, base_state[lev].nComp(),
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BCVars::base_bc);
 
         // We need to do this here because the interpolation above may leave corners unfilled
         //    when the corners need to be filled by, for example, reflection of the fine ghost
         //    cell outside the fine region but inide the domain.
         (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
    }
 
    auto& lev_new = vars_new[lev];
 
    // update if init_sounding_ideal == true
    MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
    MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
    MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
 
    const Real l_gravity = solverChoice.gravity;
    const Real l_rdOcp   = solverChoice.rdOcp;
    const bool l_moist   = (solverChoice.moisture_type != MoistureType::None);
 
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box &bx = mfi.tilebox();
        const auto &cons_arr = lev_new[Vars::cons].array(mfi);
        const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
        const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
        const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
        Array4<Real>  r_hse_arr =  r_hse.array(mfi);
        Array4<Real>  p_hse_arr =  p_hse.array(mfi);
        Array4<Real> pi_hse_arr = pi_hse.array(mfi);
        Array4<Real> th_hse_arr = th_hse.array(mfi);
 
        Array4<Real const> z_cc_arr = (solverChoice.terrain_type != TerrainType::None) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
        Array4<Real const> z_nd_arr = (solverChoice.terrain_type != TerrainType::None) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
 
        if (init_sounding_ideal)
        {
            // HSE will be initialized here, interpolated from values previously
            // calculated by calc_rho_p()
            init_bx_scalars_from_input_sounding_hse(
                bx, cons_arr,
                r_hse_arr, p_hse_arr, pi_hse_arr, th_hse_arr,
                geom[lev].data(), z_cc_arr,
                l_gravity, l_rdOcp, l_moist, input_sounding_data);
        }
        else
        {
            // HSE will be calculated later with call to initHSE
            init_bx_scalars_from_input_sounding(
                bx, cons_arr,
                geom[lev].data(), z_cc_arr,
                l_moist, input_sounding_data);
        }
 
        init_bx_velocities_from_input_sounding(
            bx, xvel_arr, yvel_arr, zvel_arr,
            geom[lev].data(), z_nd_arr,
            input_sounding_data);
 
    } //mfi
}

Here is the call graph for this function:

init_geo_wind_profile()

void ERF::init_geo_wind_profile ( const std::string  input_file, amrex::Vector< amrex::Real > &  u_geos, amrex::Gpu::DeviceVector< amrex::Real > &  u_geos_d, amrex::Vector< amrex::Real > &  v_geos, amrex::Gpu::DeviceVector< amrex::Real > &  v_geos_d, const amrex::Geometry &  lgeom, const amrex::Vector< amrex::Real > &  zlev_stag  )

private

{
    const int klo = 0;
    const int khi = lgeom.Domain().bigEnd()[AMREX_SPACEDIM-1];
    const amrex::Real dz = lgeom.CellSize()[AMREX_SPACEDIM-1];
 
    const bool grid_stretch = (zlev_stag.size() > 0);
    const Real zbot = (grid_stretch) ? zlev_stag[klo]   : lgeom.ProbLo(AMREX_SPACEDIM-1);
    const Real ztop = (grid_stretch) ? zlev_stag[khi+1] : lgeom.ProbHi(AMREX_SPACEDIM-1);
 
    amrex::Print() << "Reading geostrophic wind profile from " << input_file << std::endl;
    std::ifstream profile_reader(input_file);
    if(!profile_reader.is_open()) {
        amrex::Error("Error opening the abl_geo_wind_table\n");
    }
 
    // First, read the input data into temp vectors
    std::string line;
    Vector<Real> z_inp, Ug_inp, Vg_inp;
    Real z, Ug, Vg;
    amrex::Print() << "z  Ug  Vg" << std::endl;
    while(std::getline(profile_reader, line)) {
        std::istringstream iss(line);
        iss >> z >> Ug >> Vg;
        amrex::Print() << z << " " << Ug << " " << Vg << std::endl;
        z_inp.push_back(z);
        Ug_inp.push_back(Ug);
        Vg_inp.push_back(Vg);
        if (z >= ztop) break;
    }
 
    const int Ninp = z_inp.size();
    AMREX_ALWAYS_ASSERT(z_inp[0] <= zbot);
    AMREX_ALWAYS_ASSERT(z_inp[Ninp-1] >= ztop);
 
    // Now, interpolate vectors to the cell centers
    for (int k = 0; k <= khi; k++) {
        z = (grid_stretch) ? 0.5 * (zlev_stag[k] + zlev_stag[k+1])
                           : zbot + (k + 0.5) * dz;
        u_geos[k] = interpolate_1d(z_inp.dataPtr(), Ug_inp.dataPtr(), z, Ninp);
        v_geos[k] = interpolate_1d(z_inp.dataPtr(), Vg_inp.dataPtr(), z, Ninp);
    }
 
    // Copy from host version to device version
    Gpu::copy(Gpu::hostToDevice, u_geos.begin(), u_geos.end(), u_geos_d.begin());
    Gpu::copy(Gpu::hostToDevice, v_geos.begin(), v_geos.end(), v_geos_d.begin());
 
    profile_reader.close();
}

Here is the call graph for this function:

init_only()

void ERF::init_only	(	int	lev,
		amrex::Real	time
	)

{
    // Map the words in the inputs file to BC types, then translate
    //     those types into what they mean for each variable
    // This must be called before initHSE (where the base state is initialized)
    if (lev == 0 && init_type != InitType::Ideal) {
        init_bcs();
    }
 
    t_new[lev] = time;
    t_old[lev] = time - 1.e200;
 
    auto& lev_new = vars_new[lev];
    auto& lev_old = vars_old[lev];
 
    // Loop over grids at this level to initialize our grid data
    lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
    lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
    lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
    lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
 
    // Initialize background flow (optional)
    if (init_type == InitType::Input_Sounding) {
        // The base state is initialized by integrating vertically through the
        // input sounding, if the init_sounding_ideal flag is set; otherwise
        // it is set by initHSE()
 
        // The physbc's need the terrain but are needed for initHSE
        // We have already made the terrain in the call to init_zphys
        //    in MakeNewLevelFromScratch
        make_physbcs(lev);
 
        // Now init the base state and the data itself
        init_from_input_sounding(lev);
 
        if (init_sounding_ideal) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
                "Gravity should be on to be consistent with sounding initialization.");
        } else {
            initHSE();
        }
 
#ifdef ERF_USE_NETCDF
    } else if (init_type == InitType::Ideal || init_type == InitType::Real) {
        // The base state is initialized from WRF wrfinput data, output by
        // ideal.exe or real.exe
        init_from_wrfinput(lev);
 
        // The physbc's need the terrain but are needed for initHSE
        if (init_type == InitType::Ideal) {
            make_physbcs(lev);
            initHSE(lev);
        }
 
    } else if (init_type == InitType::Metgrid) {
        // The base state is initialized from data output by WPS metgrid;
        // we will rebalance after interpolation
        init_from_metgrid(lev);
#endif
    } else if (init_type == InitType::Uniform) {
        // Initialize a uniform background field and base state based on the
        // problem-specified reference density and temperature
 
        // The physbc's need the terrain but are needed for initHSE
        make_physbcs(lev);
 
        init_uniform(lev);
        initHSE(lev);
    } else {
        // No background flow initialization specified, initialize the
        // background field to be equal to the base state, calculated from the
        // problem-specific erf_init_dens_hse
 
        // The bc's need the terrain but are needed for initHSE
        make_physbcs(lev);
 
        // We will initialize the state from the background state so must set that first
        initHSE(lev);
        init_from_hse(lev);
    }
 
    // Add problem-specific flow features
    //
    // Notes:
    // - This calls init_custom_pert that is defined for each problem
    // - This may modify the base state
    // - The fields set by init_custom_pert are **perturbations** to the
    //   background flow set based on init_type
    init_custom(lev);
 
    // Ensure that the face-based data are the same on both sides of a periodic domain.
    // The data associated with the lower grid ID is considered the correct value.
    lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
    lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
    lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
 
   if(solverChoice.spongeChoice.sponge_type == "input_sponge"){
        input_sponge(lev);
   }
 
    // Initialize turbulent perturbation
    if (solverChoice.pert_type == PerturbationType::Source ||
        solverChoice.pert_type == PerturbationType::Direct) {
        if (lev == 0) {
            turbPert_update(lev, 0.);
            turbPert_amplitude(lev);
        }
    }
}

init_stuff()

void ERF::init_stuff ( int  lev, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm, amrex::Vector< amrex::MultiFab > &  lev_new, amrex::Vector< amrex::MultiFab > &  lev_old, amrex::MultiFab &  tmp_base_state, std::unique_ptr< amrex::MultiFab > &  tmp_zphys_nd  )

private

{
    // ********************************************************************************************
    // Base state holds r_0, pres_0, pi_0, th_0 (in that order)
    //
    // Here is where we set 3 ghost cells for the base state!
    //
    // ********************************************************************************************
    tmp_base_state.define(ba,dm,BaseState::num_comps,3);
    tmp_base_state.setVal(0.);
 
    if (solverChoice.terrain_type == TerrainType::Moving) {
        base_state_new[lev].define(ba,dm,BaseState::num_comps,base_state[lev].nGrowVect());
        base_state_new[lev].setVal(0.);
    }
 
    // ********************************************************************************************
    // Allocate terrain arrays
    // ********************************************************************************************
    if (SolverChoice::mesh_type == MeshType::StretchedDz ||
        SolverChoice::mesh_type == MeshType::VariableDz) {
        z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
 
        if (solverChoice.terrain_type == TerrainType::Moving)
        {
            detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
            detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
 
            ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
            ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
            az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
 
            ax_new[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
            ay_new[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
            az_new[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
 
            z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
        }
 
        BoxArray ba_nd(ba);
        ba_nd.surroundingNodes();
 
        // We need this to be one greater than the ghost cells to handle levels > 0
        int ngrow = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_num_diff) + 2;
        tmp_zphys_nd = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
 
        if (solverChoice.terrain_type == TerrainType::Moving) {
            z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
            z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
        }
 
    } else {
            z_phys_nd[lev] = nullptr;
            z_phys_cc[lev] = nullptr;
 
        z_phys_nd_new[lev] = nullptr;
          detJ_cc_new[lev] = nullptr;
 
        z_phys_nd_src[lev] = nullptr;
          detJ_cc_src[lev] = nullptr;
 
               z_t_rk[lev] = nullptr;
    }
 
    // We use these area arrays regardless of terrain, EB or none of the above
    detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
         ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
         ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
         az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
 
    detJ_cc[lev]->setVal(1.0);
         ax[lev]->setVal(1.0);
         ay[lev]->setVal(1.0);
         az[lev]->setVal(1.0);
 
    // ********************************************************************************************
    // Create wall distance array for RANS modeling
    // ********************************************************************************************
    if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
        walldist[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
        walldist[lev]->setVal(1e23);
    } else {
        walldist[lev] = nullptr;
    }
 
    // ********************************************************************************************
    // These are the persistent containers for the old and new data
    // ********************************************************************************************
    int ncomp;
    if (lev > 0) {
        ncomp = vars_new[lev-1][Vars::cons].nComp();
    } else {
        int n_qstate   = micro->Get_Qstate_Size();
        ncomp = NDRY + NSCALARS + n_qstate;
    }
 
    // ********************************************************************************************
    // The number of ghost cells for density must be 1 greater than that for velocity
    //     so that we can go back in forth between velocity and momentum on all faces
    // ********************************************************************************************
    int ngrow_state = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_num_diff) + 1;
    int ngrow_vels  = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_num_diff);
 
    // ********************************************************************************************
    // New solution data containers
    // ********************************************************************************************
    lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
    lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
 
    lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
    lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
 
    lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
    lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
 
    // Note that we need the ghost cells in the z-direction if we are doing any
    // kind of domain decomposition in the vertical (at level 0 or above)
    lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
    lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
 
    if (solverChoice.anelastic[lev] == 1) {
        pp_inc[lev].define(ba, dm, 1, 1);
        pp_inc[lev].setVal(0.0);
    }
 
    // ********************************************************************************************
    // These are just used for scratch in the time integrator but we might as well define them here
    // ********************************************************************************************
    rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
    rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
 
    rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
    rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
 
    rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
    rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
 
    if (lev > 0) {
        //xmom_crse_rhs[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, IntVect{0});
        //ymom_crse_rhs[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, IntVect{0});
        zmom_crse_rhs[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect{0});
    }
 
    // We do this here just so they won't be undefined in the initial FillPatch
    rU_old[lev].setVal(1.2e21);
    rV_old[lev].setVal(3.4e22);
    rW_old[lev].setVal(5.6e23);
    rU_new[lev].setVal(1.2e21);
    rV_new[lev].setVal(3.4e22);
    rW_new[lev].setVal(5.6e23);
 
    // ********************************************************************************************
    // These are just time averaged fields for diagnostics
    // ********************************************************************************************
 
    // NOTE: We are not completing a fillpach call on the time averaged data;
    //       which would copy on intersection and interpolate from coarse.
    //       Therefore, we are restarting the averaging when the ba changes,
    //       this may give poor statistics for dynamic mesh refinement.
    vel_t_avg[lev] = nullptr;
    if (solverChoice.time_avg_vel) {
        vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
        vel_t_avg[lev]->setVal(0.0);
        t_avg_cnt[lev] = 0.0;
    }
 
    // ********************************************************************************************
    // Initialize flux registers whenever we create/re-create a level
    // ********************************************************************************************
    if (solverChoice.coupling_type == CouplingType::TwoWay) {
        if (lev == 0) {
            advflux_reg[0] = nullptr;
        } else {
            int ncomp_reflux = vars_new[0][Vars::cons].nComp();
            advflux_reg[lev] = new YAFluxRegister(ba       , grids[lev-1],
                                                  dm       ,  dmap[lev-1],
                                                  geom[lev],  geom[lev-1],
                                                  ref_ratio[lev-1], lev, ncomp_reflux);
        }
    }
 
    // ********************************************************************************************
    // Define Theta_prim storage if using MOST BC
    // ********************************************************************************************
    if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::MOST) {
        Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,0));
        if (solverChoice.moisture_type != MoistureType::None) {
            Qv_prim[lev]    = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,0));
            Qr_prim[lev]    = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,0));
        } else {
            Qv_prim[lev]    = nullptr;
            Qr_prim[lev]    = nullptr;
        }
    } else {
        Theta_prim[lev] = nullptr;
        Qv_prim[lev]    = nullptr;
        Qr_prim[lev]    = nullptr;
    }
 
    // ********************************************************************************************
    // Map factors
    // ********************************************************************************************
    BoxList bl2d_mf = ba.boxList();
    for (auto& b : bl2d_mf) {
        b.setRange(2,0);
    }
    BoxArray ba2d_mf(std::move(bl2d_mf));
 
    mapfac_m[lev] = std::make_unique<MultiFab>(ba2d_mf,dm,1,3);
    mapfac_u[lev] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,3);
    mapfac_v[lev] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,3);
    if (solverChoice.test_mapfactor) {
        mapfac_m[lev]->setVal(0.5);
        mapfac_u[lev]->setVal(0.5);
        mapfac_v[lev]->setVal(0.5);
    }
    else {
        mapfac_m[lev]->setVal(1.);
        mapfac_u[lev]->setVal(1.);
        mapfac_v[lev]->setVal(1.);
    }
 
#if defined(ERF_USE_WINDFARM)
    //*********************************************************
    // Variables for Fitch model for windfarm parametrization
    //*********************************************************
    if (solverChoice.windfarm_type == WindFarmType::Fitch){
        vars_windfarm[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
    }
    if (solverChoice.windfarm_type == WindFarmType::EWP){
        vars_windfarm[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
    }
    if (solverChoice.windfarm_type == WindFarmType::SimpleAD) {
        vars_windfarm[lev].define(ba, dm, 2, ngrow_state);// dudt, dvdt
    }
    if (solverChoice.windfarm_type == WindFarmType::GeneralAD) {
        vars_windfarm[lev].define(ba, dm, 3, ngrow_state);// dudt, dvdt, dwdt
    }
        Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
        SMark[lev].define(ba, dm, 2, 1); // Free stream velocity/source term
                                                   // sampling marker in a cell - 2 components
#endif
 
 
#ifdef ERF_USE_WW3_COUPLING
    // create a new BoxArray and DistributionMapping for a MultiFab with 1 box
    BoxArray ba_onegrid(geom[lev].Domain());
    BoxList bl2d_onegrid = ba_onegrid.boxList();
    for (auto& b : bl2d_onegrid) {
        b.setRange(2,0);
    }
    BoxArray ba2d_onegrid(std::move(bl2d_onegrid));
    Vector<int> pmap;
    pmap.resize(1);
    pmap[0]=0;
    DistributionMapping dm_onegrid(ba2d_onegrid);
    dm_onegrid.define(pmap);
 
    Hwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
    Lwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
 
    BoxList bl2d_wave = ba.boxList();
    for (auto& b : bl2d_wave) {
        b.setRange(2,0);
    }
    BoxArray ba2d_wave(std::move(bl2d_wave));
 
    Hwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
    Lwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
 
    std::cout<<ba_onegrid<<std::endl;
    std::cout<<ba2d_onegrid<<std::endl;
    std::cout<<dm_onegrid<<std::endl;
#endif
 
 
#if defined(ERF_USE_RRTMGP)
    //*********************************************************
    // Radiation heating source terms
    //*********************************************************
    qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, ngrow_state);
    qheating_rates[lev]->setVal(0.);
 
    //*********************************************************
    // Radiation fluxes for coupling to LSM
    //*********************************************************
 
    // NOTE: Finer levels do not need to coincide with the bottom domain boundary
    //       at k=0. We make slabs here with the kmin for a given box. Therefore,
    //       care must be taken before applying these fluxes to an LSM model. For
 
    // Radiative fluxes for LSM
    if (solverChoice.lsm_type != LandSurfaceType::None)
    {
        BoxList m_bl = ba.boxList();
        for (auto& b : m_bl) {
            int kmin = b.smallEnd(2);
            b.setRange(2,kmin);
        }
        BoxArray m_ba(std::move(m_bl));
 
        sw_lw_fluxes[lev] = std::make_unique<MultiFab>(m_ba, dm, 5, ngrow_state); // SW direct (2), SW diffuse (2), LW
        solar_zenith[lev] = std::make_unique<MultiFab>(m_ba, dm, 2, ngrow_state);
 
        sw_lw_fluxes[lev]->setVal(0.);
        solar_zenith[lev]->setVal(0.);
    }
#endif
 
    //*********************************************************
    // Turbulent perturbation region initialization
    //*********************************************************
    // TODO: Test perturbation on multiple levels
    if (solverChoice.pert_type == PerturbationType::Source ||
        solverChoice.pert_type == PerturbationType::Direct)
    {
        if (lev == 0) {
            turbPert.init_tpi(lev, geom[lev].Domain().bigEnd(), geom[lev].CellSizeArray(), ba, dm, ngrow_state);
        }
    }
 
    //
    // Define the land mask here and set it to all land by default
    // NOTE: the logic below will BREAK if we have any grids not touching the bottom boundary
    //
    {
    lmask_lev[lev].resize(1);
    auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0;
    BoxList bl2d_mask = ba.boxList();
    for (auto& b : bl2d_mask) {
        b.setRange(2,0);
    }
    BoxArray ba2d_mask(std::move(bl2d_mask));
    lmask_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
    lmask_lev[lev][0]->setVal(1);
    lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
    }
 
    // Read in tables needed for windfarm simulations
    // fill in Nturb multifab - number of turbines in each mesh cell
    // write out the vtk files for wind turbine location and/or
    // actuator disks
    #ifdef ERF_USE_WINDFARM
        //init_windfarm(lev);
    #endif
}

init_thin_body()

void ERF::init_thin_body	(	int	lev,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

{
    //********************************************************************************************
    // Thin immersed body
    // *******************************************************************************************
#if 0
    if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
        (solverChoice.advChoice.zero_yflux.size() > 0) ||
        (solverChoice.advChoice.zero_zflux.size() > 0))
    {
        overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
        overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
    }
#endif
 
    if (solverChoice.advChoice.zero_xflux.size() > 0) {
        amrex::Print() << "Setting up thin immersed body for "
            << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
        BoxArray ba_xf(ba);
        ba_xf.surroundingNodes(0);
        thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
        thin_xforce[lev]->setVal(0.0);
        xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
        xflux_imask[lev]->setVal(1);
        for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
        {
            Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
            //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
            Box xbx = mfi.nodaltilebox(0);
            for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
                const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
                if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
                    (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
                    (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
                {
                    imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
                    amrex::AllPrint() << "  mask xface at " << faceidx << std::endl;
                }
            }
        }
    } else {
        thin_xforce[lev] = nullptr;
        xflux_imask[lev] = nullptr;
    }
 
    if (solverChoice.advChoice.zero_yflux.size() > 0) {
        amrex::Print() << "Setting up thin immersed body for "
            << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
        BoxArray ba_yf(ba);
        ba_yf.surroundingNodes(1);
        thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
        thin_yforce[lev]->setVal(0.0);
        yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
        yflux_imask[lev]->setVal(1);
        for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
        {
            Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
            //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
            Box ybx = mfi.nodaltilebox(1);
            for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
                const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
                if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
                    (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
                    (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
                {
                    imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
                    amrex::AllPrint() << "  mask yface at " << faceidx << std::endl;
                }
            }
        }
    } else {
        thin_yforce[lev] = nullptr;
        yflux_imask[lev] = nullptr;
    }
 
    if (solverChoice.advChoice.zero_zflux.size() > 0) {
        amrex::Print() << "Setting up thin immersed body for "
            << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
        BoxArray ba_zf(ba);
        ba_zf.surroundingNodes(2);
        thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
        thin_zforce[lev]->setVal(0.0);
        zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
        zflux_imask[lev]->setVal(1);
        for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
        {
            Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
            //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
            Box zbx = mfi.nodaltilebox(2);
            for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
                const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
                if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
                    (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
                    (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
                {
                    imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
                    amrex::AllPrint() << "  mask zface at " << faceidx << std::endl;
                }
            }
        }
    } else {
        thin_zforce[lev] = nullptr;
        zflux_imask[lev] = nullptr;
    }
}

init_uniform()

void ERF::init_uniform ( int  lev )

private

Use problem-specific reference density and temperature to set the background state to a uniform value.

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
    for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box &gbx = mfi.growntilebox(1);
        const auto &cons_arr = lev_new[Vars::cons].array(mfi);
        prob->init_uniform(gbx, cons_arr);
    }
}

init_zphys()

void ERF::init_zphys ( int  lev, amrex::Real  time  )

private

{
    if (SolverChoice::mesh_type == MeshType::StretchedDz ||
        SolverChoice::mesh_type == MeshType::VariableDz)
    {
        if (init_type != InitType::Real && init_type != InitType::Metgrid)
        {
            if (lev > 0) {
                //
                // First interpolate from coarser level if there is one
                // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
                //       have been pre-filled - this includes ghost cells both inside and outside
                //       the domain
                //
                InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
                                      IntVect(0,0,0), // do not fill ghost cells outside the domain
                                      *z_phys_nd[lev-1], 0, 0, 1,
                                      geom[lev-1], geom[lev],
                                      refRatio(lev-1), &node_bilinear_interp,
                                      domain_bcs_type, BCVars::cons_bc);
            }
 
            z_phys_nd[lev]->setVal(-1.e23);
            prob->init_custom_terrain(geom[lev],*z_phys_nd[lev],time);
            init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type);
 
            if (lev == 0) {
                Real zmax = z_phys_nd[0]->max(0,0,false);
                Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
                AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < 1.e-8, "Terrain is taller than domain top!");
            } // lev == 0
 
            z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
 
        } // init_type
    } // terrain
}

Here is the call graph for this function:

InitData()

void ERF::InitData

(

)

{
    BL_PROFILE_VAR("ERF::InitData()", InitData);
    InitData_pre();
    InitData_post();
    BL_PROFILE_VAR_STOP(InitData);
}

Referenced by MultiBlockContainer::InitializeBlocks(), and main().

Here is the caller graph for this function:

InitData_post()

void ERF::InitData_post

(

)

{
    if (restart_chkfile.empty()) {
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            if (init_type == InitType::Ideal) {
                Abort("We do not currently support init_type = ideal with non-constant dz");
            }
        }
 
        //
        // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
        //
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
                average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
                average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
            }
        }
 
        if (solverChoice.coupling_type == CouplingType::TwoWay) {
            AverageDown();
        }
 
        if (solverChoice.advChoice.have_zero_flux_faces)
        {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
                "Thin immersed body with refinement not currently supported.");
            if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                amrex::Print() << "NOTE: Thin immersed body with non-constant dz has not been tested." << std::endl;
            }
        }
 
#ifdef ERF_USE_PARTICLES
        if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
            for (int lev = 0; lev <= finest_level; lev++) {
                dynamic_cast<LagrangianMicrophysics&>(*micro).initParticles(z_phys_nd[lev]);
            }
        }
#endif
 
    } else { // Restart from a checkpoint
 
        restart();
 
        // Create the physbc objects for {cons, u, v, w, base state}
        // We fill the additional base state ghost cells just in case we have read the old format
        for (int lev(0); lev <= max_level; ++lev) {
            make_physbcs(lev);
            (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
        }
    }
 
#ifdef ERF_USE_PARTICLES
    /* If using a Lagrangian microphysics model, its particle container has now been
       constructed and initialized (calls to micro->Init). So, add its pointer to
       ERF::particleData and remove its name from list of unallocated particle containers. */
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
        const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
        const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
        particleData.pushBack(pc_name, pc_ptr);
        particleData.getNamesUnalloc().remove(pc_name);
    }
#endif
 
    if (input_bndry_planes) {
        // Read the "time.dat" file to know what data is available
        m_r2d->read_time_file();
 
        // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
        Real dt_dummy = 0.0;
        m_r2d->read_input_files(t_new[0],dt_dummy,m_bc_extdir_vals);
    }
 
    if (solverChoice.custom_rhotheta_forcing)
    {
        h_rhotheta_src.resize(max_level+1, Vector<Real>(0));
        d_rhotheta_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        for (int lev = 0; lev <= finest_level; lev++) {
            const int domlen = geom[lev].Domain().length(2);
            h_rhotheta_src[lev].resize(domlen, 0.0_rt);
            d_rhotheta_src[lev].resize(domlen, 0.0_rt);
            prob->update_rhotheta_sources(t_new[0],
                                          h_rhotheta_src[lev], d_rhotheta_src[lev],
                                          geom[lev], z_phys_cc[lev]);
        }
    }
 
    if (solverChoice.have_geo_wind_profile)
    {
        h_u_geos.resize(max_level+1, Vector<Real>(0));
        d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        h_v_geos.resize(max_level+1, Vector<Real>(0));
        d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        for (int lev = 0; lev <= finest_level; lev++) {
            const int domlen = geom[lev].Domain().length(2);
            h_u_geos[lev].resize(domlen, 0.0_rt);
            d_u_geos[lev].resize(domlen, 0.0_rt);
            h_v_geos[lev].resize(domlen, 0.0_rt);
            d_v_geos[lev].resize(domlen, 0.0_rt);
            if (solverChoice.custom_geostrophic_profile) {
                prob->update_geostrophic_profile(t_new[0],
                                                 h_u_geos[lev], d_u_geos[lev],
                                                 h_v_geos[lev], d_v_geos[lev],
                                                 geom[lev], z_phys_cc[lev]);
            } else {
                if (SolverChoice::mesh_type == MeshType::VariableDz) {
                    amrex::Print() << "Note: 1-D geostrophic wind profile input is not defined for real terrain" << std::endl;
                }
                init_geo_wind_profile(solverChoice.abl_geo_wind_table,
                                      h_u_geos[lev], d_u_geos[lev],
                                      h_v_geos[lev], d_v_geos[lev],
                                      geom[lev],
                                      zlevels_stag[0]);
            }
        }
    }
 
    if (solverChoice.custom_moisture_forcing)
    {
        h_rhoqt_src.resize(max_level+1, Vector<Real>(0));
        d_rhoqt_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        for (int lev = 0; lev <= finest_level; lev++) {
            const int domlen = geom[lev].Domain().length(2);
            h_rhoqt_src[lev].resize(domlen, 0.0_rt);
            d_rhoqt_src[lev].resize(domlen, 0.0_rt);
            prob->update_rhoqt_sources(t_new[0],
                                       h_rhoqt_src[lev], d_rhoqt_src[lev],
                                       geom[lev], z_phys_cc[lev]);
        }
    }
 
    if (solverChoice.custom_w_subsidence)
    {
        h_w_subsid.resize(max_level+1, Vector<Real>(0));
        d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        for (int lev = 0; lev <= finest_level; lev++) {
            const int domlen = geom[lev].Domain().length(2) + 1; // lives on z-faces
            h_w_subsid[lev].resize(domlen, 0.0_rt);
            d_w_subsid[lev].resize(domlen, 0.0_rt);
            prob->update_w_subsidence(t_new[0],
                                      h_w_subsid[lev], d_w_subsid[lev],
                                      geom[lev], z_phys_cc[lev]);
        }
    }
 
    if (solverChoice.rayleigh_damp_U ||solverChoice.rayleigh_damp_V ||
        solverChoice.rayleigh_damp_W ||solverChoice.rayleigh_damp_T)
    {
        initRayleigh();
        if (init_type == InitType::Input_Sounding)
        {
            // Overwrite ubar, vbar, and thetabar with input profiles;
            // wbar is assumed to be 0. Note: the tau coefficient set by
            // prob->erf_init_rayleigh() is still used
            bool restarting = (!restart_chkfile.empty());
            setRayleighRefFromSounding(restarting);
        }
    }
 
    // Read in sponge data from input file
    if(solverChoice.spongeChoice.sponge_type == "input_sponge")
    {
        initSponge();
        bool restarting = (!restart_chkfile.empty());
        setSpongeRefFromSounding(restarting);
    }
 
    if (is_it_time_for_action(istep[0], t_new[0], dt[0], sum_interval, sum_per)) {
        sum_integrated_quantities(t_new[0]);
    }
 
    if (solverChoice.pert_type == PerturbationType::Source ||
        solverChoice.pert_type == PerturbationType::Direct) {
        if (is_it_time_for_action(istep[0], t_new[0], dt[0], pert_interval, -1.)) {
            turbPert.debug(t_new[0]);
        }
    }
 
    // We only write the file at level 0 for now
    if (output_bndry_planes)
    {
        // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
        m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
 
        Real time = 0.;
        if (time >= bndry_output_planes_start_time) {
            bool is_moist = (micro->Get_Qstate_Size() > 0);
            m_w2d->write_planes(0, time, vars_new, is_moist);
        }
    }
 
    //
    // If we are starting from scratch, we have the option to project the initial velocity field
    //    regardless of how we initialized.
    // pp_inc is used as scratch space here; we zero it out after the projection
    //
    if (restart_chkfile == "")
    {
        if (solverChoice.project_initial_velocity) {
            Real dummy_dt = 1.0;
            for (int lev = 0; lev <= finest_level; ++lev)
            {
                project_velocities(lev, dummy_dt, vars_new[lev], pp_inc[lev]);
                pp_inc[lev].setVal(0.);
            }
        }
    }
 
    // Copy from new into old just in case
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        auto& lev_new = vars_new[lev];
        auto& lev_old = vars_old[lev];
 
        int ncomp = lev_new[Vars::cons].nComp();
 
        // ***************************************************************************
        // Physical bc's at domain boundary
        // ***************************************************************************
        IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
        IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
 
        (*physbcs_cons[lev])(lev_new[Vars::cons],0,ncomp,ngvect_cons,t_new[lev],BCVars::cons_bc,true);
        (   *physbcs_u[lev])(lev_new[Vars::xvel],0,1    ,ngvect_vels,t_new[lev],BCVars::xvel_bc,true);
        (   *physbcs_v[lev])(lev_new[Vars::yvel],0,1    ,ngvect_vels,t_new[lev],BCVars::yvel_bc,true);
        (   *physbcs_w[lev])(lev_new[Vars::zvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                             ngvect_vels,t_new[lev],BCVars::zvel_bc,true);
 
        MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,ncomp,lev_new[Vars::cons].nGrowVect());
        MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,    1,lev_new[Vars::xvel].nGrowVect());
        MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,    1,lev_new[Vars::yvel].nGrowVect());
        MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,    1,lev_new[Vars::zvel].nGrowVect());
    }
 
    // Compute the minimum dz in the domain at each level (to be used for setting the timestep)
    dz_min.resize(max_level+1);
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        dz_min[lev] = geom[lev].CellSize(2);
        if ( SolverChoice::mesh_type != MeshType::ConstantDz ) {
            dz_min[lev] *= (*detJ_cc[lev]).min(0);
        }
    }
 
    ComputeDt();
 
    // Fill ghost cells/faces
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        if (lev > 0 && cf_width >= 0) {
            Construct_ERFFillPatchers(lev);
        }
 
        auto& lev_new = vars_new[lev];
 
        //
        // Fill boundary conditions -- not sure why we need this here
        //
        bool fillset = false;
        if (lev == 0) {
            FillPatch(lev, t_new[lev],
                      {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]});
        } else {
            FillPatch(lev, t_new[lev],
                      {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
                      {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                      base_state[lev], base_state[lev],
                      fillset);
        }
 
        //
        // We do this here to make sure level (lev-1) boundary conditions are filled
        // before we interpolate to level (lev) ghost cells
        //
        if (lev < finest_level) {
            auto& lev_old = vars_old[lev];
            MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
            MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
            MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
            MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
        }
 
        //
        // We fill the ghost cell values of the base state in case it wasn't done in the initialization
        //
        base_state[lev].FillBoundary(geom[lev].periodicity());
 
        // For moving terrain only
        if (solverChoice.terrain_type == TerrainType::Moving) {
            MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
            base_state_new[lev].FillBoundary(geom[lev].periodicity());
        }
 
    }
 
    // Allow idealized cases over water, used to set lmask
    ParmParse pp("erf");
    int is_land;
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        if (pp.query("is_land", is_land, lev)) {
            if (is_land == 1) {
                amrex::Print() << "Level " << lev << " is land" << std::endl;
            } else if (is_land == 0) {
                amrex::Print() << "Level " << lev << " is water" << std::endl;
            } else {
                Error("is_land should be 0 or 1");
            }
            lmask_lev[lev][0]->setVal(is_land);
            lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
        }
    }
 
#ifdef ERF_USE_WW3_COUPLING
    int lev = 0;
    amrex::Print() <<  " About to call send_to_ww3 from ERF.cpp" << std::endl;
    send_to_ww3(lev);
    amrex::Print() <<  " About to call read_waves from ERF.cpp"  << std::endl;
    read_waves(lev);
   // send_to_ww3(lev);
#endif
 
    // Configure ABLMost params if used MostWall boundary condition
    // NOTE: we must set up the MOST routine after calling FillPatch
    //       in order to have lateral ghost cells filled (MOST + terrain interp).
    //       FillPatch does not call MOST, FillIntermediatePatch does.
    if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::MOST)
    {
        bool use_exp_most = solverChoice.use_explicit_most;
        bool use_rot_most = solverChoice.use_rotate_most;
        if (use_exp_most) {
            Print() << "Using MOST with explicitly included surface stresses" << std::endl;
            if (use_rot_most) {
                Print() << "Using MOST with surface stress rotations" << std::endl;
            }
        }
 
        m_most = std::make_unique<ABLMost>(geom, use_exp_most, use_rot_most,
                                           vars_old, Theta_prim, Qv_prim, Qr_prim, z_phys_nd,
                                           sst_lev, lmask_lev, lsm_data, lsm_flux,
                                           Hwave, Lwave, eddyDiffs_lev
#ifdef ERF_USE_NETCDF
                                           ,start_bdy_time, bdy_time_interval
#endif
                                           );
 
 
        if (restart_chkfile != "") {
            // Update surface fields if needed
            ReadCheckpointFileMOST();
        }
 
        // We now configure ABLMost params here so that we can print the averages at t=0
        // Note we don't fill ghost cells here because this is just for diagnostics
        for (int lev = 0; lev <= finest_level; ++lev)
        {
            Real time  = t_new[lev];
            IntVect ng = Theta_prim[lev]->nGrowVect();
 
            MultiFab::Copy(  *Theta_prim[lev], vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, ng);
            MultiFab::Divide(*Theta_prim[lev], vars_new[lev][Vars::cons],      Rho_comp, 0, 1, ng);
 
            if (solverChoice.moisture_type != MoistureType::None) {
                ng = Qv_prim[lev]->nGrowVect();
 
                MultiFab::Copy(  *Qv_prim[lev], vars_new[lev][Vars::cons], RhoQ1_comp, 0, 1, ng);
                MultiFab::Divide(*Qv_prim[lev], vars_new[lev][Vars::cons],   Rho_comp, 0, 1, ng);
 
                int rhoqr_comp = solverChoice.RhoQr_comp;
                if (rhoqr_comp > -1) {
                    MultiFab::Copy(  *Qr_prim[lev], vars_new[lev][Vars::cons], rhoqr_comp, 0, 1, ng);
                    MultiFab::Divide(*Qr_prim[lev], vars_new[lev][Vars::cons],   Rho_comp, 0, 1, ng);
                } else {
                    Qr_prim[lev]->setVal(0.0);
                }
            }
            m_most->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim, Qr_prim);
 
            if (restart_chkfile == "") {
                // Only do this if starting from scratch; if restarting, then
                // we don't want to call update_fluxes multiple times because
                // it will change u* and theta* from their previous values
                m_most->update_pblh(lev, vars_new, z_phys_cc[lev].get(),
                                    solverChoice.RhoQv_comp,
                                    solverChoice.RhoQc_comp,
                                    solverChoice.RhoQr_comp);
                m_most->update_fluxes(lev, time);
            }
        }
    }
 
    // Update micro vars before first plot file
    if (solverChoice.moisture_type != MoistureType::None) {
        for (int lev = 0; lev <= finest_level; ++lev) micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons]);
    }
 
    // Fill time averaged velocities before first plot file
    if (solverChoice.time_avg_vel) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
                              vars_new[lev][Vars::xvel],
                              vars_new[lev][Vars::yvel],
                              vars_new[lev][Vars::zvel]);
        }
    }
 
    // check for additional plotting variables that are available after particle containers
    // are setup.
    const std::string& pv1 = "plot_vars_1"; appendPlotVariables(pv1,plot_var_names_1);
    const std::string& pv2 = "plot_vars_2"; appendPlotVariables(pv2,plot_var_names_2);
 
    if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > 0.) )
    {
#ifdef ERF_USE_NETCDF
        if (check_type == "netcdf") {
           WriteNCCheckpointFile();
        }
#endif
        if (check_type == "native") {
           WriteCheckpointFile();
        }
        last_check_file_step = 0;
    }
 
    if ( (restart_chkfile.empty()) ||
         (!restart_chkfile.empty() && plot_file_on_restart) )
    {
        if (m_plot_int_1 > 0 || m_plot_per_1 > 0.)
        {
            WritePlotFile(1,plotfile_type_1,plot_var_names_1);
            last_plot_file_step_1 = istep[0];
        }
        if (m_plot_int_2 > 0 || m_plot_per_2 > 0.)
        {
            WritePlotFile(2,plotfile_type_2,plot_var_names_2);
            last_plot_file_step_2 = istep[0];
        }
    }
 
    // Set these up here because we need to know which MPI rank "cell" is on...
    if (pp.contains("data_log"))
    {
        int num_datalogs = pp.countval("data_log");
        datalog.resize(num_datalogs);
        datalogname.resize(num_datalogs);
        pp.queryarr("data_log",datalogname,0,num_datalogs);
        for (int i = 0; i < num_datalogs; i++)
            setRecordDataInfo(i,datalogname[i]);
    }
 
    if (restart_chkfile.empty() && profile_int > 0) {
        if (destag_profiles) {
            // all variables cell-centered
            write_1D_profiles(t_new[0]);
        } else {
            // some variables staggered
            write_1D_profiles_stag(t_new[0]);
        }
    }
 
    if (pp.contains("sample_point_log") && pp.contains("sample_point"))
    {
        int lev = 0;
 
        int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
        if (num_samplepts > 0) {
            Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
            samplepoint.resize(num_samplepts);
 
            pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
            for (int i = 0; i < num_samplepts; i++) {
                IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                samplepoint[i] = iv;
            }
        }
 
        int num_sampleptlogs = pp.countval("sample_point_log");
        AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
        if (num_sampleptlogs > 0) {
            sampleptlog.resize(num_sampleptlogs);
            sampleptlogname.resize(num_sampleptlogs);
            pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
 
            for (int i = 0; i < num_sampleptlogs; i++) {
                setRecordSamplePointInfo(i,lev,samplepoint[i],sampleptlogname[i]);
            }
        }
 
    }
 
    if (pp.contains("sample_line_log") && pp.contains("sample_line"))
    {
        int lev = 0;
 
        int num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
        if (num_samplelines > 0) {
            Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
            sampleline.resize(num_samplelines);
 
            pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
            for (int i = 0; i < num_samplelines; i++) {
                IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                sampleline[i] = iv;
            }
        }
 
        int num_samplelinelogs = pp.countval("sample_line_log");
        AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
        if (num_samplelinelogs > 0) {
            samplelinelog.resize(num_samplelinelogs);
            samplelinelogname.resize(num_samplelinelogs);
            pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
 
            for (int i = 0; i < num_samplelinelogs; i++) {
                setRecordSampleLineInfo(i,lev,sampleline[i],samplelinelogname[i]);
            }
        }
 
    }
 
    // Create object to do line and plane sampling if needed
    bool do_line = false; bool do_plane = false;
    pp.query("do_line_sampling",do_line); pp.query("do_plane_sampling",do_plane);
    if (do_line || do_plane) { data_sampler = std::make_unique<SampleData>(do_line, do_plane); }
 
#ifdef ERF_USE_EB
    bool write_eb_surface = false;
    pp.query("write_eb_surface", write_eb_surface);
    if (write_eb_surface) WriteMyEBSurface();
#endif
 
}

Here is the call graph for this function:

InitData_pre()

void ERF::InitData_pre

(

)

{
    // Initialize the start time for our CPU-time tracker
    startCPUTime = ParallelDescriptor::second();
 
    // Create the ReadBndryPlanes object so we can read boundary plane data
    // m_r2d is used by init_bcs so we must instantiate this class before
    if (input_bndry_planes) {
        Print() << "Defining r2d for the first time " << std::endl;
        m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
    }
 
    last_plot_file_step_1 = -1;
    last_plot_file_step_2 = -1;
    last_check_file_step  = -1;
 
    if (restart_chkfile.empty()) {
        // start simulation from the beginning
 
        const Real time = start_time;
        InitFromScratch(time);
    } else {
        // For initialization this is done in init_only; it is done here for restart
        init_bcs();
    }
 
    // Verify BCs are compatible with solver choice
    for (int lev(0); lev <= max_level; ++lev) {
        if ( ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
               (solverChoice.turbChoice[lev].pbl_type == PBLType::YSU)       ) &&
            phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::MOST ) {
            Abort("MYNN2.5/YSU PBL Model requires MOST at lower boundary");
        }
    }
}

initHSE() [1/2]

void ERF::initHSE ( )

private

Initialize HSE.

{
    AMREX_ALWAYS_ASSERT(!init_sounding_ideal);
    for (int lev = 0; lev <= finest_level; lev++)
    {
        initHSE(lev);
    }
}

initHSE() [2/2]

void ERF::initHSE ( int  lev )

private

Initialize density and pressure base state in hydrostatic equilibrium.

{
    // This integrates up through column to update p_hse, pi_hse, th_hse;
    // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
 
    MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
    MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
    MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
 
    bool all_boxes_touch_bottom = true;
    Box domain(geom[lev].Domain());
 
    int icomp = 0; int ncomp = BaseState::num_comps;
 
    if (lev == 0) {
        BoxArray ba(base_state[lev].boxArray());
        for (int i = 0; i < ba.size(); i++) {
            if (ba[i].smallEnd(2) != domain.smallEnd(2)) {
                all_boxes_touch_bottom = false;
            }
        }
    }
    else
    {
        //
        // We need to do this interp from coarse level in order to set the values of
        // the base state inside the domain but outside of the fine region
        //
        base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
        //
        // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
        //       have been pre-filled - this includes ghost cells both inside and outside
        //       the domain
        //
        InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                              IntVect(0,0,0), // do not fill ghost cells outside the domain
                              base_state[lev-1], icomp, icomp, ncomp,
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
 
         // We need to do this here because the interpolation above may leave corners unfilled
         //    when the corners need to be filled by, for example, reflection of the fine ghost
         //    cell outside the fine region but inide the domain.
         (*physbcs_base[lev])(base_state[lev],icomp,ncomp,base_state[lev].nGrowVect());
    }
 
    if (all_boxes_touch_bottom || lev > 0) {
 
        // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
        if (solverChoice.use_moist_background){
            prob->erf_init_dens_hse_moist(r_hse, z_phys_nd[lev], geom[lev]);
        } else {
            prob->erf_init_dens_hse(r_hse, z_phys_nd[lev], z_phys_cc[lev], geom[lev]);
        }
 
        erf_enforce_hse(lev, r_hse, p_hse, pi_hse, th_hse, z_phys_cc[lev]);
 
    } else {
 
        BoxArray ba_new(domain);
 
        ChopGrids2D(ba_new, domain, ParallelDescriptor::NProcs());
 
        DistributionMapping dm_new(ba_new);
 
        MultiFab new_base_state(ba_new, dm_new, BaseState::num_comps, base_state[lev].nGrowVect());
        new_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
                                    base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
 
        MultiFab new_r_hse (new_base_state, make_alias, BaseState::r0_comp, 1);
        MultiFab new_p_hse (new_base_state, make_alias, BaseState::p0_comp, 1);
        MultiFab new_pi_hse(new_base_state, make_alias, BaseState::pi0_comp, 1);
        MultiFab new_th_hse(new_base_state, make_alias, BaseState::th0_comp, 1);
 
        std::unique_ptr<MultiFab> new_z_phys_cc;
        std::unique_ptr<MultiFab> new_z_phys_nd;
        if (solverChoice.mesh_type != MeshType::ConstantDz) {
            new_z_phys_cc = std::make_unique<MultiFab>(ba_new,dm_new,1,1);
            new_z_phys_cc->ParallelCopy(*z_phys_cc[lev],0,0,1,1,1);
 
            BoxArray ba_new_nd(ba_new);
            ba_new_nd.surroundingNodes();
            new_z_phys_nd = std::make_unique<MultiFab>(ba_new_nd,dm_new,1,1);
            new_z_phys_nd->ParallelCopy(*z_phys_nd[lev],0,0,1,1,1);
        }
 
        // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
        if (solverChoice.use_moist_background){
            prob->erf_init_dens_hse_moist(new_r_hse, new_z_phys_nd, geom[lev]);
        } else {
            prob->erf_init_dens_hse(new_r_hse, new_z_phys_nd, new_z_phys_cc, geom[lev]);
        }
 
        erf_enforce_hse(lev, new_r_hse, new_p_hse, new_pi_hse, new_th_hse, new_z_phys_cc);
 
        // Now copy back into the original arrays
        base_state[lev].ParallelCopy(new_base_state,0,0,base_state[lev].nComp(),
                                     base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
    }
 
    //
    // Impose physical bc's on the base state -- the values outside the fine region
    //   but inside the domain have already been filled in the call above to InterpFromCoarseLevel
    //
    (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
}

Here is the call graph for this function:

initialize_integrator()

void ERF::initialize_integrator ( int  lev, amrex::MultiFab &  cons_mf, amrex::MultiFab &  vel_mf  )

private

{
    const BoxArray& ba(cons_mf.boxArray());
    const DistributionMapping& dm(cons_mf.DistributionMap());
 
    int ncomp_cons = cons_mf.nComp();
 
    // Initialize the integrator memory
    Vector<MultiFab> int_state; // integration state data structure example
    int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons));         // cons
    int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
    int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
    int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
 
    mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
    mri_integrator_mem[lev]->setNoSubstepping((solverChoice.substepping_type[lev] == SubsteppingType::None));
    mri_integrator_mem[lev]->setAnelastic(solverChoice.anelastic[lev]);
    mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
    mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
}

InitializeFromFile()

void ERF::InitializeFromFile ( )

private

InitializeLevelFromData()

void ERF::InitializeLevelFromData ( int  lev, const amrex::MultiFab &  initial_data  )

private

initializeMicrophysics()

void ERF::initializeMicrophysics ( const int &  a_nlevsmax )

private

Parameters

a_nlevsmax

number of AMR levels

{
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
 
        micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
 
    } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
#ifdef ERF_USE_PARTICLES
 
        micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
        /* Lagrangian microphysics models will have a particle container; it needs to be added
           to ERF::particleData */
        const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
        /* The particle container has not yet been constructed and initialized, so just add
           its name here for now (so that functions to set plotting variables can see it). */
        particleData.addName( pc_name );
 
#else
        Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
#endif
    }
 
    qmoist.resize(a_nlevsmax);
    return;
}

Here is the call graph for this function:

initRayleigh()

void ERF::initRayleigh ( )

private

Initialize Rayleigh damping profiles.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

{
    const int khi = geom[0].Domain().bigEnd(2);
    solverChoice.rayleigh_ztop = (solverChoice.terrain_type == TerrainType::None) ?  geom[0].ProbHi(2) : zlevels_stag[0][khi+1];
 
    h_rayleigh_ptrs.resize(max_level+1);
    d_rayleigh_ptrs.resize(max_level+1);
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        // These have 4 components: ubar, vbar, wbar, thetabar
        h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
        d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
 
        const int zlen_rayleigh = geom[lev].Domain().length(2);
 
        // Allocate space for these 1D vectors
        for (int n = 0; n < Rayleigh::nvars; n++) {
            h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
            d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
        }
 
        // Init the host vectors
        prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_nd[lev], solverChoice.rayleigh_zdamp);
 
        // Copy from host vectors to device vectors
        for (int n = 0; n < Rayleigh::nvars; n++) {
            Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
                      d_rayleigh_ptrs[lev][n].begin());
        }
    }
}

initSponge()

void ERF::initSponge ( )

private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

{
    h_sponge_ptrs.resize(max_level+1);
    d_sponge_ptrs.resize(max_level+1);
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        // These have 2 components: ubar, vbar
        h_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
        d_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
 
        const int zlen_sponge = geom[lev].Domain().length(2);
 
        // Allocate space for these 1D vectors
        for (int n = 0; n < Sponge::nvars_sponge; n++) {
            h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
            d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
        }
 
    }
}

input_sponge()

void ERF::input_sponge

(

int

lev

)

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters

lev	Integer specifying the current level

{
    // We only want to read the file once
    if (lev == 0) {
        if (input_sponge_data.input_sponge_file.empty())
            Error("input_sounding file name must be provided via input");
 
        // this will interpolate the input profiles to the nominal height levels
        // (ranging from 0 to the domain top)
        input_sponge_data.read_from_file(geom[lev], zlevels_stag[lev]);
    }
}

is_it_time_for_action()

bool ERF::is_it_time_for_action ( int  nstep, amrex::Real  time, amrex::Real  dt, int  action_interval, amrex::Real  action_per  )

static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters

nstep	Timestep number
time	Current time
dtlev	Timestep for the current level
action_interval	Interval in number of timesteps for taking action
action_per	Interval in simulation time for taking action

{
  bool int_test = (action_interval > 0 && nstep % action_interval == 0);
 
  bool per_test = false;
  if (action_per > 0.0) {
    const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
    const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
 
    if (num_per_old != num_per_new) {
      per_test = true;
    }
  }
 
  return int_test || per_test;
}

make_physbcs()

void ERF::make_physbcs ( int  lev )

private

{
    if (SolverChoice::mesh_type == MeshType::VariableDz) {
        AMREX_ALWAYS_ASSERT(z_phys_nd[lev] != nullptr);
    }
 
    physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                               m_bc_extdir_vals, m_bc_neumann_vals,
                                                               z_phys_nd[lev], use_real_bcs);
    physbcs_u[lev]    = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                            m_bc_extdir_vals, m_bc_neumann_vals,
                                                            z_phys_nd[lev], use_real_bcs, xvel_bc_data[lev].data());
    physbcs_v[lev]    = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                            m_bc_extdir_vals, m_bc_neumann_vals,
                                                            z_phys_nd[lev], use_real_bcs, yvel_bc_data[lev].data());
    physbcs_w[lev]    = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                            m_bc_extdir_vals, m_bc_neumann_vals,
                                                            solverChoice.terrain_type, z_phys_nd[lev],
                                                            use_real_bcs, zvel_bc_data[lev].data());
    physbcs_base[lev] = std::make_unique<ERFPhysBCFunct_base> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                               (solverChoice.terrain_type == TerrainType::Moving));
}

MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage	(	amrex::Vector< amrex::Real > &	h_havg,
		amrex::MultiFab &	S,
		int	n
	)

{
    // Get the number of cells in z at level 0
    int dir_z = AMREX_SPACEDIM-1;
    auto domain = geom[0].Domain();
    int size_z = domain.length(dir_z);
    int start_z = domain.smallEnd()[dir_z];
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
 
    // resize the level 0 horizontal average vectors
    h_havg.resize(size_z, 0.0_rt);
 
    // Get the cell centered data and construct sums
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(S); mfi.isValid(); ++mfi) {
        const Box& box = mfi.validbox();
        const IntVect& se = box.smallEnd();
        const IntVect& be = box.bigEnd();
 
        auto      fab_arr = S[mfi].array();
 
        FArrayBox fab_reduce(box, 1, The_Async_Arena());
        auto arr_reduce   = fab_reduce.array();
 
        ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
            arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
        });
 
        for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
            Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
            h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
        }
    }
 
    // combine sums from different MPI ranks
    ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), h_havg.size());
 
    // divide by the total number of cells we are averaging over
    for (int k = 0; k < size_z; ++k) {
        h_havg[k]     /= area_z;
    }
}

MakeHorizontalAverages()

void ERF::MakeHorizontalAverages

(

)

{
    int lev = 0;
 
    // First, average down all levels (if doing two-way coupling)
    if (solverChoice.coupling_type == CouplingType::TwoWay) {
        AverageDown();
    }
 
    MultiFab mf(grids[lev], dmap[lev], 5, 0);
 
    int zdir = 2;
    auto domain = geom[0].Domain();
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
    bool is_anelastic = (solverChoice.anelastic[lev] == 1);
 
    for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.validbox();
        auto  fab_arr = mf.array(mfi);
        auto const  hse_arr = base_state[lev].const_array(mfi);
        auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
            Real dens = cons_arr(i, j, k, Rho_comp);
            fab_arr(i, j, k, 0) = dens;
            fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
            if (!use_moisture) {
                if (is_anelastic) {
                    fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
                } else {
                    fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp));
                }
            }
        });
    }
 
    if (use_moisture)
    {
        for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.validbox();
            auto  fab_arr = mf.array(mfi);
            auto const  hse_arr = base_state[lev].const_array(mfi);
            auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
            int ncomp = vars_new[lev][Vars::cons].nComp();
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                Real dens = cons_arr(i, j, k, Rho_comp);
                if (is_anelastic) {
                    fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
                } else {
                    Real qv = cons_arr(i, j, k, RhoQ1_comp) / dens;
                    fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
                }
                fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : 0.0);
                fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : 0.0);
            });
        }
 
        Gpu::HostVector<Real> h_avg_qv          = sumToLine(mf,3,1,domain,zdir);
        Gpu::HostVector<Real> h_avg_qc          = sumToLine(mf,4,1,domain,zdir);
    }
 
    // Sum in the horizontal plane
    Gpu::HostVector<Real> h_avg_density     = sumToLine(mf,0,1,domain,zdir);
    Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
    Gpu::HostVector<Real> h_avg_pressure    = sumToLine(mf,2,1,domain,zdir);
 
    // Divide by the total number of cells we are averaging over
    int size_z = domain.length(zdir);
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    int klen = static_cast<int>(h_avg_density.size());
 
    for (int k = 0; k < klen; ++k) {
        h_havg_density[k]     /= area_z;
        h_havg_temperature[k] /= area_z;
        h_havg_pressure[k]    /= area_z;
        if (solverChoice.moisture_type != MoistureType::None)
        {
            h_havg_qc[k]          /= area_z;
            h_havg_qv[k]          /= area_z;
        }
    } // k
 
    // resize device vectors
    d_havg_density.resize(size_z, 0.0_rt);
    d_havg_temperature.resize(size_z, 0.0_rt);
    d_havg_pressure.resize(size_z, 0.0_rt);
 
    // copy host vectors to device vectors
    Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
    Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
    Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
 
    if (solverChoice.moisture_type != MoistureType::None)
    {
        d_havg_qv.resize(size_z, 0.0_rt);
        d_havg_qc.resize(size_z, 0.0_rt);
        Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
        Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
    }
}

Here is the call graph for this function:

MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

override

{
    AMREX_ALWAYS_ASSERT(lev > 0);
 
    if (verbose) {
        amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
    }
 
    //********************************************************************************************
    // This allocates all kinds of things, including but not limited to: solution arrays,
    //      terrain arrays, metric terms and base state.
    // *******************************************************************************************
    init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev], base_state[lev], z_phys_nd[lev]);
 
    t_new[lev] = time;
    t_old[lev] = time - 1.e200;
 
    // ********************************************************************************************
    // Build the data structures for terrain-related quantities
    // ********************************************************************************************
    init_zphys(lev, time);
    update_terrain_arrays(lev);
 
    //
    // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
    //
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
            average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
            average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
        }
    }
 
    // ********************************************************************************************
    // Build the data structures for canopy model (depends upon z_phys)
    // ********************************************************************************************
    if (solverChoice.do_forest_drag) { m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_nd[lev].get()); }
 
    if (solverChoice.do_terrain_drag) { m_terrain_drag[lev]->define_terrain_blank_field(ba, dm, geom[lev], z_phys_nd[lev].get()); }
    //********************************************************************************************
    // Microphysics
    // *******************************************************************************************
    int q_size  = micro->Get_Qmoist_Size(lev);
    qmoist[lev].resize(q_size);
    micro->Define(lev, solverChoice);
    if (solverChoice.moisture_type != MoistureType::None)
    {
        micro->Init(lev, vars_new[lev][Vars::cons],
                    grids[lev], Geom(lev), 0.0,
                    z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
    }
    for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
        qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
    }
 
    // ********************************************************************************************
    // Update the base state at this level by interpolation from coarser level
    // ********************************************************************************************
    //
    // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
    //       have been pre-filled - this includes ghost cells both inside and outside
    //       the domain
    //
    InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                          IntVect(0,0,0), // do not fill ghost cells outside the domain
                          base_state[lev-1], 0, 0, base_state[lev].nComp(),
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), &cell_cons_interp,
                          domain_bcs_type, BCVars::cons_bc);
 
    initHSE(lev);
 
    // ********************************************************************************************
    // Build the data structures for calculating diffusive/turbulent terms
    // ********************************************************************************************
    update_diffusive_arrays(lev, ba, dm);
 
    // *****************************************************************************************************
    // Initialize the boundary conditions (after initializing the terrain but before calling FillCoarsePatch
    // *****************************************************************************************************
    make_physbcs(lev);
 
    // ********************************************************************************************
    // Fill data at the new level by interpolation from the coarser level
    // Note that internal to FillCoarsePatch we will convert velocity to momentum,
    //      then interpolate momentum, then convert momentum back to velocity
    // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
    // ********************************************************************************************
    FillCoarsePatch(lev, time);
 
    // ********************************************************************************************
    // Initialize the integrator class
    // ********************************************************************************************
    dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
    initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
 
    // ********************************************************************************************
    // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
    // ********************************************************************************************
    if (lev > 0 && cf_width >= 0) {
        Construct_ERFFillPatchers(lev);
           Define_ERFFillPatchers(lev);
    }
 
#ifdef ERF_USE_PARTICLES
    // particleData.Redistribute();
#endif
}

MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

override

{
    BoxArray ba;
    DistributionMapping dm;
    Box domain(Geom(0).Domain());
    if (lev == 0 && restart_chkfile.empty() &&
        (max_grid_size[0][0] >= domain.length(0)) &&
        (max_grid_size[0][1] >= domain.length(1)) &&
        ba_in.size() != ParallelDescriptor::NProcs())
    {
        // We only decompose in z if max_grid_size_z indicates we should
        bool decompose_in_z = (max_grid_size[0][2] < domain.length(2));
 
        ba = ERFPostProcessBaseGrids(Geom(0).Domain(),decompose_in_z);
        dm = DistributionMapping(ba);
    } else {
        ba = ba_in;
        dm = dm_in;
    }
 
    // Define grids[lev] to be ba
    SetBoxArray(lev, ba);
 
    // Define dmap[lev] to be dm
    SetDistributionMap(lev, dm);
 
    if (verbose) {
        amrex::Print() <<" BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
    }
 
    if (lev == 0) init_bcs();
 
#ifdef ERF_USE_EB
    m_factory[lev] = makeEBFabFactory(geom[lev], grids[lev], dmap[lev],
                                      {nghost_eb_basic(),
                                       nghost_eb_volume(),
                                       nghost_eb_full()},
                                       EBSupport::full);
#else
    m_factory[lev] = std::make_unique<FArrayBoxFactory>();
#endif
 
    auto& lev_new = vars_new[lev];
    auto& lev_old = vars_old[lev];
 
    //********************************************************************************************
    // This allocates all kinds of things, including but not limited to: solution arrays,
    //      terrain arrays, metric terms and base state.
    // *******************************************************************************************
    init_stuff(lev, ba, dm, lev_new, lev_old, base_state[lev], z_phys_nd[lev]);
 
    //********************************************************************************************
    // Land Surface Model
    // *******************************************************************************************
    int lsm_size  = lsm.Get_Data_Size();
    lsm_data[lev].resize(lsm_size);
    lsm_flux[lev].resize(lsm_size);
    lsm.Define(lev, solverChoice);
    if (solverChoice.lsm_type != LandSurfaceType::None)
    {
        lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
    }
    for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
        lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
        lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
    }
 
    // ********************************************************************************************
    // Build the data structures for calculating diffusive/turbulent terms
    // ********************************************************************************************
    update_diffusive_arrays(lev, ba, dm);
 
    // ********************************************************************************************
    // Build the data structures for holding sea surface temps
    // ********************************************************************************************
    sst_lev[lev].resize(1);     sst_lev[lev][0] = nullptr;
 
    // ********************************************************************************************
    // Thin immersed body
    // *******************************************************************************************
    init_thin_body(lev, ba, dm);
 
    // ********************************************************************************************
    // Initialize the integrator class
    // ********************************************************************************************
    initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
 
    // ********************************************************************************************
    // Initialize the data itself
    // If (init_type == InitType::Real) then we are initializing terrain and the initial data in
    //                                  the same call so we must call init_only before update_terrain_arrays
    // If (init_type != InitType::Real) then we want to initialize the terrain before the initial data
    //                                  since we may need to use the grid information before constructing
    //                                  initial idealized data
    // ********************************************************************************************
    if (restart_chkfile.empty()) {
        if ((init_type == InitType::Real) || (init_type == InitType::Metgrid)) {
            init_only(lev, start_time);
            init_zphys(lev, time);
            update_terrain_arrays(lev);
            make_physbcs(lev);
        } else {
            init_zphys(lev, time);
            update_terrain_arrays(lev);
            // Note that for init_type != InitType::Real or InitType::Metgrid,
            // make_physbcs is called inside init_only
            init_only(lev, start_time);
        }
    }
 
     // Read in tables needed for windfarm simulations
    // fill in Nturb multifab - number of turbines in each mesh cell
    // write out the vtk files for wind turbine location and/or
    // actuator disks
    #ifdef ERF_USE_WINDFARM
        init_windfarm(lev);
    #endif
    // ********************************************************************************************
    // Build the data structures for canopy model (depends upon z_phys)
    // ********************************************************************************************
    if (solverChoice.do_forest_drag) { m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_nd[lev].get()); }
 
    if (solverChoice.do_terrain_drag) { m_terrain_drag[lev]->define_terrain_blank_field(ba, dm, geom[lev], z_phys_nd[lev].get()); }
 
    //********************************************************************************************
    // Create wall distance field for RANS model (depends upon z_phys)
    // *******************************************************************************************
    if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
        poisson_wall_dist(lev);
    }
 
    //********************************************************************************************
    // Microphysics
    // *******************************************************************************************
    int q_size  = micro->Get_Qmoist_Size(lev);
    qmoist[lev].resize(q_size);
    micro->Define(lev, solverChoice);
    if (solverChoice.moisture_type != MoistureType::None)
    {
        micro->Init(lev, vars_new[lev][Vars::cons],
                    grids[lev], Geom(lev), 0.0,
                    z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
    }
    for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
        qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
    }
 
    // ********************************************************************************************
    // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
    // ********************************************************************************************
    if (lev > 0 && cf_width >= 0) {
        Construct_ERFFillPatchers(lev);
           Define_ERFFillPatchers(lev);
    }
 
#ifdef ERF_USE_PARTICLES
    if (restart_chkfile.empty()) {
        if (lev == 0) {
            initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd);
        } else {
            particleData.Redistribute();
        }
    }
#endif
}

Here is the call graph for this function:

NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )

inlineprivatenoexcept

    {
        return datalog.size();
    }

NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )

inlineprivatenoexcept

    {
        return samplelinelog.size();
    }

NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )

inlineprivatenoexcept

    {
        return sampleline.size();
    }

NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )

inlineprivatenoexcept

    {
        return sampleptlog.size();
    }

NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )

inlineprivatenoexcept

    {
        return samplepoint.size();
    }

operator=() [1/2]

ERF& ERF::operator= ( const ERF &  other )

delete

operator=() [2/2]

ERF& ERF::operator= ( ERF &&  other )

deletenoexcept

ParameterSanityChecks()

void ERF::ParameterSanityChecks ( )

private

{
    AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt[0] > 0.);
 
    // We don't allow use_real_bcs to be true if init_type is not either InitType::Real or InitType::Metgrid
    AMREX_ALWAYS_ASSERT(!use_real_bcs || ((init_type == InitType::Real) || (init_type == InitType::Metgrid)) );
 
    AMREX_ALWAYS_ASSERT(real_width >= 0);
    AMREX_ALWAYS_ASSERT(real_set_width >= 0);
    AMREX_ALWAYS_ASSERT(real_width >= real_set_width);
 
    if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
        Abort("You must set cf_width >= cf_set_width >= 0");
    }
    if (max_level > 0 && cf_set_width > 0) {
        for (int lev = 1; lev <= max_level; lev++) {
            if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
                cf_set_width%ref_ratio[lev-1][1] != 0 ||
                cf_set_width%ref_ratio[lev-1][2] != 0 ) {
                Abort("You must set cf_width to be a multiple of ref_ratio");
            }
        }
    }
 
    // If fixed_mri_dt_ratio is set, it must be even
    if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
    {
        Abort("If you specify fixed_mri_dt_ratio, it must be even");
    }
 
    for (int lev = 0; lev <= max_level; lev++)
    {
        // We ignore fixed_fast_dt if not substepping
        if (solverChoice.substepping_type[lev] == SubsteppingType::None) {
            fixed_fast_dt[lev] = -1.0;
        }
 
        // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
        if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio <= 0)
        {
            Real eps = 1.e-12;
            int ratio = static_cast<int>( ( (1.0+eps) * fixed_dt[lev] ) / fixed_fast_dt[lev] );
            if (fixed_dt[lev] / fixed_fast_dt[lev] != ratio)
            {
                Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
            }
        }
 
        // If all three are specified, they must be consistent
        if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. &&  fixed_mri_dt_ratio > 0)
        {
            if (fixed_dt[lev] / fixed_fast_dt[lev] != fixed_mri_dt_ratio)
            {
                Abort("Dt is over-specfied");
            }
        }
    } // lev
 
    if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
        Abort("For two-way coupling you must set cf_width = 0");
    }
}

PlotFileName()

std::string ERF::PlotFileName ( int  lev ) const

private

PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string >  plot_var_names )

staticprivate

{
    Vector<std::string> names;
 
    names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
 
    return names;
 
}

poisson_wall_dist()

void ERF::poisson_wall_dist

(

int

lev

)

Calculate wall distances using the Poisson equation

See Tucker, P. G. (2003). Differential equation-based wall distance computation for DES and RANS. Journal of Computational Physics, 190(1), 229–248. https://doi.org/10.1016/S0021-9991(03)00272-9

{
    BL_PROFILE("ERF::poisson_wall_dist()");
 
    auto const& geomdata = geom[lev];
 
    if (solverChoice.mesh_type == MeshType::ConstantDz) {
// Comment this out to test the wall dist calc in the trivial case:
//#if 0
        Print() << "Directly calculating direct wall distance for constant dz" << std::endl;
        const Real* prob_lo = geomdata.ProbLo();
        const Real* dx = geomdata.CellSize();
        for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.validbox();
            auto dist_arr = walldist[lev]->array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                dist_arr(i, j, k) = prob_lo[2] + (k + 0.5) * dx[2];
            });
        }
        return;
//#endif
    } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
        // TODO: Handle this trivial case
        Error("Wall dist calc not implemented with grid stretching yet");
    } else {
        // TODO
        Error("Wall dist calc not implemented over terrain yet");
    }
 
    Print() << "Calculating Poisson wall distance" << std::endl;
 
    // Make sure the solver only sees the levels over which we are solving
    BoxArray nba = walldist[lev]->boxArray();
    nba.surroundingNodes();
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(nba);
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(walldist[lev]->DistributionMap());
 
    Vector<MultiFab> rhs;
    Vector<MultiFab> phi;
 
#ifdef ERF_USE_EB
    Error("Wall dist calc not implemented for EB";
#else
    rhs.resize(1);   rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
    phi.resize(1);   phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
#endif
 
    rhs[0].setVal(-1.0);
 
    // Define an overset mask to set dirichlet nodes on walls
    iMultiFab mask(ba_tmp[0], dm_tmp[0], 1, 0);
    Vector<const iMultiFab*> overset_mask = {&mask};
 
    auto const dom_lo = lbound(geom[lev].Domain());
    auto const dom_hi = ubound(geom[lev].Domain());
 
    // ****************************************************************************
    // Initialize phi
    // (It is essential that we do this in order to fill the corners; this is
    // used if we include blanking.)
    // ****************************************************************************
    phi[0].setVal(0.0);
 
    // ****************************************************************************
    // Interior boundaries are marked with phi=0
    // ****************************************************************************
    // Overset mask is 0/1: 1 means the node is an unknown. 0 means it's known.
    mask.setVal(1);
    if (solverChoice.advChoice.have_zero_flux_faces) {
        Warning("Poisson distance is inaccurate for bodies in open domains that are small compared to the domain size, skipping...");
        walldist[lev]->setVal(1e34);
        return;
#if 0
        Gpu::DeviceVector<IntVect> xfacelist, yfacelist, zfacelist;
 
        xfacelist.resize(solverChoice.advChoice.zero_xflux.size());
        yfacelist.resize(solverChoice.advChoice.zero_yflux.size());
        zfacelist.resize(solverChoice.advChoice.zero_zflux.size());
 
        if (xfacelist.size() > 0) {
            Gpu::copy(amrex::Gpu::hostToDevice,
                      solverChoice.advChoice.zero_xflux.begin(),
                      solverChoice.advChoice.zero_xflux.end(),
                      xfacelist.begin());
            Print() << "  masking interior xfaces" << std::endl;
        }
        if (yfacelist.size() > 0) {
            Gpu::copy(amrex::Gpu::hostToDevice,
                      solverChoice.advChoice.zero_yflux.begin(),
                      solverChoice.advChoice.zero_yflux.end(),
                      yfacelist.begin());
            Print() << "  masking interior yfaces" << std::endl;
        }
        if (zfacelist.size() > 0) {
            Gpu::copy(amrex::Gpu::hostToDevice,
                      solverChoice.advChoice.zero_zflux.begin(),
                      solverChoice.advChoice.zero_zflux.end(),
                      zfacelist.begin());
            Print() << "  masking interior zfaces" << std::endl;
        }
 
        for (MFIter mfi(phi[0]); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.validbox();
 
            auto phi_arr  = phi[0].array(mfi);
            auto mask_arr = mask.array(mfi);
 
            if (xfacelist.size() > 0) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    for (int iface=0; iface < xfacelist.size(); ++iface) {
                        if ((i == xfacelist[iface][0]) &&
                            (j == xfacelist[iface][1]) &&
                            (k == xfacelist[iface][2]))
                        {
                            mask_arr(i, j  , k  ) = 0;
                            mask_arr(i, j  , k+1) = 0;
                            mask_arr(i, j+1, k  ) = 0;
                            mask_arr(i, j+1, k+1) = 0;
                        }
                    }
                });
            }
 
            if (yfacelist.size() > 0) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    for (int iface=0; iface < yfacelist.size(); ++iface) {
                        if ((i == yfacelist[iface][0]) &&
                            (j == yfacelist[iface][1]) &&
                            (k == yfacelist[iface][2]))
                        {
                            mask_arr(i  , j, k  ) = 0;
                            mask_arr(i  , j, k+1) = 0;
                            mask_arr(i+1, j, k  ) = 0;
                            mask_arr(i+1, j, k+1) = 0;
                        }
                    }
                });
            }
 
            if (zfacelist.size() > 0) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    for (int iface=0; iface < zfacelist.size(); ++iface) {
                        if ((i == xfacelist[iface][0]) &&
                            (j == xfacelist[iface][1]) &&
                            (k == xfacelist[iface][2]))
                        {
                            mask_arr(i  , j  , k) = 0;
                            mask_arr(i  , j+1, k) = 0;
                            mask_arr(i+1, j  , k) = 0;
                            mask_arr(i+1, j+1, k) = 0;
                        }
                    }
                });
            }
        }
#endif
    }
 
    // ****************************************************************************
    // Setup BCs, with solid domain boundaries being dirichlet
    // We assume that the zlo boundary corresponds to the land surface
    // ****************************************************************************
    amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bc3d_lo, bc3d_hi;
    Orientation zlo(Direction::z, Orientation::low);
    bool havewall{false};
    for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
        if (geom[0].isPeriodic(dir)) {
            bc3d_lo[dir] = LinOpBCType::Periodic;
            bc3d_hi[dir] = LinOpBCType::Periodic;
        } else {
            bc3d_lo[dir] = LinOpBCType::Neumann;
            bc3d_hi[dir] = LinOpBCType::Neumann;
        }
    }
    if ( ( phys_bc_type[zlo] == ERF_BC::MOST                               ) ||
         ( phys_bc_type[zlo] == ERF_BC::no_slip_wall                       ) )/*||
         ((phys_bc_type[zlo] == ERF_BC::slip_wall) && (dom_hi.z > dom_lo.z)) )*/
    {
        Print() << "  Poisson zlo BC is dirichlet" << std::endl;
        bc3d_lo[2] = LinOpBCType::Dirichlet;
        havewall = true;
    }
    Print() << "  bc lo : " << bc3d_lo << std::endl;
    Print() << "  bc hi : " << bc3d_hi << std::endl;
 
    if (!solverChoice.advChoice.have_zero_flux_faces && !havewall) {
        Error("No solid boundaries in the computational domain");
    }
 
    LPInfo info;
/* Nodal solver cannot have hidden dimensions */
#if 0
    // Allow a hidden direction if the domain is one cell wide
    if (dom_lo.x == dom_hi.x) {
        info.setHiddenDirection(0);
        Print() << "  domain is 2D in yz" << std::endl;
    } else if (dom_lo.y == dom_hi.y) {
        info.setHiddenDirection(1);
        Print() << "  domain is 2D in xz" << std::endl;
    } else if (dom_lo.z == dom_hi.z) {
        info.setHiddenDirection(2);
        Print() << "  domain is 2D in xy" << std::endl;
    }
#endif
 
    // ****************************************************************************
    // Solve nodal masked Poisson problem with MLMG
    // TODO: different solver for terrain?
    // ****************************************************************************
    const Real reltol = solverChoice.poisson_reltol;
    const Real abstol = solverChoice.poisson_abstol;
 
    Real sigma = 1.0;
    MLNodeLaplacian mlpoisson(geom_tmp, ba_tmp, dm_tmp, info, {}, sigma);
 
    mlpoisson.setDomainBC(bc3d_lo, bc3d_hi);
 
    if (lev > 0) {
        mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
    }
 
    mlpoisson.setLevelBC(0, nullptr);
 
    mlpoisson.setOversetMask(0, mask);
 
    // Solve
    MLMG mlmg(mlpoisson);
    int max_iter = 100;
    mlmg.setMaxIter(max_iter);
 
    mlmg.setVerbose(mg_verbose);
    mlmg.setBottomVerbose(0);
 
    mlmg.solve(GetVecOfPtrs(phi),
               GetVecOfConstPtrs(rhs),
               reltol, abstol);
 
    // Now overwrite with periodic fill outside domain and fine-fine fill inside
    phi[0].FillBoundary(geom[lev].periodicity());
 
    // ****************************************************************************
    // Compute grad(phi) to get distances
    // - Note that phi is nodal and walldist is cell-centered
    // - TODO: include terrain metrics for dphi/dz
    // ****************************************************************************
    for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.validbox();
 
        const auto invCellSize = geomdata.InvCellSizeArray();
 
        auto const& phi_arr = phi[0].const_array(mfi);
        auto dist_arr = walldist[lev]->array(mfi);
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
            Real dpdx{0}, dpdy{0}, dpdz{0};
 
            // dphi/dx
            if (dom_lo.x != dom_hi.x) {
                dpdx = 0.25 * invCellSize[0] * (
                        (phi_arr(i+1, j  , k  ) - phi_arr(i, j  , k  ))
                      + (phi_arr(i+1, j  , k+1) - phi_arr(i, j  , k+1))
                      + (phi_arr(i+1, j+1, k  ) - phi_arr(i, j+1, k  ))
                      + (phi_arr(i+1, j+1, k+1) - phi_arr(i, j+1, k+1)) );
            }
 
            // dphi/dy
            if (dom_lo.y != dom_hi.y) {
                dpdy = 0.25 * invCellSize[1] * (
                        (phi_arr(i  , j+1, k  ) - phi_arr(i  , j, k  ))
                      + (phi_arr(i  , j+1, k+1) - phi_arr(i  , j, k+1))
                      + (phi_arr(i+1, j+1, k  ) - phi_arr(i+1, j, k  ))
                      + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j, k+1)) );
            }
 
            // dphi/dz
            if (dom_lo.z != dom_hi.z) {
                dpdz = 0.25 * invCellSize[2] * (
                        (phi_arr(i  , j  , k+1) - phi_arr(i  , j  , k))
                      + (phi_arr(i  , j+1, k+1) - phi_arr(i  , j+1, k))
                      + (phi_arr(i+1, j  , k+1) - phi_arr(i+1, j  , k))
                      + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j+1, k)) );
            }
 
            Real dp_dot_dp = dpdx*dpdx + dpdy*dpdy + dpdz*dpdz;
            Real phi_avg = 0.125 * (
                    phi_arr(i  , j  , k  ) + phi_arr(i  , j  , k+1) + phi_arr(i  , j+1, k  ) + phi_arr(i  , j+1, k+1)
                  + phi_arr(i+1, j  , k  ) + phi_arr(i+1, j  , k+1) + phi_arr(i+1, j+1, k  ) + phi_arr(i+1, j+1, k+1) );
            dist_arr(i, j, k) = -std::sqrt(dp_dot_dp) + std::sqrt(dp_dot_dp + 2*phi_avg);
 
            // DEBUG: output phi instead
            //dist_arr(i, j, k) = phi_arr(i, j, k);
        });
    }
}

post_timestep()

void ERF::post_timestep	(	int	nstep,
		amrex::Real	time,
		amrex::Real	dt_lev
	)

{
    BL_PROFILE("ERF::post_timestep()");
 
#ifdef ERF_USE_PARTICLES
    particleData.Redistribute();
#endif
 
    if (solverChoice.coupling_type == CouplingType::TwoWay)
    {
        int ncomp = vars_new[0][Vars::cons].nComp();
        for (int lev = finest_level-1; lev >= 0; lev--)
        {
            // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
            // m is the map scale factor at cell centers
            // Here we pre-divide (rho S) by m^2 before refluxing
            for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
                const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
                if (solverChoice.mesh_type == MeshType::ConstantDz) {
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
                    });
                } else {
                    const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
                    });
                }
            } // mfi
 
            // This call refluxes all "slow" cell-centered variables
            // (i.e. not density or (rho theta) or velocities) from the lev/lev+1 interface onto lev
            getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 2, 2, ncomp-2);
 
            // Here we multiply (rho S) by m^2 after refluxing
            for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
                const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
                if (solverChoice.mesh_type == MeshType::ConstantDz) {
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
                    });
                } else {
                    const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)) / detJ_arr(i,j,k);
                    });
                }
            } // mfi
 
            // We need to do this before anything else because refluxing changes the
            // values of coarse cells underneath fine grids with the assumption they'll
            // be over-written by averaging down
            int src_comp;
            if (solverChoice.anelastic[lev]) {
                src_comp = 1;
            } else {
                src_comp = 0;
            }
            int num_comp = ncomp - src_comp;
            AverageDownTo(lev,src_comp,num_comp);
        }
    }
 
    if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
        sum_integrated_quantities(time);
    }
 
    if (solverChoice.pert_type == PerturbationType::Source ||
        solverChoice.pert_type == PerturbationType::Direct) {
        if (is_it_time_for_action(nstep, time, dt_lev0, pert_interval, -1.)) {
            turbPert.debug(time);
        }
    }
 
    if (profile_int > 0 && (nstep+1) % profile_int == 0) {
        if (destag_profiles) {
            // all variables cell-centered
            write_1D_profiles(time);
        } else {
            // some variables staggered
            write_1D_profiles_stag(time);
        }
    }
 
    if (output_1d_column) {
#ifdef ERF_USE_NETCDF
      if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
      {
         int lev_column = 0;
         for (int lev = finest_level; lev >= 0; lev--)
         {
            Real dx_lev = geom[lev].CellSize(0);
            Real dy_lev = geom[lev].CellSize(1);
            int i_lev = static_cast<int>(std::floor(column_loc_x / dx_lev));
            int j_lev = static_cast<int>(std::floor(column_loc_y / dy_lev));
            if (grids[lev].contains(IntVect(i_lev,j_lev,0))) lev_column = lev;
         }
         writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
      }
#else
      Abort("To output 1D column files ERF must be compiled with NetCDF");
#endif
    }
 
    if (output_bndry_planes)
    {
      if (is_it_time_for_action(istep[0], time, dt_lev0, bndry_output_planes_interval, bndry_output_planes_per) &&
          time >= bndry_output_planes_start_time)
      {
         bool is_moist = (micro->Get_Qstate_Size() > 0);
         m_w2d->write_planes(istep[0], time, vars_new, is_moist);
      }
    }
 
    // Write plane/line sampler data
    if (is_it_time_for_action(nstep, time, dt_lev0, sampler_interval, sampler_per) && (data_sampler) ) {
        data_sampler->get_sample_data(geom, vars_new);
        data_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
    }
 
    // Moving terrain
    if ( solverChoice.terrain_type == TerrainType::Moving )
    {
      for (int lev = finest_level; lev >= 0; lev--)
      {
        // Copy z_phs_nd and detJ_cc at end of timestep
        MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
        MultiFab::Copy(  *detJ_cc[lev],   *detJ_cc_new[lev], 0, 0, 1,   detJ_cc[lev]->nGrowVect());
        MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
 
        make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
      }
    }
} // post_timestep

Here is the call graph for this function:

post_update()

void ERF::post_update ( amrex::MultiFab &  state_mf, amrex::Real  time, const amrex::Geometry &  geom  )

private

print_banner()

void ERF::print_banner ( MPI_Comm  comm, std::ostream &  out  )

static

                     : " << msg << std::endl;
}
 
void ERF::print_banner (MPI_Comm comm, std::ostream& out)
{
#ifdef AMREX_USE_MPI
    int irank = 0;
    int num_ranks = 1;
    MPI_Comm_size(comm, &num_ranks);
    MPI_Comm_rank(comm, &irank);
 
    // Only root process does the printing
    if (irank != 0) return;
#else
    amrex::ignore_unused(comm);
#endif
 
    auto etime = std::chrono::system_clock::now();
    auto etimet = std::chrono::system_clock::to_time_t(etime);
#ifndef _WIN32
    char time_buf[64];
    ctime_r(&etimet, time_buf);
    const std::string tstamp(time_buf);
#else
    char* time_buf = new char[64];
    ctime_s(time_buf, 64, &etimet);
    const std::string tstamp(time_buf);
#endif
 
    const char* githash1 = amrex::buildInfoGetGitHash(1);
    const char* githash2 = amrex::buildInfoGetGitHash(2);
 
    // clang-format off
    out << dbl_line
        << "                ERF (https://github.com/erf-model/ERF)"
        << std::endl << std::endl
        << "  ERF Git SHA      :: " << githash1 << std::endl
        << "  AMReX Git SHA    :: " << githash2 << std::endl
        << "  AMReX version    :: " << amrex::Version() << std::endl << std::endl
        << "  Exec. time       :: " << tstamp
        << "  Build time       :: " << amrex::buildInfoGetBuildDate() << std::endl
        << "  C++ compiler     :: " << amrex::buildInfoGetComp()
        << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
        << "  MPI              :: "
#ifdef AMREX_USE_MPI
        << "ON    (Num. ranks = " << num_ranks << ")" << std::endl
#else
        << "OFF " << std::endl
#endif
        << "  GPU              :: "
#ifdef AMREX_USE_GPU
        << "ON    "
#if defined(AMREX_USE_CUDA)
        << "(Backend: CUDA)"
#elif defined(AMREX_USE_HIP)
        << "(Backend: HIP)"
#elif defined(AMREX_USE_SYCL)
        << "(Backend: SYCL)"
#endif
        << std::endl
#else
        << "OFF" << std::endl
#endif
        << "  OpenMP           :: "
#ifdef AMREX_USE_OMP
        << "ON    (Num. threads = " << omp_get_max_threads() << ")" << std::endl
#else
        << "OFF" << std::endl
#endif
        << std::endl;
 

Referenced by main().

Here is the caller graph for this function:

print_error()

void ERF::print_error ( MPI_Comm  comm, const std::string &  msg  )

static

        :
    input_file   : Input file with simulation settings
 
Optional:
    param=value  : Overrides for parameters during runtime
)doc" << std::endl;
}
 
void ERF::print_error (MPI_Comm comm, const std::string& msg)
{
#ifdef AMREX_USE_MPI
    int irank = 0;
    int num_ranks = 1;
    MPI_Comm_size(comm, &num_ranks);
    MPI_Comm_rank(comm, &irank);
 

Referenced by main().

Here is the caller graph for this function:

print_summary()

static void ERF::print_summary ( std::ostream &  )

static

print_tpls()

void ERF::print_tpls ( std::ostream &  out )

static

                      ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
        << dash_line << std::endl;
    // clang-format on
}
 
void ERF::print_tpls (std::ostream& out)
{
    amrex::Vector<std::string> tpls;
 
#ifdef ERF_USE_NETCDF
    tpls.push_back(std::string("NetCDF    ") + NC_VERSION);
#endif
#ifdef AMREX_USE_SUNDIALS
    tpls.push_back(std::string("SUNDIALS     ") + SUNDIALS_VERSION);
#endif
 
    if (!tpls.empty()) {
        out << "  Enabled third-party libraries: ";
        for (const auto& val : tpls) {

print_usage()

void ERF::print_usage ( MPI_Comm  comm, std::ostream &  out  )

static

{
#ifdef AMREX_USE_MPI
    int irank = 0;
    int num_ranks = 1;
    MPI_Comm_size(comm, &num_ranks);
    MPI_Comm_rank(comm, &irank);
 
    // Only root process does the printing
    if (irank != 0) return;
#else
    amrex::ignore_unused(comm);
#endif
 
    out << R"doc(Usage:
    ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

project_velocities()

void ERF::project_velocities	(	int	lev,
		amrex::Real	dt,
		amrex::Vector< amrex::MultiFab > &	vars,
		amrex::MultiFab &	p
	)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level 0.

{
    BL_PROFILE("ERF::project_velocities()");
 
#ifndef ERF_USE_EB
    auto const dom_lo = lbound(geom[lev].Domain());
    auto const dom_hi = ubound(geom[lev].Domain());
#endif
 
    // Make sure the solver only sees the levels over which we are solving
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(mom_mf[Vars::cons].boxArray());
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap());
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
 
    MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
 
    Vector<MultiFab> rhs;
    Vector<MultiFab> phi;
 
#ifdef ERF_USE_EB
    rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0, MFInfo(), Factory(lev));
    phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1, MFInfo(), Factory(lev));
#else
    rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
    phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
#endif
 
    auto dxInv = geom[lev].InvCellSizeArray();
 
    //
    // ****************************************************************************
    // Now convert the rho0w MultiFab to hold Omega rather than rhow
    // ****************************************************************************
    //
#ifndef ERF_USE_EB
    if (solverChoice.mesh_type == MeshType::VariableDz)
    {
        for ( MFIter mfi(rhs[0],TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Array4<Real const>& rho0u_arr = mom_mf[IntVars::xmom].const_array(mfi);
            const Array4<Real const>& rho0v_arr = mom_mf[IntVars::ymom].const_array(mfi);
            const Array4<Real      >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
 
            const Array4<Real const>&     z_nd = z_phys_nd[lev]->const_array(mfi);
            //
            // Define Omega from (rho0 W) but store it in the same array
            //
            Box tbz = mfi.nodaltilebox(2);
            ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                if (k > dom_lo.z && k <= dom_hi.z) {
                    Real rho0w = rho0w_arr(i,j,k);
                    rho0w_arr(i,j,k) = OmegaFromW(i,j,k,rho0w,rho0u_arr,rho0v_arr,z_nd,dxInv);
                } else {
                    rho0w_arr(i,j,k) = Real(0.0);
                }
            });
        } // mfi
    }
#endif
 
    // ****************************************************************************
    // Compute divergence which will form RHS
    // Note that we replace "rho0w" with the contravariant momentum, Omega
    // ****************************************************************************
    Array<MultiFab const*, AMREX_SPACEDIM> rho0_u_const;
    rho0_u_const[0] = &mom_mf[IntVars::xmom];
    rho0_u_const[1] = &mom_mf[IntVars::ymom];
    rho0_u_const[2] = &mom_mf[IntVars::zmom];
 
    compute_divergence(lev, rhs[0], rho0_u_const, geom_tmp[0]);
 
    Real rhsnorm = rhs[0].norm0();
 
    if (mg_verbose > 0) {
        Print() << "Max/L2 norm of divergence before solve at level " << lev << " : " << rhsnorm << " " << rhs[0].norm2() << std::endl;
    }
 
    // ****************************************************************************
    //
    // No need to build the solver if RHS == 0
    //
    if (rhsnorm <= solverChoice.poisson_abstol) return;
    // ****************************************************************************
 
    // ****************************************************************************
    // Initialize phi to 0
    // (It is essential that we do this in order to fill the corners; these are never
    //  used but the Saxpy requires the values to be initialized.)
    // ****************************************************************************
    phi[0].setVal(0.0);
 
    Real start_step = static_cast<Real>(ParallelDescriptor::second());
 
    // ****************************************************************************
    // Allocate fluxes
    // ****************************************************************************
    Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
    fluxes.resize(1);
    for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
#ifdef ERF_USE_EB
        fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0, MFInfo(), Factory(lev));
#else
        fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
#endif
    }
 
    // ****************************************************************************
    // Choose the solver and solve
    // ****************************************************************************
 
    // ****************************************************************************
    // EB
    // ****************************************************************************
#ifdef ERF_USE_EB
    solve_with_EB_mlmg(lev, rhs, phi, fluxes);
#else
 
#ifdef ERF_USE_FFT
        Box my_region(ba_tmp[0].minimalBox());
        bool boxes_make_rectangle = (my_region.numPts() == ba_tmp[0].numPts());
#endif
 
    // ****************************************************************************
    // No terrain or grid stretching
    // ****************************************************************************
    if (solverChoice.mesh_type == MeshType::ConstantDz) {
#ifdef ERF_USE_FFT
        if (use_fft) {
            if (boxes_make_rectangle) {
                solve_with_fft(lev, rhs[0], phi[0], fluxes[0]);
            } else {
                amrex::Warning("FFT won't work unless the boxArray covers the domain: defaulting to MLMG");
                solve_with_mlmg(lev, rhs, phi, fluxes);
            }
        } else {
            solve_with_mlmg(lev, rhs, phi, fluxes);
        }
#else
        if (use_fft) {
            amrex::Warning("You set use_fft=true but didn't build with USE_FFT = TRUE; defaulting to MLMG");
        }
        solve_with_mlmg(lev, rhs, phi, fluxes);
#endif
    } // No terrain or grid stretching
 
    // ****************************************************************************
    // Grid stretching (flat terrain)
    // ****************************************************************************
    else if (solverChoice.mesh_type == MeshType::StretchedDz) {
#ifndef ERF_USE_FFT
        amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT solver");
#else
        if (!boxes_make_rectangle) {
            amrex::Abort("FFT won't work unless the boxArray covers the domain");
        } else {
            if (!use_fft) {
                amrex::Warning("Using FFT even though you didn't set use_fft to true; it's the best choice");
            }
            solve_with_fft(lev, rhs[0], phi[0], fluxes[0]);
        }
#endif
    } // grid stretching
 
    // ****************************************************************************
    // General terrain
    // ****************************************************************************
    else if (solverChoice.mesh_type == MeshType::VariableDz) {
#ifdef ERF_USE_FFT
        if (use_fft)
        {
            amrex::Warning("FFT solver does not work for general terrain: switching to FFT-preconditioned GMRES");
        }
        if (!boxes_make_rectangle) {
            amrex::Abort("FFT preconditioner for GMRES won't work unless the boxArray covers the domain");
        } else {
            solve_with_gmres(lev, rhs, phi, fluxes);
        }
#else
        amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT preconditioner for GMRES");
#endif
    } // general terrain
 
#endif // not EB
 
    // ****************************************************************************
    // Print time in solve
    // ****************************************************************************
    Real end_step = static_cast<Real>(ParallelDescriptor::second());
    if (mg_verbose > 0) {
        amrex::Print() << "Time in solve " << end_step - start_step << std::endl;
    }
 
    // ****************************************************************************
    // Subtract dt grad(phi) from the momenta (rho0u, rho0v, Omega)
    // ****************************************************************************
    MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0);
    MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0);
    MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0);
 
    //
    // This call is only to verify the divergence after the solve
    // It is important we do this before computing the rho0w_arr from Omega back to rho0w
    //
    // ****************************************************************************
    // THIS IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE
    // ****************************************************************************
    //
    if (mg_verbose > 0)
    {
        compute_divergence(lev, rhs[0], rho0_u_const, geom_tmp[0]);
 
        Print() << "Max/L2 norm of divergence  after solve at level " << lev << " : " << rhs[0].norm0() << " " << rhs[0].norm2() << std::endl;
 
#if 0
         // FOR DEBUGGING ONLY
         for ( MFIter mfi(rhs[0],TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
            const Array4<Real const>& rhs_arr = rhs[0].const_array(mfi);
            Box bx = mfi.validbox();
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                if (std::abs(rhs_arr(i,j,k)) > 1.e-10) {
                    amrex::Print() << "RHS AT " << IntVect(i,j,k) << " " << rhs_arr(i,j,k) << std::endl;
                }
            });
         } // mfi
#endif
 
    } // mg_verbose
 
 
    //
    // ****************************************************************************
    // Now convert the rho0w MultiFab back to holding (rho0w) rather than Omega
    // ****************************************************************************
    //
    if (solverChoice.mesh_type == MeshType::VariableDz)
    {
        for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
             Box tbz = mfi.nodaltilebox(2);
             const Array4<Real      >& rho0u_arr = mom_mf[IntVars::xmom].array(mfi);
             const Array4<Real      >& rho0v_arr = mom_mf[IntVars::ymom].array(mfi);
             const Array4<Real      >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
             const Array4<Real const>&      z_nd = z_phys_nd[lev]->const_array(mfi);
             ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                 Real omega = rho0w_arr(i,j,k);
                 rho0w_arr(i,j,k) = WFromOmega(i,j,k,omega,rho0u_arr,rho0v_arr,z_nd,dxInv);
             });
        } // mfi
    }
 
    // ****************************************************************************
    // Update pressure variable with phi -- note that phi is dt * change in pressure
    // ****************************************************************************
    MultiFab::Saxpy(pmf, 1.0/l_dt, phi[0],0,0,1,1);
    pmf.FillBoundary(geom[lev].periodicity());
}

Here is the call graph for this function:

project_velocities_tb()

void ERF::project_velocities_tb	(	int	lev,
		amrex::Real	dt,
		amrex::Vector< amrex::MultiFab > &	vars,
		amrex::MultiFab &	p
	)

Project the single-level velocity field to enforce incompressibility with a thin body

{
    BL_PROFILE("ERF::project_velocities_tb()");
    AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
 
    // Make sure the solver only sees the levels over which we are solving
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(vmf[Vars::cons].boxArray());
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(vmf[Vars::cons].DistributionMap());
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
 
    // Use the default settings
    LPInfo info;
    std::unique_ptr<MLPoisson> p_mlpoisson;
#if 0
    if (overset_imask[0]) {
        // Add overset mask around thin body
        p_mlpoisson = std::make_unique<MLPoisson>(geom, grids, dmap, GetVecOfConstPtrs(overset_imask), info);
    }
    else
#endif
    {
        // Use the default settings
        p_mlpoisson = std::make_unique<MLPoisson>(geom_tmp, ba_tmp, dm_tmp, info);
    }
 
    auto bclo = get_projection_bc(Orientation::low);
    auto bchi = get_projection_bc(Orientation::high);
    bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true;
    p_mlpoisson->setDomainBC(bclo, bchi);
 
    if (lev > 0) {
        p_mlpoisson->setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
    }
 
    p_mlpoisson->setLevelBC(0, nullptr);
 
    Vector<MultiFab> rhs;
    Vector<MultiFab> phi;
    Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
    Vector<Array<MultiFab,AMREX_SPACEDIM> > deltaf; // f^* - f^{n-1}
    Vector<Array<MultiFab,AMREX_SPACEDIM> > u_plus_dtdf; // u + dt*deltaf
 
    // Used to pass array of const MFs to ComputeDivergence
    Array<MultiFab const*, AMREX_SPACEDIM> u;
 
    rhs.resize(1);
    phi.resize(1);
    fluxes.resize(1);
    deltaf.resize(1);
    u_plus_dtdf.resize(1);
 
    rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
    phi[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
    rhs[0].setVal(0.0);
    phi[0].setVal(0.0);
 
    for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
             fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
        u_plus_dtdf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
 
             deltaf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
             deltaf[0][idim].setVal(0.0); // start with f^* == f^{n-1}
    }
 
#if 0
    // DEBUG
    u[0] = &(vmf[Vars::xvel]);
    u[1] = &(vmf[Vars::yvel]);
    u[2] = &(vmf[Vars::zvel]);
    computeDivergence(rhs[0], u, geom[0]);
    Print() << "Max norm of divergence before solve at level 0 : " << rhs[0].norm0() << std::endl;
#endif
 
    for (int itp = 0; itp < solverChoice.ncorr; ++itp)
    {
        // Calculate u + dt*deltaf
        for (int idim = 0; idim < 3; ++idim) {
            MultiFab::Copy(u_plus_dtdf[0][idim], deltaf[0][idim], 0, 0, 1, 0);
            u_plus_dtdf[0][0].mult(-l_dt,0,1,0);
        }
        MultiFab::Add(u_plus_dtdf[0][0], vmf[Vars::xvel], 0, 0, 1, 0);
        MultiFab::Add(u_plus_dtdf[0][1], vmf[Vars::yvel], 0, 0, 1, 0);
        MultiFab::Add(u_plus_dtdf[0][2], vmf[Vars::zvel], 0, 0, 1, 0);
 
        u[0] = &(u_plus_dtdf[0][0]);
        u[1] = &(u_plus_dtdf[0][1]);
        u[2] = &(u_plus_dtdf[0][2]);
        computeDivergence(rhs[0], u, geom_tmp[0]);
 
#if 0
        // DEBUG
        if (itp==0) {
            for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real const>& divU = rhs[0].const_array(mfi);
                const Array4<Real const>& uarr = vmf[Vars::xvel].const_array(mfi);
                const Array4<Real const>& varr = vmf[Vars::yvel].const_array(mfi);
                const Array4<Real const>& warr = vmf[Vars::zvel].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    if ((i>=120) && (i<=139) && (j==0) && ((k>=127)&&(k<=128))) {
                        amrex::AllPrint() << "before project div"<<IntVect(i,j,k)<<" = "<< divU(i,j,k)
                            << " u: " << uarr(i,j,k) << " " << uarr(i+1,j,k)
                            << " v: " << varr(i,j,k) << " " << varr(i,j+1,k)
                            << " w: " << warr(i,j,k) << " " << warr(i,j,k+1)
                            << std::endl;
                    }
                });
            }
        }
#endif
 
        // If all Neumann BCs, adjust RHS to make sure we can converge
        if (need_adjust_rhs) {
            Real offset = volWgtSumMF(lev, rhs[0], 0, *mapfac_m[lev], false, false);
            // amrex::Print() << "Poisson solvability offset = " << offset << std::endl;
            rhs[0].plus(-offset, 0, 1);
        }
 
        // Initialize phi to 0
        phi[0].setVal(0.0);
 
        MLMG mlmg(*p_mlpoisson);
        int max_iter = 100;
        mlmg.setMaxIter(max_iter);
 
        mlmg.setVerbose(mg_verbose);
        //mlmg.setBottomVerbose(mg_verbose);
 
        // solve for dt*p
        mlmg.solve(GetVecOfPtrs(phi),
                   GetVecOfConstPtrs(rhs),
                   solverChoice.poisson_reltol,
                   solverChoice.poisson_abstol);
 
        mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
 
        // Calculate new intermediate body force with updated gradp
        if (thin_xforce[lev]) {
            MultiFab::Copy(   deltaf[0][0], fluxes[0][0], 0, 0, 1, 0);
            ApplyInvertedMask(deltaf[0][0], *xflux_imask[0]);
        }
        if (thin_yforce[lev]) {
            MultiFab::Copy(   deltaf[0][1], fluxes[0][1], 0, 0, 1, 0);
            ApplyInvertedMask(deltaf[0][1], *yflux_imask[0]);
        }
        if (thin_zforce[lev]) {
            MultiFab::Copy(   deltaf[0][2], fluxes[0][2], 0, 0, 1, 0);
            ApplyInvertedMask(deltaf[0][2], *zflux_imask[0]);
        }
 
        // DEBUG
        //        for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
        //        {
        //            const Box& bx = mfi.tilebox();
        //            const Array4<Real const>& dfz_arr = deltaf[0][2].const_array(mfi);
        //            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        //            {
        //                if ((i>=120) && (i<=139) && (j==0) && (k==128)) {
        //                    amrex::AllPrint()
        //                        << " piter" << itp
        //                        << " dfz"<<IntVect(i,j,k)<<" = "<< dfz_arr(i,j,k)
        //                        << std::endl;
        //                }
        //            });
        //        }
 
        // Update pressure variable with phi -- note that phi is change in pressure, not the full pressure
        MultiFab::Saxpy(pmf, 1.0, phi[0],0,0,1,0);
 
        // Subtract grad(phi) from the velocity components
        Real beta = 1.0;
        MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
        MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
        MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
        if (thin_xforce[lev]) {
            ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
        }
        if (thin_yforce[lev]) {
            ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
        }
        if (thin_zforce[lev]) {
            ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
        }
    } // itp: pressure-force iterations
 
    // Subtract grad(phi) from the velocity components
//    Real beta = 1.0;
//    for (int ilev = lev_min; ilev <= lev_max; ++ilev) {
//        MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
//        MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
//        MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
//        if (thin_xforce[lev]) {
//            ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
//        }
//        if (thin_yforce[lev]) {
//            ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
//        }
//        if (thin_zforce[lev]) {
//            ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
//        }
//    }
 
#if 0
    // Confirm that the velocity is now divergence free
    u[0] = &(vmf[Vars::xvel]);
    u[1] = &(vmf[Vars::yvel]);
    u[2] = &(vmf[Vars::zvel]);
    computeDivergence(rhs[0], u, geom_tmp[0]);
    Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
 
#endif
}

Here is the call graph for this function:

projection_has_dirichlet()

bool ERF::projection_has_dirichlet

(

amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM >

bcs

)

const

{
    for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
        if (bcs[dir] == LinOpBCType::Dirichlet) return true;
    }
    return false;
}

ReadCheckpointFile()

void ERF::ReadCheckpointFile

(

)

ERF function for reading data from a checkpoint file during restart.

{
    Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
 
    // Header
    std::string File(restart_chkfile + "/Header");
 
    VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
 
    Vector<char> fileCharPtr;
    ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
    std::string fileCharPtrString(fileCharPtr.dataPtr());
    std::istringstream is(fileCharPtrString, std::istringstream::in);
 
    std::string line, word;
 
    int chk_ncomp_cons, chk_ncomp;
 
    // read in title line
    std::getline(is, line);
 
    // read in finest_level
    is >> finest_level;
    GotoNextLine(is);
 
    // read the number of components
    // for each variable we store
 
    // conservative, cell-centered vars
    is >> chk_ncomp_cons;
    GotoNextLine(is);
 
    // x-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // y-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // z-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // read in array of istep
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            istep[i++] = std::stoi(word);
        }
    }
 
    // read in array of dt
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            dt[i++] = std::stod(word);
        }
    }
 
    // read in array of t_new
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            t_new[i++] = std::stod(word);
        }
    }
 
    for (int lev = 0; lev <= finest_level; ++lev) {
        // read in level 'lev' BoxArray from Header
        BoxArray ba;
        ba.readFrom(is);
        GotoNextLine(is);
 
        // create a distribution mapping
        DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
 
        MakeNewLevelFromScratch (lev, t_new[lev], ba, dm);
    }
 
    // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
    // NOTE: Data is written over ncomp, so check that we match the header file
    int ncomp_cons = vars_new[0][Vars::cons].nComp();
 
    // NOTE: QKE was removed so this is for backward compatibility
    AMREX_ASSERT((chk_ncomp_cons==ncomp_cons) || ((chk_ncomp_cons-1)==ncomp_cons));
 
    // read in the MultiFab data
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        // NOTE: For backward compatibility (chk file has QKE)
        if ((chk_ncomp_cons-1)==ncomp_cons) {
            MultiFab cons(grids[lev],dmap[lev],chk_ncomp_cons,0);
            VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
 
            // Copy up to RhoKE_comp
            MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,(RhoKE_comp+1),0);
 
            // Only if we have a PBL model do we need to copy QKE is src to KE in dst
            if (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) {
                MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+1),RhoKE_comp,1,0);
                vars_new[lev][Vars::cons].mult(0.5,RhoKE_comp,1,0);
            }
 
            // Copy other components
            int ncomp_remainder = ncomp_cons - (RhoKE_comp + 1);
            MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+2),(RhoKE_comp+1),ncomp_remainder,0);
 
            vars_new[lev][Vars::cons].setBndry(1.0e34);
        } else {
            MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
            VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
            MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
            vars_new[lev][Vars::cons].setBndry(1.0e34);
        }
 
        MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
        VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
        MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
        vars_new[lev][Vars::xvel].setBndry(1.0e34);
 
        MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
        VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
        MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
        vars_new[lev][Vars::yvel].setBndry(1.0e34);
 
        MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
        VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
        MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
        vars_new[lev][Vars::zvel].setBndry(1.0e34);
 
        // Note that we read the ghost cells of the base state (unlike above)
 
        // The original base state only had 3 components and 1 ghost cell -- we read this
        // here to be consistent with the old style
        IntVect ng_base; int ncomp_base;
        bool read_old_base_state = true;
        if (read_old_base_state) {
               ng_base = IntVect{1};
            ncomp_base = 3;
        } else {
               ng_base = base_state[lev].nGrowVect();
            ncomp_base = base_state[lev].nComp();
        }
        MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
        VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
 
        MultiFab::Copy(base_state[lev],base,0,0,ncomp_base,ng_base);
 
        if (read_old_base_state) {
            for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // We only compute theta_0 on valid cells since we will impose domain BC's after restart
                const Box& bx = mfi.tilebox();
                Array4<Real> const& fab = base_state[lev].array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    fab(i,j,k,BaseState::th0_comp) = getRhoThetagivenP(fab(i,j,k,BaseState::p0_comp))
                                                     / fab(i,j,k,BaseState::r0_comp);
                });
            }
        }
        base_state[lev].FillBoundary(geom[lev].periodicity());
 
        if (SolverChoice::mesh_type != MeshType::ConstantDz)  {
           // Note that we also read the ghost cells of z_phys_nd
           IntVect ng = z_phys_nd[lev]->nGrowVect();
           MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
           VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
           MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
           update_terrain_arrays(lev);
        }
 
        // Read in the moisture model restart variables
        std::vector<int> qmoist_indices(0);
        std::vector<std::string> qmoist_names(0);
        micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
        int qmoist_nvar = qmoist_indices.size();
        for (int var = 0; var < qmoist_nvar; var++) {
            IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
            const int ncomp_moist = 1;
            MultiFab moist_vars(grids[lev],dmap[lev],ncomp_moist,ng_moist);
            VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", qmoist_names[var]));
            MultiFab::Copy(*(qmoist[lev][qmoist_indices[var]]),moist_vars,0,0,ncomp_moist,ng_moist);
        }
 
#if defined(ERF_USE_WINDFARM)
        if(solverChoice.windfarm_type == WindFarmType::Fitch or
           solverChoice.windfarm_type == WindFarmType::EWP or
           solverChoice.windfarm_type == WindFarmType::SimpleAD){
            IntVect ng = Nturb[lev].nGrowVect();
            MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
            VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
            MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
        }
#endif
 
        if (solverChoice.lsm_type != LandSurfaceType::None) {
            for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
                BoxArray ba = lsm_data[lev][mvar]->boxArray();
                DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
                IntVect ng = lsm_data[lev][mvar]->nGrowVect();
                int nvar = lsm_data[lev][mvar]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmVars"));
                MultiFab::Copy(*(lsm_data[lev][mvar]),lsm_vars,0,0,nvar,ng);
            }
        }
 
        // Note that we read the ghost cells of the mapfactors
        BoxList bl2d = grids[lev].boxList();
        for (auto& b : bl2d) {
            b.setRange(2,0);
        }
        BoxArray ba2d(std::move(bl2d));
 
        {
        IntVect ng = mapfac_m[lev]->nGrowVect();
        MultiFab mf_m(ba2d,dmap[lev],1,ng);
        VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
        MultiFab::Copy(*mapfac_m[lev],mf_m,0,0,1,ng);
 
        ng = mapfac_u[lev]->nGrowVect();
        MultiFab mf_u(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,ng);
        VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
        MultiFab::Copy(*mapfac_u[lev],mf_u,0,0,1,ng);
 
        ng = mapfac_v[lev]->nGrowVect();
        MultiFab mf_v(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,ng);
        VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
        MultiFab::Copy(*mapfac_v[lev],mf_v,0,0,1,ng);
        }
    }
 
#ifdef ERF_USE_PARTICLES
    restartTracers((ParGDBBase*)GetParGDB(),restart_chkfile);
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
        dynamic_cast<LagrangianMicrophysics&>(*micro).restartParticles((ParGDBBase*)GetParGDB(),restart_chkfile);
    }
#endif
 
#ifdef ERF_USE_NETCDF
    // Read bdy_data files
    if ((init_type==InitType::Real) || (init_type==InitType::Metgrid)) {
        int ioproc = ParallelDescriptor::IOProcessorNumber();  // I/O rank
        int num_time;
        int num_var;
        Vector<Box> bx_v;
        if (ParallelDescriptor::IOProcessor()) {
            // Open header file and read from it
            std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
            bdy_h_file >> num_time;
            bdy_h_file >> num_var;
            bdy_h_file >> start_bdy_time;
            bdy_h_file >> bdy_time_interval;
            bdy_h_file >> real_width;
            bx_v.resize(4*num_var);
            for (int ivar(0); ivar<num_var; ++ivar) {
                bdy_h_file >> bx_v[4*ivar  ];
                bdy_h_file >> bx_v[4*ivar+1];
                bdy_h_file >> bx_v[4*ivar+2];
                bdy_h_file >> bx_v[4*ivar+3];
            }
 
            // IO size the FABs
            bdy_data_xlo.resize(num_time);
            bdy_data_xhi.resize(num_time);
            bdy_data_ylo.resize(num_time);
            bdy_data_yhi.resize(num_time);
            for (int itime(0); itime<num_time; ++itime) {
                bdy_data_xlo[itime].resize(num_var);
                bdy_data_xhi[itime].resize(num_var);
                bdy_data_ylo[itime].resize(num_var);
                bdy_data_yhi[itime].resize(num_var);
                for (int ivar(0); ivar<num_var; ++ivar) {
                    bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
                    bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
                    bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
                    bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
                }
            }
 
            // Open data file and read from it
            std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
            for (int itime(0); itime<num_time; ++itime) {
                for (int ivar(0); ivar<num_var; ++ivar) {
                    bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
                }
            }
        } // IO
 
        // Broadcast the data
        ParallelDescriptor::Barrier();
        ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
        ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
        ParallelDescriptor::Bcast(&real_width,1,ioproc);
        ParallelDescriptor::Bcast(&num_time,1,ioproc);
        ParallelDescriptor::Bcast(&num_var,1,ioproc);
 
        // Everyone size their boxes
        bx_v.resize(4*num_var);
 
        ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
 
        // Everyone but IO size their FABs
        if (!ParallelDescriptor::IOProcessor()) {
          bdy_data_xlo.resize(num_time);
          bdy_data_xhi.resize(num_time);
          bdy_data_ylo.resize(num_time);
          bdy_data_yhi.resize(num_time);
          for (int itime(0); itime<num_time; ++itime) {
            bdy_data_xlo[itime].resize(num_var);
            bdy_data_xhi[itime].resize(num_var);
            bdy_data_ylo[itime].resize(num_var);
            bdy_data_yhi[itime].resize(num_var);
            for (int ivar(0); ivar<num_var; ++ivar) {
              bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
              bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
              bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
              bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
            }
          }
        }
 
        for (int itime(0); itime<num_time; ++itime) {
            for (int ivar(0); ivar<num_var; ++ivar) {
                ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
            }
        }
    } // init real
#endif
}

Here is the call graph for this function:

ReadCheckpointFileMOST()

void ERF::ReadCheckpointFileMOST

(

)

ERF function for reading additional data for MOST from a checkpoint file during restart.

This is called after the ABLMost object is instantiated.

{
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        // Note that we read the ghost cells
        BoxList bl2d = grids[lev].boxList();
        for (auto& b : bl2d) {
            b.setRange(2,0);
        }
        BoxArray ba2d(std::move(bl2d));
 
        if (m_most->have_variable_sea_roughness())  {
            amrex::Print() << "Reading variable surface roughness" << std::endl;
            IntVect ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab z0_in(ba2d,dmap[lev],1,ng);
            VisMF::Read(z0_in, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z0"));
            auto z0 = const_cast<FArrayBox*>(m_most->get_z0(lev));
            for (amrex::MFIter mfi(z0_in); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.growntilebox();
                z0->copy<RunOn::Host>(z0_in[mfi], bx);
            }
        }
    }
}

ReadParameters()

void ERF::ReadParameters ( )

private

{
    {
        ParmParse pp;  // Traditionally, max_step and stop_time do not have prefix.
        pp.query("max_step", max_step);
        pp.query("stop_time", stop_time);
 
        pp.query("start_time", start_time); // This is optional, it defaults to 0
    }
 
    ParmParse pp(pp_prefix);
    ParmParse pp_amr("amr");
    {
        // The type of the file we restart from
        pp.query("restart_type", restart_type);
 
        pp.query("regrid_int", regrid_int);
        pp.query("check_file", check_file);
        pp.query("check_type", check_type);
 
        // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
        //    for those or "erf.restart" / "erf.m_check_int" with the former taking
        //    precedence if both are specified
        pp.query("check_int", m_check_int);
        pp.query("check_per", m_check_per);
        pp_amr.query("check_int", m_check_int);
        pp_amr.query("check_per", m_check_per);
 
        pp.query("restart", restart_chkfile);
        pp_amr.query("restart", restart_chkfile);
 
        // Verbosity
        pp.query("v", verbose);
        pp.query("mg_v", mg_verbose);
        pp.query("use_fft", use_fft);
#ifndef ERF_USE_FFT
        if (use_fft) {
            amrex::Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
        }
#endif
 
        // Frequency of diagnostic output
        pp.query("sum_interval", sum_interval);
        pp.query("sum_period"  , sum_per);
 
        pp.query("pert_interval", pert_interval);
 
        // Time step controls
        pp.query("cfl", cfl);
        pp.query("substepping_cfl", sub_cfl);
        pp.query("init_shrink", init_shrink);
        pp.query("change_max", change_max);
 
        fixed_dt.resize(max_level+1,-1.);
        fixed_fast_dt.resize(max_level+1,-1.);
 
        pp.query("fixed_dt", fixed_dt[0]);
        pp.query("fixed_fast_dt", fixed_fast_dt[0]);
 
        for (int lev = 1; lev <= max_level; lev++)
        {
            fixed_dt[lev]      = fixed_dt[lev-1]     / static_cast<Real>(MaxRefRatio(lev-1));
            fixed_fast_dt[lev] = fixed_fast_dt[lev-1] / static_cast<Real>(MaxRefRatio(lev-1));
        }
 
        pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
 
        // How to initialize
        init_type = InitType::None;
        pp.query_enum_case_insensitive("init_type",init_type);
 
        // Should we use the bcs we've read in from wrfbdy or metgrid files?
        // We default to yes if we have them, but the user can override that option
        use_real_bcs = ( (init_type == InitType::Real) || (init_type == InitType::Metgrid) );
        pp.query("use_real_bcs",use_real_bcs);
 
        // We use this to keep track of how many boxes we read in from WRF initialization
        num_files_at_level.resize(max_level+1,0);
 
        // We use this to keep track of how many boxes are specified thru the refinement indicators
        num_boxes_at_level.resize(max_level+1,0);
            boxes_at_level.resize(max_level+1);
 
        // We always have exactly one file at level 0
        num_boxes_at_level[0] = 1;
        boxes_at_level[0].resize(1);
        boxes_at_level[0][0] = geom[0].Domain();
 
#ifdef ERF_USE_NETCDF
        nc_init_file.resize(max_level+1);
 
        // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
        //        but we always have exactly one file at level 0
        for (int lev = 0; lev <= max_level; lev++)
        {
            const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
            if (pp.contains(nc_file_names.c_str()))
            {
                int num_files = pp.countval(nc_file_names.c_str());
                num_files_at_level[lev] = num_files;
                nc_init_file[lev].resize(num_files);
                pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
                for (int j = 0; j < num_files; j++)
                    Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
            }
        }
 
        // NetCDF wrfbdy lateral boundary file
        pp.query("nc_bdy_file", nc_bdy_file);
#endif
 
        // Flag to trigger initialization from input_sounding like WRF's ideal.exe
        pp.query("init_sounding_ideal", init_sounding_ideal);
 
        // Options for vertical interpolation of met_em*.nc data.
        pp.query("metgrid_debug_quiescent",  metgrid_debug_quiescent);
        pp.query("metgrid_debug_isothermal", metgrid_debug_isothermal);
        pp.query("metgrid_debug_dry",        metgrid_debug_dry);
        pp.query("metgrid_debug_psfc",       metgrid_debug_psfc);
        pp.query("metgrid_debug_msf",        metgrid_debug_msf);
        pp.query("metgrid_interp_theta",     metgrid_interp_theta);
        pp.query("metgrid_basic_linear",     metgrid_basic_linear);
        pp.query("metgrid_use_below_sfc",    metgrid_use_below_sfc);
        pp.query("metgrid_use_sfc",          metgrid_use_sfc);
        pp.query("metgrid_retain_sfc",       metgrid_retain_sfc);
        pp.query("metgrid_proximity",        metgrid_proximity);
        pp.query("metgrid_order",            metgrid_order);
        pp.query("metgrid_force_sfc_k",      metgrid_force_sfc_k);
 
        // Set default to FullState for now ... later we will try Perturbation
        interpolation_type = StateInterpType::FullState;
        pp.query_enum_case_insensitive("interpolation_type"  ,interpolation_type);
 
        PlotFileType plotfile_type_temp = PlotFileType::None;
        pp.query_enum_case_insensitive("plotfile_type"  ,plotfile_type_temp);
        pp.query_enum_case_insensitive("plotfile_type_1",plotfile_type_1);
        pp.query_enum_case_insensitive("plotfile_type_2",plotfile_type_2);
        //
        // This option is for backward consistency -- if only plotfile_type is set,
        //     then it will be used for both 1 and 2 if and only if they are not set
        //
        // Default is native amrex if no type is specified
        //
        if (plotfile_type_temp == PlotFileType::None) {
            if (plotfile_type_1 == PlotFileType::None) {
                plotfile_type_1  = PlotFileType::Amrex;
            }
            if (plotfile_type_2 == PlotFileType::None) {
                plotfile_type_2  = PlotFileType::Amrex;
            }
        } else {
            if (plotfile_type_1 == PlotFileType::None) {
                plotfile_type_1  = plotfile_type_temp;
            } else {
                amrex::Abort("You must set either plotfile_type or plotfile_type_1, not both");
            }
            if (plotfile_type_2 == PlotFileType::None) {
                plotfile_type_2  = plotfile_type_temp;
            } else {
                amrex::Abort("You must set either plotfile_type or plotfile_type_2, not both");
            }
        }
#ifndef ERF_USE_NETCDF
        if (plotfile_type_1 == PlotFileType::Netcdf ||
            plotfile_type_2 == PlotFileType::Netcdf) {
            amrex::Abort("Plotfile type = Netcdf is not allowed without USE_NETCDF = TRUE");
        }
#endif
 
        pp.query("plot_file_1",   plot_file_1);
        pp.query("plot_file_2",   plot_file_2);
        pp.query("plot_int_1" , m_plot_int_1);
        pp.query("plot_int_2" , m_plot_int_2);
        pp.query("plot_per_1",  m_plot_per_1);
        pp.query("plot_per_2",  m_plot_per_2);
 
        pp.query("expand_plotvars_to_unif_rr",m_expand_plotvars_to_unif_rr);
 
        pp.query("plot_face_vels",m_plot_face_vels);
 
        if ( (m_plot_int_1 > 0 && m_plot_per_1 > 0) ||
             (m_plot_int_2 > 0 && m_plot_per_2 > 0.) ) {
            Abort("Must choose only one of plot_int or plot_per");
        }
 
        pp.query("profile_int", profile_int);
        pp.query("destag_profiles", destag_profiles);
 
        pp.query("plot_lsm", plot_lsm);
#ifdef ERF_USE_RRTMGP
        pp.query("plot_rad", plot_rad);
#endif
 
        pp.query("output_1d_column", output_1d_column);
        pp.query("column_per", column_per);
        pp.query("column_interval", column_interval);
        pp.query("column_loc_x", column_loc_x);
        pp.query("column_loc_y", column_loc_y);
        pp.query("column_file_name", column_file_name);
 
        // Sampler output frequency
        pp.query("sampler_per", sampler_per);
        pp.query("sampler_interval", sampler_interval);
 
        // Specify information about outputting planes of data
        pp.query("output_bndry_planes", output_bndry_planes);
        pp.query("bndry_output_planes_interval", bndry_output_planes_interval);
        pp.query("bndry_output_planes_per", bndry_output_planes_per);
        pp.query("bndry_output_start_time", bndry_output_planes_start_time);
 
        // Specify whether ingest boundary planes of data
        pp.query("input_bndry_planes", input_bndry_planes);
 
        // Query the set and total widths for wrfbdy interior ghost cells
        pp.query("real_width", real_width);
        pp.query("real_set_width", real_set_width);
 
        // Query the set and total widths for crse-fine interior ghost cells
        pp.query("cf_width", cf_width);
        pp.query("cf_set_width", cf_set_width);
 
        // Query the canopy model file name
        std::string forestfile;
        solverChoice.do_forest_drag = pp.query("forest_file", forestfile);
        if (solverChoice.do_forest_drag) {
            for (int lev = 0; lev <= max_level; ++lev) {
                m_forest_drag[lev] = std::make_unique<ForestDrag>(forestfile);
            }
        }
 
        //Query the terrain file name
        std::string terrainfile;
        solverChoice.do_terrain_drag = pp.query("terrain_file", terrainfile);
        if (solverChoice.do_terrain_drag) {
            for (int lev = 0; lev <= max_level; ++lev) {
                m_terrain_drag[lev] = std::make_unique<TerrainDrag>(terrainfile);
            }
        }
 
        // AmrMesh iterate on grids?
        bool iterate(true);
        pp_amr.query("iterate_grids",iterate);
        if (!iterate) SetIterateToFalse();
    }
 
#ifdef ERF_USE_PARTICLES
    readTracersParams();
#endif
 
#ifdef ERF_USE_MULTIBLOCK
    solverChoice.pp_prefix = pp_prefix;
#endif
 
    solverChoice.init_params(max_level);
 
    // No moving terrain with init real (we must do this after init_params
    //    because that is where we set terrain_type
    if (init_type == InitType::Real && solverChoice.terrain_type == TerrainType::Moving) {
        Abort("Moving terrain is not supported with init real");
    }
 
    // What type of land surface model to use
    // NOTE: Must be checked after init_params
    if (solverChoice.lsm_type == LandSurfaceType::SLM) {
        lsm.SetModel<SLM>();
        Print() << "SLM land surface model!\n";
    } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
        lsm.SetModel<MM5>();
        Print() << "MM5 land surface model!\n";
#ifdef ERF_USE_NOAH
    } else if (solverChoice.lsm_type == LandSurfaceType::NOAH) {
        lsm.SetModel<NOAH>();
        Print() << "NOAH land surface model!\n";
#endif
    } else if (solverChoice.lsm_type == LandSurfaceType::None) {
        lsm.SetModel<NullSurf>();
        Print() << "Null land surface model!\n";
    } else {
        Abort("Dont know this LandSurfaceType!") ;
    }
 
    if (verbose > 0) {
        solverChoice.display(max_level);
    }
 
    ParameterSanityChecks();
}

Here is the call graph for this function:

refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )

private

Function to define the refinement criteria based on user input

{
    if (max_level > 0)
    {
        ParmParse pp(pp_prefix);
        Vector<std::string> refinement_indicators;
        pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
 
        for (int i=0; i<refinement_indicators.size(); ++i)
        {
            std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
 
            ParmParse ppr(ref_prefix);
            RealBox realbox;
            int lev_for_box;
 
            int num_real_lo = ppr.countval("in_box_lo");
            int num_indx_lo = ppr.countval("in_box_lo_indices");
 
            if ( !((num_real_lo == AMREX_SPACEDIM && num_indx_lo == 0) ||
                   (num_indx_lo == AMREX_SPACEDIM && num_real_lo == 0) ||
                   (num_indx_lo ==              0 && num_real_lo == 0)) )
            {
                amrex::Abort("Must only specify box for refinement using real OR index space");
            }
 
            if (num_real_lo > 0) {
                std::vector<Real> rbox_lo(3), rbox_hi(3);
                ppr.get("max_level",lev_for_box);
                if (lev_for_box <= max_level)
                {
                    if (n_error_buf[0] != IntVect::TheZeroVector()) {
                        amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
                    }
 
                    const Real* plo = geom[lev_for_box].ProbLo();
                    const Real* phi = geom[lev_for_box].ProbHi();
 
                    ppr.getarr("in_box_lo",rbox_lo,0,AMREX_SPACEDIM);
                    ppr.getarr("in_box_hi",rbox_hi,0,AMREX_SPACEDIM);
 
                    if (rbox_lo[0] < plo[0]) rbox_lo[0] = plo[0];
                    if (rbox_lo[1] < plo[1]) rbox_lo[1] = plo[1];
                    if (rbox_hi[0] > phi[0]) rbox_hi[0] = phi[0];
                    if (rbox_hi[1] > phi[1]) rbox_hi[1] = phi[1];
 
                    realbox = RealBox(&(rbox_lo[0]),&(rbox_hi[0]));
 
                    Print() << "Realbox read in and intersected laterally with domain is " << realbox << std::endl;
 
                    num_boxes_at_level[lev_for_box] += 1;
 
                    int ilo, jlo, klo;
                    int ihi, jhi, khi;
                    const auto* dx  = geom[lev_for_box].CellSize();
                    ilo = static_cast<int>((rbox_lo[0] - plo[0])/dx[0]);
                    jlo = static_cast<int>((rbox_lo[1] - plo[1])/dx[1]);
                    ihi = static_cast<int>((rbox_hi[0] - plo[0])/dx[0]-1);
                    jhi = static_cast<int>((rbox_hi[1] - plo[1])/dx[1]-1);
                    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                        // Search for k indices corresponding to nominal grid
                        // AGL heights
                        const Box& domain = geom[lev_for_box].Domain();
                        klo = domain.smallEnd(2) - 1;
                        khi = domain.smallEnd(2) - 1;
 
                        if (rbox_lo[2] < zlevels_stag[lev_for_box][domain.smallEnd(2)])
                        {
                            klo = domain.smallEnd(2);
                        }
                        else
                        {
                            for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
                                if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
                                    klo = k-1;
                                    break;
                                }
                            }
                        }
                        AMREX_ASSERT(klo >= domain.smallEnd(2));
 
                        if (rbox_hi[2] > zlevels_stag[lev_for_box][domain.bigEnd(2)+1])
                        {
                            khi = domain.bigEnd(2);
                        }
                        else
                        {
                            for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
                                if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
                                    khi = k-1;
                                    break;
                                }
                            }
                        }
                        AMREX_ASSERT((khi <= domain.bigEnd(2)) && (khi > klo));
 
                        // Need to update realbox because tagging is based on
                        // the initial _un_deformed grid
                        realbox = RealBox(plo[0]+ ilo   *dx[0], plo[1]+ jlo   *dx[1], plo[2]+ klo   *dx[2],
                                          plo[0]+(ihi+1)*dx[0], plo[1]+(jhi+1)*dx[1], plo[2]+(khi+1)*dx[2]);
                    } else {
                        klo = static_cast<int>((rbox_lo[2] - plo[2])/dx[2]);
                        khi = static_cast<int>((rbox_hi[2] - plo[2])/dx[2]-1);
                    }
 
                    Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
                    if ( (ilo%ref_ratio[lev_for_box-1][0] != 0) || ((ihi+1)%ref_ratio[lev_for_box-1][0] != 0) ||
                         (jlo%ref_ratio[lev_for_box-1][1] != 0) || ((jhi+1)%ref_ratio[lev_for_box-1][1] != 0) ||
                         (klo%ref_ratio[lev_for_box-1][2] != 0) || ((khi+1)%ref_ratio[lev_for_box-1][2] != 0) )
                    {
                        amrex::Print() << "Box : " << bx << std::endl;
                        amrex::Print() << "RealBox : " << realbox << std::endl;
                        amrex::Print() << "ilo, ihi+1, jlo, jhi+1, klo, khi+1 by ref_ratio : "
                                       << ilo%ref_ratio[lev_for_box-1][0] << " " << (ihi+1)%ref_ratio[lev_for_box-1][0] << " "
                                       << jlo%ref_ratio[lev_for_box-1][1] << " " << (jhi+1)%ref_ratio[lev_for_box-1][1] << " "
                                       << klo%ref_ratio[lev_for_box-1][2] << " " << (khi+1)%ref_ratio[lev_for_box-1][2] << std::endl;
                        amrex::Error("Fine box is not legit with this ref_ratio");
                    }
                    boxes_at_level[lev_for_box].push_back(bx);
                    Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                } // lev
                if (init_type == InitType::Real || init_type == InitType::Metgrid) {
                    if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                        amrex::Error("Number of boxes doesn't match number of input files");
 
                    }
                }
 
            } else if (num_indx_lo > 0) {
 
                std::vector<int> box_lo(3), box_hi(3);
                ppr.get("max_level",lev_for_box);
                if (lev_for_box <= max_level)
                {
                    if (n_error_buf[0] != IntVect::TheZeroVector()) {
                        amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
                    }
 
                    ppr.getarr("in_box_lo_indices",box_lo,0,AMREX_SPACEDIM);
                    ppr.getarr("in_box_hi_indices",box_hi,0,AMREX_SPACEDIM);
 
                    Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
                    amrex::Print() << "BOX " << bx << std::endl;
 
                    const auto* dx  = geom[lev_for_box].CellSize();
                    const Real* plo = geom[lev_for_box].ProbLo();
                    realbox = RealBox(plo[0]+ box_lo[0]   *dx[0], plo[1]+ box_lo[1]   *dx[1], plo[2]+ box_lo[2]   *dx[2],
                                      plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
 
                    Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
                    num_boxes_at_level[lev_for_box] += 1;
 
                    if ( (box_lo[0]%ref_ratio[lev_for_box-1][0] != 0) || ((box_hi[0]+1)%ref_ratio[lev_for_box-1][0] != 0) ||
                         (box_lo[1]%ref_ratio[lev_for_box-1][1] != 0) || ((box_hi[1]+1)%ref_ratio[lev_for_box-1][1] != 0) ||
                         (box_lo[2]%ref_ratio[lev_for_box-1][2] != 0) || ((box_hi[2]+1)%ref_ratio[lev_for_box-1][2] != 0) )
                         amrex::Error("Fine box is not legit with this ref_ratio");
                    boxes_at_level[lev_for_box].push_back(bx);
                    Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                } // lev
                if (init_type == InitType::Real || init_type == InitType::Metgrid) {
                    if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                        amrex::Error("Number of boxes doesn't match number of input files");
 
                    }
                }
            }
 
            AMRErrorTagInfo info;
 
            if (realbox.ok()) {
                info.SetRealBox(realbox);
            }
            if (ppr.countval("start_time") > 0) {
                Real ref_min_time; ppr.get("start_time",ref_min_time);
                info.SetMinTime(ref_min_time);
            }
            if (ppr.countval("end_time") > 0) {
                Real ref_max_time; ppr.get("end_time",ref_max_time);
                info.SetMaxTime(ref_max_time);
            }
            if (ppr.countval("max_level") > 0) {
                int ref_max_level; ppr.get("max_level",ref_max_level);
                info.SetMaxLevel(ref_max_level);
            }
 
            if (ppr.countval("value_greater")) {
                int num_val = ppr.countval("value_greater");
                Vector<Real> value(num_val);
                ppr.getarr("value_greater",value,0,num_val);
                std::string field; ppr.get("field_name",field);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
            }
            else if (ppr.countval("value_less")) {
                int num_val = ppr.countval("value_less");
                Vector<Real> value(num_val);
                ppr.getarr("value_less",value,0,num_val);
                std::string field; ppr.get("field_name",field);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
            }
            else if (ppr.countval("adjacent_difference_greater")) {
                int num_val = ppr.countval("adjacent_difference_greater");
                Vector<Real> value(num_val);
                ppr.getarr("adjacent_difference_greater",value,0,num_val);
                std::string field; ppr.get("field_name",field);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
            }
            else if (realbox.ok())
            {
                ref_tags.push_back(AMRErrorTag(info));
            } else {
                Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
            }
        } // loop over criteria
    } // if max_level > 0
}

Here is the call graph for this function:

remake_zphys()

void ERF::remake_zphys ( int  lev, std::unique_ptr< amrex::MultiFab > &  temp_zphys_nd  )

private

{
    if (lev > 0 && SolverChoice::mesh_type == MeshType::VariableDz)
    {
        //
        // First interpolate from coarser level
        // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
        //       have been pre-filled - this includes ghost cells both inside and outside
        //       the domain
        //
        InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
                              IntVect(0,0,0), // do not fill ghost cells outside the domain
                              *z_phys_nd[lev-1], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &node_bilinear_interp,
                              domain_bcs_type, BCVars::cons_bc);
 
        // This recomputes the fine values using the bottom terrain at the fine resolution,
        //    and also fills values of z_phys_nd outside the domain
        init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type);
 
        std::swap(temp_zphys_nd, z_phys_nd[lev]);
 
    } // use_terrain && lev > 0
}

Here is the call graph for this function:

RemakeLevel()

void ERF::RemakeLevel ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

override

{
    if (verbose) {
        amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
    }
 
    AMREX_ALWAYS_ASSERT(lev > 0);
    AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::Moving);
 
    BoxArray            ba_old(vars_new[lev][Vars::cons].boxArray());
    DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
 
    if (verbose) {
        amrex::Print() <<"               OLD BA AT LEVEL " << lev << " " << ba_old << std::endl;
    }
 
    int     ncomp_cons  = vars_new[lev][Vars::cons].nComp();
    IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
 
    int ngrow_vels  = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_num_diff);
 
    Vector<MultiFab> temp_lev_new(Vars::NumTypes);
    Vector<MultiFab> temp_lev_old(Vars::NumTypes);
    MultiFab temp_base_state;
 
    std::unique_ptr<MultiFab> temp_zphys_nd;
 
    //********************************************************************************************
    // This allocates all kinds of things, including but not limited to: solution arrays,
    //      terrain arrays and metrics, and base state.
    // *******************************************************************************************
    init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old, temp_base_state, temp_zphys_nd);
 
    // ********************************************************************************************
    // Build the data structures for terrain-related quantities
    // ********************************************************************************************
    remake_zphys(lev, temp_zphys_nd);
    update_terrain_arrays(lev);
 
    //
    // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
    //
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
            average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
            average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
        }
    }
 
    // ********************************************************************************************
    // Build the data structures for canopy model (depends upon z_phys)
    // ********************************************************************************************
    if (solverChoice.do_forest_drag) { m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_nd[lev].get()); }
 
    if (solverChoice.do_terrain_drag) { m_terrain_drag[lev]->define_terrain_blank_field(ba, dm, geom[lev], z_phys_nd[lev].get()); }
    // *****************************************************************************************************
    // Create the physbcs objects (after initializing the terrain but before calling FillCoarsePatch
    // *****************************************************************************************************
    make_physbcs(lev);
 
    // ********************************************************************************************
    // Update the base state at this level by interpolation from coarser level AND copy
    //    from previous (pre-regrid) base_state array
    // ********************************************************************************************
    Interpolater* mapper = &cell_cons_interp;
 
    Vector<MultiFab*> fmf = {&base_state[lev  ], &base_state[lev  ]};
    Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
    Vector<Real> ftime    = {time, time};
    Vector<Real> ctime    = {time, time};
 
    // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
    FillPatchTwoLevels(temp_base_state, temp_base_state.nGrowVect(), IntVect(0,0,0),
                       time, cmf, ctime, fmf, ftime,
                       0, 0, temp_base_state.nComp(), geom[lev-1], geom[lev],
                       refRatio(lev-1), mapper, domain_bcs_type,
                       BCVars::base_bc);
 
    // Impose bc's outside the domain
    (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
 
    // *************************************************************************************************
    // This will fill the temporary MultiFabs with data from vars_new
    // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
    // NOTE: we must create the new base state before calling FillPatch because we will
    //       interpolate perturbational quantities
    // *************************************************************************************************
    FillPatch(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
                          &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
                         {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                         base_state[lev], temp_base_state, false);
 
    // Now swap the pointers since we needed both old and new in the FillPatch
    std::swap(temp_base_state, base_state[lev]);
 
    // ********************************************************************************************
    // Copy from new into old just in case
    // ********************************************************************************************
    MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
    MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0,    1,ngrow_vels);
    MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0,    1,ngrow_vels);
    MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0,    1,IntVect(ngrow_vels,ngrow_vels,0));
 
    // ********************************************************************************************
    // Now swap the pointers
    // ********************************************************************************************
    for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
        std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
        std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
    }
 
    t_new[lev] = time;
    t_old[lev] = time - 1.e200;
 
    // ********************************************************************************************
    // Build the data structures for calculating diffusive/turbulent terms
    // ********************************************************************************************
    update_diffusive_arrays(lev, ba, dm);
 
    //********************************************************************************************
    // Microphysics
    // *******************************************************************************************
    int q_size = micro->Get_Qmoist_Size(lev);
    qmoist[lev].resize(q_size);
    micro->Define(lev, solverChoice);
    if (solverChoice.moisture_type != MoistureType::None)
    {
        micro->Init(lev, vars_new[lev][Vars::cons],
                    grids[lev], Geom(lev), 0.0,
                    z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
    }
    for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
        qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
    }
 
    // ********************************************************************************************
    // Initialize the integrator class
    // ********************************************************************************************
    initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
 
    // We need to re-define the FillPatcher if the grids have changed
    if (lev > 0 && cf_width >= 0) {
        bool ba_changed = (ba != ba_old);
        bool dm_changed = (dm != dm_old);
        if (ba_changed || dm_changed) {
          Define_ERFFillPatchers(lev);
        }
    }
 
#ifdef ERF_USE_PARTICLES
    particleData.Redistribute();
#endif
}

restart()

void ERF::restart

(

)

{
    // TODO: This could be deleted since ba/dm are not created yet?
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        auto& lev_new = vars_new[lev];
        auto& lev_old = vars_old[lev];
        lev_new[Vars::cons].setVal(0.); lev_old[Vars::cons].setVal(0.);
        lev_new[Vars::xvel].setVal(0.); lev_old[Vars::xvel].setVal(0.);
        lev_new[Vars::yvel].setVal(0.); lev_old[Vars::yvel].setVal(0.);
        lev_new[Vars::zvel].setVal(0.); lev_old[Vars::zvel].setVal(0.);
    }
 
#ifdef ERF_USE_NETCDF
    if (restart_type == "netcdf") {
       ReadNCCheckpointFile();
    }
#endif
    if (restart_type == "native") {
       ReadCheckpointFile();
    }
 
    // We set this here so that we don't over-write the checkpoint file we just started from
    last_check_file_step = istep[0];
}

sample_lines()

void ERF::sample_lines	(	int	lev,
		amrex::Real	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters

lev	Current level
time	Current time
cell	IntVect containing the x,y-dimension indices to sample along z
mf	MultiFab from which we sample the data

{
    int datwidth = 14;
    int datprecision = 6;
 
    int ifile = 0;
 
    const int ncomp = mf.nComp(); // cell-centered state vars
 
    MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
    average_face_to_cellcenter(mf_vels, 0,
                               Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
 
    //
    // Sample the data at a line (in direction "dir") in space
    // In this case we sample in the vertical direction so dir = 2
    // The "k" value of "cell" is ignored
    //
    int dir = 2;
    MultiFab my_line       = get_line_data(mf,              dir, cell);
    MultiFab my_line_vels  = get_line_data(mf_vels,         dir, cell);
    MultiFab my_line_tau11 = get_line_data(*Tau11_lev[lev], dir, cell);
    MultiFab my_line_tau12 = get_line_data(*Tau12_lev[lev], dir, cell);
    MultiFab my_line_tau13 = get_line_data(*Tau13_lev[lev], dir, cell);
    MultiFab my_line_tau22 = get_line_data(*Tau22_lev[lev], dir, cell);
    MultiFab my_line_tau23 = get_line_data(*Tau23_lev[lev], dir, cell);
    MultiFab my_line_tau33 = get_line_data(*Tau33_lev[lev], dir, cell);
 
    for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
    {
        // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
 
        std::ostream& sample_log = SampleLineLog(ifile);
        if (sample_log.good()) {
          sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
          const auto& my_line_arr = my_line[0].const_array();
          const auto& my_line_vels_arr = my_line_vels[0].const_array();
          const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
          const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
          const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
          const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
          const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
          const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
          const Box&  my_box = my_line[0].box();
          const int klo = my_box.smallEnd(2);
          const int khi = my_box.bigEnd(2);
          int i = cell[0];
          int j = cell[1];
          for (int n = 0; n < ncomp; n++) {
              for (int k = klo; k <= khi; k++) {
                  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
              }
          }
          for (int n = 0; n < AMREX_SPACEDIM; n++) {
              for (int k = klo; k <= khi; k++) {
                  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
              }
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
          }
          sample_log << std::endl;
        } // if good
    } // mfi
}

sample_points()

void ERF::sample_points	(	int	lev,
		amrex::Real	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling MultiFab data at a specified cell index.

Parameters

lev	Level for the associated MultiFab data
time	Current time
cell	IntVect containing the indexes for the cell where we want to sample
mf	MultiFab from which we wish to sample data

{
    int datwidth = 14;
 
    int ifile = 0;
 
    //
    // Sample the data at a single point in space
    //
    int ncomp = mf.nComp();
    Vector<Real> my_point = get_cell_data(mf, cell);
 
    if (!my_point.empty()) {
 
        // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
 
        std::ostream& sample_log = SamplePointLog(ifile);
        if (sample_log.good()) {
          sample_log << std::setw(datwidth) << time;
          for (int i = 0; i < ncomp; ++i)
          {
              sample_log << std::setw(datwidth) << my_point[i];
          }
          sample_log << std::endl;
        } // if good
    } // only write from processor that holds the cell
}

SampleLine()

amrex::IntVect& ERF::SampleLine ( int  i )

inlineprivate

    {
        return sampleline[i];
    }

SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int  i )

inlineprivate

    {
        return *samplelinelog[i];
    }

SampleLineLogName()

std::string ERF::SampleLineLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith samplelinelog file.

{ return samplelinelogname[i]; }

SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int  i )

inlineprivate

    {
        return samplepoint[i];
    }

SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int  i )

inlineprivate

    {
        return *sampleptlog[i];
    }

SamplePointLogName()

std::string ERF::SamplePointLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith sampleptlog file.

{ return sampleptlogname[i]; }

setPlotVariables()

void ERF::setPlotVariables ( const std::string &  pp_plot_var_names, amrex::Vector< std::string > &  plot_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    if (pp.contains(pp_plot_var_names.c_str()))
    {
        std::string nm;
 
        int nPltVars = pp.countval(pp_plot_var_names.c_str());
 
        for (int i = 0; i < nPltVars; i++)
        {
            pp.get(pp_plot_var_names.c_str(), nm, i);
 
            // Add the named variable to our list of plot variables
            // if it is not already in the list
            if (!containerHasElement(plot_var_names, nm)) {
                plot_var_names.push_back(nm);
            }
        }
    } else {
        //
        // The default is to add none of the variables to the list
        //
        plot_var_names.clear();
    }
 
    // Get state variables in the same order as we define them,
    // since they may be in any order in the input list
    Vector<std::string> tmp_plot_names;
 
    for (int i = 0; i < cons_names.size(); ++i) {
        if ( containerHasElement(plot_var_names, cons_names[i]) ) {
            if ( (solverChoice.moisture_type == MoistureType::SAM) || (cons_names[i] != "rhoQ4" &&
                                                                       cons_names[i] != "rhoQ5" &&
                                                                       cons_names[i] != "rhoQ6") )
            {
                tmp_plot_names.push_back(cons_names[i]);
            } // moisture_type
        }
    }
 
    // check for velocity since it's not in cons_names
    // if we are asked for any velocity component, we will need them all
    if (containerHasElement(plot_var_names, "x_velocity") ||
        containerHasElement(plot_var_names, "y_velocity") ||
        containerHasElement(plot_var_names, "z_velocity")) {
        tmp_plot_names.push_back("x_velocity");
        tmp_plot_names.push_back("y_velocity");
        tmp_plot_names.push_back("z_velocity");
    }
 
    //
    // If the model we are running doesn't have the variable listed in the inputs file,
    //     just ignore it rather than aborting
    //
    for (int i = 0; i < derived_names.size(); ++i) {
        if ( containerHasElement(plot_var_names, derived_names[i]) ) {
            if ( SolverChoice::mesh_type != MeshType::ConstantDz ||
                (derived_names[i] != "z_phys" && derived_names[i] != "detJ") ) {
                if ( (solverChoice.moisture_type == MoistureType::SAM ||
                      solverChoice.moisture_type == MoistureType::SAM_NoIce) ||
                     (derived_names[i] != "qi" &&
                      derived_names[i] != "qsnow" &&
                      derived_names[i] != "qgraup" &&
                      derived_names[i] != "snow_accum" &&
                      derived_names[i] != "graup_accum") )
                {
                    if ( (solverChoice.moisture_type != MoistureType::None)      ||
                         (derived_names[i] != "qv" && derived_names[i] != "qc" and
                          derived_names[i] != "num_turb" and derived_names[i] != "SMark0" and
                          derived_names[i] != "SMark1") ) {
                        tmp_plot_names.push_back(derived_names[i]);
                    }
                } // moisture_type
            } // use_terrain?
        } // hasElement
    }
 
#ifdef ERF_USE_WINDFARM
    for (int i = 0; i < derived_names.size(); ++i) {
        if ( containerHasElement(plot_var_names, derived_names[i]) ) {
            if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP) {
                if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0") {
                    tmp_plot_names.push_back(derived_names[i]);
                }
            }
            if( solverChoice.windfarm_type == WindFarmType::SimpleAD or
                solverChoice.windfarm_type == WindFarmType::GeneralAD ) {
                if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0" or derived_names[i] == "SMark1") {
                    tmp_plot_names.push_back(derived_names[i]);
                }
            }
        }
    }
#endif
 
#ifdef ERF_USE_PARTICLES
    const auto& particles_namelist( particleData.getNamesUnalloc() );
    for (auto it = particles_namelist.cbegin(); it != particles_namelist.cend(); ++it) {
        std::string tmp( (*it)+"_count" );
        if (containerHasElement(plot_var_names, tmp) ) {
            tmp_plot_names.push_back(tmp);
        }
    }
#endif
 
    plot_var_names = tmp_plot_names;
}

Here is the call graph for this function:

setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool  restarting )

private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters

[in]

restarting

Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

{
    // If we are restarting then we haven't read the input_sounding file yet
    //    so we need to read it here
    // TODO: should we store this information in the checkpoint file instead?
    if (restarting) {
        input_sounding_data.resize_arrays();
        for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
            input_sounding_data.read_from_file(geom[0], zlevels_stag[0], n);
        }
    }
 
    const Real* z_inp_sound     = input_sounding_data.z_inp_sound[0].dataPtr();
    const Real* U_inp_sound     = input_sounding_data.U_inp_sound[0].dataPtr();
    const Real* V_inp_sound     = input_sounding_data.V_inp_sound[0].dataPtr();
    const Real* theta_inp_sound = input_sounding_data.theta_inp_sound[0].dataPtr();
    const int   inp_sound_size  = input_sounding_data.size(0);
 
    int refine_fac{1};
    for (int lev = 0; lev <= finest_level; lev++)
    {
        const int klo = geom[lev].Domain().smallEnd(2);
        const int khi = geom[lev].Domain().bigEnd(2);
        const int Nz = khi - klo + 1;
 
        Vector<Real> zcc(Nz);
        Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
                                 zlevels_stag[0].begin()+khi/refine_fac+2);
        expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
#if 0
        amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
        for (int k = 0; k < zlevels_sub.size(); k++) { amrex::AllPrint() << zlevels_sub[k] << " "; }
        amrex::AllPrint() << " --> ";
        for (int k = 0; k < Nz; k++) { amrex::AllPrint() << zcc[k] << " "; }
        amrex::AllPrint() << std::endl;
#endif
 
        for (int k = 0; k < Nz; k++)
        {
            h_rayleigh_ptrs[lev][Rayleigh::ubar][k]     = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
            h_rayleigh_ptrs[lev][Rayleigh::vbar][k]     = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
            h_rayleigh_ptrs[lev][Rayleigh::wbar][k]     = Real(0.0);
            h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
        }
 
        // Copy from host version to device version
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
 
        refine_fac *= ref_ratio[lev][2];
    }
}

Here is the call graph for this function:

setRecordDataInfo()

void ERF::setRecordDataInfo ( int  i, const std::string &  filename  )

inlineprivate

    {
        if (amrex::ParallelDescriptor::IOProcessor())
        {
            datalog[i] = std::make_unique<std::fstream>();
            datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
            if (!datalog[i]->good()) {
                amrex::FileOpenFailed(filename);
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
    }

setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo ( int  i, int  lev, amrex::IntVect &  cell, const std::string &  filename  )

inlineprivate

    {
        amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
        for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
        {
            const amrex::Box& bx = mfi.validbox();
            if (bx.contains(cell)) {
                samplelinelog[i] = std::make_unique<std::fstream>();
                samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                if (!samplelinelog[i]->good()) {
                    amrex::FileOpenFailed(filename);
                }
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
    }

setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo ( int  i, int  lev, amrex::IntVect &  cell, const std::string &  filename  )

inlineprivate

    {
        amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
        for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
        {
            const amrex::Box& bx = mfi.validbox();
            if (bx.contains(cell)) {
                sampleptlog[i] = std::make_unique<std::fstream>();
                sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                if (!sampleptlog[i]->good()) {
                    amrex::FileOpenFailed(filename);
                }
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
    }

setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool  restarting )

private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters

[in]

restarting

Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

{
    // If we are restarting then we haven't read the input_sponge file yet
    //    so we need to read it here
    // TODO: should we store this information in the checkpoint file instead?
    if (restarting) {
        input_sponge_data.read_from_file(geom[0], zlevels_stag[0]);
    }
 
    const Real* z_inp_sponge     = input_sponge_data.z_inp_sponge.dataPtr();
    const Real* U_inp_sponge     = input_sponge_data.U_inp_sponge.dataPtr();
    const Real* V_inp_sponge     = input_sponge_data.V_inp_sponge.dataPtr();
    const int   inp_sponge_size  = input_sponge_data.size();
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        const int khi = geom[lev].Domain().bigEnd()[2];
        Vector<Real> zcc(khi+1);
 
        if (z_phys_cc[lev]) {
            // use_terrain=1
            // calculate the damping strength based on the max height at each k
            reduce_to_max_per_height(zcc, z_phys_cc[lev]);
        } else {
            const auto *const prob_lo = geom[lev].ProbLo();
            const auto *const dx = geom[lev].CellSize();
            for (int k = 0; k <= khi; k++)
            {
                zcc[k] = prob_lo[2] + (k+0.5) * dx[2];
            }
        }
 
        for (int k = 0; k <= khi; k++)
        {
            h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
            h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
        }
 
        // Copy from host version to device version
        Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
                         d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
        Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
                         d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
    }
}

Here is the call graph for this function:

solve_with_gmres()

void ERF::solve_with_gmres	(	int	lev,
		amrex::Vector< amrex::MultiFab > &	rhs,
		amrex::Vector< amrex::MultiFab > &	p,
		amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &	fluxes
	)

Solve the Poisson equation using FFT-preconditioned GMRES

{
#ifdef ERF_USE_FFT
    BL_PROFILE("ERF::solve_with_gmres()");
 
    Real reltol = solverChoice.poisson_reltol;
    Real abstol = solverChoice.poisson_abstol;
 
    amrex::GMRES<MultiFab, TerrainPoisson> gmsolver;
 
    TerrainPoisson tp(geom[lev], rhs[0].boxArray(), rhs[0].DistributionMap(), stretched_dz_d[lev],
                      z_phys_nd[lev].get(), domain_bc_type);
 
    gmsolver.define(tp);
 
    gmsolver.setVerbose(mg_verbose);
 
    tp.usePrecond(true);
 
    gmsolver.solve(phi[0], rhs[0], reltol, abstol);
 
    tp.getFluxes(phi[0], fluxes[0]);
#else
    amrex::ignore_unused(lev, rhs, phi, fluxes);
#endif
}

solve_with_mlmg()

void ERF::solve_with_mlmg	(	int	lev,
		amrex::Vector< amrex::MultiFab > &	rhs,
		amrex::Vector< amrex::MultiFab > &	p,
		amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &	fluxes
	)

Solve the Poisson equation using MLMG Note that the level may or may not be level 0.

{
    BL_PROFILE("ERF::solve_with_mlmg()");
 
    auto const dom_lo = lbound(geom[lev].Domain());
    auto const dom_hi = ubound(geom[lev].Domain());
 
    LPInfo info;
    // Allow a hidden direction if the domain is one cell wide in any lateral direction
    if (dom_lo.x == dom_hi.x) {
        info.setHiddenDirection(0);
    } else if (dom_lo.y == dom_hi.y) {
        info.setHiddenDirection(1);
    }
 
    // Make sure the solver only sees the levels over which we are solving
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(rhs[0].boxArray());
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(rhs[0].DistributionMap());
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
 
    auto bclo = get_projection_bc(Orientation::low);
    auto bchi = get_projection_bc(Orientation::high);
 
    // amrex::Print() << "BCLO " << bclo[0] << " " << bclo[1] << " " << bclo[2] << std::endl;
    // amrex::Print() << "BCHI " << bchi[0] << " " << bchi[1] << " " << bchi[2] << std::endl;
 
    Real reltol = solverChoice.poisson_reltol;
    Real abstol = solverChoice.poisson_abstol;
 
    // ****************************************************************************
    // Multigrid solve
    // ****************************************************************************
 
    MLPoisson mlpoisson(geom_tmp, ba_tmp, dm_tmp, info);
    mlpoisson.setDomainBC(bclo, bchi);
    if (lev > 0) {
        mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
    }
    mlpoisson.setLevelBC(0, nullptr);
 
    MLMG mlmg(mlpoisson);
    int max_iter = 100;
    mlmg.setMaxIter(max_iter);
 
    mlmg.setVerbose(mg_verbose);
    mlmg.setBottomVerbose(0);
 
    mlmg.solve(GetVecOfPtrs(phi),
               GetVecOfConstPtrs(rhs),
               reltol, abstol);
    mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
 
    phi[0].FillBoundary(geom[lev].periodicity());
 
    // ****************************************************************************
    // Impose bc's on pprime
    // ****************************************************************************
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(phi[0],TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        Array4<Real> const& pp_arr  = phi[0].array(mfi);
        Box const& bx    = mfi.tilebox();
        auto const bx_lo = lbound(bx);
        auto const bx_hi = ubound(bx);
        if (bx_lo.x == dom_lo.x) {
            auto bc_type = domain_bc_type[Orientation(0,Orientation::low)];
            if (bc_type == "Outflow" || bc_type == "Open") {
                ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    pp_arr(i-1,j,k) = -pp_arr(i,j,k);
                });
            } else {
                ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    pp_arr(i-1,j,k) = pp_arr(i,j,k);
                });
            }
        }
        if (bx_lo.y == dom_lo.y) {
            auto bc_type = domain_bc_type[Orientation(1,Orientation::low)];
            if (bc_type == "Outflow" || bc_type == "Open") {
                ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    pp_arr(i,j-1,k) = -pp_arr(i,j,k);
                });
            } else {
                ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    pp_arr(i,j-1,k) = pp_arr(i,j,k);
                });
            }
        }
        if (bx_lo.z == dom_lo.z) {
            ParallelFor(makeSlab(bx,2,dom_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j,k-1) = pp_arr(i,j,k);
            });
        }
        if (bx_hi.x == dom_hi.x) {
            auto bc_type = domain_bc_type[Orientation(0,Orientation::high)];
            if (bc_type == "Outflow" || bc_type == "Open") {
                ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    pp_arr(i+1,j,k) = -pp_arr(i,j,k);
                });
            } else {
                ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    pp_arr(i+1,j,k) = pp_arr(i,j,k);
                });
            }
        }
        if (bx_hi.y == dom_hi.y) {
            auto bc_type = domain_bc_type[Orientation(1,Orientation::high)];
            if (bc_type == "Outflow" || bc_type == "Open") {
                ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    pp_arr(i,j+1,k) = -pp_arr(i,j,k);
                });
            } else {
                ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    pp_arr(i,j+1,k) = pp_arr(i,j,k);
                });
            }
        }
        if (bx_hi.z == dom_hi.z) {
            ParallelFor(makeSlab(bx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j,k+1) = pp_arr(i,j,k);
            });
        }
    } // mfi
 
    // Now overwrite with periodic fill outside domain and fine-fine fill inside
    phi[0].FillBoundary(geom[lev].periodicity());
}

sum_integrated_quantities()

void ERF::sum_integrated_quantities

(

amrex::Real

time

)

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters

time	Current time

{
    BL_PROFILE("ERF::sum_integrated_quantities()");
 
    if (verbose <= 0)
      return;
 
    int datwidth = 14;
    int datprecision = 6;
 
    // Single level sum
    Real mass_sl;
 
    // Multilevel sums
    Real mass_ml = 0.0;
    Real rhth_ml = 0.0;
    Real scal_ml = 0.0;
 
#if 1
    mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,*mapfac_m[0],true,false);
    for (int lev = 0; lev <= finest_level; lev++) {
        mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,*mapfac_m[lev],true,true);
    }
#else
    for (int lev = 0; lev <= finest_level; lev++) {
        MultiFab pert_dens(vars_new[lev][Vars::cons].boxArray(),
                           vars_new[lev][Vars::cons].DistributionMap(),
                           1,0);
        MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
        for ( MFIter mfi(pert_dens,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real      >& pert_dens_arr = pert_dens.array(mfi);
            const Array4<Real const>&         S_arr = vars_new[lev][Vars::cons].const_array(mfi);
            const Array4<Real const>&        r0_arr = r_hse.const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                pert_dens_arr(i, j, k, 0) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
            });
        }
        if (lev == 0) {
            mass_sl = volWgtSumMF(0,pert_dens,0,*mapfac_m[0],true,false);
        }
        mass_ml += volWgtSumMF(lev,pert_dens,0,*mapfac_m[lev],true,true);
    } // lev
#endif
 
    Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,*mapfac_m[0],true,false);
    Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,*mapfac_m[0],true,false);
 
    for (int lev = 0; lev <= finest_level; lev++) {
        rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,*mapfac_m[lev],true,true);
        scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,*mapfac_m[lev],true,true);
    }
 
    Gpu::HostVector<Real> h_avg_ustar; h_avg_ustar.resize(1);
    Gpu::HostVector<Real> h_avg_tstar; h_avg_tstar.resize(1);
    Gpu::HostVector<Real> h_avg_olen; h_avg_olen.resize(1);
    if ((m_most != nullptr) && (NumDataLogs() > 0)) {
        Box domain = geom[0].Domain();
        int zdir = 2;
        h_avg_ustar = sumToLine(*m_most->get_u_star(0),0,1,domain,zdir);
        h_avg_tstar = sumToLine(*m_most->get_t_star(0),0,1,domain,zdir);
        h_avg_olen  = sumToLine(*m_most->get_olen(0),0,1,domain,zdir);
 
        // Divide by the total number of cells we are averaging over
        Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
        h_avg_ustar[0] /= area_z;
        h_avg_tstar[0] /= area_z;
        h_avg_olen[0]  /= area_z;
 
    } else {
        h_avg_ustar[0] = 0.;
        h_avg_tstar[0] = 0.;
        h_avg_olen[0]  = 0.;
    }
 
    const int nfoo = 6;
    Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mass_ml,rhth_ml,scal_ml};
#ifdef AMREX_LAZY
    Lazy::QueueReduction([=]() mutable {
#endif
    ParallelDescriptor::ReduceRealSum(
        foo, nfoo, ParallelDescriptor::IOProcessorNumber());
 
      if (ParallelDescriptor::IOProcessor()) {
        int i = 0;
        mass_sl = foo[i++];
        rhth_sl = foo[i++];
        scal_sl = foo[i++];
        mass_ml = foo[i++];
        rhth_ml = foo[i++];
        scal_ml = foo[i++];
 
        Print() << '\n';
        if (finest_level ==  0) {
#if 1
           Print() << "TIME= " << time << "     MASS          = " << mass_sl << '\n';
#else
           Print() << "TIME= " << time << " PERT MASS         = " << mass_sl << '\n';
#endif
           Print() << "TIME= " << time << " RHO THETA         = " << rhth_sl << '\n';
           Print() << "TIME= " << time << " RHO SCALAR        = " << scal_sl << '\n';
        } else {
#if 1
           Print() << "TIME= " << time << "      MASS   SL/ML = " << mass_sl << " " << mass_ml << '\n';
#else
           Print() << "TIME= " << time << " PERT MASS   SL/ML = " << mass_sl << " " << mass_ml << '\n';
#endif
           Print() << "TIME= " << time << " RHO THETA   SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
           Print() << "TIME= " << time << " RHO SCALAR  SL/ML = " << scal_sl << " " << scal_ml << '\n';
        }
 
        // The first data log only holds scalars
        if (NumDataLogs() > 0)
        {
            int n_d = 0;
            std::ostream& data_log1 = DataLog(n_d);
            if (data_log1.good()) {
                if (time == 0.0) {
                    data_log1 << std::setw(datwidth) << "          time";
                    data_log1 << std::setw(datwidth) << "          u_star";
                    data_log1 << std::setw(datwidth) << "          t_star";
                    data_log1 << std::setw(datwidth) << "          olen";
                    data_log1 << std::endl;
                } // time = 0
 
              // Write the quantities at this time
              data_log1 << std::setw(datwidth) << time;
              data_log1 << std::setw(datwidth) << std::setprecision(datprecision)
                        << h_avg_ustar[0];
              data_log1 << std::setw(datwidth) << std::setprecision(datprecision)
                        << h_avg_tstar[0];
              data_log1 << std::setw(datwidth) << std::setprecision(datprecision)
                        << h_avg_olen[0];
              data_log1 << std::endl;
            } // if good
        } // loop over i
      } // if IOProcessor
#ifdef AMREX_LAZY
    });
#endif
 
    // This is just an alias for convenience
    int lev = 0;
    if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
        for (int i = 0; i < NumSamplePoints(); ++i)
        {
            sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
        }
    }
    if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
        for (int i = 0; i < NumSampleLines(); ++i)
        {
            sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
        }
    }
}

timeStep()

void ERF::timeStep ( int  lev, amrex::Real  time, int  iteration  )

private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	iteration	time step counter

{
    //
    // We need to FillPatch the coarse level before assessing whether to regrid
    // We have not done the swap yet so we fill the "new" which will become the "old"
    //
    MultiFab& S_new = vars_new[lev][Vars::cons];
    MultiFab& U_new = vars_new[lev][Vars::xvel];
    MultiFab& V_new = vars_new[lev][Vars::yvel];
    MultiFab& W_new = vars_new[lev][Vars::zvel];
 
    //
    // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
    //
    if (lev == 0) {
        FillPatch(lev, time, {&S_new, &U_new, &V_new, &W_new});
    } else if (lev < finest_level) {
        FillPatch(lev, time, {&S_new, &U_new, &V_new, &W_new},
                             {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                             base_state[lev], base_state[lev]);
    }
 
    if (regrid_int > 0)  // We may need to regrid
    {
        // help keep track of whether a level was already regridded
        // from a coarser level call to regrid
        static Vector<int> last_regrid_step(max_level+1, 0);
 
        // regrid changes level "lev+1" so we don't regrid on max_level
        // also make sure we don't regrid fine levels again if
        // it was taken care of during a coarser regrid
        if (lev < max_level)
        {
            if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
            {
                // regrid could add newly refine levels (if finest_level < max_level)
                // so we save the previous finest level index
                int old_finest = finest_level;
 
                regrid(lev, time);
 
#ifdef ERF_USE_PARTICLES
                if (finest_level != old_finest) {
                    particleData.Redistribute();
                }
#endif
 
                // mark that we have regridded this level already
                for (int k = lev; k <= finest_level; ++k) {
                    last_regrid_step[k] = istep[k];
                }
 
                // if there are newly created levels, set the time step
                for (int k = old_finest+1; k <= finest_level; ++k) {
                    dt[k] = dt[k-1] / MaxRefRatio(k-1);
                }
            } // if
        } // lev
    }
 
    // Update what we call "old" and "new" time
    t_old[lev] = t_new[lev];
    t_new[lev] += dt[lev];
 
    if (Verbose()) {
        amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
        amrex::Print() << "ADVANCE from time = " << t_old[lev] << " to " << t_new[lev]
                       << " with dt = " << dt[lev] << std::endl;
    }
 
#ifdef ERF_USE_WW3_COUPLING
    amrex::Print() <<  " About to call send_to_ww3 from ERF_Timestep" << std::endl;
    send_to_ww3(lev);
    amrex::Print() <<  " About to call read_waves from ERF_Timestep"  << std::endl;
    read_waves(lev);
    //send_to_ww3(lev);
    //read_waves(lev);
    //send_to_ww3(lev);
#endif
 
    // Advance a single level for a single time step
    Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
 
    ++istep[lev];
 
    if (Verbose()) {
        amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
        amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
    }
 
    if (lev < finest_level)
    {
        // recursive call for next-finer level
        for (int i = 1; i <= nsubsteps[lev+1]; ++i)
        {
            Real strt_time_for_fine = time + (i-1)*dt[lev+1];
            timeStep(lev+1, strt_time_for_fine, i);
        }
    }
 
    if (verbose && lev == 0) {
        if (solverChoice.moisture_type != MoistureType::None) {
            amrex::Print() << "Cloud fraction " << time << "  " << cloud_fraction(time) << std::endl;
        }
    }
}

turbPert_amplitude()

void ERF::turbPert_amplitude ( const int  lev )

private

{
    // Accessing data
    auto& lev_new = vars_new[lev];
 
    // Creating local data
    int ncons = lev_new[Vars::cons].nComp();
    MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
 
    // Defining BoxArray type
    auto m_ixtype = cons_data.boxArray().ixType();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
        const Box &bx  = mfi.validbox();
        const auto &cons_pert_arr = cons_data.array(mfi); // Address of perturbation array
        const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell.array(mfi); // per-cell perturbation stored in structure
 
        turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cons_pert_arr, pert_cell);
    } // mfi
}

Here is the call graph for this function:

turbPert_update()

void ERF::turbPert_update ( const int  lev, const amrex::Real  dt  )

private

{
    // Accessing data
    auto& lev_new = vars_new[lev];
 
    // Create aliases to state data to pass to calc_tpi_update
    int ncons = lev_new[Vars::cons].nComp();
    MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
    MultiFab xvel_data(lev_new[Vars::xvel], make_alias, 0, 1);
    MultiFab yvel_data(lev_new[Vars::yvel], make_alias, 0, 1);
 
    // This logic is done once then stored within ERF_TurbPertStruct.H
    turbPert.pt_type = -1;
    if (solverChoice.pert_type == PerturbationType::Source) {
        turbPert.pt_type = 0;
    } else if (solverChoice.pert_type == PerturbationType::Direct) {
        turbPert.pt_type = 1;
    }
    AMREX_ALWAYS_ASSERT(turbPert.pt_type >= 0);
 
    // Computing perturbation update time
    turbPert.calc_tpi_update(lev, local_dt, xvel_data, yvel_data, cons_data);
 
    Print() << "Successfully initialized turbulent perturbation update time and amplitude with type: "<< turbPert.pt_type <<"\n";
}

update_diffusive_arrays()

void ERF::update_diffusive_arrays ( int  lev, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

private

{
    // ********************************************************************************************
    // Diffusive terms
    // ********************************************************************************************
    bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None);
    bool l_use_diff    = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
                           (solverChoice.turbChoice[lev].les_type        !=       LESType::None) ||
                           (solverChoice.turbChoice[lev].pbl_type        !=       PBLType::None) );
    bool l_use_kturb   = ( (solverChoice.turbChoice[lev].les_type        != LESType::None)   ||
                           (solverChoice.turbChoice[lev].pbl_type        != PBLType::None) );
    bool l_use_ddorf   = (solverChoice.turbChoice[lev].les_type       == LESType::Deardorff);
    bool l_use_moist   = (  solverChoice.moisture_type != MoistureType::None  );
 
    BoxArray ba12 = convert(ba, IntVect(1,1,0));
    BoxArray ba13 = convert(ba, IntVect(1,0,1));
    BoxArray ba23 = convert(ba, IntVect(0,1,1));
 
    if (l_use_diff) {
        //
        // NOTE: We require ghost cells in the vertical when allowing grids that don't
        //       cover the entire vertical extent of the domain at this level
        //
        Tau11_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
        Tau22_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
        Tau33_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
        Tau12_lev[lev] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
        Tau13_lev[lev] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
        Tau23_lev[lev] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
        if (l_use_terrain) {
            Tau21_lev[lev] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
            Tau31_lev[lev] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
            Tau32_lev[lev] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
        } else {
            Tau21_lev[lev] = nullptr;
            Tau31_lev[lev] = nullptr;
            Tau32_lev[lev] = nullptr;
        }
        SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
        SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
        SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
        SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
        SFS_hfx1_lev[lev]->setVal(0.);
        SFS_hfx2_lev[lev]->setVal(0.);
        SFS_hfx3_lev[lev]->setVal(0.);
        SFS_diss_lev[lev]->setVal(0.);
        if (l_use_moist) {
            SFS_q1fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
            SFS_q2fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
            SFS_q1fx3_lev[lev]->setVal(0.0);
            SFS_q2fx3_lev[lev]->setVal(0.0);
            if (solverChoice.use_rotate_most) {
                SFS_q1fx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
                SFS_q1fx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
                SFS_q1fx1_lev[lev]->setVal(0.0);
                SFS_q1fx2_lev[lev]->setVal(0.0);
            } else {
                SFS_q1fx1_lev[lev] = nullptr;
                SFS_q1fx2_lev[lev] = nullptr;
            }
        } else {
            SFS_q1fx1_lev[lev] = nullptr;
            SFS_q1fx2_lev[lev] = nullptr;
            SFS_q1fx3_lev[lev] = nullptr;
            SFS_q2fx3_lev[lev] = nullptr;
        }
    } else {
        Tau11_lev[lev] = nullptr; Tau22_lev[lev] = nullptr; Tau33_lev[lev] = nullptr;
        Tau12_lev[lev] = nullptr; Tau21_lev[lev] = nullptr;
        Tau13_lev[lev] = nullptr; Tau31_lev[lev] = nullptr;
        Tau23_lev[lev] = nullptr; Tau32_lev[lev] = nullptr;
        SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
        SFS_diss_lev[lev] = nullptr;
    }
 
    if (l_use_kturb) {
        eddyDiffs_lev[lev] = std::make_unique<MultiFab>(ba, dm, EddyDiff::NumDiffs, 2);
        eddyDiffs_lev[lev]->setVal(0.0);
        if(l_use_ddorf) {
            SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
        } else {
            SmnSmn_lev[lev] = nullptr;
        }
    } else {
        eddyDiffs_lev[lev] = nullptr;
        SmnSmn_lev[lev]    = nullptr;
    }
}

update_terrain_arrays()

void ERF::update_terrain_arrays ( int  lev )

private

{
    if (SolverChoice::mesh_type == MeshType::StretchedDz ||
        SolverChoice::mesh_type == MeshType::VariableDz) {
        make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
        make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
        make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
    }
}

Here is the call graph for this function:

volWgtSumMF()

Real ERF::volWgtSumMF	(	int	lev,
		const amrex::MultiFab &	mf,
		int	comp,
		const amrex::MultiFab &	mapfac,
		bool	local,
		bool	finemask
	)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters

lev	Current level
mf	MultiFab on which we do the volume weighted sum
comp	Index of the component we want to sum
local	Boolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)
finemask	If a finer level is available, determines whether we mask fine data

{
    BL_PROFILE("ERF::volWgtSumMF()");
 
    Real sum = 0.0;
    MultiFab tmp(grids[lev], dmap[lev], 1, 0);
    MultiFab::Copy(tmp, mf, comp, 0, 1, 0);
 
    // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
    // m is the map scale factor at cell centers
    for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        const Array4<      Real>    tmp_arr =    tmp.array(mfi);
        const Array4<const Real> mapfac_arr = mapfac.const_array(mfi);
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            tmp_arr(i,j,k) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
        });
    } // mfi
 
    if (lev < finest_level && finemask) {
        const MultiFab& mask = build_fine_mask(lev+1);
        MultiFab::Multiply(tmp, mask, 0, 0, 1, 0);
    }
 
    MultiFab volume(grids[lev], dmap[lev], 1, 0);
    auto const& dx = geom[lev].CellSizeArray();
    Real cell_vol = dx[0]*dx[1]*dx[2];
    volume.setVal(cell_vol);
    if (solverChoice.mesh_type != MeshType::ConstantDz) {
        MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
    }
    sum = MultiFab::Dot(tmp, 0, volume, 0, 1, 0, local);
 
    if (!local) {
      ParallelDescriptor::ReduceRealSum(sum);
    }
 
    return sum;
}

write_1D_profiles()

void ERF::write_1D_profiles

(

amrex::Real

time

)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters

time	Current time

{
    BL_PROFILE("ERF::write_1D_profiles()");
 
    int datwidth = 14;
    int datprecision = 6;
    int timeprecision = 13; // e.g., 1-yr LES: 31,536,000 s with dt ~ 0.01 ==> min prec = 10
 
    if (NumDataLogs() > 1)
    {
        // Define the 1d arrays we will need
        Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
        Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
        Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
        Gpu::HostVector<Real> h_avg_wthv;
        Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
        Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
        Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
        Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
        Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
        Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
 
        if (NumDataLogs() > 1) {
            derive_diag_profiles(time,
                                 h_avg_u, h_avg_v, h_avg_w,
                                 h_avg_rho, h_avg_th, h_avg_ksgs,
                                 h_avg_Kmv, h_avg_Khv,
                                 h_avg_qv, h_avg_qc, h_avg_qr,
                                 h_avg_wqv, h_avg_wqc, h_avg_wqr,
                                 h_avg_qi, h_avg_qs, h_avg_qg,
                                 h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                 h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                 h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                 h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
                                 h_avg_wthv);
        }
 
        if (NumDataLogs() > 3 && time > 0.) {
            derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                   h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                   h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
                                   h_avg_sgsdiss);
        }
 
        int hu_size =  h_avg_u.size();
 
        auto const& dx = geom[0].CellSizeArray();
        if (ParallelDescriptor::IOProcessor()) {
            if (NumDataLogs() > 1) {
                std::ostream& data_log1 = DataLog(1);
                if (data_log1.good()) {
                  // Write the quantities at this time
                  for (int k = 0; k < hu_size; k++) {
                      Real z;
                      if (zlevels_stag[0].size() > 1) {
                          z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                      } else {
                          z = (k + 0.5)* dx[2];
                      }
                      data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_u[k]   << " " << h_avg_v[k]   << " " << h_avg_w[k]     << " "
                                << h_avg_rho[k] << " " << h_avg_th[k]  << " " << h_avg_ksgs[k] << " "
                                << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
                                << h_avg_qv[k]  << " " << h_avg_qc[k]  << " " << h_avg_qr[k]    << " "
                                << h_avg_qi[k]  << " " << h_avg_qs[k]  << " " << h_avg_qg[k]
                                << std::endl;
                  } // loop over z
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 2) {
                std::ostream& data_log2 = DataLog(2);
                if (data_log2.good()) {
                  // Write the perturbational quantities at this time
                  for (int k = 0; k < hu_size; k++) {
                      Real z;
                      if (zlevels_stag[0].size() > 1) {
                          z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                      } else {
                          z = (k + 0.5)* dx[2];
                      }
                      Real thv = h_avg_th[k] * (1 + 0.61*h_avg_qv[k] - h_avg_qc[k] - h_avg_qr[k]);
                      data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]  << " "
                                << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]  << " "
                                << h_avg_uw[k]   - h_avg_u[k]*h_avg_w[k]  << " "
                                << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]  << " "
                                << h_avg_vw[k]   - h_avg_v[k]*h_avg_w[k]  << " "
                                << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]  << " "
                                << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k] << " "
                                << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k] << " "
                                << h_avg_wth[k]  - h_avg_w[k]*h_avg_th[k] << " "
                                << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
                                // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                //                        -   <u_i u_i> * <u_j>
                                //                        - 2*<u_i> * <u_i u_j>
                                //                        + 2*<u_i>*<u_i> * <u_j>
                                << h_avg_uiuiu[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
                                 - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
                                   << " " // (u'_i u'_i)u'
                                << h_avg_uiuiv[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
                                 - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
                                   << " " // (u'_i u'_i)v'
                                << h_avg_uiuiw[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
                                 - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                   << " " // (u'_i u'_i)w'
                                << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k] << " "
                                << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k] << " "
                                << h_avg_pw[k]   - h_avg_p[k]*h_avg_w[k] << " "
                                << h_avg_wqv[k]  - h_avg_qv[k]*h_avg_w[k] << " "
                                << h_avg_wqc[k]  - h_avg_qc[k]*h_avg_w[k] << " "
                                << h_avg_wqr[k]  - h_avg_qr[k]*h_avg_w[k] << " "
                                << h_avg_wthv[k] - h_avg_w[k]*thv
                                << std::endl;
                  } // loop over z
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 3 && time > 0.) {
                std::ostream& data_log3 = DataLog(3);
                if (data_log3.good()) {
                  // Write the average stresses
                  for (int k = 0; k < hu_size; k++) {
                      Real z;
                      if (zlevels_stag[0].size() > 1) {
                          z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                      } else {
                          z = (k + 0.5)* dx[2];
                      }
                      data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_tau11[k]  << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                << h_avg_tau22[k]  << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                << h_avg_sgshfx[k] << " "
                                << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
                                << h_avg_sgsdiss[k]
                                << std::endl;
                  } // loop over z
                } // if good
            } // if (NumDataLogs() > 3)
        } // if IOProcessor
    } // if (NumDataLogs() > 1)
}

write_1D_profiles_stag()

void ERF::write_1D_profiles_stag

(

amrex::Real

time

)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Quantities are output at their native grid locations. Therefore, w and associated flux quantities <(•)'w'>, tau13, and tau23 (where '•' includes u, v, p, theta, ...) will be output at staggered heights (i.e., coincident with z faces) rather than cell-center heights to avoid performing additional averaging. Unstaggered (i.e., cell-centered) quantities are output alongside staggered quantities at the lower cell faces in the log file; these quantities will have a zero value at the big end, corresponding to k=Nz+1.

Parameters

time	Current time

{
    BL_PROFILE("ERF::write_1D_profiles()");
 
    int datwidth = 14;
    int datprecision = 6;
    int timeprecision = 13; // e.g., 1-yr LES: 31,536,000 s with dt ~ 0.01 ==> min prec = 10
 
    if (NumDataLogs() > 1)
    {
        // Define the 1d arrays we will need
        Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
        Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
        Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
        Gpu::HostVector<Real> h_avg_wthv;
        Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
        Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
        Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
        Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
        Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
        Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
 
        if (NumDataLogs() > 1) {
            derive_diag_profiles_stag(time,
                                      h_avg_u, h_avg_v, h_avg_w,
                                      h_avg_rho, h_avg_th, h_avg_ksgs,
                                      h_avg_Kmv, h_avg_Khv,
                                      h_avg_qv, h_avg_qc, h_avg_qr,
                                      h_avg_wqv, h_avg_wqc, h_avg_wqr,
                                      h_avg_qi, h_avg_qs, h_avg_qg,
                                      h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                      h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                      h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                      h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
                                      h_avg_wthv);
        }
 
        if (NumDataLogs() > 3 && time > 0.) {
            derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                        h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                        h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
                                        h_avg_sgsdiss);
        }
 
        int unstag_size =  h_avg_w.size() - 1; // _un_staggered heights
 
        auto const& dx = geom[0].CellSizeArray();
        if (ParallelDescriptor::IOProcessor()) {
            if (NumDataLogs() > 1) {
                std::ostream& data_log1 = DataLog(1);
                if (data_log1.good()) {
                  // Write the quantities at this time
                  for (int k = 0; k < unstag_size; k++) {
                      Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                      data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_u[k]   << " " << h_avg_v[k]   << " " << h_avg_w[k]     << " "
                                << h_avg_rho[k] << " " << h_avg_th[k]  << " " << h_avg_ksgs[k] << " "
                                << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
                                << h_avg_qv[k]  << " " << h_avg_qc[k]  << " " << h_avg_qr[k]    << " "
                                << h_avg_qi[k]  << " " << h_avg_qs[k]  << " " << h_avg_qg[k]
                                << std::endl;
                  } // loop over z
                  // Write top face values
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                            << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                            << 0 << " " << 0 << " " << h_avg_w[unstag_size+1] << " "
                            << 0 << " " << 0 << " " << 0 << " " // rho, theta, ksgs
                            << 0 << " " << 0 << " "             // Kmv, Khv
                            << 0 << " " << 0 << " " << 0 << " " // qv, qc, qr
                            << 0 << " " << 0 << " " << 0        // qi, qs, qg
                            << std::endl;
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 2) {
                std::ostream& data_log2 = DataLog(2);
                if (data_log2.good()) {
                  // Write the perturbational quantities at this time
                  // For surface values (k=0), assume w = uw = vw = ww = 0
                  Real w_cc  = h_avg_w[1] / 2;  // w at first cell center
                  Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
                  Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
                  Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
                  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                            << std::setw(datwidth) << std::setprecision(datprecision) << 0 << " "
                            << h_avg_uu[0]   - h_avg_u[0]*h_avg_u[0]   << " " // u'u'
                            << h_avg_uv[0]   - h_avg_u[0]*h_avg_v[0]   << " " // u'v'
                            << 0                                       << " " // u'w'
                            << h_avg_vv[0]   - h_avg_v[0]*h_avg_v[0]   << " " // v'v'
                            << 0                                       << " " // v'w'
                            << 0                                       << " " // w'w'
                            << h_avg_uth[0]  - h_avg_u[0]*h_avg_th[0]  << " " // u'th'
                            << h_avg_vth[0]  - h_avg_v[0]*h_avg_th[0]  << " " // v'th'
                            << 0                                       << " " // w'th'
                            << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
                            << h_avg_uiuiu[0]
                             - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
                             - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
                             + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
                               << " " // (u'_i u'_i)u'
                            << h_avg_uiuiv[0]
                             - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
                             - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
                             + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
                               << " " // (u'_i u'_i)v'
                            << 0 << " " // (u'_i u'_i)w'
                            << h_avg_pu[0]   - h_avg_p[0]*h_avg_u[0]   << " " // p'u'
                            << h_avg_pv[0]   - h_avg_p[0]*h_avg_v[0]   << " " // p'v'
                            << 0                                       << " " // p'w'
                            << 0                                       << " " // qv'w'
                            << 0                                       << " " // qc'w'
                            << 0                                       << " " // qr'w'
                            << 0                                              // thv'w'
                            << std::endl;
 
                  // For internal values, interpolate scalar quantities to faces
                  for (int k = 1; k < unstag_size; k++) {
                      Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                      Real uface  = 0.5*(h_avg_u[k]  + h_avg_u[k-1]);
                      Real vface  = 0.5*(h_avg_v[k]  + h_avg_v[k-1]);
                      Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]);
                      Real pface  = 0.5*(h_avg_p[k]  + h_avg_p[k-1]);
                      Real qvface = 0.5*(h_avg_qv[k] + h_avg_qv[k-1]);
                      Real qcface = 0.5*(h_avg_qc[k] + h_avg_qc[k-1]);
                      Real qrface = 0.5*(h_avg_qr[k] + h_avg_qr[k-1]);
                      Real uuface = 0.5*(h_avg_uu[k] + h_avg_uu[k-1]);
                      Real vvface = 0.5*(h_avg_vv[k] + h_avg_vv[k-1]);
                      Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
                      w_cc   = 0.5*(h_avg_w[k-1]  + h_avg_w[k]);
                      uw_cc  = 0.5*(h_avg_uw[k-1] + h_avg_uw[k]);
                      vw_cc  = 0.5*(h_avg_vw[k-1] + h_avg_vw[k]);
                      ww_cc  = 0.5*(h_avg_ww[k-1] + h_avg_ww[k]);
                      data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]   << " " // u'u'
                                << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]   << " " // u'v'
                                << h_avg_uw[k]   -      uface*h_avg_w[k]   << " " // u'w'
                                << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]   << " " // v'v'
                                << h_avg_vw[k]   -      vface*h_avg_w[k]   << " " // v'w'
                                << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]   << " " // w'w'
                                << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k]  << " " // u'th'
                                << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k]  << " " // v'th'
                                << h_avg_wth[k]  - h_avg_w[k]*thface       << " " // w'th'
                                << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
                                // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                //                        -   <u_i u_i> * <u_j>
                                //                        - 2*<u_i> * <u_i u_j>
                                //                        + 2*<u_i>*<u_i> * <u_j>
                                << h_avg_uiuiu[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
                                 - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
                                   << " " // cell-centered (u'_i u'_i)u'
                                << h_avg_uiuiv[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
                                 - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
                                   << " " // cell-centered (u'_i u'_i)v'
                                << h_avg_uiuiw[k]
                                 - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                                 - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                 + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                   << " " // face-centered (u'_i u'_i)w'
                                << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k]   << " " // cell-centered p'u'
                                << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k]   << " " // cell-centered p'v'
                                << h_avg_pw[k]   -      pface*h_avg_w[k]   << " " // face-centered p'w'
                                << h_avg_wqv[k]  -     qvface*h_avg_w[k]   << " "
                                << h_avg_wqc[k]  -     qcface*h_avg_w[k]   << " "
                                << h_avg_wqr[k]  -     qrface*h_avg_w[k]   << " "
                                << h_avg_wthv[k] -    thvface*h_avg_w[k]
                                << std::endl;
                  } // loop over z
 
                  // Write top face values, extrapolating scalar quantities
                  const int k = unstag_size;
                  Real uface  = 1.5*h_avg_u[k-1]  - 0.5*h_avg_u[k-2];
                  Real vface  = 1.5*h_avg_v[k-1]  - 0.5*h_avg_v[k-2];
                  Real thface = 1.5*h_avg_th[k-1] - 0.5*h_avg_th[k-2];
                  Real pface  = 1.5*h_avg_p[k-1]  - 0.5*h_avg_p[k-2];
                  Real qvface = 1.5*h_avg_qv[k-1] - 0.5*h_avg_qv[k-2];
                  Real qcface = 1.5*h_avg_qc[k-1] - 0.5*h_avg_qc[k-2];
                  Real qrface = 1.5*h_avg_qr[k-1] - 0.5*h_avg_qr[k-2];
                  Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2];
                  Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2];
                  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                            << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                            << 0                                     << " " // u'u'
                            << 0                                     << " " // u'v'
                            << h_avg_uw[k]   -      uface*h_avg_w[k] << " " // u'w'
                            << 0                                     << " " // v'v'
                            << h_avg_vw[k]   -      vface*h_avg_w[k] << " " // v'w'
                            << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k] << " " // w'w'
                            << 0                                     << " " // u'th'
                            << 0                                     << " " // v'th'
                            << h_avg_wth[k]  -     thface*h_avg_w[k] << " " // w'th'
                            << 0                                     << " " // th'th'
                            << 0                                     << " " // (u'_i u'_i)u'
                            << 0                                     << " " // (u'_i u'_i)v'
                            << h_avg_uiuiw[k]
                             - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                             - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                             + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                               << " " // (u'_i u'_i)w'
                            << 0                                     << " " // pu'
                            << 0                                     << " " // pv'
                            << h_avg_pw[k]   -      pface*h_avg_w[k] << " " // pw'
                            << h_avg_wqv[k]  -     qvface*h_avg_w[k] << " "
                            << h_avg_wqc[k]  -     qcface*h_avg_w[k] << " "
                            << h_avg_wqr[k]  -     qrface*h_avg_w[k] << " "
                            << h_avg_wthv[k] -    thvface*h_avg_w[k]
                            << std::endl;
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 3 && time > 0.) {
                std::ostream& data_log3 = DataLog(3);
                if (data_log3.good()) {
                  // Write the average stresses
                  for (int k = 0; k < unstag_size; k++) {
                      Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                      data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_tau11[k]  << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                << h_avg_tau22[k]  << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                << h_avg_sgshfx[k] << " "
                                << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
                                << h_avg_sgsdiss[k]
                                << std::endl;
                  } // loop over z
                  // Write top face values
                  Real NANval = 0.0;
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                            << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                            << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
                            << NANval << " " << h_avg_tau23[unstag_size] << " " << NANval << " "
                            << h_avg_sgshfx[unstag_size] << " "
                            << h_avg_sgsq1fx[unstag_size] << " " << h_avg_sgsq2fx[unstag_size] << " "
                            << NANval
                            << std::endl;
                } // if good
            } // if (NumDataLogs() > 3)
        } // if IOProcessor
    } // if (NumDataLogs() > 1)
}

writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream &  os )

static

{
    std::string PrettyLine = std::string(78, '=') + "\n";
    std::string OtherLine = std::string(78, '-') + "\n";
    std::string SkipSpace = std::string(8, ' ');
 
    // build information
    os << PrettyLine;
    os << " ERF Build Information\n";
    os << PrettyLine;
 
    os << "build date:    " << buildInfoGetBuildDate() << "\n";
    os << "build machine: " << buildInfoGetBuildMachine() << "\n";
    os << "build dir:     " << buildInfoGetBuildDir() << "\n";
    os << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
 
    os << "\n";
 
    os << "COMP:          " << buildInfoGetComp() << "\n";
    os << "COMP version:  " << buildInfoGetCompVersion() << "\n";
 
    os << "C++ compiler:  " << buildInfoGetCXXName() << "\n";
    os << "C++ flags:     " << buildInfoGetCXXFlags() << "\n";
 
    os << "\n";
 
    os << "Link flags:    " << buildInfoGetLinkFlags() << "\n";
    os << "Libraries:     " << buildInfoGetLibraries() << "\n";
 
    os << "\n";
 
    for (int n = 1; n <= buildInfoGetNumModules(); n++) {
      os << buildInfoGetModuleName(n) << ": "
         << buildInfoGetModuleVal(n) << "\n";
    }
 
    os << "\n";
    const char* githash1 = buildInfoGetGitHash(1);
    const char* githash2 = buildInfoGetGitHash(2);
    if (strlen(githash1) > 0) {
      os << "ERF       git hash: " << githash1 << "\n";
    }
    if (strlen(githash2) > 0) {
      os << "AMReX       git hash: " << githash2 << "\n";
    }
 
    const char* buildgithash = buildInfoGetBuildGitHash();
    const char* buildgitname = buildInfoGetBuildGitName();
    if (strlen(buildgithash) > 0) {
      os << buildgitname << " git hash: " << buildgithash << "\n";
    }
 
    os << "\n";
    os << " ERF Compile time variables: \n";
 
    os << "\n";
    os << " ERF Defines: \n";
#ifdef _OPENMP
    os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
       << std::endl;
#else
    os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
       << std::endl;
#endif
 
#ifdef MPI_VERSION
    os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
       << MPI_VERSION << std::endl;
#else
    os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
       << "UNDEFINED" << std::endl;
#endif
 
#ifdef MPI_SUBVERSION
    os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
       << MPI_SUBVERSION << std::endl;
#else
    os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
       << "UNDEFINED" << std::endl;
#endif
 
    os << "\n\n";
}

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

WriteCheckpointFile()

void ERF::WriteCheckpointFile

(

)

const

ERF function for writing a checkpoint file.

{
    // chk00010            write a checkpoint file with this root directory
    // chk00010/Header     this contains information you need to save (e.g., finest_level, t_new, etc.) and also
    //                     the BoxArrays at each level
    // chk00010/Level_0/
    // chk00010/Level_1/
    // etc.                these subdirectories will hold the MultiFab data at each level of refinement
 
    // checkpoint file name, e.g., chk00010
    const std::string& checkpointname = Concatenate(check_file,istep[0],5);
 
    Print() << "Writing native checkpoint " << checkpointname << "\n";
 
    const int nlevels = finest_level+1;
 
    // ---- prebuild a hierarchy of directories
    // ---- dirName is built first.  if dirName exists, it is renamed.  then build
    // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
    // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
    // ---- after all directories are built
    // ---- ParallelDescriptor::IOProcessor() creates the directories
    PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
 
    int ncomp_cons = vars_new[0][Vars::cons].nComp();
 
    // write Header file
    if (ParallelDescriptor::IOProcessor()) {
 
       std::string HeaderFileName(checkpointname + "/Header");
       VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
       std::ofstream HeaderFile;
       HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
       HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                               std::ofstream::trunc |
                                               std::ofstream::binary);
       if( ! HeaderFile.good()) {
           FileOpenFailed(HeaderFileName);
       }
 
       HeaderFile.precision(17);
 
       // write out title line
       HeaderFile << "Checkpoint file for ERF\n";
 
       // write out finest_level
       HeaderFile << finest_level << "\n";
 
       // write the number of components
       // for each variable we store
 
       // conservative, cell-centered vars
       HeaderFile << ncomp_cons << "\n";
 
       // x-velocity on faces
       HeaderFile << 1 << "\n";
 
       // y-velocity on faces
       HeaderFile << 1 << "\n";
 
       // z-velocity on faces
       HeaderFile << 1 << "\n";
 
       // write out array of istep
       for (int i = 0; i < istep.size(); ++i) {
           HeaderFile << istep[i] << " ";
       }
       HeaderFile << "\n";
 
       // write out array of dt
       for (int i = 0; i < dt.size(); ++i) {
           HeaderFile << dt[i] << " ";
       }
       HeaderFile << "\n";
 
       // write out array of t_new
       for (int i = 0; i < t_new.size(); ++i) {
           HeaderFile << t_new[i] << " ";
       }
       HeaderFile << "\n";
 
       // write the BoxArray at each level
       for (int lev = 0; lev <= finest_level; ++lev) {
           boxArray(lev).writeOn(HeaderFile);
           HeaderFile << '\n';
       }
   }
 
    // write the MultiFab data to, e.g., chk00010/Level_0/
    // Here we make copies of the MultiFab with no ghost cells
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
        MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
        VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
 
        MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
        MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
        VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
 
        MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
        MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
        VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
 
        MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
        MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
        VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
 
        // Note that we write the ghost cells of the base state (unlike above)
        // For backward compatibility we only write the first components and 1 ghost cell
        IntVect ng_base; int ncomp_base;
        bool write_old_base_state = true;
        if (write_old_base_state) {
            ng_base    = IntVect{1};
            ncomp_base = 3;
        } else {
            ng_base    = base_state[lev].nGrowVect();
            ncomp_base = base_state[lev].nComp();
        }
        MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
        MultiFab::Copy(base,base_state[lev],0,0,ncomp_base,ng_base);
        VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
 
        if (SolverChoice::mesh_type != MeshType::ConstantDz)  {
            // Note that we also write the ghost cells of z_phys_nd
            IntVect ng = z_phys_nd[lev]->nGrowVect();
            MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
            MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
            VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
        }
 
        // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
        // Write the moisture model restart variables
        std::vector<int> qmoist_indices(0);
        std::vector<std::string> qmoist_names(0);
        micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
        int qmoist_nvar = qmoist_indices.size();
        for (int var = 0; var < qmoist_nvar; var++) {
           IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
           const int ncomp = 1;
           MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
           MultiFab::Copy(moist_vars,*(qmoist[lev][qmoist_indices[var]]),0,0,ncomp,ng_moist);
           VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", qmoist_names[var]));
        }
 
#if defined(ERF_USE_WINDFARM)
        if(solverChoice.windfarm_type == WindFarmType::Fitch or
           solverChoice.windfarm_type == WindFarmType::EWP or
           solverChoice.windfarm_type == WindFarmType::SimpleAD){
            IntVect ng_turb = Nturb[lev].nGrowVect();
            MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng_turb);
            MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng_turb);
            VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
        }
#endif
 
        if (solverChoice.lsm_type != LandSurfaceType::None) {
            for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
                BoxArray ba = lsm_data[lev][mvar]->boxArray();
                DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
                IntVect ng = lsm_data[lev][mvar]->nGrowVect();
                int nvar = lsm_data[lev][mvar]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                MultiFab::Copy(lsm_vars,*(lsm_data[lev][mvar]),0,0,nvar,ng);
                VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmVars"));
            }
        }
 
        // Note that we also write the ghost cells of the mapfactors (2D)
        BoxList bl2d = grids[lev].boxList();
        for (auto& b : bl2d) {
            b.setRange(2,0);
        }
        BoxArray ba2d(std::move(bl2d));
 
        IntVect ng = mapfac_m[lev]->nGrowVect();
        MultiFab mf_m(ba2d,dmap[lev],1,ng);
        MultiFab::Copy(mf_m,*mapfac_m[lev],0,0,1,ng);
        VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_m"));
 
        ng = mapfac_u[lev]->nGrowVect();
        MultiFab mf_u(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,ng);
        MultiFab::Copy(mf_u,*mapfac_u[lev],0,0,1,ng);
        VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_u"));
 
        ng = mapfac_v[lev]->nGrowVect();
        MultiFab mf_v(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,ng);
        MultiFab::Copy(mf_v,*mapfac_v[lev],0,0,1,ng);
        VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_v"));
 
        if (m_most && m_most->have_variable_sea_roughness())  {
            amrex::Print() << "Writing variable surface roughness" << std::endl;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab z0(ba2d,dmap[lev],1,ng);
            for (amrex::MFIter mfi(z0); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.growntilebox();
                z0[mfi].copy<RunOn::Host>(*(m_most->get_z0(lev)), bx);
            }
            VisMF::Write(z0, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z0"));
        }
    }
 
#ifdef ERF_USE_PARTICLES
   particleData.Checkpoint(checkpointname);
#endif
 
#ifdef ERF_USE_NETCDF
   // Write bdy_data files
   if (ParallelDescriptor::IOProcessor() && ((init_type==InitType::Real) || (init_type==InitType::Metgrid))) {
 
     // Vector dimensions
     int num_time = bdy_data_xlo.size();
     int num_var  = bdy_data_xlo[0].size();
 
     // Open header file and write to it
     std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
     bdy_h_file << std::setprecision(1) << std::fixed;
     bdy_h_file << num_time << "\n";
     bdy_h_file << num_var  << "\n";
     bdy_h_file << start_bdy_time << "\n";
     bdy_h_file << bdy_time_interval << "\n";
     bdy_h_file << real_width << "\n";
     for (int ivar(0); ivar<num_var; ++ivar) {
       bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
       bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
       bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
       bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
     }
 
     // Open data file and write to it
     std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
     for (int itime(0); itime<num_time; ++itime) {
       for (int ivar(0); ivar<num_var; ++ivar) {
         bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
         bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
         bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
         bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
       }
     }
   }
#endif
 
}

WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain	(	std::ostream &	HeaderFile,
		int	nlevels,
		const amrex::Vector< amrex::BoxArray > &	bArray,
		const amrex::Vector< std::string > &	varnames,
		const amrex::Vector< amrex::Geometry > &	my_geom,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const amrex::Vector< amrex::IntVect > &	my_ref_ratio,
		const std::string &	versionName,
		const std::string &	levelPrefix,
		const std::string &	mfPrefix
	)		const

{
    AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
    AMREX_ALWAYS_ASSERT(nlevels <= my_ref_ratio.size()+1);
    AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
 
    HeaderFile.precision(17);
 
    // ---- this is the generic plot file type name
    HeaderFile << versionName << '\n';
 
    HeaderFile << varnames.size() << '\n';
 
    for (int ivar = 0; ivar < varnames.size(); ++ivar) {
        HeaderFile << varnames[ivar] << "\n";
    }
    HeaderFile << AMREX_SPACEDIM << '\n';
    HeaderFile << my_time << '\n';
    HeaderFile << finest_level << '\n';
    for (int i = 0; i < AMREX_SPACEDIM; ++i) {
        HeaderFile << my_geom[0].ProbLo(i) << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i < AMREX_SPACEDIM; ++i) {
        HeaderFile << my_geom[0].ProbHi(i) << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i < finest_level; ++i) {
        HeaderFile << my_ref_ratio[i][0] << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i <= finest_level; ++i) {
        HeaderFile << my_geom[i].Domain() << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i <= finest_level; ++i) {
        HeaderFile << level_steps[i] << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i <= finest_level; ++i) {
        for (int k = 0; k < AMREX_SPACEDIM; ++k) {
            HeaderFile << my_geom[i].CellSize()[k] << ' ';
        }
        HeaderFile << '\n';
    }
    HeaderFile << (int) my_geom[0].Coord() << '\n';
    HeaderFile << "0\n";
 
    for (int level = 0; level <= finest_level; ++level) {
        HeaderFile << level << ' ' << bArray[level].size() << ' ' << my_time << '\n';
        HeaderFile << level_steps[level] << '\n';
 
        const IntVect& domain_lo = my_geom[level].Domain().smallEnd();
        for (int i = 0; i < bArray[level].size(); ++i)
        {
            // Need to shift because the RealBox ctor we call takes the
            // physical location of index (0,0,0).  This does not affect
            // the usual cases where the domain index starts with 0.
            const Box& b = shift(bArray[level][i], -domain_lo);
            RealBox loc = RealBox(b, my_geom[level].CellSize(), my_geom[level].ProbLo());
            for (int n = 0; n < AMREX_SPACEDIM; ++n) {
                HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
            }
        }
 
        HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
    }
    HeaderFile << "1" << "\n";
    HeaderFile << "3" << "\n";
    HeaderFile << "amrexvec_nu_x" << "\n";
    HeaderFile << "amrexvec_nu_y" << "\n";
    HeaderFile << "amrexvec_nu_z" << "\n";
    std::string mf_nodal_prefix = "Nu_nd";
    for (int level = 0; level <= finest_level; ++level) {
        HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
    }
}

writeJobInfo()

void ERF::writeJobInfo

(

const std::string &

dir

)

const

{
    // job_info file with details about the run
    std::ofstream jobInfoFile;
    std::string FullPathJobInfoFile = dir;
    FullPathJobInfoFile += "/job_info";
    jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out);
 
    std::string PrettyLine = "==================================================="
      "============================\n";
    std::string OtherLine = "----------------------------------------------------"
      "----------------------------\n";
    std::string SkipSpace = "        ";
 
    // job information
    jobInfoFile << PrettyLine;
    jobInfoFile << " ERF Job Information\n";
    jobInfoFile << PrettyLine;
 
    jobInfoFile << "inputs file: " << inputs_name << "\n\n";
 
    jobInfoFile << "number of MPI processes: "
                << ParallelDescriptor::NProcs() << "\n";
#ifdef _OPENMP
    jobInfoFile << "number of threads:       " << omp_get_max_threads() << "\n";
#endif
 
    jobInfoFile << "\n";
    jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
                << getCPUTime() / 3600.0;
 
    jobInfoFile << "\n\n";
 
    // plotfile information
    jobInfoFile << PrettyLine;
    jobInfoFile << " Plotfile Information\n";
    jobInfoFile << PrettyLine;
 
    time_t now = time(nullptr);
 
    // Convert now to tm struct for local timezone
    tm* localtm = localtime(&now);
    jobInfoFile << "output data / time: " << asctime(localtm);
 
    std::string currentDir = FileSystem::CurrentPath();
    jobInfoFile << "output dir:         " << currentDir << "\n";
 
    jobInfoFile << "\n\n";
 
    // build information
    jobInfoFile << PrettyLine;
    jobInfoFile << " Build Information\n";
    jobInfoFile << PrettyLine;
 
    jobInfoFile << "build date:    " << buildInfoGetBuildDate() << "\n";
    jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
    jobInfoFile << "build dir:     " << buildInfoGetBuildDir() << "\n";
    jobInfoFile << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
 
    jobInfoFile << "\n";
 
    jobInfoFile << "COMP:          " << buildInfoGetComp() << "\n";
    jobInfoFile << "COMP version:  " << buildInfoGetCompVersion() << "\n";
 
    jobInfoFile << "\n";
 
    for (int n = 1; n <= buildInfoGetNumModules(); n++) {
      jobInfoFile << buildInfoGetModuleName(n) << ": "
                  << buildInfoGetModuleVal(n) << "\n";
    }
 
    jobInfoFile << "\n";
 
    const char* githash1 = buildInfoGetGitHash(1);
    const char* githash2 = buildInfoGetGitHash(2);
    if (strlen(githash1) > 0) {
      jobInfoFile << "ERF       git hash: " << githash1 << "\n";
    }
    if (strlen(githash2) > 0) {
      jobInfoFile << "AMReX       git hash: " << githash2 << "\n";
    }
 
    const char* buildgithash = buildInfoGetBuildGitHash();
    const char* buildgitname = buildInfoGetBuildGitName();
    if (strlen(buildgithash) > 0) {
      jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
    }
 
    jobInfoFile << "\n\n";
 
    // grid information
    jobInfoFile << PrettyLine;
    jobInfoFile << " Grid Information\n";
    jobInfoFile << PrettyLine;
 
    int f_lev = finest_level;
 
    for (int i = 0; i <= f_lev; i++) {
      jobInfoFile << " level: " << i << "\n";
      jobInfoFile << "   number of boxes = " << grids[i].size() << "\n";
      jobInfoFile << "   maximum zones   = ";
      for (int n = 0; n < AMREX_SPACEDIM; n++) {
        jobInfoFile << geom[i].Domain().length(n) << " ";
      }
      jobInfoFile << "\n\n";
    }
 
    jobInfoFile << " Boundary conditions\n";
 
    jobInfoFile << "   -x: " << domain_bc_type[0] << "\n";
    jobInfoFile << "   +x: " << domain_bc_type[3] << "\n";
    jobInfoFile << "   -y: " << domain_bc_type[1] << "\n";
    jobInfoFile << "   +y: " << domain_bc_type[4] << "\n";
    jobInfoFile << "   -z: " << domain_bc_type[2] << "\n";
    jobInfoFile << "   +z: " << domain_bc_type[5] << "\n";
 
    jobInfoFile << "\n\n";
 
    // runtime parameters
    jobInfoFile << PrettyLine;
    jobInfoFile << " Inputs File Parameters\n";
    jobInfoFile << PrettyLine;
 
    ParmParse::dumpTable(jobInfoFile, true);
    jobInfoFile.close();
}

Here is the call graph for this function:

WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain	(	const std::string &	plotfilename,
		int	nlevels,
		const amrex::Vector< const amrex::MultiFab * > &	mf,
		const amrex::Vector< const amrex::MultiFab * > &	mf_nd,
		const amrex::Vector< std::string > &	varnames,
		const amrex::Vector< amrex::Geometry > &	my_geom,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const amrex::Vector< amrex::IntVect > &	my_ref_ratio,
		const std::string &	versionName = "HyperCLaw-V1.1",
		const std::string &	levelPrefix = "Level_",
		const std::string &	mfPrefix = "Cell",
		const amrex::Vector< std::string > &	extra_dirs = amrex::Vector<std::string>()
	)		const

{
    BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
 
    AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
    AMREX_ALWAYS_ASSERT(nlevels <= rr.size()+1);
    AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
    AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
 
    bool callBarrier(false);
    PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
    if (!extra_dirs.empty()) {
        for (const auto& d : extra_dirs) {
            const std::string ed = plotfilename+"/"+d;
            PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
        }
    }
    ParallelDescriptor::Barrier();
 
    if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
        Vector<BoxArray> boxArrays(nlevels);
        for(int level(0); level < boxArrays.size(); ++level) {
            boxArrays[level] = mf[level]->boxArray();
        }
 
        auto f = [=]() {
            VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
            std::string HeaderFileName(plotfilename + "/Header");
            std::ofstream HeaderFile;
            HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
            HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                                    std::ofstream::trunc |
                                                    std::ofstream::binary);
            if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
            WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
                                                  my_geom, time, level_steps, rr, versionName,
                                                  levelPrefix, mfPrefix);
        };
 
        if (AsyncOut::UseAsyncOut()) {
            AsyncOut::Submit(std::move(f));
        } else {
            f();
        }
    }
 
    std::string mf_nodal_prefix = "Nu_nd";
    for (int level = 0; level <= finest_level; ++level)
    {
        if (AsyncOut::UseAsyncOut()) {
            VisMF::AsyncWrite(*mf[level],
                              MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
                              true);
            VisMF::AsyncWrite(*mf_nd[level],
                              MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
                              true);
        } else {
            const MultiFab* data;
            std::unique_ptr<MultiFab> mf_tmp;
            if (mf[level]->nGrowVect() != 0) {
                mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
                                                    mf[level]->DistributionMap(),
                                                    mf[level]->nComp(), 0, MFInfo(),
                                                    mf[level]->Factory());
                MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
                data = mf_tmp.get();
            } else {
                data = mf[level];
            }
            VisMF::Write(*data        , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
            VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
        }
    }
}

writeNow()

bool ERF::writeNow	(	const amrex::Real	cur_time,
		const amrex::Real	dt,
		const int	nstep,
		const int	plot_int,
		const amrex::Real	plot_per
	)

{
    bool write_now = false;
 
    if ( plot_int > 0 && (nstep % plot_int == 0) ) {
        write_now = true;
 
    } else if (plot_per > 0.0) {
 
        // Check to see if we've crossed a plot_per interval by comparing
        // the number of intervals that have elapsed for both the current
        // time and the time at the beginning of this timestep.
 
        const Real eps = std::numeric_limits<Real>::epsilon() * Real(10.0) * std::abs(cur_time);
 
        int num_per_old = static_cast<int>(std::floor((cur_time-eps-dt_lev) / plot_per));
        int num_per_new = static_cast<int>(std::floor((cur_time-eps       ) / plot_per));
 
        // Before using these, however, we must test for the case where we're
        // within machine epsilon of the next interval. In that case, increment
        // the counter, because we have indeed reached the next plot_per interval
        // at this point.
 
        const Real next_plot_time = (num_per_old + 1) * plot_per;
 
        if ((num_per_new == num_per_old) && std::abs(cur_time - next_plot_time) <= eps)
        {
            num_per_new += 1;
        }
 
        // Similarly, we have to account for the case where the old time is within
        // machine epsilon of the beginning of this interval, so that we don't double
        // count that time threshold -- we already plotted at that time on the last timestep.
 
        if ((num_per_new != num_per_old) && std::abs((cur_time - dt_lev) - next_plot_time) <= eps)
            num_per_old += 1;
 
        if (num_per_old != num_per_new)
            write_now = true;
    }
    return write_now;
}

WritePlotFile()

void ERF::WritePlotFile	(	int	which,
		PlotFileType	plotfile_type,
		amrex::Vector< std::string >	plot_var_names
	)

{
    const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
    const int ncomp_mf = varnames.size();
 
    int ncomp_cons = vars_new[0][Vars::cons].nComp();
 
    if (ncomp_mf == 0) return;
 
    // We Fillpatch here because some of the derived quantities require derivatives
    //     which require ghost cells to be filled.  We do not need to call FillPatcher
    //     because we don't need to set interior fine points.
    // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
 
    // Level 0 FilLPatch
    FillPatch(0, t_new[0], {&vars_new[0][Vars::cons], &vars_new[0][Vars::xvel],
                            &vars_new[0][Vars::yvel], &vars_new[0][Vars::zvel]});
 
    for (int lev = 1; lev <= finest_level; ++lev) {
        bool fillset = false;
        FillPatch(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
                                    &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
                                   {&vars_new[lev][Vars::cons],
                                    &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                                    base_state[lev], base_state[lev], fillset);
    }
 
    // Get qmoist pointers if using moisture
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
    for (int lev = 0; lev <= finest_level; ++lev) {
        for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
            qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
        }
    }
 
    // Vector of MultiFabs for cell-centered data
    Vector<MultiFab> mf(finest_level+1);
    for (int lev = 0; lev <= finest_level; ++lev) {
        mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
    }
 
    // Vector of MultiFabs for nodal data
    Vector<MultiFab> mf_nd(finest_level+1);
    if ( SolverChoice::mesh_type != MeshType::ConstantDz) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
            mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
            mf_nd[lev].setVal(0.);
        }
    }
 
    // Vector of MultiFabs for face-centered velocity
    Vector<MultiFab> mf_u(finest_level+1);
    Vector<MultiFab> mf_v(finest_level+1);
    Vector<MultiFab> mf_w(finest_level+1);
    if (m_plot_face_vels) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            BoxArray grid_stag_u(grids[lev]); grid_stag_u.surroundingNodes(0);
            BoxArray grid_stag_v(grids[lev]); grid_stag_v.surroundingNodes(1);
            BoxArray grid_stag_w(grids[lev]); grid_stag_w.surroundingNodes(2);
            mf_u[lev].define(grid_stag_u, dmap[lev], 1, 0);
            mf_v[lev].define(grid_stag_v, dmap[lev], 1, 0);
            mf_w[lev].define(grid_stag_w, dmap[lev], 1, 0);
            MultiFab::Copy(mf_u[lev],vars_new[lev][Vars::xvel],0,0,1,0);
            MultiFab::Copy(mf_v[lev],vars_new[lev][Vars::yvel],0,0,1,0);
            MultiFab::Copy(mf_w[lev],vars_new[lev][Vars::zvel],0,0,1,0);
        }
    }
 
    // Array of MultiFabs for cell-centered velocity
    Vector<MultiFab> mf_cc_vel(finest_level+1);
 
    if (containerHasElement(plot_var_names, "x_velocity" ) ||
        containerHasElement(plot_var_names, "y_velocity" ) ||
        containerHasElement(plot_var_names, "z_velocity" ) ||
        containerHasElement(plot_var_names, "magvel"     ) ||
        containerHasElement(plot_var_names, "vorticity_x") ||
        containerHasElement(plot_var_names, "vorticity_y") ||
        containerHasElement(plot_var_names, "vorticity_z") ) {
 
        for (int lev = 0; lev <= finest_level; ++lev) {
            mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
            average_face_to_cellcenter(mf_cc_vel[lev],0,
                                       Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                                &vars_new[lev][Vars::yvel],
                                                                &vars_new[lev][Vars::zvel]});
        } // lev
    } // if (vel or vort)
 
    // We need ghost cells if computing vorticity
    if ( containerHasElement(plot_var_names, "vorticity_x")||
         containerHasElement(plot_var_names, "vorticity_y") ||
         containerHasElement(plot_var_names, "vorticity_z") )
    {
        amrex::Interpolater* mapper = &cell_cons_interp;
        for (int lev = 1; lev <= finest_level; ++lev)
        {
            Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
            Vector<Real> ftime    = {t_new[lev], t_new[lev]};
            Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
            Vector<Real> ctime    = {t_new[lev], t_new[lev]};
 
            amrex::FillPatchTwoLevels(mf_cc_vel[lev], t_new[lev], cmf, ctime, fmf, ftime,
                                      0, 0, AMREX_SPACEDIM, geom[lev-1], geom[lev],
                                      null_bc_for_fill, 0, null_bc_for_fill, 0, refRatio(lev-1),
                                      mapper, domain_bcs_type, 0);
        } // lev
 
        // Impose bc's at domain boundaries at all levels
        FillBdyCCVels(mf_cc_vel);
    } // if (vort)
 
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        int mf_comp = 0;
 
        // First, copy any of the conserved state variables into the output plotfile
        for (int i = 0; i < cons_names.size(); ++i) {
            if (containerHasElement(plot_var_names, cons_names[i])) {
                MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,0);
                mf_comp++;
            }
        }
 
        // Next, check for velocities
        if (containerHasElement(plot_var_names, "x_velocity")) {
            MultiFab::Copy(mf[lev], mf_cc_vel[lev], 0, mf_comp, 1, 0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "y_velocity")) {
            MultiFab::Copy(mf[lev], mf_cc_vel[lev], 1, mf_comp, 1, 0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "z_velocity")) {
            MultiFab::Copy(mf[lev], mf_cc_vel[lev], 2, mf_comp, 1, 0);
            mf_comp += 1;
        }
 
        // Finally, check for any derived quantities and compute them, inserting
        // them into our output multifab
        auto calculate_derived = [&](const std::string& der_name,
                                     MultiFab& src_mf,
                                     decltype(derived::erf_dernull)& der_function)
        {
            if (containerHasElement(plot_var_names, der_name)) {
                MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    auto& dfab = dmf[mfi];
                    auto& sfab = src_mf[mfi];
                    der_function(bx, dfab, 0, 1, sfab, Geom(lev), t_new[0], nullptr, lev);
                }
 
                mf_comp++;
            }
        };
 
        bool ismoist = (solverChoice.moisture_type != MoistureType::None);
 
        // Note: All derived variables must be computed in order of "derived_names" defined in ERF.H
        calculate_derived("soundspeed",  vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
        if (ismoist) {
            calculate_derived("temp",        vars_new[lev][Vars::cons], derived::erf_dermoisttemp);
        } else {
            calculate_derived("temp",        vars_new[lev][Vars::cons], derived::erf_dertemp);
        }
        calculate_derived("theta",       vars_new[lev][Vars::cons], derived::erf_dertheta);
        calculate_derived("KE",          vars_new[lev][Vars::cons], derived::erf_derKE);
        calculate_derived("scalar",      vars_new[lev][Vars::cons], derived::erf_derscalar);
        calculate_derived("vorticity_x", mf_cc_vel[lev]           , derived::erf_dervortx);
        calculate_derived("vorticity_y", mf_cc_vel[lev]           , derived::erf_dervorty);
        calculate_derived("vorticity_z", mf_cc_vel[lev]           , derived::erf_dervortz);
        calculate_derived("magvel"     , mf_cc_vel[lev]           , derived::erf_dermagvel);
 
        if (containerHasElement(plot_var_names, "divU"))
        {
            MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
            Array<MultiFab const*, AMREX_SPACEDIM> u;
            u[0] = &(vars_new[lev][Vars::xvel]);
            u[1] = &(vars_new[lev][Vars::yvel]);
            u[2] = &(vars_new[lev][Vars::zvel]);
            compute_divergence (lev, dmf, u, geom[lev]);
            mf_comp += 1;
        }
 
        MultiFab  r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
        MultiFab  p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
        MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
        if (containerHasElement(plot_var_names, "pres_hse"))
        {
            MultiFab::Copy(mf[lev],p_hse,0,mf_comp,1,0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "dens_hse"))
        {
            MultiFab::Copy(mf[lev],r_hse,0,mf_comp,1,0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "theta_hse"))
        {
            MultiFab::Copy(mf[lev],th_hse,0,mf_comp,1,0);
            mf_comp += 1;
        }
 
        if (containerHasElement(plot_var_names, "pressure"))
        {
            if (solverChoice.anelastic[lev] == 1) {
                MultiFab::Copy(mf[lev], p_hse, 0, mf_comp, 1, 0);
            } else
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real      >& derdat = mf[lev].array(mfi);
                    const Array4<Real const>&  S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                    const int ncomp = vars_new[lev][Vars::cons].nComp();
 
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
                        const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                        derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p);
                    });
               }
            } // not anelastic
            mf_comp += 1;
        } // pressure
 
        if (containerHasElement(plot_var_names, "pert_pres"))
        {
            if (solverChoice.anelastic[lev] == 1) {
                MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0);
            } else
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
                    const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                    const int ncomp = vars_new[lev][Vars::cons].nComp();
 
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
                        const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                        derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p) - p0_arr(i,j,k);
                    });
                }
            } // not anelastic
            mf_comp += 1;
        }
 
        if (containerHasElement(plot_var_names, "pert_dens"))
        {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf[lev].array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
                });
            }
            mf_comp ++;
        }
 
        if (containerHasElement(plot_var_names, "eq_pot_temp"))
        {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf[lev].array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                const int ncomp = vars_new[lev][Vars::cons].nComp();
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
                    Real qc = (use_moisture && (ncomp > RhoQ2_comp)) ? S_arr(i,j,k,RhoQ2_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
                    Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                    Real pressure = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp), qv);
                    Real fac = Cp_d + Cp_l*(qv + qc);
                    Real pv = erf_esatw(T)*100.0;
 
                    derdat(i, j, k, mf_comp) = T*std::pow((pressure - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
                });
            }
            mf_comp ++;
        }
 
        if (containerHasElement(plot_var_names, "terrain_IB_mask"))
        {
            MultiFab* terrain_blank = m_terrain_drag[lev]->get_terrain_blank_field();
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf[lev].array(mfi);
                const Array4<Real const>& src = terrain_blank->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i, j, k, mf_comp) = src(i,j,k);
                });
            }
            mf_comp ++;
        }
 
#ifdef ERF_USE_WINDFARM
        if (containerHasElement(plot_var_names, "num_turb") and
            (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
            solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD))
        {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf[lev].array(mfi);
                const Array4<Real const>& Nturb_array = Nturb[lev].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i, j, k, mf_comp) = Nturb_array(i,j,k,0);
                });
            }
            mf_comp ++;
        }
 
        if( containerHasElement(plot_var_names, "SMark0") and
           (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
            solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) ) {
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf[lev].array(mfi);
                const Array4<Real const>& SMark_array = SMark[lev].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i, j, k, mf_comp) = SMark_array(i,j,k,0);
                });
            }
            mf_comp ++;
        }
 
         if(containerHasElement(plot_var_names, "SMark1") and
           (solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD)) {
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf[lev].array(mfi);
                const Array4<Real const>& SMark_array = SMark[lev].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i, j, k, mf_comp) = SMark_array(i,j,k,1);
                });
            }
            mf_comp ++;
        }
 
#endif
 
        int klo = geom[lev].Domain().smallEnd(2);
        int khi = geom[lev].Domain().bigEnd(2);
 
        if (containerHasElement(plot_var_names, "dpdx"))
        {
            auto dxInv = geom[lev].InvCellSizeArray();
            MultiFab pres(vars_new[lev][Vars::cons].boxArray(), vars_new[lev][Vars::cons].DistributionMap(), 1, 1);
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // First define pressure on grown box
                const Box& gbx = mfi.growntilebox(1);
                const Array4<Real      > &   p_arr  = pres.array(mfi);
                const Array4<Real const> & hse_arr  = base_state[lev].const_array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                if (solverChoice.anelastic[lev] == 1) {
                    ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                        p_arr(i,j,k) = hse_arr(i,j,k,1);
                    });
                } else {
                    ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                        p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp));
                    });
                }
            }
            pres.FillBoundary(geom[lev].periodicity());
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // Now compute pressure gradient on valid box
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf[lev].array(mfi);
                const Array4<Real> & p_arr  = pres.array(mfi);
 
                if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                    const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
 
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
 
                        // Pgrad at lower I face
                        Real met_h_xi_lo   = Compute_h_xi_AtIface  (i, j, k, dxInv, z_nd);
                        Real met_h_zeta_lo = Compute_h_zeta_AtIface(i, j, k, dxInv, z_nd);
                        Real gp_xi_lo = dxInv[0] * (p_arr(i,j,k) - p_arr(i-1,j,k));
                        Real gp_zeta_on_iface_lo;
                        if(k == klo) {
                            gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
                                p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
                              - p_arr(i-1,j,k  ) - p_arr(i,j,k  ) );
                        } else if (k == khi) {
                            gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
                                p_arr(i-1,j,k  ) + p_arr(i,j,k  )
                              - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
                        } else {
                            gp_zeta_on_iface_lo = 0.25 * dxInv[2] * (
                                p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
                              - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
                        }
                        Real gpx_lo = gp_xi_lo - (met_h_xi_lo/ met_h_zeta_lo) * gp_zeta_on_iface_lo;
 
                        // Pgrad at higher I face
                        Real met_h_xi_hi   = Compute_h_xi_AtIface  (i+1, j, k, dxInv, z_nd);
                        Real met_h_zeta_hi = Compute_h_zeta_AtIface(i+1, j, k, dxInv, z_nd);
                        Real gp_xi_hi = dxInv[0] * (p_arr(i+1,j,k) - p_arr(i,j,k));
                        Real gp_zeta_on_iface_hi;
                        if(k == klo) {
                            gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
                                p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
                                - p_arr(i+1,j,k  ) - p_arr(i,j,k  ) );
                        } else if (k == khi) {
                            gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
                                p_arr(i+1,j,k  ) + p_arr(i,j,k  )
                                - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
                        } else {
                            gp_zeta_on_iface_hi = 0.25 * dxInv[2] * (
                                p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
                                - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
                        }
                        Real gpx_hi = gp_xi_hi - (met_h_xi_hi/ met_h_zeta_hi) * gp_zeta_on_iface_hi;
 
                        // Average P grad to CC
                        derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_lo + gpx_hi);
                    });
                } else {
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i ,j ,k, mf_comp) = 0.5 * (p_arr(i+1,j,k) - p_arr(i-1,j,k)) * dxInv[0];
                    });
                }
            } // mfi
            mf_comp ++;
        } // dpdx
 
        if (containerHasElement(plot_var_names, "dpdy"))
        {
            auto dxInv = geom[lev].InvCellSizeArray();
 
            MultiFab pres(vars_new[lev][Vars::cons].boxArray(), vars_new[lev][Vars::cons].DistributionMap(), 1, 1);
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // First define pressure on grown box
                const Box& gbx = mfi.growntilebox(1);
                const Array4<Real      > & p_arr    = pres.array(mfi);
                const Array4<Real const> & hse_arr  = base_state[lev].const_array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                if (solverChoice.anelastic[lev] == 1) {
                    ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                        p_arr(i,j,k) = hse_arr(i,j,k,1);
                    });
                } else {
                    ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                        p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp));
                    });
                }
            }
            pres.FillBoundary(geom[lev].periodicity());
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // Now compute pressure gradient on valid box
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf[lev].array(mfi);
                const Array4<Real> & p_arr  = pres.array(mfi);
 
                if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                    const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
 
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
 
                        Real met_h_eta_lo  = Compute_h_eta_AtJface (i, j, k, dxInv, z_nd);
                        Real met_h_zeta_lo = Compute_h_zeta_AtJface(i, j, k, dxInv, z_nd);
                        Real gp_eta_lo = dxInv[1] * (p_arr(i,j,k) - p_arr(i,j-1,k));
                        Real gp_zeta_on_jface_lo;
                        if (k == klo) {
                            gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
                                p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
                                - p_arr(i,j,k  ) - p_arr(i,j-1,k  ) );
                        } else if (k == khi) {
                            gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
                                p_arr(i,j,k  ) + p_arr(i,j-1,k  )
                                - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
                        } else {
                            gp_zeta_on_jface_lo = 0.25 * dxInv[2] * (
                                p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
                                - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
                        }
                        Real gpy_lo = gp_eta_lo - (met_h_eta_lo / met_h_zeta_lo) * gp_zeta_on_jface_lo;
 
                        Real met_h_eta_hi  = Compute_h_eta_AtJface (i, j+1, k, dxInv, z_nd);
                        Real met_h_zeta_hi = Compute_h_zeta_AtJface(i, j+1, k, dxInv, z_nd);
                        Real gp_eta_hi = dxInv[1] * (p_arr(i,j+1,k) - p_arr(i,j,k));
                        Real gp_zeta_on_jface_hi;
                        if (k == klo) {
                            gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
                                p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
                                - p_arr(i,j+1,k  ) - p_arr(i,j,k  ) );
                        } else if (k == khi) {
                            gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
                                p_arr(i,j+1,k  ) + p_arr(i,j,k  )
                                - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
                        } else {
                            gp_zeta_on_jface_hi = 0.25 * dxInv[2] * (
                                p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
                                - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
                        }
                        Real gpy_hi = gp_eta_hi - (met_h_eta_hi / met_h_zeta_hi) * gp_zeta_on_jface_hi;
 
                        derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_lo + gpy_hi);
                    });
                } else {
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i ,j ,k, mf_comp) = 0.5 * (p_arr(i,j+1,k) - p_arr(i,j-1,k)) * dxInv[1];
                    });
                }
            } // mf
            mf_comp ++;
        } // dpdy
 
        if (containerHasElement(plot_var_names, "pres_hse_x"))
        {
            auto dxInv = geom[lev].InvCellSizeArray();
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real      >&  derdat = mf[lev].array(mfi);
                const Array4<Real const>&   p_arr = p_hse.const_array(mfi);
 
                //USE_TERRAIN POSSIBLE ISSUE HERE
                const Array4<Real const>& z_nd  = z_phys_nd[lev]->const_array(mfi);
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    Real met_h_xi_lo   = Compute_h_xi_AtIface  (i, j, k, dxInv, z_nd);
                    Real met_h_zeta_lo = Compute_h_zeta_AtIface(i, j, k, dxInv, z_nd);
                    Real gp_xi_lo = dxInv[0] * (p_arr(i,j,k) - p_arr(i-1,j,k));
                    Real gp_zeta_on_iface_lo;
                    if (k == klo) {
                      gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
                                                              p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
                                                            - p_arr(i-1,j,k  ) - p_arr(i,j,k  ) );
                    } else if (k == khi) {
                      gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
                                                              p_arr(i-1,j,k  ) + p_arr(i,j,k  )
                                                            - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
                    } else {
                      gp_zeta_on_iface_lo = 0.25 * dxInv[2] * (
                                                               p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
                                                             - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
                    }
                    Real gpx_lo = gp_xi_lo - (met_h_xi_lo/ met_h_zeta_lo) * gp_zeta_on_iface_lo;
 
                    Real met_h_xi_hi   = Compute_h_xi_AtIface  (i+1, j, k, dxInv, z_nd);
                    Real met_h_zeta_hi = Compute_h_zeta_AtIface(i+1, j, k, dxInv, z_nd);
                    Real gp_xi_hi = dxInv[0] * (p_arr(i+1,j,k) - p_arr(i,j,k));
                    Real gp_zeta_on_iface_hi;
                    if (k == klo) {
                      gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
                                                              p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
                                                            - p_arr(i+1,j,k  ) - p_arr(i,j,k  ) );
                    } else if (k == khi) {
                      gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
                                                              p_arr(i+1,j,k  ) + p_arr(i,j,k  )
                                                            - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
                    } else {
                      gp_zeta_on_iface_hi = 0.25 * dxInv[2] * (
                                                               p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
                                                             - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
                    }
                    Real gpx_hi = gp_xi_hi - (met_h_xi_hi/ met_h_zeta_hi) * gp_zeta_on_iface_hi;
 
                    derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_lo + gpx_hi);
                });
            }
            mf_comp += 1;
        } // pres_hse_x
 
        if (containerHasElement(plot_var_names, "pres_hse_y"))
        {
            auto dxInv = geom[lev].InvCellSizeArray();
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real      >& derdat = mf[lev].array(mfi);
                const Array4<Real const>&   p_arr = p_hse.const_array(mfi);
                const Array4<Real const>& z_nd    = z_phys_nd[lev]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    Real met_h_eta_lo  = Compute_h_eta_AtJface (i, j, k, dxInv, z_nd);
                    Real met_h_zeta_lo = Compute_h_zeta_AtJface(i, j, k, dxInv, z_nd);
                    Real gp_eta_lo = dxInv[1] * (p_arr(i,j,k) - p_arr(i,j-1,k));
                    Real gp_zeta_on_jface_lo;
                    if (k == klo) {
                      gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
                                                              p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
                                                            - p_arr(i,j,k  ) - p_arr(i,j-1,k  ) );
                    } else if (k == khi) {
                      gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
                                                              p_arr(i,j,k  ) + p_arr(i,j-1,k  )
                                                            - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
                    } else {
                      gp_zeta_on_jface_lo = 0.25 * dxInv[2] * (
                                                               p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
                                                             - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
                    }
                    Real gpy_lo = gp_eta_lo - (met_h_eta_lo / met_h_zeta_lo) * gp_zeta_on_jface_lo;
 
                    Real met_h_eta_hi  = Compute_h_eta_AtJface (i, j+1, k, dxInv, z_nd);
                    Real met_h_zeta_hi = Compute_h_zeta_AtJface(i, j+1, k, dxInv, z_nd);
                    Real gp_eta_hi = dxInv[1] * (p_arr(i,j+1,k) - p_arr(i,j,k));
                    Real gp_zeta_on_jface_hi;
                    if (k == klo) {
                      gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
                                                              p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
                                                            - p_arr(i,j+1,k  ) - p_arr(i,j,k  ) );
                    } else if (k == khi) {
                      gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
                                                              p_arr(i,j+1,k  ) + p_arr(i,j,k  )
                                                            - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
                    } else {
                      gp_zeta_on_jface_hi = 0.25 * dxInv[2] * (
                                                               p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
                                                             - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
                    }
                    Real gpy_hi = gp_eta_hi - (met_h_eta_hi / met_h_zeta_hi) * gp_zeta_on_jface_hi;
 
                    derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_lo + gpy_hi);
                });
            }
            mf_comp += 1;
        } // pres_hse_y
 
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            if (containerHasElement(plot_var_names, "z_phys"))
            {
                MultiFab::Copy(mf[lev],*z_phys_cc[lev],0,mf_comp,1,0);
                mf_comp ++;
            }
 
            if (containerHasElement(plot_var_names, "detJ"))
            {
                MultiFab::Copy(mf[lev],*detJ_cc[lev],0,mf_comp,1,0);
                mf_comp ++;
            }
        } // use_terrain
 
        if (containerHasElement(plot_var_names, "mapfac")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf[lev].array(mfi);
                const Array4<Real>& mf_m   = mapfac_m[lev]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                   derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
                });
            }
            mf_comp ++;
        }
 
#ifdef ERF_USE_NETCDF
        if (use_real_bcs) {
            if (containerHasElement(plot_var_names, "lat_m")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = lat_m[lev]->array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = data(i,j,0);
                    });
                }
                mf_comp ++;
            } // lat_m
            if (containerHasElement(plot_var_names, "lon_m")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = lon_m[lev]->array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = data(i,j,0);
                    });
                }
                mf_comp ++;
            } // lon_m
        } // use_real_bcs
#endif
 
 
        if (solverChoice.time_avg_vel) {
            if (containerHasElement(plot_var_names, "u_t_avg")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                    const Real norm = t_avg_cnt[lev];
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        derdat(i ,j ,k, mf_comp) = data(i,j,k,0) / norm;
                    });
                }
                mf_comp ++;
            }
 
            if (containerHasElement(plot_var_names, "v_t_avg")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                    const Real norm = t_avg_cnt[lev];
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        derdat(i ,j ,k, mf_comp) = data(i,j,k,1) / norm;
                    });
                }
                mf_comp ++;
            }
 
            if (containerHasElement(plot_var_names, "w_t_avg")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                    const Real norm = t_avg_cnt[lev];
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        derdat(i ,j ,k, mf_comp) = data(i,j,k,2) / norm;
                    });
                }
                mf_comp ++;
            }
 
            if (containerHasElement(plot_var_names, "umag_t_avg")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                    const Real norm = t_avg_cnt[lev];
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        derdat(i ,j ,k, mf_comp) = data(i,j,k,3) / norm;
                    });
                }
                mf_comp ++;
            }
        }
 
        if (containerHasElement(plot_var_names, "nut")) {
            MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
            MultiFab cmf(vars_new[lev][Vars::cons], make_alias, 0, 1); // to provide rho only
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                auto       prim = dmf[mfi].array();
                auto const cons = cmf[mfi].const_array();
                auto const diff = (*eddyDiffs_lev[lev])[mfi].const_array();
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    const Real rho = cons(i, j, k, Rho_comp);
                    const Real Kmv = diff(i, j, k, EddyDiff::Mom_v);
                    prim(i,j,k) = Kmv / rho;
                });
            }
 
            mf_comp++;
        }
 
        if (containerHasElement(plot_var_names, "Kmv")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_v,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "Kmh")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "Khv")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_v,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "Khh")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "Lturb")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Turb_lengthscale,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "walldist")) {
            MultiFab::Copy(mf[lev],*walldist[lev],0,mf_comp,1,0);
            mf_comp ++;
        }
 
        // TODO: The size of the q variables can vary with different
        //       moisture models. Therefore, certain components may
        //       reside at different indices. For example, Kessler is
        //       warm but precipitating. This puts qp at index 3.
        //       However, SAM is cold and precipitating so qp is index 4.
        //       Need to built an external enum struct or a better pathway.
 
        // NOTE: Protect against accessing non-existent data
        if (use_moisture) {
            int n_qstate   = micro->Get_Qstate_Size();
 
            // Non-precipitating components
            //--------------------------------------------------------------------------
            if(containerHasElement(plot_var_names, "qt"))
            {
                int n_start = RhoQ1_comp; // qv
                int n_end   = RhoQ2_comp; // qc
                if (n_qstate > 3) n_end = RhoQ3_comp; // qi
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
                for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
                    MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                }
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qv") && (n_qstate >= 1))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ1_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qc") && (n_qstate >= 2))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ2_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qi") && (n_qstate >= 4))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ3_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            // Precipitating components
            //--------------------------------------------------------------------------
            if(containerHasElement(plot_var_names, "qp") && (n_qstate >= 3))
            {
                int n_start = (n_qstate > 3) ? RhoQ4_comp : RhoQ3_comp;
                int n_end   = ncomp_cons - 1;
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
                for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
                    MultiFab::Add(  mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                }
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qrain") && (n_qstate >= 3))
            {
                int n_start = (n_qstate > 3) ? RhoQ4_comp : RhoQ3_comp;
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], n_start , mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qsnow") && (n_qstate >= 5))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ5_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons],   Rho_comp, mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qgraup") && (n_qstate >= 6))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ6_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons],   Rho_comp, mf_comp, 1, 0);
                mf_comp += 1;
            }
 
        if (containerHasElement(plot_var_names, "qsat"))
        {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf[lev].array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    Real       qv = S_arr(i,j,k,RhoQ1_comp) / S_arr(i,j,k,Rho_comp);
                    Real       T  = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                    Real pressure = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp), qv) * Real(0.01);
                    erf_qsatw(T, pressure, derdat(i,j,k,mf_comp));
                });
            }
            mf_comp ++;
        }
 
        if(solverChoice.moisture_type == MoistureType::Kessler){
            if (containerHasElement(plot_var_names, "rain_accum"))
            {
                MultiFab::Copy(mf[lev],*(qmoist[lev][4]),0,mf_comp,1,0);
                mf_comp += 1;
            }
        }
        else if(solverChoice.moisture_type == MoistureType::SAM)
        {
            if (containerHasElement(plot_var_names, "rain_accum"))
            {
                MultiFab::Copy(mf[lev],*(qmoist[lev][8]),0,mf_comp,1,0);
                mf_comp += 1;
            }
            if (containerHasElement(plot_var_names, "snow_accum"))
            {
                MultiFab::Copy(mf[lev],*(qmoist[lev][9]),0,mf_comp,1,0);
                mf_comp += 1;
            }
            if (containerHasElement(plot_var_names, "graup_accum"))
            {
                MultiFab::Copy(mf[lev],*(qmoist[lev][10]),0,mf_comp,1,0);
                mf_comp += 1;
            }
        }
        }
 
#ifdef ERF_USE_PARTICLES
        const auto& particles_namelist( particleData.getNames() );
        for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++) {
            if (containerHasElement(plot_var_names, std::string(particles_namelist[i]+"_count"))) {
                MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
                temp_dat.setVal(0);
                particleData[particles_namelist[i]]->Increment(temp_dat, lev);
                MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                mf_comp += 1;
            }
        }
 
        Vector<std::string> particle_mesh_plot_names(0);
        particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
        for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
            std::string plot_var_name(particle_mesh_plot_names[i]);
            if (containerHasElement(plot_var_names, plot_var_name) ) {
                MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
                temp_dat.setVal(0);
                particleData.GetMeshPlotVar(plot_var_name, temp_dat, lev);
                MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                mf_comp += 1;
            }
        }
#endif
 
#ifdef ERF_USE_EB
        if (containerHasElement(plot_var_names, "volfrac")) {
            MultiFab::Copy(mf[lev], EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
            mf_comp += 1;
        }
#endif
 
#ifdef ERF_COMPUTE_ERROR
        // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
        if (containerHasElement(plot_var_names, "xvel_err") ||
            containerHasElement(plot_var_names, "yvel_err") ||
            containerHasElement(plot_var_names, "zvel_err"))
        {
            //
            // Moving terrain ANALYTICAL
            //
            Real H           = geom[lev].ProbHi()[2];
            Real Ampl        = 0.16;
            Real wavelength  = 100.;
            Real kp          = 2. * PI / wavelength;
            Real g           = CONST_GRAV;
            Real omega       = std::sqrt(g * kp);
            Real omega_t     = omega * t_new[lev];
 
            const auto dx = geom[lev].CellSizeArray();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.validbox();
                Box xbx(bx); xbx.surroundingNodes(0);
                const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
                const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
 
                const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
 
                ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    Real x = i * dx[0];
                    Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
 
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
                    z -= z_base;
 
                    Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
 
                    xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
                });
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    Real x   = (i + 0.5) * dx[0];
                    Real z   = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
 
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
                    z -= z_base;
 
                    Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
 
                    zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
                });
            }
 
            MultiFab temp_mf(mf[lev].boxArray(), mf[lev].DistributionMap(), AMREX_SPACEDIM, 0);
            average_face_to_cellcenter(temp_mf,0,
                Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
 
            if (containerHasElement(plot_var_names, "xvel_err")) {
                MultiFab::Copy(mf[lev],temp_mf,0,mf_comp,1,0);
                mf_comp += 1;
            }
            if (containerHasElement(plot_var_names, "yvel_err")) {
                MultiFab::Copy(mf[lev],temp_mf,1,mf_comp,1,0);
                mf_comp += 1;
            }
            if (containerHasElement(plot_var_names, "zvel_err")) {
                MultiFab::Copy(mf[lev],temp_mf,2,mf_comp,1,0);
                mf_comp += 1;
            }
 
            // Now restore the velocities to what they were
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.validbox();
                Box xbx(bx); xbx.surroundingNodes(0);
 
                const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
                const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
 
                const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
 
                ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    Real x = i * dx[0];
                    Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
 
                    z -= z_base;
 
                    Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                    xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
                });
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    Real x   = (i + 0.5) * dx[0];
                    Real z   = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
 
                    z -= z_base;
                    Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
 
                    zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
                });
            }
        } // end xvel_err, yvel_err, zvel_err
 
        if (containerHasElement(plot_var_names, "pp_err"))
        {
            // Moving terrain ANALYTICAL
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf[lev].array(mfi);
                const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
 
                const auto dx = geom[lev].CellSizeArray();
                const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
                const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
 
                Real H           = geom[lev].ProbHi()[2];
                Real Ampl        = 0.16;
                Real wavelength  = 100.;
                Real kp          = 2. * PI / wavelength;
                Real g           = CONST_GRAV;
                Real omega       = std::sqrt(g * kp);
                Real omega_t     = omega * t_new[lev];
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                    derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta) - p0_arr(i,j,k);
 
                    Real rho_hse     = r0_arr(i,j,k);
 
                    Real x   = (i + 0.5) * dx[0];
                    Real z   = 0.125 * ( z_nd(i,j,k  ) + z_nd(i+1,j,k  ) + z_nd(i,j+1,k  ) + z_nd(i+1,j+1,k  )
                                        +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
 
                    z -= z_base;
                    Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                    Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
                                              std::sin(kp * x - omega_t) * r0_arr(i,j,k);
 
                    derdat(i,j,k,mf_comp) -= pprime_exact;
                });
            }
            mf_comp += 1;
        }
#endif
 
#ifdef ERF_USE_RRTMGP
    if (containerHasElement(plot_var_names, "qsrc_sw")) {
        MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 0, mf_comp, 1, 0);
        mf_comp += 1;
    }
    if (containerHasElement(plot_var_names, "qsrc_lw")) {
        MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 1, mf_comp, 1, 0);
        mf_comp += 1;
    }
#endif
    }
 
#ifdef ERF_USE_EB
    for (int lev = 0; lev <= finest_level; ++lev) {
        EB_set_covered(mf[lev], 0.0);
    }
#endif
 
    // Fill terrain distortion MF
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        for (int lev(0); lev <= finest_level; ++lev) {
            MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
            Real dz = Geom()[lev].CellSizeArray()[2];
            for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                Array4<      Real> mf_arr = mf_nd[lev].array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    mf_arr(i,j,k,2) -= k * dz;
                });
            }
        }
    }
 
    std::string plotfilename;
    std::string plotfilenameU;
    std::string plotfilenameV;
    std::string plotfilenameW;
    if (which == 1) {
       plotfilename = Concatenate(plot_file_1, istep[0], 5);
       plotfilenameU = Concatenate(plot_file_1+"U", istep[0], 5);
       plotfilenameV = Concatenate(plot_file_1+"V", istep[0], 5);
       plotfilenameW = Concatenate(plot_file_1+"W", istep[0], 5);
    } else if (which == 2) {
       plotfilename = Concatenate(plot_file_2, istep[0], 5);
       plotfilenameU = Concatenate(plot_file_2+"U", istep[0], 5);
       plotfilenameV = Concatenate(plot_file_2+"V", istep[0], 5);
       plotfilenameW = Concatenate(plot_file_2+"W", istep[0], 5);
    }
 
    // LSM writes it's own data
    if (which==1 && plot_lsm) {
        lsm.Plot_Lsm_Data(t_new[0], istep, refRatio());
    }
 
#ifdef ERF_USE_RRTMGP
    // write additional RRTMGP data
    // TODO: currently single level only
    if (which==1 && plot_rad) {
        rad.writePlotfile(plot_file_1, t_new[0], istep[0]);
    }
#endif
 
    // Single level
    if (finest_level == 0)
    {
        if (plotfile_type == PlotFileType::Amrex)
        {
            Print() << "Writing native plotfile " << plotfilename << "\n";
            if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                   GetVecOfConstPtrs(mf),
                                                   GetVecOfConstPtrs(mf_nd),
                                                   varnames,
                                                   Geom(), t_new[0], istep, refRatio());
            } else {
                WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                        GetVecOfConstPtrs(mf),
                                        varnames,
                                        Geom(), t_new[0], istep, refRatio());
            }
            writeJobInfo(plotfilename);
 
            if (m_plot_face_vels) {
                Print() << "Writing face velocities" << std::endl;
                WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
                                        GetVecOfConstPtrs(mf_u),
                                        {"x_velocity_stag"},
                                        Geom(), t_new[0], istep, refRatio());
                WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
                                        GetVecOfConstPtrs(mf_v),
                                        {"y_velocity_stag"},
                                        Geom(), t_new[0], istep, refRatio());
                WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
                                        GetVecOfConstPtrs(mf_w),
                                        {"z_velocity_stag"},
                                        Geom(), t_new[0], istep, refRatio());
            }
 
#ifdef ERF_USE_PARTICLES
            particleData.writePlotFile(plotfilename);
#endif
#ifdef ERF_USE_NETCDF
        } else if (plotfile_type == PlotFileType::Netcdf) {
             int lev   = 0;
             int l_which = 0;
             writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep, t_new[0]);
#endif
        } else {
            // Here we assume the plotfile_type is PlotFileType::None
            Print() << "Writing no plotfile since plotfile_type is none" << std::endl;
        }
 
    } else { // Multilevel
 
        if (plotfile_type == PlotFileType::Amrex) {
 
            int lev0 = 0;
            int desired_ratio = std::max(std::max(ref_ratio[lev0][0],ref_ratio[lev0][1]),ref_ratio[lev0][2]);
            bool any_ratio_one = ( ( (ref_ratio[lev0][0] == 1) || (ref_ratio[lev0][1] == 1) ) ||
                                     (ref_ratio[lev0][2] == 1) );
            for (int lev = 1; lev < finest_level; lev++) {
                any_ratio_one = any_ratio_one ||
                                     ( ( (ref_ratio[lev][0] == 1) || (ref_ratio[lev][1] == 1) ) ||
                                         (ref_ratio[lev][2] == 1) );
            }
 
            if (any_ratio_one && m_expand_plotvars_to_unif_rr)
            {
                Vector<IntVect>   r2(finest_level);
                Vector<Geometry>  g2(finest_level+1);
                Vector<MultiFab> mf2(finest_level+1);
 
                mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
 
                // Copy level 0 as is
                MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
 
                // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
                Array<int,AMREX_SPACEDIM> periodicity =
                             {Geom()[lev0].isPeriodic(0),Geom()[lev0].isPeriodic(1),Geom()[lev0].isPeriodic(2)};
                g2[lev0].define(Geom()[lev0].Domain(),&(Geom()[lev0].ProbDomain()),0,periodicity.data());
 
                r2[0] = IntVect(desired_ratio/ref_ratio[lev0][0],
                                desired_ratio/ref_ratio[lev0][1],
                                desired_ratio/ref_ratio[lev0][2]);
 
                for (int lev = 1; lev <= finest_level; ++lev) {
                    if (lev > 1) {
                        r2[lev-1][0] = r2[lev-2][0] * desired_ratio / ref_ratio[lev-1][0];
                        r2[lev-1][1] = r2[lev-2][1] * desired_ratio / ref_ratio[lev-1][1];
                        r2[lev-1][2] = r2[lev-2][2] * desired_ratio / ref_ratio[lev-1][2];
                    }
 
                    mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
 
                    // Set the new problem domain
                    Box d2(Geom()[lev].Domain());
                    d2.refine(r2[lev-1]);
 
                    g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
                }
 
                // Do piecewise interpolation of mf into mf2
                for (int lev = 1; lev <= finest_level; ++lev) {
                    Interpolater* mapper_c = &pc_interp;
                    InterpFromCoarseLevel(mf2[lev], t_new[lev], mf[lev],
                                          0, 0, ncomp_mf,
                                          geom[lev], g2[lev],
                                          null_bc_for_fill, 0, null_bc_for_fill, 0,
                                          r2[lev-1], mapper_c, domain_bcs_type, 0);
                }
 
                // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
                Vector<IntVect> rr(finest_level);
                for (int lev = 0; lev < finest_level; ++lev) {
                    rr[lev] = IntVect(desired_ratio);
                }
 
               Print() << "Writing plotfile " << plotfilename << "\n";
               if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                   WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                      GetVecOfConstPtrs(mf2),
                                                      GetVecOfConstPtrs(mf_nd),
                                                      varnames,
                                                      g2, t_new[0], istep, rr);
               } else {
                   WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                           GetVecOfConstPtrs(mf2), varnames,
                                           g2, t_new[0], istep, rr);
               }
 
            } else {
               if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                    WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                       GetVecOfConstPtrs(mf),
                                                       GetVecOfConstPtrs(mf_nd),
                                                       varnames,
                                                       geom, t_new[0], istep, ref_ratio);
                } else {
                    WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                            GetVecOfConstPtrs(mf), varnames,
                                            geom, t_new[0], istep, ref_ratio);
                }
                if (m_plot_face_vels) {
                    Print() << "Writing face velocities" << std::endl;
                    WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
                                            GetVecOfConstPtrs(mf_u),
                                            {"x_velocity_stag"},
                                            geom, t_new[0], istep, ref_ratio);
                    WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
                                            GetVecOfConstPtrs(mf_v),
                                            {"y_velocity_stag"},
                                            geom, t_new[0], istep, ref_ratio);
                    WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
                                            GetVecOfConstPtrs(mf_w),
                                            {"z_velocity_stag"},
                                            geom, t_new[0], istep, ref_ratio);
                }
            } // ref_ratio test
 
            writeJobInfo(plotfilename);
 
#ifdef ERF_USE_PARTICLES
            particleData.writePlotFile(plotfilename);
#endif
 
#ifdef ERF_USE_NETCDF
        } else if (plotfile_type == PlotFileType::Netcdf) {
             for (int lev = 0; lev <= finest_level; ++lev) {
                 for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
                     writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep, t_new[0]);
                 }
             }
#endif
        }
    } // end multi-level
}

Here is the call graph for this function:

Member Data Documentation

advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg

private

Referenced by getAdvFluxReg().

ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax

private

ax_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_new

private

ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src

private

ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay

private

ay_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_new

private

ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src

private

az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az

private

az_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_new

private

az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src

private

base_state

amrex::Vector<amrex::MultiFab> ERF::base_state

private

base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new

private

bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1

staticprivate

bndry_output_planes_per

Real ERF::bndry_output_planes_per = -1.0

staticprivate

bndry_output_planes_start_time

Real ERF::bndry_output_planes_start_time = 0.0

staticprivate

boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level

private

cf_set_width

int ERF::cf_set_width {0}

private

cf_width

int ERF::cf_width {0}

private

cfl

Real ERF::cfl = 0.8

staticprivate

change_max

Real ERF::change_max = 1.1

staticprivate

check_file

std::string ERF::check_file {"chk"}

private

check_type

std::string ERF::check_type {"native"}

private

column_file_name

std::string ERF::column_file_name = "column_data.nc"

staticprivate

column_interval

int ERF::column_interval = -1

staticprivate

column_loc_x

Real ERF::column_loc_x = 0.0

staticprivate

column_loc_y

Real ERF::column_loc_y = 0.0

staticprivate

column_per

Real ERF::column_per = -1.0

staticprivate

cons_names

const amrex::Vector<std::string> ERF::cons_names

private

Initial value:

{"density", "rhotheta", "rhoKE", "rhoadv_0",
                                                     "rhoQ1", "rhoQ2", "rhoQ3",
                                                     "rhoQ4", "rhoQ5", "rhoQ6"}

d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density

private

d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure

private

d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc

private

d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv

private

d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature

private

d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs

private

d_rhoqt_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhoqt_src

private

d_rhotheta_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhotheta_src

private

d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs

private

d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos

private

d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos

private

d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid

private

data_sampler

std::unique_ptr<SampleData> ERF::data_sampler = nullptr

private

datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog

private

Referenced by DataLog(), NumDataLogs(), and setRecordDataInfo().

datalogname

amrex::Vector<std::string> ERF::datalogname

private

Referenced by DataLogName().

derived_names

const amrex::Vector<std::string> ERF::derived_names

private

destag_profiles

bool ERF::destag_profiles = true

private

detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc

private

detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new

private

detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src

private

domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type

private

domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type

private

domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d

private

dt

amrex::Vector<amrex::Real> ERF::dt

private

dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio

private

dz_min

amrex::Vector<amrex::Real> ERF::dz_min

private

eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev

private

fine_mask

amrex::MultiFab ERF::fine_mask

private

finished_wave

bool ERF::finished_wave = false

private

fixed_dt

amrex::Vector<amrex::Real> ERF::fixed_dt

private

fixed_fast_dt

amrex::Vector<amrex::Real> ERF::fixed_fast_dt

private

fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0

staticprivate

FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c

private

FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u

private

FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v

private

FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w

private

h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density

private

h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure

private

h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc

private

h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv

private

h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature

private

h_rayleigh_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs

private

h_rhoqt_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhoqt_src

private

h_rhotheta_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhotheta_src

private

h_sponge_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs

private

h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos

private

h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos

private

h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid

private

Hwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave

private

Hwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave_onegrid

private

init_shrink

Real ERF::init_shrink = 1.0

staticprivate

init_sounding_ideal

bool ERF::init_sounding_ideal = false

staticprivate

init_type

InitType ERF::init_type

staticprivate

input_bndry_planes

int ERF::input_bndry_planes = 0

staticprivate

input_sounding_data

InputSoundingData ERF::input_sounding_data

private

input_sponge_data

InputSpongeData ERF::input_sponge_data

private

interpolation_type

StateInterpType ERF::interpolation_type

staticprivate

istep

amrex::Vector<int> ERF::istep

private

last_check_file_step

int ERF::last_check_file_step

private

last_plot_file_step_1

int ERF::last_plot_file_step_1

private

last_plot_file_step_2

int ERF::last_plot_file_step_2

private

lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev

private

lsm

LandSurface ERF::lsm

private

lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data

private

lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux

private

Lwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave

private

Lwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave_onegrid

private

m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_extdir_vals

private

m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_neumann_vals

private

m_check_int

int ERF::m_check_int = -1

private

m_check_per

amrex::Real ERF::m_check_per = -1.0

private

m_expand_plotvars_to_unif_rr

bool ERF::m_expand_plotvars_to_unif_rr = false

private

m_factory

amrex::Vector<std::unique_ptr<amrex::FabFactory<amrex::FArrayBox> > > ERF::m_factory

private

m_forest_drag

amrex::Vector<std::unique_ptr<ForestDrag> > ERF::m_forest_drag

private

m_most

std::unique_ptr<ABLMost> ERF::m_most = nullptr

private

m_plot_face_vels

bool ERF::m_plot_face_vels = false

private

m_plot_int_1

int ERF::m_plot_int_1 = -1

private

m_plot_int_2

int ERF::m_plot_int_2 = -1

private

m_plot_per_1

amrex::Real ERF::m_plot_per_1 = -1.0

private

m_plot_per_2

amrex::Real ERF::m_plot_per_2 = -1.0

private

m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr

private

m_terrain_drag

amrex::Vector<std::unique_ptr<TerrainDrag> > ERF::m_terrain_drag

private

m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr

private

mapfac_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mapfac_m

private

mapfac_u

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mapfac_u

private

mapfac_v

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mapfac_v

private

max_step

int ERF::max_step = std::numeric_limits<int>::max()

private

metgrid_basic_linear

bool ERF::metgrid_basic_linear {false}

private

metgrid_debug_dry

bool ERF::metgrid_debug_dry {false}

private

metgrid_debug_isothermal

bool ERF::metgrid_debug_isothermal {false}

private

metgrid_debug_msf

bool ERF::metgrid_debug_msf {false}

private

metgrid_debug_psfc

bool ERF::metgrid_debug_psfc {false}

private

metgrid_debug_quiescent

bool ERF::metgrid_debug_quiescent {false}

private

metgrid_force_sfc_k

int ERF::metgrid_force_sfc_k {0}

private

metgrid_interp_theta

bool ERF::metgrid_interp_theta {false}

private

metgrid_order

int ERF::metgrid_order {2}

private

metgrid_proximity

amrex::Real ERF::metgrid_proximity {1000.0}

private

metgrid_retain_sfc

bool ERF::metgrid_retain_sfc {false}

private

metgrid_use_below_sfc

bool ERF::metgrid_use_below_sfc {true}

private

metgrid_use_sfc

bool ERF::metgrid_use_sfc {true}

private

mg_verbose

int ERF::mg_verbose = 0

staticprivate

micro

std::unique_ptr<Microphysics> ERF::micro

private

mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem

private

nc_bdy_file

std::string ERF::nc_bdy_file

staticprivate

nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}

staticprivate

ng_dens_hse

int ERF::ng_dens_hse

staticprivate

ng_pres_hse

int ERF::ng_pres_hse

staticprivate

nsubsteps

amrex::Vector<int> ERF::nsubsteps

private

num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level

private

num_files_at_level

amrex::Vector<int> ERF::num_files_at_level

private

output_1d_column

int ERF::output_1d_column = 0

staticprivate

output_bndry_planes

int ERF::output_bndry_planes = 0

staticprivate

pert_interval

int ERF::pert_interval = -1

staticprivate

phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type

private

physbcs_base

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_base> > ERF::physbcs_base

private

physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons

private

physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u

private

physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v

private

physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w

private

plot_file_1

std::string ERF::plot_file_1 {"plt_1_"}

private

plot_file_2

std::string ERF::plot_file_2 {"plt_2_"}

private

plot_file_on_restart

int ERF::plot_file_on_restart = 1

private

plot_lsm

bool ERF::plot_lsm = false

private

plot_var_names_1

amrex::Vector<std::string> ERF::plot_var_names_1

private

plot_var_names_2

amrex::Vector<std::string> ERF::plot_var_names_2

private

plotfile_type_1

PlotFileType ERF::plotfile_type_1 = PlotFileType::None

staticprivate

plotfile_type_2

PlotFileType ERF::plotfile_type_2 = PlotFileType::None

staticprivate

pp_inc

amrex::Vector<amrex::MultiFab> ERF::pp_inc

private

pp_prefix

std::string ERF::pp_prefix {"erf"}

previousCPUTimeUsed

Real ERF::previousCPUTimeUsed = 0.0

staticprivate

Referenced by getCPUTime().

prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr

private

profile_int

int ERF::profile_int = -1

private

qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist

private

Qr_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qr_prim

private

Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim

private

real_set_width

int ERF::real_set_width {0}

private

real_width

int ERF::real_width {0}

private

ref_tags

Vector< AMRErrorTag > ERF::ref_tags

staticprivate

regrid_int

int ERF::regrid_int = -1

private

restart_chkfile

std::string ERF::restart_chkfile = ""

private

restart_type

std::string ERF::restart_type {"native"}

private

rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new

private

rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old

private

rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new

private

rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old

private

rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new

private

rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old

private

sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline

private

Referenced by NumSampleLines(), and SampleLine().

samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog

private

Referenced by NumSampleLineLogs(), SampleLineLog(), and setRecordSampleLineInfo().

samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname

private

Referenced by SampleLineLogName().

samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint

private

Referenced by NumSamplePoints(), and SamplePoint().

sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog

private

Referenced by NumSamplePointLogs(), SamplePointLog(), and setRecordSamplePointInfo().

sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname

private

Referenced by SamplePointLogName().

sampler_interval

int ERF::sampler_interval = -1

private

sampler_per

amrex::Real ERF::sampler_per = -1.0

private

SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev

private

SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev

private

SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev

private

SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev

private

SFS_q1fx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx1_lev

private

SFS_q1fx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx2_lev

private

SFS_q1fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx3_lev

private

SFS_q2fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q2fx3_lev

private

SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev

private

solverChoice

SolverChoice ERF::solverChoice

staticprivate

sponge_type

std::string ERF::sponge_type

staticprivate

sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev

private

start_time

amrex::Real ERF::start_time = 0.0

private

startCPUTime

Real ERF::startCPUTime = 0.0

staticprivate

Referenced by getCPUTime().

stop_time

amrex::Real ERF::stop_time = std::numeric_limits<amrex::Real>::max()

private

stretched_dz_d

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::stretched_dz_d

private

stretched_dz_h

amrex::Vector<amrex::Vector<amrex::Real> > ERF::stretched_dz_h

private

sub_cfl

Real ERF::sub_cfl = 1.0

staticprivate

sum_interval

int ERF::sum_interval = -1

staticprivate

sum_per

Real ERF::sum_per = -1.0

staticprivate

t_avg_cnt

amrex::Vector<amrex::Real> ERF::t_avg_cnt

private

t_new

amrex::Vector<amrex::Real> ERF::t_new

private

t_old

amrex::Vector<amrex::Real> ERF::t_old

private

Tau11_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau11_lev

private

Tau12_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau12_lev

private

Tau13_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau13_lev

private

Tau21_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau21_lev

private

Tau22_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau22_lev

private

Tau23_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau23_lev

private

Tau31_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau31_lev

private

Tau32_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau32_lev

private

Tau33_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau33_lev

private

Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim

private

thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce

private

thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce

private

thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce

private

turbPert

TurbulentPerturbation ERF::turbPert

private

use_fft

bool ERF::use_fft = false

staticprivate

use_real_bcs

bool ERF::use_real_bcs

staticprivate

vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new

private

Referenced by MultiBlockContainer::FillPatchBlocks(), MultiBlockContainer::SetBlockCommMetaData(), and MultiBlockContainer::SetBoxLists().

vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old

private

vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg

private

verbose

int ERF::verbose = 0

staticprivate

walldist

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::walldist

private

xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask

private

xvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::xvel_bc_data

private

yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask

private

yvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::yvel_bc_data

private

z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc

private

z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd

private

z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new

private

z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src

private

z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk

private

zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask

private

zlevels_stag

amrex::Vector<amrex::Vector<amrex::Real> > ERF::zlevels_stag

private

zmom_crse_rhs

amrex::Vector<amrex::MultiFab> ERF::zmom_crse_rhs

private

zvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::zvel_bc_data

private

---

The documentation for this class was generated from the following files:

• Source/ERF.H
• Source/BoundaryConditions/ERF_BoundaryConditionsBndryReg.cpp
• Source/BoundaryConditions/ERF_FillBdyCCVels.cpp
• Source/BoundaryConditions/ERF_FillCoarsePatch.cpp
• Source/BoundaryConditions/ERF_FillIntermediatePatch.cpp
• Source/ERF.cpp
• Source/ERF_MakeNewArrays.cpp
• Source/ERF_MakeNewLevel.cpp
• Source/ERF_Tagging.cpp
• Source/Initialization/ERF_Init1D.cpp
• Source/Initialization/ERF_InitBCs.cpp
• Source/Initialization/ERF_InitCustom.cpp
• Source/Initialization/ERF_InitFromHSE.cpp
• Source/Initialization/ERF_InitFromInputSounding.cpp
• Source/Initialization/ERF_InitGeowind.cpp
• Source/Initialization/ERF_InitRayleigh.cpp
• Source/Initialization/ERF_InitSponge.cpp
• Source/Initialization/ERF_InitTurbPert.cpp
• Source/Initialization/ERF_InitUniform.cpp
• Source/IO/ERF_Checkpoint.cpp
• Source/IO/ERF_ConsoleIO.cpp
• Source/IO/ERF_Plotfile.cpp
• Source/IO/ERF_Write1DProfiles.cpp
• Source/IO/ERF_Write1DProfiles_stag.cpp
• Source/IO/ERF_WriteJobInfo.cpp
• Source/IO/ERF_WriteScalarProfiles.cpp
• Source/LinearSolvers/ERF_ComputeDivergence.cpp
• Source/LinearSolvers/ERF_PoissonSolve.cpp
• Source/LinearSolvers/ERF_PoissonSolve_tb.cpp
• Source/LinearSolvers/ERF_PoissonWallDist.cpp
• Source/LinearSolvers/ERF_SolveWithGMRES.cpp
• Source/LinearSolvers/ERF_SolveWithMLMG.cpp
• Source/TimeIntegration/ERF_Advance.cpp
• Source/TimeIntegration/ERF_AdvanceDycore.cpp
• Source/TimeIntegration/ERF_AdvanceLSM.cpp
• Source/TimeIntegration/ERF_AdvanceMicrophysics.cpp
• Source/TimeIntegration/ERF_ComputeTimestep.cpp
• Source/TimeIntegration/ERF_TimeStep.cpp
• Source/Utils/ERF_AverageDown.cpp