#include <ERF.H>

Inheritance diagram for ERF:

Collaboration diagram for ERF:

[legend]

Public Member Functions
	ERF ()

	~ERF () override

	ERF (ERF &&) noexcept=delete

ERF &	operator= (ERF &&other) noexcept=delete

	ERF (const ERF &other)=delete

ERF &	operator= (const ERF &other)=delete

void	Evolve ()

void	ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override

void	InitData ()

void	init_only (int lev, amrex::Real time)

void	restart ()

bool	writeNow (const amrex::Real cur_time, const amrex::Real dt, const int nstep, const int plot_int, const amrex::Real plot_per)

void	post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev)

void	sum_integrated_quantities (amrex::Real time)

void	write_1D_profiles (amrex::Real time)

void	write_1D_profiles_stag (amrex::Real time)

void	FillBdyCCVels (amrex::Vector< amrex::MultiFab > &mf_cc_vel)

void	sample_points (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)

void	sample_lines (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)

void	derive_diag_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw)

void	derive_diag_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw)

void	derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)

void	derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)

amrex::Real	volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, const amrex::MultiFab &mapfac, bool local, bool finemask)

void	MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override

void	RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override

void	ClearLevel (int lev) override

void	MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override

amrex::Real	estTimeStep (int lev, long &dt_fast_ratio) const

void	advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)

void	advance_microphysics (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)

void	advance_lsm (int lev, amrex::MultiFab &, const amrex::Real &dt_advance)

amrex::MultiFab &	build_fine_mask (int lev)

void	MakeHorizontalAverages ()

void	MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)

void	derive_upwp (amrex::Vector< amrex::Real > &h_havg)

void	WritePlotFile (int which, amrex::Vector< std::string > plot_var_names)

void	WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, amrex::Real time, const amrex::Vector< int > &level_steps, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const

void	WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, amrex::Real time, const amrex::Vector< int > &level_steps, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const

void	erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, std::unique_ptr< amrex::MultiFab > &z_cc)

void	init_from_input_sounding (int lev)

void	input_sponge (int lev)

void	init_from_hse (int lev)

void	fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)

void	AverageDownTo (int crse_lev, int scomp, int ncomp)

void	writeJobInfo (const std::string &dir) const

Static Public Member Functions
static bool	is_it_time_for_action (int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)

static void	writeBuildInfo (std::ostream &os)

static void	print_banner (MPI_Comm, std::ostream &)

static void	print_usage (MPI_Comm, std::ostream &)

static void	print_error (MPI_Comm, const std::string &msg)

static void	print_summary (std::ostream &)

static void	print_tpls (std::ostream &)

Public Attributes
std::string	pp_prefix {"erf"}

Private Member Functions
void	ReadParameters ()

void	AverageDown ()

void	update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)

void	update_terrain_arrays (int lev, amrex::Real time)

void	Construct_ERFFillPatchers (int lev)

void	Define_ERFFillPatchers (int lev)

void	init1DArrays ()

void	init_bcs ()

void	init_custom (int lev)

void	init_uniform (int lev)

void	init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old)

void	initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)

void	initialize_bcs (int lev)

void	initializeMicrophysics (const int &)

void	FillPatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, bool fillset=true, bool cons_only=false)

void	FillPatchMoistVars (int lev, amrex::MultiFab &mf)

void	FillIntermediatePatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons, bool allow_most_bcs=true)

void	FillCoarsePatch (int lev, amrex::Real time)

void	timeStep (int lev, amrex::Real time, int iteration)

void	Advance (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)

void	initHSE ()
	Initialize HSE. More...

void	initHSE (int lev)

void	initRayleigh ()
	Initialize Rayleigh damping profiles. More...

void	initSponge ()
	Initialize sponge profiles. More...

void	setRayleighRefFromSounding (bool restarting)
	Set Rayleigh mean profiles from input sounding. More...

void	setSpongeRefFromSounding (bool restarting)
	Set sponge mean profiles from input sounding. More...

void	ComputeDt ()

std::string	PlotFileName (int lev) const

void	setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)

void	appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)

void	WriteCheckpointFile () const

void	ReadCheckpointFile ()

void	InitializeFromFile ()

void	InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)

void	post_update (amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)

void	fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)

void	refinement_criteria_setup ()

AMREX_FORCE_INLINE amrex::YAFluxRegister *	getAdvFluxReg (int lev)

AMREX_FORCE_INLINE std::ostream &	DataLog (int i)

AMREX_FORCE_INLINE int	NumDataLogs () noexcept

AMREX_FORCE_INLINE std::ostream &	SamplePointLog (int i)

AMREX_FORCE_INLINE int	NumSamplePointLogs () noexcept

AMREX_FORCE_INLINE std::ostream &	SampleLineLog (int i)

AMREX_FORCE_INLINE int	NumSampleLineLogs () noexcept

amrex::IntVect &	SamplePoint (int i)

AMREX_FORCE_INLINE int	NumSamplePoints () noexcept

amrex::IntVect &	SampleLine (int i)

AMREX_FORCE_INLINE int	NumSampleLines () noexcept

void	setRecordDataInfo (int i, const std::string &filename)

void	setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)

void	setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)

std::string	DataLogName (int i) const noexcept
	The filename of the ith datalog file. More...

std::string	SamplePointLogName (int i) const noexcept
	The filename of the ith sampleptlog file. More...

std::string	SampleLineLogName (int i) const noexcept
	The filename of the ith samplelinelog file. More...

amrex::FabFactory< amrex::FArrayBox > const &	Factory (int lev) const noexcept

Static Private Member Functions
static amrex::Vector< std::string >	PlotFileVarNames (amrex::Vector< std::string > plot_var_names)

static void	GotoNextLine (std::istream &is)

static AMREX_FORCE_INLINE int	ComputeGhostCells (const AdvChoice &advChoice, bool use_num_diff)

static amrex::Real	getCPUTime ()

Private Attributes
InputSoundingData	input_sounding_data

InputSpongeData	input_sponge_data

amrex::Vector< int >	min_k_at_level

amrex::Vector< int >	max_k_at_level

std::unique_ptr< ProblemBase >	prob = nullptr

amrex::Vector< int >	num_boxes_at_level

amrex::Vector< int >	num_files_at_level

amrex::Vector< amrex::Vector< amrex::Box > >	boxes_at_level

amrex::Vector< int >	istep

amrex::Vector< int >	nsubsteps

amrex::Vector< amrex::Real >	t_new

amrex::Vector< amrex::Real >	t_old

amrex::Vector< amrex::Real >	dt

amrex::Vector< long >	dt_mri_ratio

amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_new

amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_old

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vel_t_avg

amrex::Vector< amrex::Real >	t_avg_cnt

amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > >	mri_integrator_mem

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > >	physbcs_cons

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > >	physbcs_u

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > >	physbcs_v

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > >	physbcs_w

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w_no_terrain > >	physbcs_w_no_terrain

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Theta_prim

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Qv_prim

amrex::Vector< amrex::MultiFab >	rU_old

amrex::Vector< amrex::MultiFab >	rU_new

amrex::Vector< amrex::MultiFab >	rV_old

amrex::Vector< amrex::MultiFab >	rV_new

amrex::Vector< amrex::MultiFab >	rW_old

amrex::Vector< amrex::MultiFab >	rW_new

std::unique_ptr< Microphysics >	micro

amrex::Vector< amrex::Vector< amrex::MultiFab * > >	qmoist

amrex::Vector< amrex::MultiFab >	Nturb

amrex::Vector< amrex::MultiFab >	vars_fitch

amrex::Vector< amrex::MultiFab >	vars_ewp

amrex::Real	hub_height

amrex::Real	rotor_dia

amrex::Real	thrust_coeff_standing

amrex::Real	nominal_power

amrex::Vector< amrex::Real >	wind_speed

amrex::Vector< amrex::Real >	thrust_coeff

amrex::Vector< amrex::Real >	power

LandSurface	lsm

amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_data

amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_flux

int	cf_width {0}

int	cf_set_width {0}

amrex::Vector< ERFFillPatcher >	FPr_c

amrex::Vector< ERFFillPatcher >	FPr_u

amrex::Vector< ERFFillPatcher >	FPr_v

amrex::Vector< ERFFillPatcher >	FPr_w

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau11_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau22_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau33_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau12_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau21_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau13_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau31_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau23_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Tau32_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	eddyDiffs_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SmnSmn_lev

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	sst_lev

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	lmask_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx1_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx2_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx3_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_diss_lev

amrex::Vector< amrex::Real >	zlevels_stag

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_cc

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_new

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_new

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax_new

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay_new

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az_new

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_t_rk

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mapfac_m

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mapfac_u

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mapfac_v

amrex::Vector< amrex::MultiFab >	base_state

amrex::Vector< amrex::MultiFab >	base_state_new

amrex::Vector< amrex::YAFluxRegister * >	advflux_reg

amrex::Vector< amrex::BCRec >	domain_bcs_type

amrex::Gpu::DeviceVector< amrex::BCRec >	domain_bcs_type_d

amrex::Array< std::string, 2 *AMREX_SPACEDIM >	domain_bc_type

amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NVAR_max >	m_bc_extdir_vals

amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NVAR_max >	m_bc_neumann_vals

amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 >	phys_bc_type

amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	xflux_imask

amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	yflux_imask

amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	zflux_imask

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_xforce

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_yforce

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_zforce

int	last_plot_file_step_1

int	last_plot_file_step_2

int	last_check_file_step

int	plot_file_on_restart = 1

int	max_step = std::numeric_limits<int>::max()

amrex::Real	start_time = 0.0

amrex::Real	stop_time = std::numeric_limits<amrex::Real>::max()

std::string	restart_chkfile = ""

int	regrid_int = -1

std::string	plot_file_1 {"plt_1_"}

std::string	plot_file_2 {"plt_2_"}

int	m_plot_int_1 = -1

int	m_plot_int_2 = -1

amrex::Real	m_plot_per_1 = -1.0

amrex::Real	m_plot_per_2 = -1.0

bool	plot_lsm = false

int	profile_int = -1

bool	cc_profiles = true

std::string	check_file {"chk"}

std::string	check_type {"native"}

std::string	restart_type {"native"}

int	m_check_int = -1

amrex::Real	m_check_per = -1.0

amrex::Vector< std::string >	plot_var_names_1

amrex::Vector< std::string >	plot_var_names_2

const amrex::Vector< std::string >	cons_names

const amrex::Vector< std::string >	derived_names

int	real_width {0}

int	real_set_width {0}

amrex::Vector< amrex::Vector< amrex::Real > >	h_rhotheta_src

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_rhotheta_src

amrex::Vector< amrex::Vector< amrex::Real > >	h_rhoqt_src

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_rhoqt_src

amrex::Vector< amrex::Vector< amrex::Real > >	h_w_subsid

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_w_subsid

amrex::Vector< amrex::Vector< amrex::Real > >	h_u_geos

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_u_geos

amrex::Vector< amrex::Vector< amrex::Real > >	h_v_geos

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_v_geos

amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_rayleigh_ptrs

amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_sponge_ptrs

amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_rayleigh_ptrs

amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_sponge_ptrs

amrex::Vector< amrex::Real >	h_havg_density

amrex::Vector< amrex::Real >	h_havg_temperature

amrex::Vector< amrex::Real >	h_havg_pressure

amrex::Vector< amrex::Real >	h_havg_qv

amrex::Vector< amrex::Real >	h_havg_qc

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_density

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_temperature

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_pressure

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qv

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qc

std::unique_ptr< WriteBndryPlanes >	m_w2d = nullptr

std::unique_ptr< ReadBndryPlanes >	m_r2d = nullptr

std::unique_ptr< ABLMost >	m_most = nullptr

amrex::MultiFab	fine_mask

amrex::Real	dz_min

amrex::Vector< std::unique_ptr< std::fstream > >	datalog

amrex::Vector< std::string >	datalogname

amrex::Vector< std::unique_ptr< std::fstream > >	sampleptlog

amrex::Vector< std::string >	sampleptlogname

amrex::Vector< amrex::IntVect >	samplepoint

amrex::Vector< std::unique_ptr< std::fstream > >	samplelinelog

amrex::Vector< std::string >	samplelinelogname

amrex::Vector< amrex::IntVect >	sampleline

amrex::Vector< std::unique_ptr< amrex::FabFactory< amrex::FArrayBox > > >	m_factory

Static Private Attributes
static amrex::Real	cfl = 0.8

static amrex::Real	init_shrink = 1.0

static amrex::Real	change_max = 1.1

static amrex::Real	fixed_dt = -1.0

static amrex::Real	fixed_fast_dt = -1.0

static int	fixed_mri_dt_ratio = 0

static SolverChoice	solverChoice

static int	verbose = 0

static int	sum_interval = -1

static amrex::Real	sum_per = -1.0

static std::string	plotfile_type = "amrex"

static std::string	init_type

static std::string	sponge_type

static bool	use_real_bcs

static amrex::Vector< amrex::Vector< std::string > >	nc_init_file = {{""}}

static std::string	nc_bdy_file

static std::string	input_sounding_file = "input_sounding"

static std::string	input_sponge_file = "input_sponge_file.txt"

static bool	init_sounding_ideal = false

static int	output_1d_column = 0

static int	column_interval = -1

static amrex::Real	column_per = -1.0

static amrex::Real	column_loc_x = 0.0

static amrex::Real	column_loc_y = 0.0

static std::string	column_file_name = "column_data.nc"

static int	output_bndry_planes = 0

static int	bndry_output_planes_interval = -1

static amrex::Real	bndry_output_planes_per = -1.0

static amrex::Real	bndry_output_planes_start_time = 0.0

static int	input_bndry_planes = 0

static int	ng_dens_hse

static int	ng_pres_hse

static amrex::Vector< amrex::AMRErrorTag >	ref_tags

static amrex::Real	startCPUTime = 0.0

static amrex::Real	previousCPUTimeUsed = 0.0

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

◆ ERF() [1/3]

ERF::ERF ( )

 {
     if (ParallelDescriptor::IOProcessor()) {
         const char* erf_hash = buildInfoGetGitHash(1);
         const char* amrex_hash = buildInfoGetGitHash(2);
         const char* buildgithash = buildInfoGetBuildGitHash();
         const char* buildgitname = buildInfoGetBuildGitName();
  
         if (strlen(erf_hash) > 0) {
           Print() << "\n"
                          << "ERF git hash: " << erf_hash << "\n";
         }
         if (strlen(amrex_hash) > 0) {
           Print() << "AMReX git hash: " << amrex_hash << "\n";
         }
         if (strlen(buildgithash) > 0) {
           Print() << buildgitname << " git hash: " << buildgithash << "\n";
         }
  
         Print() << "\n";
     }
  
     int nlevs_max = max_level + 1;
  
 #ifdef ERF_USE_WINDFARM
     Nturb.resize(nlevs_max);
     vars_fitch.resize(nlevs_max);
     vars_ewp.resize(nlevs_max);
 #endif
  
 #if defined(ERF_USE_RRTMGP)
     qheating_rates.resize(nlevs_max);
 #endif
  
     // NOTE: size lsm before readparams (chooses the model at all levels)
     lsm.ReSize(nlevs_max);
     lsm_data.resize(nlevs_max);
     lsm_flux.resize(nlevs_max);
  
     ReadParameters();
     initializeMicrophysics(nlevs_max);
  
     const std::string& pv1 = "plot_vars_1"; setPlotVariables(pv1,plot_var_names_1);
     const std::string& pv2 = "plot_vars_2"; setPlotVariables(pv2,plot_var_names_2);
  
     // Initialize staggered vertical levels for grid stretching or terrain.
  
     if (solverChoice.use_terrain) {
         init_zlevels(zlevels_stag,
                      geom[0],
                      solverChoice.grid_stretching_ratio,
                      solverChoice.zsurf,
                      solverChoice.dz0);
  
         int nz = geom[0].Domain().length(2) + 1; // staggered
         if (std::fabs(zlevels_stag[nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
             Print() << "WARNING: prob_hi[2]=" << geom[0].ProbHi(2)
                 << " does not match highest requested z level " << zlevels_stag[nz-1]
                 << std::endl;
         }
         if (std::fabs(zlevels_stag[0]-geom[0].ProbLo(2)) > 1.0e-4) {
             Print() << "WARNING: prob_lo[2]=" << geom[0].ProbLo(2)
                 << " does not match lowest requested level " << zlevels_stag[0]
                 << std::endl;
         }
  
         // Redefine the problem domain here?
     }
  
     prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
  
     // Geometry on all levels has been defined already.
  
     // No valid BoxArray and DistributionMapping have been defined.
     // But the arrays for them have been resized.
  
     istep.resize(nlevs_max, 0);
     nsubsteps.resize(nlevs_max, 1);
     for (int lev = 1; lev <= max_level; ++lev) {
         nsubsteps[lev] = MaxRefRatio(lev-1);
     }
  
     t_new.resize(nlevs_max, 0.0);
     t_old.resize(nlevs_max, -1.e100);
     dt.resize(nlevs_max, 1.e100);
     dt_mri_ratio.resize(nlevs_max, 1);
  
     vars_new.resize(nlevs_max);
     vars_old.resize(nlevs_max);
  
     rU_new.resize(nlevs_max);
     rV_new.resize(nlevs_max);
     rW_new.resize(nlevs_max);
  
     rU_old.resize(nlevs_max);
     rV_old.resize(nlevs_max);
     rW_old.resize(nlevs_max);
  
     for (int lev = 0; lev < nlevs_max; ++lev) {
         vars_new[lev].resize(Vars::NumTypes);
         vars_old[lev].resize(Vars::NumTypes);
     }
  
     // Time integrator
     mri_integrator_mem.resize(nlevs_max);
  
     // Physical boundary conditions
     physbcs_cons.resize(nlevs_max);
     physbcs_u.resize(nlevs_max);
     physbcs_v.resize(nlevs_max);
     physbcs_w.resize(nlevs_max);
     physbcs_w_no_terrain.resize(nlevs_max);
  
     advflux_reg.resize(nlevs_max);
  
     // Stresses
     Tau11_lev.resize(nlevs_max); Tau22_lev.resize(nlevs_max); Tau33_lev.resize(nlevs_max);
     Tau12_lev.resize(nlevs_max); Tau21_lev.resize(nlevs_max);
     Tau13_lev.resize(nlevs_max); Tau31_lev.resize(nlevs_max);
     Tau23_lev.resize(nlevs_max); Tau32_lev.resize(nlevs_max);
     SFS_hfx1_lev.resize(nlevs_max); SFS_hfx2_lev.resize(nlevs_max); SFS_hfx3_lev.resize(nlevs_max);
     SFS_diss_lev.resize(nlevs_max);
     eddyDiffs_lev.resize(nlevs_max);
     SmnSmn_lev.resize(nlevs_max);
  
     // Sea surface temps
     sst_lev.resize(nlevs_max);
     lmask_lev.resize(nlevs_max);
  
     // Metric terms
     z_phys_nd.resize(nlevs_max);
     z_phys_cc.resize(nlevs_max);
     detJ_cc.resize(nlevs_max);
     ax.resize(nlevs_max);
     ay.resize(nlevs_max);
     az.resize(nlevs_max);
  
     z_phys_nd_new.resize(nlevs_max);
     detJ_cc_new.resize(nlevs_max);
     ax_new.resize(nlevs_max);
     ay_new.resize(nlevs_max);
     az_new.resize(nlevs_max);
  
     z_phys_nd_src.resize(nlevs_max);
     detJ_cc_src.resize(nlevs_max);
     ax_src.resize(nlevs_max);
     ay_src.resize(nlevs_max);
     az_src.resize(nlevs_max);
  
     z_t_rk.resize(nlevs_max);
  
     // Mapfactors
     mapfac_m.resize(nlevs_max);
     mapfac_u.resize(nlevs_max);
     mapfac_v.resize(nlevs_max);
  
     // Thin immersed body
     xflux_imask.resize(nlevs_max);
     yflux_imask.resize(nlevs_max);
     zflux_imask.resize(nlevs_max);
     //overset_imask.resize(nlevs_max);
     thin_xforce.resize(nlevs_max);
     thin_yforce.resize(nlevs_max);
     thin_zforce.resize(nlevs_max);
  
     // Base state
     base_state.resize(nlevs_max);
     base_state_new.resize(nlevs_max);
  
     // Theta prim for MOST
     Theta_prim.resize(nlevs_max);
  
     // Qv prim for MOST
     Qv_prim.resize(nlevs_max);
  
     // Time averaged velocity field
     vel_t_avg.resize(nlevs_max);
     t_avg_cnt.resize(nlevs_max);
  
 #ifdef ERF_USE_NETCDF
     // Size lat long arrays if using netcdf
     lat_m.resize(nlevs_max);
     lon_m.resize(nlevs_max);
 #endif
  
     // Initialize tagging criteria for mesh refinement
     refinement_criteria_setup();
  
     // We have already read in the ref_Ratio (via amr.ref_ratio =) but we need to enforce
     //     that there is no refinement in the vertical so we test on that here.
     for (int lev = 0; lev < max_level; ++lev)
     {
        Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
           ref_ratio[lev][0]  << " " << ref_ratio[lev][1]  <<  " " << ref_ratio[lev][2] << std::endl;
  
        if (ref_ratio[lev][2] != 1)
        {
            Error("We don't allow refinement in the vertical -- make sure to set ref_ratio = 1 in z");
        }
     }
  
     // We define m_factory even with no EB
     m_factory.resize(max_level+1);
  
 #ifdef AMREX_USE_EB
     // We will create each of these in MakeNewLevel.../RemakeLevel
  
     // This is needed before initializing level MultiFabs
     MakeEBGeometry();
 #endif
 }

Here is the call graph for this function:

◆ ~ERF()

ERF::~ERF ( )

overridedefault

◆ ERF() [2/3]

ERF::ERF ( ERF && )

deletenoexcept

◆ ERF() [3/3]

ERF::ERF ( const ERF & other )

delete

Member Function Documentation

◆ Advance()

void ERF::Advance	(	int	lev,
		amrex::Real	time,
		amrex::Real	dt_lev,
		int	iteration,
		int	ncycle
	)

private

Function that advances the solution at one level for a single time step – this does some preliminaries then calls erf_advance

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	dt_lev	time step for this time advance

 {
     BL_PROFILE("ERF::Advance()");
  
     // We must swap the pointers so the previous step's "new" is now this step's "old"
     std::swap(vars_old[lev], vars_new[lev]);
  
     MultiFab& S_old = vars_old[lev][Vars::cons];
     MultiFab& S_new = vars_new[lev][Vars::cons];
  
     MultiFab& U_old = vars_old[lev][Vars::xvel];
     MultiFab& V_old = vars_old[lev][Vars::yvel];
     MultiFab& W_old = vars_old[lev][Vars::zvel];
  
     MultiFab& U_new = vars_new[lev][Vars::xvel];
     MultiFab& V_new = vars_new[lev][Vars::yvel];
     MultiFab& W_new = vars_new[lev][Vars::zvel];
  
  
     // configure ABLMost params if used MostWall boundary condition
     if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::MOST) {
         if (m_most) {
             IntVect ng = Theta_prim[lev]->nGrowVect();
             MultiFab::Copy(  *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
             MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp     , 0, 1, ng);
             if (solverChoice.moisture_type != MoistureType::None) {
                 ng = Qv_prim[lev]->nGrowVect();
                 MultiFab::Copy(  *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
                 MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp  , 0, 1, ng);
             }
             // NOTE: std::swap above causes the field ptrs to be out of date.
             //       Reassign the field ptrs for MAC avg computation.
             m_most->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim);
             m_most->update_fluxes(lev, time);
         }
     }
  
     // We need to set these because otherwise in the first call to erf_advance we may
     //    read uninitialized data on ghost values in setting the bc's on the velocities
     U_new.setVal(1.e34,U_new.nGrowVect());
     V_new.setVal(1.e34,V_new.nGrowVect());
     W_new.setVal(1.e34,W_new.nGrowVect());
  
     FillPatch(lev, time, {&S_old, &U_old, &V_old, &W_old},
                          {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]});
  
     if (solverChoice.moisture_type != MoistureType::None) {
         // TODO: This is only qv
         if (qmoist[lev].size() > 0) FillPatchMoistVars(lev, *(qmoist[lev][0]));
     }
  
 #if defined(ERF_USE_WINDFARM)
     // Update with the Fitch source terms
     if (solverChoice.windfarm_type == WindFarmType::Fitch) {
         fitch_advance(lev, Geom(lev), dt_lev, S_old,
                       U_old, V_old, W_old, vars_fitch[lev], Nturb[lev]);
     }
     else if (solverChoice.windfarm_type == WindFarmType::EWP) {
         ewp_advance(lev, Geom(lev), dt_lev, S_old,
                       U_old, V_old, W_old, vars_ewp[lev], Nturb[lev]);
     }
  
 #endif
  
     const BoxArray&            ba = S_old.boxArray();
     const DistributionMapping& dm = S_old.DistributionMap();
  
     int nvars = S_old.nComp();
  
     // Source array for conserved cell-centered quantities -- this will be filled
     //     in the call to make_sources in TI_slow_rhs_fun.H
     MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0.0);
  
     // Source arrays for momenta -- these will be filled
     //     in the call to make_mom_sources in TI_slow_rhs_fun.H
     MultiFab xmom_source(ba,dm,nvars,1); xmom_source.setVal(0.0);
     MultiFab ymom_source(ba,dm,nvars,1); ymom_source.setVal(0.0);
     MultiFab zmom_source(ba,dm,nvars,1); zmom_source.setVal(0.0);
  
     // We don't need to call FillPatch on cons_mf because we have fillpatch'ed S_old above
     MultiFab cons_mf(ba,dm,nvars,S_old.nGrowVect());
     MultiFab::Copy(cons_mf,S_old,0,0,S_old.nComp(),S_old.nGrowVect());
  
     amrex::Vector<MultiFab> state_old;
     amrex::Vector<MultiFab> state_new;
  
     // **************************************************************************************
     // Here we define state_old and state_new which are to be advanced
     // **************************************************************************************
     // Initial solution
     // Note that "old" and "new" here are relative to each RK stage.
     state_old.push_back(MultiFab(cons_mf    , amrex::make_alias, 0, nvars)); // cons
     state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0,     1)); // xmom
     state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0,     1)); // ymom
     state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0,     1)); // zmom
  
     // Final solution
     // state_new at the end of the last RK stage holds the t^{n+1} data
     state_new.push_back(MultiFab(S_new      , amrex::make_alias, 0, nvars)); // cons
     state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0,     1)); // xmom
     state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0,     1)); // ymom
     state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0,     1)); // zmom
  
     // **************************************************************************************
     // Update the dycore
     // **************************************************************************************
     advance_dycore(lev, state_old, state_new,
                    U_old, V_old, W_old,
                    U_new, V_new, W_new,
                    cc_source, xmom_source, ymom_source, zmom_source,
                    Geom(lev), dt_lev, time);
  
     // **************************************************************************************
     // Update the microphysics (moisture)
     // **************************************************************************************
     advance_microphysics(lev, S_new, dt_lev, iteration, time);
  
     // **************************************************************************************
     // Update the land surface model
     // **************************************************************************************
     advance_lsm(lev, S_new, dt_lev);
  
 #if defined(ERF_USE_RRTMGP)
     // **************************************************************************************
     // Update the radiation
     // **************************************************************************************
     advance_radiation(lev, S_new, dt_lev);
 #endif
  
 #ifdef ERF_USE_PARTICLES
     // **************************************************************************************
     // Update the particle positions
     // **************************************************************************************
    evolveTracers( lev, dt_lev, vars_new, z_phys_nd );
 #endif
  
     // **************************************************************************************
     // Register old and new coarse data if we are at a level less than the finest level
     // **************************************************************************************
     if (lev < finest_level)
     {
         if (cf_width > 0) {
             // We must fill the ghost cells of these so that the parallel copy works correctly
             state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
             state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
             FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
                                           {time, time + dt_lev});
         }
  
         if (cf_width >= 0) {
             // We must fill the ghost cells of these so that the parallel copy works correctly
             state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
             state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
             FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
                                           {time, time + dt_lev});
  
             state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
             state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
             FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
                                           {time, time + dt_lev});
  
             state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
             state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
             FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
                                           {time, time + dt_lev});
         }
     }
  
     // ***********************************************************************************************
     // Update the time averaged velocities if they are requested
     // ***********************************************************************************************
     if (solverChoice.time_avg_vel) {
         Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
     }
 }

Here is the call graph for this function:

◆ advance_dycore()

void ERF::advance_dycore	(	int	level,
		amrex::Vector< amrex::MultiFab > &	state_old,
		amrex::Vector< amrex::MultiFab > &	state_new,
		amrex::MultiFab &	xvel_old,
		amrex::MultiFab &	yvel_old,
		amrex::MultiFab &	zvel_old,
		amrex::MultiFab &	xvel_new,
		amrex::MultiFab &	yvel_new,
		amrex::MultiFab &	zvel_new,
		amrex::MultiFab &	source,
		amrex::MultiFab &	xmom_src,
		amrex::MultiFab &	ymom_src,
		amrex::MultiFab &	zmom_src,
		amrex::Geometry	fine_geom,
		amrex::Real	dt,
		amrex::Real	time
	)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters

[in]	level	level of refinement (coarsest level is 0)
[in]	state_old	old-time conserved variables
[in]	state_new	new-time conserved variables
[in]	xvel_old	old-time x-component of velocity
[in]	yvel_old	old-time y-component of velocity
[in]	zvel_old	old-time z-component of velocity
[in]	xvel_new	new-time x-component of velocity
[in]	yvel_new	new-time y-component of velocity
[in]	zvel_new	new-time z-component of velocity
[in]	cc_src	source term for conserved variables
[in]	xmom_src	source term for x-momenta
[in]	ymom_src	source term for y-momenta
[in]	zmom_src	source term for z-momenta
[in]	fine_geom	container for geometry information at current level
[in]	dt_advance	time step for this time advance
[in]	old_time	old time for this time advance

 {
     BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
  
     const Box& domain = fine_geom.Domain();
  
     DiffChoice dc    = solverChoice.diffChoice;
     TurbChoice tc    = solverChoice.turbChoice[level];
     SpongeChoice sc  = solverChoice.spongeChoice;
  
     MultiFab r_hse (base_state[level], make_alias, 0, 1); // r_0 is first  component
     MultiFab p_hse (base_state[level], make_alias, 1, 1); // p_0 is second component
     MultiFab pi_hse(base_state[level], make_alias, 2, 1); // pi_0 is second component
  
     // These pointers are used in the MRI utility functions
     MultiFab* r0  = &r_hse;
     MultiFab* p0  = &p_hse;
     MultiFab* pi0 = &pi_hse;
  
     Real* dptr_rhotheta_src = solverChoice.custom_rhotheta_forcing ? d_rhotheta_src[level].data() : nullptr;
     Real* dptr_rhoqt_src    = solverChoice.custom_moisture_forcing ? d_rhoqt_src[level].data()    : nullptr;
     Real* dptr_wbar_sub     = solverChoice.custom_w_subsidence     ? d_w_subsid[level].data()     : nullptr;
  
     Vector<Real*> d_rayleigh_ptrs_at_lev;
     d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
     bool rayleigh_damp_any = (solverChoice.rayleigh_damp_U ||solverChoice.rayleigh_damp_V ||
                               solverChoice.rayleigh_damp_W ||solverChoice.rayleigh_damp_T);
     d_rayleigh_ptrs_at_lev[Rayleigh::tau]      =              rayleigh_damp_any ? d_rayleigh_ptrs[level][Rayleigh::tau ].data() : nullptr;
     d_rayleigh_ptrs_at_lev[Rayleigh::ubar]     = solverChoice.rayleigh_damp_U   ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
     d_rayleigh_ptrs_at_lev[Rayleigh::vbar]     = solverChoice.rayleigh_damp_V   ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
     d_rayleigh_ptrs_at_lev[Rayleigh::wbar]     = solverChoice.rayleigh_damp_W   ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
     d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.rayleigh_damp_T   ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
  
     Vector<Real*> d_sponge_ptrs_at_lev;
     if(sc.sponge_type=="input_sponge")
     {
         d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
         d_sponge_ptrs_at_lev[Sponge::ubar_sponge]  =  d_sponge_ptrs[level][Sponge::ubar_sponge].data();
         d_sponge_ptrs_at_lev[Sponge::vbar_sponge]  =  d_sponge_ptrs[level][Sponge::vbar_sponge].data();
     }
  
     bool l_use_terrain = solverChoice.use_terrain;
     bool l_use_diff    = ( (dc.molec_diff_type != MolecDiffType::None) ||
                            (tc.les_type        !=       LESType::None) ||
                            (tc.pbl_type        !=       PBLType::None) );
     bool l_use_kturb   = ( (tc.les_type != LESType::None)   ||
                            (tc.pbl_type != PBLType::None) );
  
     const bool use_most = (m_most != nullptr);
     amrex::ignore_unused(use_most);
  
     const BoxArray& ba            = state_old[IntVars::cons].boxArray();
     const BoxArray& ba_z          = zvel_old.boxArray();
     const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
  
     int num_prim = state_old[IntVars::cons].nComp() - 1;
  
     MultiFab    S_prim  (ba  , dm, num_prim,          state_old[IntVars::cons].nGrowVect());
     MultiFab  pi_stage  (ba  , dm,        1,          state_old[IntVars::cons].nGrowVect());
     MultiFab fast_coeffs(ba_z, dm,        5,          0);
     MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
     MultiFab* SmnSmn    = SmnSmn_lev[level].get();
  
     // **************************************************************************************
     // Compute strain for use in slow RHS, Smagorinsky model, and MOST
     // **************************************************************************************
     {
     BL_PROFILE("erf_advance_strain");
     if (l_use_diff) {
  
         const BCRec* bc_ptr_h = domain_bcs_type.data();
         const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
  
 #ifdef _OPENMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
         for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
         {
             Box bxcc  = mfi.growntilebox(IntVect(1,1,0));
             Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
             Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
             Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
  
             const Array4<const Real> & u = xvel_old.array(mfi);
             const Array4<const Real> & v = yvel_old.array(mfi);
             const Array4<const Real> & w = zvel_old.array(mfi);
  
             Array4<Real> tau11 = Tau11_lev[level].get()->array(mfi);
             Array4<Real> tau22 = Tau22_lev[level].get()->array(mfi);
             Array4<Real> tau33 = Tau33_lev[level].get()->array(mfi);
             Array4<Real> tau12 = Tau12_lev[level].get()->array(mfi);
             Array4<Real> tau13 = Tau13_lev[level].get()->array(mfi);
             Array4<Real> tau23 = Tau23_lev[level].get()->array(mfi);
  
             Array4<Real> tau21  = l_use_terrain ? Tau21_lev[level].get()->array(mfi) : Array4<Real>{};
             Array4<Real> tau31  = l_use_terrain ? Tau31_lev[level].get()->array(mfi) : Array4<Real>{};
             Array4<Real> tau32  = l_use_terrain ? Tau32_lev[level].get()->array(mfi) : Array4<Real>{};
             const Array4<const Real>& z_nd = l_use_terrain ? z_phys_nd[level]->const_array(mfi) : Array4<const Real>{};
  
             const Array4<const Real> mf_m = mapfac_m[level]->array(mfi);
             const Array4<const Real> mf_u = mapfac_u[level]->array(mfi);
             const Array4<const Real> mf_v = mapfac_v[level]->array(mfi);
  
             if (l_use_terrain) {
                 ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                 u, v, w,
                                 tau11, tau22, tau33,
                                 tau12, tau13,
                                 tau21, tau23,
                                 tau31, tau32,
                                 z_nd, detJ_cc[level]->const_array(mfi), bc_ptr_h, dxInv,
                                 mf_m, mf_u, mf_v);
             } else {
                 ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                 u, v, w,
                                 tau11, tau22, tau33,
                                 tau12, tau13, tau23,
                                 bc_ptr_h, dxInv,
                                 mf_m, mf_u, mf_v);
             }
         } // mfi
     } // l_use_diff
     } // profile
  
     MultiFab Omega (state_old[IntVars::zmom].boxArray(),dm,1,1);
  
 #include "TI_utils.H"
  
     // Additional SFS quantities, calculated once per timestep
     MultiFab* Hfx1 = SFS_hfx1_lev[level].get();
     MultiFab* Hfx2 = SFS_hfx2_lev[level].get();
     MultiFab* Hfx3 = SFS_hfx3_lev[level].get();
     MultiFab* Diss = SFS_diss_lev[level].get();
  
     // *************************************************************************
     // Calculate cell-centered eddy viscosity & diffusivities
     //
     // Notes -- we fill all the data in ghost cells before calling this so
     //    that we can fill the eddy viscosity in the ghost regions and
     //    not have to call a boundary filler on this data itself
     //
     // LES - updates both horizontal and vertical eddy viscosity components
     // PBL - only updates vertical eddy viscosity components so horizontal
     //       components come from the LES model or are left as zero.
     // *************************************************************************
     if (l_use_kturb)
     {
         // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
         const BCRec* bc_ptr_d = domain_bcs_type_d.data();
         ComputeTurbulentViscosity(xvel_old, yvel_old,
                                   *Tau11_lev[level].get(), *Tau22_lev[level].get(), *Tau33_lev[level].get(),
                                   *Tau12_lev[level].get(), *Tau13_lev[level].get(), *Tau23_lev[level].get(),
                                   state_old[IntVars::cons],
                                   *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
                                   fine_geom, *mapfac_u[level], *mapfac_v[level],
                                   z_phys_nd[level], tc, solverChoice.gravity,
                                   m_most, level, bc_ptr_d);
     }
  
     // ***********************************************************************************************
     // Update user-defined source terms -- these are defined once per time step (not per RK stage)
     // ***********************************************************************************************
     if (solverChoice.custom_rhotheta_forcing) {
         prob->update_rhotheta_sources(old_time,
                                       h_rhotheta_src[level], d_rhotheta_src[level],
                                       fine_geom, z_phys_cc[level]);
     }
  
     if (solverChoice.custom_moisture_forcing) {
         prob->update_rhoqt_sources(old_time,
                                    h_rhoqt_src[level], d_rhoqt_src[level],
                                    fine_geom, z_phys_cc[level]);
     }
  
     if (solverChoice.custom_geostrophic_profile) {
         prob->update_geostrophic_profile(old_time,
                                    h_u_geos[level], d_u_geos[level],
                                    h_v_geos[level], d_v_geos[level],
                                    fine_geom, z_phys_cc[level]);
     }
  
     // ***********************************************************************************************
     // Convert old velocity available on faces to old momentum on faces to be used in time integration
     // ***********************************************************************************************
     MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
  
     //
     // This is an optimization since we won't need more than one ghost
     // cell of momentum in the integrator if not using NumDiff
     //
     IntVect ngu = (solverChoice.use_NumDiff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
     IntVect ngv = (solverChoice.use_NumDiff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
     IntVect ngw = (solverChoice.use_NumDiff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
     VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
                        state_old[IntVars::xmom],
                        state_old[IntVars::ymom],
                        state_old[IntVars::zmom],
                        domain, domain_bcs_type);
  
     MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
     MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
     MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
  
     bool fast_only = false;
     bool vel_and_mom_synced = true;
  
     apply_bcs(state_old, old_time,
               state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
               fast_only, vel_and_mom_synced);
     cons_to_prim(state_old[IntVars::cons], state_old[IntVars::cons].nGrow());
  
 #include "TI_no_substep_fun.H"
 #include "TI_slow_rhs_fun.H"
 #include "TI_fast_rhs_fun.H"
  
     // ***************************************************************************************
     // Setup the integrator and integrate for a single timestep
     // **************************************************************************************
     MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
  
     // Define rhs and 'post update' utility function that is called after calculating
     // any state data (e.g. at RK stages or at the end of a timestep)
     mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
     mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
     mri_integrator.set_pre_update (pre_update_fun);
     mri_integrator.set_post_update(post_update_fun);
  
 #ifdef ERF_USE_POISSON_SOLVE
     if (solverChoice.incompressible) {
         mri_integrator.set_slow_rhs_inc(slow_rhs_fun_inc);
     }
 #endif
  
     mri_integrator.set_fast_rhs(fast_rhs_fun);
     mri_integrator.set_slow_fast_timestep_ratio(fixed_mri_dt_ratio > 0 ? fixed_mri_dt_ratio : dt_mri_ratio[level]);
     mri_integrator.set_no_substep(no_substep_fun);
  
     mri_integrator.advance(state_old, state_new, old_time, dt_advance);
  
     if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
 }

Here is the call graph for this function:

◆ advance_lsm()

void ERF::advance_lsm	(	int	lev,
		amrex::MultiFab &	,
		const amrex::Real &	dt_advance
	)

 {
     if (solverChoice.lsm_type != LandSurfaceType::None) {
         lsm.Advance(lev, dt_advance);
     }
 }

◆ advance_microphysics()

void ERF::advance_microphysics	(	int	lev,
		amrex::MultiFab &	cons_in,
		const amrex::Real &	dt_advance,
		const int &	iteration,
		const amrex::Real &	time
	)

 {
     if (solverChoice.moisture_type != MoistureType::None) {
         micro->Update_Micro_Vars_Lev(lev, cons);
         micro->Advance(lev, dt_advance, iteration, time, solverChoice, vars_new, z_phys_nd);
         micro->Update_State_Vars_Lev(lev, cons);
     }
 }

◆ appendPlotVariables()

void ERF::appendPlotVariables	(	const std::string &	pp_plot_var_names,
		amrex::Vector< std::string > &	plot_var_names
	)

private

 {
     ParmParse pp(pp_prefix);
  
     Vector<std::string> plot_var_names(0);
     if (pp.contains(pp_plot_var_names.c_str())) {
         std::string nm;
         int nPltVars = pp.countval(pp_plot_var_names.c_str());
         for (int i = 0; i < nPltVars; i++) {
             pp.get(pp_plot_var_names.c_str(), nm, i);
             // Add the named variable to our list of plot variables
             // if it is not already in the list
             if (!containerHasElement(plot_var_names, nm)) {
                 plot_var_names.push_back(nm);
             }
         }
     }
  
     Vector<std::string> tmp_plot_names(0);
 #ifdef ERF_USE_PARTICLES
     Vector<std::string> particle_mesh_plot_names;
     particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
     for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
         std::string tmp(particle_mesh_plot_names[i]);
         if (containerHasElement(plot_var_names, tmp) ) {
             tmp_plot_names.push_back(tmp);
         }
     }
 #endif
  
     for (int i = 0; i < tmp_plot_names.size(); i++) {
         a_plot_var_names.push_back( tmp_plot_names[i] );
     }
  
     // Finally, check to see if we found all the requested variables
     for (const auto& plot_name : plot_var_names) {
         if (!containerHasElement(a_plot_var_names, plot_name)) {
              if (amrex::ParallelDescriptor::IOProcessor()) {
                  Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
              }
         }
     }
 }

Here is the call graph for this function:

◆ AverageDown()

void ERF::AverageDown ( )

private

 {
     AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
     int  src_comp = 0;
     int  num_comp = vars_new[0][Vars::cons].nComp();
     for (int lev = finest_level-1; lev >= 0; --lev)
     {
         AverageDownTo(lev,src_comp,num_comp);
     }
 }

◆ AverageDownTo()

void ERF::AverageDownTo	(	int	crse_lev,
		int	scomp,
		int	ncomp
	)

 {
     AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
  
     // ******************************************************************************************
     // First do cell-centered quantities
     // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
     // m is the map scale factor at cell centers
     // Here we pre-divide (rho S) by m^2 before average down
     // ******************************************************************************************
     for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
       for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.tilebox();
         const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
         const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
         if (solverChoice.use_terrain) {
             const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
             ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
             {
                 cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
             });
         } else {
             ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
             {
                 cons_arr(i,j,k,scomp+n) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
             });
         }
       } // mfi
     } // lev
  
     average_down(vars_new[crse_lev+1][Vars::cons],
                  vars_new[crse_lev  ][Vars::cons],
                  scomp, ncomp, refRatio(crse_lev));
  
     // Here we multiply (rho S) by m^2 after average down
     for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
       for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.tilebox();
         const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
         const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
         if (solverChoice.use_terrain) {
             const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
             ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
             {
                 cons_arr(i,j,k,scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)) / detJ_arr(i,j,k);
             });
         } else {
             ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
             {
                 cons_arr(i,j,k,scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
             });
         }
       } // mfi
     } // lev
  
     // ******************************************************************************************
     // Now average down momenta.
     // Note that vars_new holds velocities not momenta, but we want to do conservative
     //    averaging so we first convert to momentum, then average down, then convert
     //    back to velocities -- only on the valid region
     // ******************************************************************************************
     for (int lev = crse_lev; lev <= crse_lev+1; lev++)
     {
         // FillBoundary for density so we can go back and forth between velocity and momentum
         vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
  
         VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
                            vars_new[lev][Vars::yvel], IntVect(0,0,0),
                            vars_new[lev][Vars::zvel], IntVect(0,0,0),
                            vars_new[lev][Vars::cons],
                              rU_new[lev],
                              rV_new[lev],
                              rW_new[lev],
                            Geom(lev).Domain(),
                            domain_bcs_type);
     }
  
     average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
     average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
     average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
  
     for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
         MomentumToVelocity(vars_new[lev][Vars::xvel],
                            vars_new[lev][Vars::yvel],
                            vars_new[lev][Vars::zvel],
                            vars_new[lev][Vars::cons],
                              rU_new[lev],
                              rV_new[lev],
                              rW_new[lev],
                            Geom(lev).Domain(),
                            domain_bcs_type);
     }
 }

Here is the call graph for this function:

◆ build_fine_mask()

MultiFab & ERF::build_fine_mask ( int level )

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters

level Fine level index which masks underlying coarser data

 {
     // Mask for zeroing covered cells
     AMREX_ASSERT(level > 0);
  
     const BoxArray& cba = grids[level-1];
     const DistributionMapping& cdm = dmap[level-1];
  
     // TODO -- we should make a vector of these a member of ERF class
     fine_mask.define(cba, cdm, 1, 0, MFInfo());
     fine_mask.setVal(1.0);
  
     BoxArray fba = grids[level];
     iMultiFab ifine_mask = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
  
     const auto  fma =  fine_mask.arrays();
     const auto ifma = ifine_mask.arrays();
     ParallelFor(fine_mask, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
     {
        fma[bno](i,j,k) = ifma[bno](i,j,k);
     });
  
     return fine_mask;
 }

◆ ClearLevel()

void ERF::ClearLevel ( int lev )

override

 {
     for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
         vars_new[lev][var_idx].clear();
         vars_old[lev][var_idx].clear();
     }
  
     base_state[lev].clear();
  
     rU_new[lev].clear();
     rU_old[lev].clear();
     rV_new[lev].clear();
     rV_old[lev].clear();
     rW_new[lev].clear();
     rW_old[lev].clear();
  
     // Clears the integrator memory
     mri_integrator_mem[lev].reset();
  
     // Clears the physical boundary condition routines
     physbcs_cons[lev].reset();
     physbcs_u[lev].reset();
     physbcs_v[lev].reset();
     physbcs_w[lev].reset();
     physbcs_w_no_terrain[lev].reset();
  
     // Clears the flux register array
     advflux_reg[lev]->reset();
 }

◆ ComputeDt()

void ERF::ComputeDt ( )

private

Function that calls estTimeStep for each level

 {
     Vector<Real> dt_tmp(finest_level+1);
  
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
     }
  
     ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
  
     Real dt_0 = dt_tmp[0];
     int n_factor = 1;
     for (int lev = 0; lev <= finest_level; ++lev) {
         dt_tmp[lev] = amrex::min(dt_tmp[lev], change_max*dt[lev]);
         n_factor *= nsubsteps[lev];
         dt_0 = amrex::min(dt_0, n_factor*dt_tmp[lev]);
     }
  
     // Limit dt's by the value of stop_time.
     const Real eps = 1.e-3*dt_0;
     if (t_new[0] + dt_0 > stop_time - eps) {
         dt_0 = stop_time - t_new[0];
     }
  
     dt[0] = dt_0;
     for (int lev = 1; lev <= finest_level; ++lev) {
         dt[lev] = dt[lev-1] / nsubsteps[lev];
     }
 }

◆ ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells	(	const AdvChoice &	advChoice,
		bool	use_num_diff
	)

inlinestaticprivate

     {
         if (use_num_diff)
         {
             return 3;
         } else {
             if (
                  (advChoice.dycore_horiz_adv_type    == AdvType::Centered_6th)
               || (advChoice.dycore_vert_adv_type     == AdvType::Centered_6th)
               || (advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
               || (advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
               || (advChoice.dryscal_horiz_adv_type   == AdvType::Centered_6th)
               || (advChoice.dryscal_vert_adv_type    == AdvType::Centered_6th) )
             { return 3; }
             else if (
                  (advChoice.dycore_horiz_adv_type    == AdvType::Upwind_5th)
               || (advChoice.dycore_vert_adv_type     == AdvType::Upwind_5th)
               || (advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
               || (advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
               || (advChoice.dryscal_horiz_adv_type   == AdvType::Upwind_5th)
               || (advChoice.dryscal_vert_adv_type    == AdvType::Upwind_5th) )
             { return 3; }
             else if (
                   (advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5)
                || (advChoice.dryscal_vert_adv_type    == AdvType::Weno_5)
                || (advChoice.moistscal_horiz_adv_type == AdvType::Weno_5)
                || (advChoice.moistscal_vert_adv_type  == AdvType::Weno_5)
                || (advChoice.moistscal_horiz_adv_type == AdvType::Weno_5Z)
                || (advChoice.moistscal_vert_adv_type  == AdvType::Weno_5Z)
                || (advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5Z)
                || (advChoice.dryscal_vert_adv_type    == AdvType::Weno_5Z) )
             { return 3; }
             else
             { return 2; }
         }
     }

◆ Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int lev )

private

 {
     auto& fine_new = vars_new[lev];
     auto& crse_new = vars_new[lev-1];
     auto& ba_fine  = fine_new[Vars::cons].boxArray();
     auto& ba_crse  = crse_new[Vars::cons].boxArray();
     auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
     auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
  
     int ncomp = vars_new[lev][Vars::cons].nComp();
  
     FPr_c.emplace_back(ba_fine, dm_fine, geom[lev]  ,
                        ba_crse, dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
     FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
     FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
     FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
 }

◆ DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int i )

inlineprivate

     {
         return *datalog[i];
     }

◆ DataLogName()

std::string ERF::DataLogName ( int i ) const

inlineprivatenoexcept

The filename of the ith datalog file.

1143 { return datalogname[i]; }

ERF::datalogname

amrex::Vector< std::string > datalogname

Definition: ERF.H:1132

◆ Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int lev )

private

 {
     auto& fine_new = vars_new[lev];
     auto& crse_new = vars_new[lev-1];
     auto& ba_fine  = fine_new[Vars::cons].boxArray();
     auto& ba_crse  = crse_new[Vars::cons].boxArray();
     auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
     auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
  
     int ncomp = fine_new[Vars::cons].nComp();
  
     FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev]  ,
                         ba_crse, dm_crse, geom[lev-1],
                         -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
     FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                         convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                         -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
     FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                         convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                         -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
     FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                         convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                         -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
 }

◆ derive_diag_profiles()

void ERF::derive_diag_profiles	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw
	)

Computes the profiles for diagnostic quantities.

Parameters

h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_uiuiu	Profile for u_iu_iu triple product on Host
h_avg_uiuiv	Profile for u_iu_iv triple product on Host
h_avg_uiuiw	Profile for u_iu_iw triple product on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host

 {
     // We assume that this is always called at level 0
     int lev = 0;
  
     bool l_use_KE  = (solverChoice.turbChoice[lev].les_type == LESType::Deardorff);
     bool l_use_QKE = solverChoice.turbChoice[lev].use_QKE && solverChoice.turbChoice[lev].advect_QKE;
  
     // This will hold rho, theta, ksgs, uu, uv, uw, vv, vw, ww, uth, vth, wth,
     //                  0      1     2   3   4   5   6   7   8    9   10   11
     //                thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr
     //                  12     13     14     15 16  17  18  19  20  21  22  23   24   25
     MultiFab mf_out(grids[lev], dmap[lev], 26, 0);
  
     MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
  
     MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
     MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
     MultiFab  w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
  
     average_face_to_cellcenter(mf_vels,0,
             Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
  
     int zdir = 2;
     auto domain = geom[0].Domain();
  
     // Sum in the horizontal plane
     h_avg_u  = sumToLine(mf_vels ,0,1,domain,zdir);
     h_avg_v  = sumToLine(mf_vels ,1,1,domain,zdir);
     h_avg_w  = sumToLine(mf_vels ,2,1,domain,zdir);
  
     int hu_size =  h_avg_u.size();
  
     // Divide by the total number of cells we are averaging over
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     for (int k = 0; k < hu_size; ++k) {
         h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
     }
  
     int nvars = vars_new[lev][Vars::cons].nComp();
     MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
  
     MultiFab p_hse (base_state[lev], make_alias, 1, 1); // p_0  is second component
  
     bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
  
     for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
         const Array4<Real>& fab_arr = mf_out.array(mfi);
         const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
         const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
         const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
         const Array4<Real>& cons_arr = mf_cons.array(mfi);
         const Array4<Real>&   p0_arr = p_hse.array(mfi);
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 1) = theta;
             Real ksgs = 0.0;
             if (l_use_KE)
                 ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
             else if (l_use_QKE)
                 ksgs = cons_arr(i,j,k,RhoQKE_comp) / cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 2) = ksgs;
             fab_arr(i, j, k, 3) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
             fab_arr(i, j, k, 4) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
             fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // u*w
             fab_arr(i, j, k, 6) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
             fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // v*w
             fab_arr(i, j, k, 8) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // w*w
             fab_arr(i, j, k, 9) = u_cc_arr(i,j,k) * theta; // u*th
             fab_arr(i, j, k,10) = v_cc_arr(i,j,k) * theta; // v*th
             fab_arr(i, j, k,11) = w_cc_arr(i,j,k) * theta; // w*th
             fab_arr(i, j, k,12) = theta * theta;           // th*th
             Real uiui = fab_arr(i,j,k,3) + fab_arr(i,j,k,6) + fab_arr(i,j,k,8);
             fab_arr(i, j, k,13) = uiui * u_cc_arr(i,j,k);   // (ui*ui)*u
             fab_arr(i, j, k,14) = uiui * v_cc_arr(i,j,k);   // (ui*ui)*v
             fab_arr(i, j, k,15) = uiui * w_cc_arr(i,j,k);   // (ui*ui)*w
  
             if (!use_moisture) {
                 Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                 p -= p0_arr(i,j,k);
                 fab_arr(i, j, k,16) = p;                       // p'
                 fab_arr(i, j, k,17) = p * u_cc_arr(i,j,k);     // p'*u
                 fab_arr(i, j, k,18) = p * v_cc_arr(i,j,k);     // p'*v
                 fab_arr(i, j, k,19) = p * w_cc_arr(i,j,k);     // p'*w
                 fab_arr(i, j, k,20) = 0.;  // qv
                 fab_arr(i, j, k,21) = 0.;  // qc
                 fab_arr(i, j, k,22) = 0.;  // qr
                 fab_arr(i, j, k,23) = 0.;  // w*qv
                 fab_arr(i, j, k,24) = 0.;  // w*qc
                 fab_arr(i, j, k,25) = 0.;  // w*qr
             }
         });
     } // mfi
  
     if (use_moisture)
     {
         int RhoQr_comp;
         int n_qstate = micro->Get_Qstate_Size();
         if (n_qstate > 3) {
             RhoQr_comp = RhoQ4_comp;
         } else {
             RhoQr_comp = RhoQ3_comp;
         }
  
         for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             const Box& bx = mfi.tilebox();
             const Array4<Real>& fab_arr  = mf_out.array(mfi);
             const Array4<Real>& cons_arr = mf_cons.array(mfi);
             const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
             const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
             const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
             const Array4<Real>&   p0_arr = p_hse.array(mfi);
             const Array4<Real>&   qv_arr = qmoist[0][0]->array(mfi); // TODO: Is this written only on lev 0?
  
             ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
             {
                 Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv_arr(i,j,k));
  
                 p -= p0_arr(i,j,k);
                 fab_arr(i, j, k,16) = p;                       // p'
                 fab_arr(i, j, k,17) = p * u_cc_arr(i,j,k);     // p'*u
                 fab_arr(i, j, k,18) = p * v_cc_arr(i,j,k);     // p'*v
                 fab_arr(i, j, k,19) = p * w_cc_arr(i,j,k);     // p'*w
                 fab_arr(i, j, k,20) = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);  // qv
                 fab_arr(i, j, k,21) = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);  // qc
                 fab_arr(i, j, k,22) = cons_arr(i,j,k,RhoQr_comp) / cons_arr(i,j,k,Rho_comp);  // qr
                 fab_arr(i, j, k,23) = w_cc_arr(i,j,k) * cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);  // w*qv
                 fab_arr(i, j, k,24) = w_cc_arr(i,j,k) * cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);  // w*qc
                 fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * cons_arr(i,j,k,RhoQr_comp) / cons_arr(i,j,k,Rho_comp);  // w*qr
             });
         } // mfi
     } // use_moisture
  
     h_avg_rho  = sumToLine(mf_out, 0,1,domain,zdir);
     h_avg_th   = sumToLine(mf_out, 1,1,domain,zdir);
     h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
     h_avg_uu   = sumToLine(mf_out, 3,1,domain,zdir);
     h_avg_uv   = sumToLine(mf_out, 4,1,domain,zdir);
     h_avg_uw   = sumToLine(mf_out, 5,1,domain,zdir);
     h_avg_vv   = sumToLine(mf_out, 6,1,domain,zdir);
     h_avg_vw   = sumToLine(mf_out, 7,1,domain,zdir);
     h_avg_ww   = sumToLine(mf_out, 8,1,domain,zdir);
     h_avg_uth  = sumToLine(mf_out, 9,1,domain,zdir);
     h_avg_vth  = sumToLine(mf_out,10,1,domain,zdir);
     h_avg_wth  = sumToLine(mf_out,11,1,domain,zdir);
     h_avg_thth = sumToLine(mf_out,12,1,domain,zdir);
     h_avg_uiuiu= sumToLine(mf_out,13,1,domain,zdir);
     h_avg_uiuiv= sumToLine(mf_out,14,1,domain,zdir);
     h_avg_uiuiw= sumToLine(mf_out,15,1,domain,zdir);
     h_avg_p    = sumToLine(mf_out,16,1,domain,zdir);
     h_avg_pu   = sumToLine(mf_out,17,1,domain,zdir);
     h_avg_pv   = sumToLine(mf_out,18,1,domain,zdir);
     h_avg_pw   = sumToLine(mf_out,19,1,domain,zdir);
     h_avg_qv   = sumToLine(mf_out,20,1,domain,zdir);
     h_avg_qc   = sumToLine(mf_out,21,1,domain,zdir);
     h_avg_qr   = sumToLine(mf_out,22,1,domain,zdir);
     h_avg_wqv  = sumToLine(mf_out,23,1,domain,zdir);
     h_avg_wqc  = sumToLine(mf_out,24,1,domain,zdir);
     h_avg_wqr  = sumToLine(mf_out,25,1,domain,zdir);
  
     // Divide by the total number of cells we are averaging over
     int h_avg_u_size = static_cast<int>(h_avg_u.size());
     for (int k = 0; k < h_avg_u_size; ++k) {
         h_avg_rho[k] /= area_z;  h_avg_ksgs[k] /= area_z;
         h_avg_th[k]  /= area_z;  h_avg_thth[k] /= area_z;
         h_avg_uu[k]  /= area_z;  h_avg_uv[k]   /= area_z;  h_avg_uw[k]  /= area_z;
         h_avg_vv[k]  /= area_z;  h_avg_vw[k]   /= area_z;  h_avg_ww[k]  /= area_z;
         h_avg_uth[k] /= area_z;  h_avg_vth[k]  /= area_z;  h_avg_wth[k] /= area_z;
         h_avg_uiuiu[k] /= area_z;
         h_avg_uiuiv[k] /= area_z;
         h_avg_uiuiw[k] /= area_z;
         h_avg_p[k]   /= area_z;
         h_avg_pu[k]  /= area_z;  h_avg_pv[k]   /= area_z;  h_avg_pw[k]  /= area_z;
         h_avg_qv[k]  /= area_z;  h_avg_qc[k]   /= area_z;  h_avg_qr[k]  /= area_z;
         h_avg_wqv[k] /= area_z;  h_avg_wqc[k]  /= area_z;  h_avg_wqr[k] /= area_z;
     }
 }

Here is the call graph for this function:

◆ derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw
	)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters

h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_uiuiu	Profile for u_iu_iu triple product on Host
h_avg_uiuiv	Profile for u_iu_iv triple product on Host
h_avg_uiuiw	Profile for u_iu_iw triple product on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host

 {
     // We assume that this is always called at level 0
     int lev = 0;
  
     bool l_use_KE  = (solverChoice.turbChoice[lev].les_type == LESType::Deardorff);
     bool l_use_QKE = solverChoice.turbChoice[lev].use_QKE && solverChoice.turbChoice[lev].advect_QKE;
  
     // This will hold rho, theta, ksgs, uu, uv, vv, uth, vth, thth, uiuiu, uiuiv, p, pu, pv
     //         index:   0      1     2   3   4   5    6    7     8      9     10  11 12  13
     MultiFab mf_out(grids[lev], dmap[lev], 14, 0);
  
     // This will hold uw, vw, ww, wth, uiuiw, pw (note: uiui == u_i*u_i = u*u + v*v + w*w)
     //         index:  0   1   2    3      4   5
     MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 6, 0);
  
     // This is only used to average u and v
     MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
  
     MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
     MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
     MultiFab  w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers
  
     int zdir = 2;
     auto domain = geom[0].Domain();
     Box stag_domain = domain;
     stag_domain.convert(IntVect(0,0,1));
  
     int nvars = vars_new[lev][Vars::cons].nComp();
     MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
  
     MultiFab p_hse (base_state[lev], make_alias, 1, 1); // p_0  is second component
  
     bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
  
     for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
         const Array4<Real>& fab_arr = mf_out.array(mfi);
         const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
         const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
         const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
         const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
         const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
         const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
         const Array4<Real>& cons_arr = mf_cons.array(mfi);
         const Array4<Real>&   p0_arr = p_hse.array(mfi);
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             u_cc_arr(i,j,k) = 0.5 * (u_arr(i,j,k) + u_arr(i+1,j  ,k));
             v_cc_arr(i,j,k) = 0.5 * (v_arr(i,j,k) + v_arr(i  ,j+1,k));
  
             Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 1) = theta;
             Real ksgs = 0.0;
             if (l_use_KE)
                 ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
             else if (l_use_QKE)
                 ksgs = cons_arr(i,j,k,RhoQKE_comp) / cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 2) = ksgs;
             fab_arr(i, j, k, 3) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
             fab_arr(i, j, k, 4) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
             fab_arr(i, j, k, 5) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
             fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * theta;             // u*th
             fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * theta;             // v*th
             fab_arr(i, j, k, 8) = theta * theta;                       // th*th
  
             Real wcc = 0.5 * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
             Real uiui = fab_arr(i,j,k,3) + fab_arr(i,j,k,5) + wcc*wcc;
             fab_arr(i, j, k,9 ) = uiui * u_cc_arr(i,j,k);           // (ui*ui)*u
             fab_arr(i, j, k,10) = uiui * v_cc_arr(i,j,k);           // (ui*ui)*v
  
             if (!use_moisture) {
                 Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                 p -= p0_arr(i,j,k);
                 fab_arr(i, j, k,11) = p;                       // p'
                 fab_arr(i, j, k,12) = p * u_cc_arr(i,j,k);     // p'*u
                 fab_arr(i, j, k,13) = p * v_cc_arr(i,j,k);     // p'*v
             }
         });
  
         const Box& zbx = mfi.tilebox(IntVect(0,0,1));
         ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             // average to z faces (first to cell centers, then in z)
             Real uface = 0.25 * ( u_arr(i  ,j,k) + u_arr(i  ,j,k-1)
                                 + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
             Real vface = 0.25 * ( v_arr(i,j  ,k) + v_arr(i,j  ,k-1)
                                 + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
             Real theta0 = cons_arr(i,j,k  ,RhoTheta_comp) / cons_arr(i,j,k  ,Rho_comp);
             Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
             Real thface = 0.5*(theta0 + theta1);
             fab_arr_stag(i,j,k,0) =           uface * w_fc_arr(i,j,k); // u*w
             fab_arr_stag(i,j,k,1) =           vface * w_fc_arr(i,j,k); // v*w
             fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
             fab_arr_stag(i,j,k,3) =          thface * w_fc_arr(i,j,k); // th*w
             Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
             fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
             if (!use_moisture) {
                 Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp)) - p0_arr(i,j,k  );
                 Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
                 Real pface = 0.5 * (p0 + p1);
                 fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k);       // p'*w
             }
         });
  
     } // mfi
  
     if (use_moisture)
     {
         for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             const Box& bx = mfi.tilebox();
             const Array4<Real>& fab_arr  = mf_out.array(mfi);
             const Array4<Real>& fab_arr_stag  = mf_out_stag.array(mfi);
             const Array4<Real>& cons_arr = mf_cons.array(mfi);
             const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
             const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
             const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
             const Array4<Real>&   p0_arr = p_hse.array(mfi);
             const Array4<Real>&   qv_arr = qmoist[0][0]->array(mfi); // TODO: Is this written only on lev 0?
  
             ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
             {
                 Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv_arr(i,j,k));
                 p -= p0_arr(i,j,k);
                 fab_arr(i, j, k,11) = p;                       // p'
                 fab_arr(i, j, k,12) = p * u_cc_arr(i,j,k);     // p'*u
                 fab_arr(i, j, k,13) = p * v_cc_arr(i,j,k);     // p'*v
             });
  
             const Box& zbx = mfi.tilebox(IntVect(0,0,1));
             ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
             {
                 Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp), qv_arr(i,j,k  )) - p0_arr(i,j,k  );
                 Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv_arr(i,j,k-1)) - p0_arr(i,j,k-1);
                 Real pface = 0.5 * (p0 + p1);
                 fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p'*w
             });
         } // mfi
     } // use_moisture
  
     // Sum in the horizontal plane
     h_avg_u  = sumToLine(u_cc,0,1,     domain,zdir);
     h_avg_v  = sumToLine(v_cc,0,1,     domain,zdir);
     h_avg_w  = sumToLine(w_fc,0,1,stag_domain,zdir);
  
     h_avg_rho  = sumToLine(mf_out, 0,1,domain,zdir);
     h_avg_th   = sumToLine(mf_out, 1,1,domain,zdir);
     h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
     h_avg_uu   = sumToLine(mf_out, 3,1,domain,zdir);
     h_avg_uv   = sumToLine(mf_out, 4,1,domain,zdir);
     h_avg_vv   = sumToLine(mf_out, 5,1,domain,zdir);
     h_avg_uth  = sumToLine(mf_out, 6,1,domain,zdir);
     h_avg_vth  = sumToLine(mf_out, 7,1,domain,zdir);
     h_avg_thth = sumToLine(mf_out, 8,1,domain,zdir);
     h_avg_uiuiu= sumToLine(mf_out, 9,1,stag_domain,zdir);
     h_avg_uiuiv= sumToLine(mf_out,10,1,stag_domain,zdir);
     h_avg_p    = sumToLine(mf_out,11,1,domain,zdir);
     h_avg_pu   = sumToLine(mf_out,12,1,domain,zdir);
     h_avg_pv   = sumToLine(mf_out,13,1,domain,zdir);
  
     h_avg_uw   = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
     h_avg_vw   = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
     h_avg_ww   = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
     h_avg_wth  = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
     h_avg_uiuiw= sumToLine(mf_out_stag,4,1,stag_domain,zdir);
     h_avg_pw   = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
  
     // Divide by the total number of cells we are averaging over
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
     for (int k = 0; k < unstag_size; ++k) {
         h_avg_u[k]     /= area_z;
         h_avg_v[k]     /= area_z;
         h_avg_rho[k]   /= area_z;
         h_avg_ksgs[k]  /= area_z;
         h_avg_th[k]    /= area_z;
         h_avg_thth[k]  /= area_z;
         h_avg_uu[k]    /= area_z;
         h_avg_uv[k]    /= area_z;
         h_avg_vv[k]    /= area_z;
         h_avg_uth[k]   /= area_z;
         h_avg_vth[k]   /= area_z;
         h_avg_uiuiu[k] /= area_z;
         h_avg_uiuiv[k] /= area_z;
         h_avg_p[k]     /= area_z;
         h_avg_pu[k]    /= area_z;
         h_avg_pv[k]    /= area_z;
     }
  
     for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
         h_avg_w[k]     /= area_z;
         h_avg_uw[k]    /= area_z;
         h_avg_vw[k]    /= area_z;
         h_avg_ww[k]    /= area_z;
         h_avg_wth[k]   /= area_z;
         h_avg_uiuiw[k] /= area_z;
         h_avg_pw[k]    /= area_z;
     }
 }

Here is the call graph for this function:

◆ derive_stress_profiles()

void ERF::derive_stress_profiles	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

 {
     int lev = 0;
  
     // This will hold the stress tensor components
     MultiFab mf_out(grids[lev], dmap[lev], 8, 0);
  
     for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
         const Array4<Real>& fab_arr = mf_out.array(mfi);
  
         // NOTE: These are from the last RK stage...
         const Array4<const Real>& tau11_arr = Tau11_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau12_arr = Tau12_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau13_arr = Tau13_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau22_arr = Tau22_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau23_arr = Tau23_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau33_arr = Tau33_lev[lev]->const_array(mfi);
  
         // These should be re-calculated during ERF_slow_rhs_post
         // -- just vertical SFS kinematic heat flux for now
         //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
         //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
         const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
         const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             fab_arr(i, j, k, 0) = tau11_arr(i,j,k);
             fab_arr(i, j, k, 1) = 0.25 * ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                          + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) );
             fab_arr(i, j, k, 2) = 0.25 * ( tau13_arr(i,j,k  ) + tau13_arr(i+1,j,k)
                                          + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) );
             fab_arr(i, j, k, 3) = tau22_arr(i,j,k);
             fab_arr(i, j, k, 4) = 0.25 * ( tau23_arr(i,j,k  ) + tau23_arr(i,j+1,k)
                                          + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) );
             fab_arr(i, j, k, 5) = tau33_arr(i,j,k);
             fab_arr(i, j, k, 6) =  hfx3_arr(i,j,k);
             fab_arr(i, j, k, 7) =  diss_arr(i,j,k);
         });
     }
  
     int zdir = 2;
     auto domain = geom[0].Domain();
  
     h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
     h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
     h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
     h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
     h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
     h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
     h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
     h_avg_diss  = sumToLine(mf_out,7,1,domain,zdir);
  
     int ht_size =  h_avg_tau11.size();
  
     // Divide by the total number of cells we are averaging over
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     for (int k = 0; k < ht_size; ++k) {
         h_avg_tau11[k] /= area_z; h_avg_tau12[k] /= area_z; h_avg_tau13[k] /= area_z;
         h_avg_tau22[k] /= area_z; h_avg_tau23[k] /= area_z; h_avg_tau33[k] /= area_z;
         h_avg_hfx3[k] /= area_z;
         h_avg_diss[k] /= area_z;
     }
 }

◆ derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

 {
     int lev = 0;
  
     // This will hold the stress tensor components
     MultiFab mf_out(grids[lev], dmap[lev], 8, 0);
  
     // This will hold Tau13 and Tau23
     MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 2, 0);
  
     for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
         const Array4<Real>& fab_arr = mf_out.array(mfi);
         const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
  
         // NOTE: These are from the last RK stage...
         const Array4<const Real>& tau11_arr = Tau11_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau12_arr = Tau12_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau13_arr = Tau13_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau22_arr = Tau22_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau23_arr = Tau23_lev[lev]->const_array(mfi);
         const Array4<const Real>& tau33_arr = Tau33_lev[lev]->const_array(mfi);
  
         // These should be re-calculated during ERF_slow_rhs_post
         // -- just vertical SFS kinematic heat flux for now
         //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
         //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
         const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
         const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             fab_arr(i, j, k, 0) = tau11_arr(i,j,k);
             fab_arr(i, j, k, 1) = 0.25 * ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                          + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) );
 //          fab_arr(i, j, k, 2) = 0.25 * ( tau13_arr(i,j,k  ) + tau13_arr(i+1,j,k)
 //                                       + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) );
             fab_arr(i, j, k, 3) = tau22_arr(i,j,k);
 //          fab_arr(i, j, k, 4) = 0.25 * ( tau23_arr(i,j,k  ) + tau23_arr(i,j+1,k)
 //                                       + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) );
             fab_arr(i, j, k, 5) = tau33_arr(i,j,k);
             fab_arr(i, j, k, 6) =  hfx3_arr(i,j,k);
             fab_arr(i, j, k, 7) =  diss_arr(i,j,k);
         });
  
         const Box& zbx = mfi.tilebox(IntVect(0,0,1));
         ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             fab_arr_stag(i,j,k,0) = 0.5*(tau13_arr(i,j,k) + tau13_arr(i+1,j  ,k));
             fab_arr_stag(i,j,k,1) = 0.5*(tau23_arr(i,j,k) + tau23_arr(i  ,j+1,k));
         });
     }
  
     int zdir = 2;
     auto domain = geom[0].Domain();
     Box stag_domain = domain;
     stag_domain.convert(IntVect(0,0,1));
  
     h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
     h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
 //  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
     h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
 //  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
     h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
     h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
     h_avg_diss  = sumToLine(mf_out,7,1,domain,zdir);
  
     h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
     h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
  
     int ht_size =  h_avg_tau11.size(); // _un_staggered
  
     // Divide by the total number of cells we are averaging over
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     for (int k = 0; k < ht_size; ++k) {
         h_avg_tau11[k] /= area_z; h_avg_tau12[k] /= area_z;
         h_avg_tau22[k] /= area_z; h_avg_tau33[k] /= area_z;
         h_avg_hfx3[k] /= area_z;
         h_avg_diss[k] /= area_z;
     }
     for (int k = 0; k < ht_size+1; ++k) { // staggered heights
         h_avg_tau13[k] /= area_z;
         h_avg_tau23[k] /= area_z;
     }
 }

◆ derive_upwp()

void ERF::derive_upwp ( amrex::Vector< amrex::Real > & h_havg )

◆ erf_enforce_hse()

void ERF::erf_enforce_hse	(	int	lev,
		amrex::MultiFab &	dens,
		amrex::MultiFab &	pres,
		amrex::MultiFab &	pi,
		std::unique_ptr< amrex::MultiFab > &	z_cc
	)

Enforces hydrostatic equilibrium when using terrain.

Parameters

[in]	lev	Integer specifying the current level
[out]	dens	MultiFab storing base state density
[out]	pres	MultiFab storing base state pressure
[out]	pi	MultiFab storing base state Exner function
[in]	z_cc	Pointer to MultiFab storing cell centered z-coordinates

 {
     Real l_gravity = solverChoice.gravity;
     bool l_use_terrain = solverChoice.use_terrain;
  
     const auto geomdata = geom[lev].data();
     const Real dz = geomdata.CellSize(2);
  
     const Box& domain = geom[lev].Domain();
  
     for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
     {
         // Create a flat box with same horizontal extent but only one cell in vertical
         const Box& tbz = mfi.nodaltilebox(2);
         int klo = tbz.smallEnd(2);
         int khi = tbz.bigEnd(2);
  
         Box b2d = tbz; // Copy constructor
         b2d.grow(0,1); b2d.grow(1,1); // Grow by one in the lateral directions
         b2d.setRange(2,0);
  
         // We integrate to the first cell (and below) by using rho in this cell
         // If gravity == 0 this is constant pressure
         // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
         // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
         // Note ng_pres_hse = 1
  
        // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
        // Note that gravity is positive
  
         Array4<Real>  rho_arr = dens.array(mfi);
         Array4<Real> pres_arr = pres.array(mfi);
         Array4<Real>   pi_arr =   pi.array(mfi);
         Array4<Real> zcc_arr;
         if (l_use_terrain) {
            zcc_arr = z_cc->array(mfi);
         }
  
         const Real rdOcp = solverChoice.rdOcp;
  
         ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
         {
             // Set value at surface from Newton iteration for rho
             if (klo == 0) {
                 // Physical height of the terrain at cell center
                 Real hz;
                 if (l_use_terrain) {
                     hz = zcc_arr(i,j,klo);
                 } else {
                     hz = 0.5*dz;
                 }
  
                 pres_arr(i,j,klo  ) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
                 pi_arr(i,j,klo  ) = getExnergivenP(pres_arr(i,j,klo  ), rdOcp);
  
                 // Set ghost cell with dz and rho at boundary
                 pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
                 pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
  
             } else {
                 // If klo > 0, we need to use the value of pres_arr(i,j,klo-1) which was
                 //    filled from FillPatch-ing it.
                 Real dz_loc;
                 if (l_use_terrain) {
                     dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
                 } else {
                     dz_loc = dz;
                 }
                 Real dens_interp = 0.5*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
                 pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
  
                 pi_arr(i,j,klo  ) = getExnergivenP(pres_arr(i,j,klo  ), rdOcp);
                 pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
             }
  
             Real dens_interp;
             if (l_use_terrain) {
                 for (int k = klo+1; k <= khi; k++) {
                     Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
                     dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                     pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
                     pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                 }
             } else {
                 for (int k = klo+1; k <= khi; k++) {
                     dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                     pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
                     pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                 }
             }
         });
  
         int domlo_x = domain.smallEnd(0); int domhi_x = domain.bigEnd(0);
         int domlo_y = domain.smallEnd(1); int domhi_y = domain.bigEnd(1);
  
         if (pres[mfi].box().smallEnd(0) < domlo_x)
         {
             Box bx = mfi.nodaltilebox(2);
             bx.setSmall(0,domlo_x-1);
             bx.setBig(0,domlo_x-1);
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                 pres_arr(i,j,k) = pres_arr(domlo_x,j,k);
                 pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
             });
         }
  
         if (pres[mfi].box().bigEnd(0) > domhi_x)
         {
             Box bx = mfi.nodaltilebox(2);
             bx.setSmall(0,domhi_x+1);
             bx.setBig(0,domhi_x+1);
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                 pres_arr(i,j,k) = pres_arr(domhi_x,j,k);
                 pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
             });
         }
  
         if (pres[mfi].box().smallEnd(1) < domlo_y)
         {
             Box bx = mfi.nodaltilebox(2);
             bx.setSmall(1,domlo_y-1);
             bx.setBig(1,domlo_y-1);
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                 pres_arr(i,j,k) = pres_arr(i,domlo_y,k);
                 pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
             });
         }
  
         if (pres[mfi].box().bigEnd(1) > domhi_y)
         {
             Box bx = mfi.nodaltilebox(2);
             bx.setSmall(1,domhi_y+1);
             bx.setBig(1,domhi_y+1);
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                 pres_arr(i,j,k) = pres_arr(i,domhi_y,k);
                 pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
             });
         }
     }
  
     dens.FillBoundary(geom[lev].periodicity());
     pres.FillBoundary(geom[lev].periodicity());
 }

Here is the call graph for this function:

◆ ErrorEst()

void ERF::ErrorEst	(	int	lev,
		amrex::TagBoxArray &	tags,
		amrex::Real	time,
		int	ngrow
	)

override

Function to tag cells for refinement – this overrides the pure virtual function in AmrCore

Parameters

[in]	levc	level of refinement at which we tag cells (0 is coarsest level)
[out]	tags	array of tagged cells
[in]	time	current time

 {
     const int clearval = TagBox::CLEAR;
     const int   tagval = TagBox::SET;
  
     for (int j=0; j < ref_tags.size(); ++j)
     {
         std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 0);
  
         // This allows dynamic refinement based on the value of the density
         if (ref_tags[j].Field() == "density")
         {
             MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,0);
  
         // This allows dynamic refinement based on the value of qv
         } else if ( ref_tags[j].Field() == "qv" ) {
             MultiFab qv(vars_new[levc][Vars::cons],make_alias,0,RhoQ1_comp+1);
             MultiFab::Copy(  *mf, qv, RhoQ1_comp, 0, 1, 0);
             MultiFab::Divide(*mf, qv, Rho_comp  , 0, 1, 0);
  
  
         // This allows dynamic refinement based on the value of qc
         } else if (ref_tags[j].Field() == "qc" ) {
             MultiFab qc(vars_new[levc][Vars::cons],make_alias,0,RhoQ2_comp+1);
             MultiFab::Copy(  *mf, qc, RhoQ2_comp, 0, 1, 0);
             MultiFab::Divide(*mf, qc, Rho_comp  , 0, 1, 0);
  
  
         // This allows dynamic refinement based on the value of the scalar/pressure/theta
         } else if ( (ref_tags[j].Field() == "scalar"  ) ||
                     (ref_tags[j].Field() == "pressure") ||
                     (ref_tags[j].Field() == "theta"   ) )
         {
             for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 auto& dfab = (*mf)[mfi];
                 auto& sfab = vars_new[levc][Vars::cons][mfi];
                 if (ref_tags[j].Field() == "scalar") {
                     derived::erf_derscalar(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                 } else if (ref_tags[j].Field() == "theta") {
                     derived::erf_dertheta(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                 }
             } // mfi
 #ifdef ERF_USE_PARTICLES
         } else {
             //
             // This allows dynamic refinement based on the number of particles per cell
             //
             // Note that we must count all the particles in levels both at and above the current,
             //      since otherwise, e.g., if the particles are all at level 1, counting particles at
             //      level 0 will not trigger refinement when regridding so level 1 will disappear,
             //      then come back at the next regridding
             //
             const auto& particles_namelist( particleData.getNames() );
             mf->setVal(0.0);
             for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
             {
                 std::string tmp_string(particles_namelist[i]+"_count");
                 IntVect rr = IntVect::TheUnitVector();
                 if (ref_tags[j].Field() == tmp_string) {
                     for (int lev = levc; lev <= finest_level; lev++)
                     {
                         MultiFab temp_dat(grids[lev], dmap[lev], 1, 0); temp_dat.setVal(0);
                         particleData[particles_namelist[i]]->IncrementWithTotal(temp_dat, lev);
  
                         MultiFab temp_dat_crse(grids[levc], dmap[levc], 1, 0); temp_dat_crse.setVal(0);
  
                         if (lev == levc) {
                             MultiFab::Copy(*mf, temp_dat, 0, 0, 1, 0);
                         } else {
                             for (int d = 0; d < AMREX_SPACEDIM; d++) {
                                 rr[d] *= ref_ratio[levc][d];
                             }
                             average_down(temp_dat, temp_dat_crse, 0, 1, rr);
                             MultiFab::Add(*mf, temp_dat_crse, 0, 0, 1, 0);
                         }
                     }
                 }
             }
 #endif
         }
  
         ref_tags[j](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
     } // loop over j
 }

Here is the call graph for this function:

◆ estTimeStep()

Real ERF::estTimeStep	(	int	level,
		long &	dt_fast_ratio
	)		const

Function that calls estTimeStep for each level

Parameters

[in]	level	level of refinement (coarsest level i 0)
[out]	dt_fast_ratio	ratio of slow to fast time step

 {
     BL_PROFILE("ERF::estTimeStep()");
  
     Real estdt_comp = 1.e20;
     Real estdt_lowM = 1.e20;
  
     auto const dxinv = geom[level].InvCellSizeArray();
     auto const dzinv = 1.0 / dz_min;
  
     MultiFab const& S_new = vars_new[level][Vars::cons];
  
     MultiFab ccvel(grids[level],dmap[level],3,0);
  
     average_face_to_cellcenter(ccvel,0,
                                Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
                                                         &vars_new[level][Vars::yvel],
                                                         &vars_new[level][Vars::zvel]});
  
     int l_no_substepping = solverChoice.no_substepping;
  
 #ifdef ERF_USE_EB
     EBFArrayBoxFactory ebfact = EBFactory(level);
     const MultiFab& detJ = ebfact.getVolFrac();
 #endif
  
 #ifdef ERF_USE_EB
     Real estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
        [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                   Array4<Real const> const& s,
                                   Array4<Real const> const& vf,
                                   Array4<Real const> const& u) -> Real
 #else
     Real estdt_comp_inv = ReduceMax(S_new, ccvel, 0,
        [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                   Array4<Real const> const& s,
                                   Array4<Real const> const& u) -> Real
 #endif
        {
            Real new_comp_dt = -1.e100;
            amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
            {
 #ifdef ERF_USE_EB
                if (vf(i,j,k) > 0.)
 #endif
                {
                    const Real rho      = s(i, j, k, Rho_comp);
                    const Real rhotheta = s(i, j, k, RhoTheta_comp);
  
                    // NOTE: even when moisture is present,
                    //       we only use the partial pressure of the dry air
                    //       to compute the soundspeed
                    Real pressure = getPgivenRTh(rhotheta);
                    Real c = std::sqrt(Gamma * pressure / rho);
  
                    // If we are not doing the acoustic substepping, then the z-direction contributes
                    //    to the computation of the time step
                    if (l_no_substepping) {
                        new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                 ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                 ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
  
                    // If we are     doing the acoustic substepping, then the z-direction does not contribute
                    //    to the computation of the time step
                    } else {
                        new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                 ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                    }
                }
            });
            return new_comp_dt;
        });
  
     ParallelDescriptor::ReduceRealMax(estdt_comp_inv);
     estdt_comp = cfl / estdt_comp_inv;
  
      Real estdt_lowM_inv = ReduceMax(ccvel, 0,
        [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                   Array4<Real const> const& u) -> Real
        {
            Real new_lm_dt = -1.e100;
            Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
            {
                new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
                                       ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
                                       ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
            });
            return new_lm_dt;
        });
  
      ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
      if (estdt_lowM_inv > 0.0_rt)
          estdt_lowM = cfl / estdt_lowM_inv;
  
      if (verbose) {
          if (fixed_dt <= 0.0) {
              Print() << "Using cfl = " << cfl << std::endl;
              Print() << "Fast  dt at level " << level << ":  " << estdt_comp << std::endl;
              if (estdt_lowM_inv > 0.0_rt) {
                  Print() << "Slow  dt at level " << level << ":  " << estdt_lowM << std::endl;
              } else {
                  Print() << "Slow  dt at level " << level << ": undefined " << std::endl;
              }
          }
  
          if (fixed_dt > 0.0) {
              Print() << "Based on cfl of 1.0 " << std::endl;
              Print() << "Fast  dt at level " << level << " would be:  " << estdt_comp/cfl << std::endl;
              if (estdt_lowM_inv > 0.0_rt) {
                  Print() << "Slow  dt at level " << level << " would be:  " << estdt_lowM/cfl << std::endl;
              } else {
                  Print() << "Slow  dt at level " << level << " would be undefined " << std::endl;
              }
              Print() << "Fixed dt at level " << level << "       is:  " << fixed_dt << std::endl;
              if (fixed_fast_dt > 0.0) {
                  Print() << "Fixed fast dt at level " << level << "       is:  " << fixed_fast_dt << std::endl;
              }
          }
      }
  
      if (fixed_dt > 0. && fixed_fast_dt > 0.) {
          dt_fast_ratio = static_cast<long>( fixed_dt / fixed_fast_dt );
      } else if (fixed_dt > 0.) {
          dt_fast_ratio = static_cast<long>( std::ceil((fixed_dt/estdt_comp)) );
      } else {
          dt_fast_ratio = (estdt_lowM_inv > 0.0) ? static_cast<long>( std::ceil((estdt_lowM/estdt_comp)) ) : 1;
      }
  
      // Force time step ratio to be an even value
      if (solverChoice.force_stage1_single_substep) {
          if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
      } else {
          if ( dt_fast_ratio%6 != 0) {
              Print() << "mri_dt_ratio = " << dt_fast_ratio
                             << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
              dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/6.0) * 6);
          }
      }
  
      if (verbose)
          Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
  
      if (fixed_dt > 0.0) {
          return fixed_dt;
      } else {
          if (l_no_substepping) {
              return estdt_comp;
          } else {
              return estdt_lowM;
          }
      }
 }

Here is the call graph for this function:

◆ Evolve()

void ERF::Evolve ( )

 {
     BL_PROFILE_VAR("ERF::Evolve()", evolve);
  
     Real cur_time = t_new[0];
  
     // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
     //      for finer levels (with or without subcycling)
     for (int step = istep[0]; step < max_step && cur_time < stop_time; ++step)
     {
         Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
  
         ComputeDt();
  
         // Make sure we have read enough of the boundary plane data to make it through this timestep
         if (input_bndry_planes)
         {
             m_r2d->read_input_files(cur_time,dt[0],m_bc_extdir_vals);
         }
  
         int lev = 0;
         int iteration = 1;
         timeStep(lev, cur_time, iteration);
  
         cur_time  += dt[0];
  
         Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
                 << " DT = " << dt[0]  << std::endl;
  
         post_timestep(step, cur_time, dt[0]);
  
         if (writeNow(cur_time, dt[0], step+1, m_plot_int_1, m_plot_per_1)) {
             last_plot_file_step_1 = step+1;
             WritePlotFile(1,plot_var_names_1);
         }
         if (writeNow(cur_time, dt[0], step+1, m_plot_int_2, m_plot_per_2)) {
             last_plot_file_step_2 = step+1;
             WritePlotFile(2,plot_var_names_2);
         }
  
         if (writeNow(cur_time, dt[0], step+1, m_check_int, m_check_per)) {
             last_check_file_step = step+1;
 #ifdef ERF_USE_NETCDF
             if (check_type == "netcdf") {
                WriteNCCheckpointFile();
             }
 #endif
             if (check_type == "native") {
                WriteCheckpointFile();
             }
         }
  
 #ifdef AMREX_MEM_PROFILING
         {
             std::ostringstream ss;
             ss << "[STEP " << step+1 << "]";
             MemProfiler::report(ss.str());
         }
 #endif
  
         if (cur_time >= stop_time - 1.e-6*dt[0]) break;
     }
  
     // Write plotfiles at final time
     if ( (m_plot_int_1 > 0 || m_plot_per_1 > 0.) && istep[0] > last_plot_file_step_1 ) {
         WritePlotFile(1,plot_var_names_1);
     }
     if ( (m_plot_int_2 > 0 || m_plot_per_2 > 0.) && istep[0] > last_plot_file_step_2) {
         WritePlotFile(2,plot_var_names_2);
     }
  
     if ( (m_check_int > 0 || m_check_per > 0.) && istep[0] > last_check_file_step) {
 #ifdef ERF_USE_NETCDF
         if (check_type == "netcdf") {
            WriteNCCheckpointFile();
         }
 #endif
         if (check_type == "native") {
            WriteCheckpointFile();
         }
     }
  
     BL_PROFILE_VAR_STOP(evolve);
 }

Referenced by main().

Here is the caller graph for this function:

◆ Factory()

amrex::FabFactory<amrex::FArrayBox> const& ERF::Factory ( int lev ) const

inlineprivatenoexcept

1154 { return *m_factory[lev]; }

◆ fill_from_bndryregs()

void ERF::fill_from_bndryregs	(	const amrex::Vector< amrex::MultiFab * > &	mfs,
		amrex::Real	time
	)

 {
     //
     // We now assume that if we read in on one face, we read in on all faces
     //
     AMREX_ALWAYS_ASSERT(m_r2d);
  
     int lev = 0;
     const Box& domain = geom[lev].Domain();
  
     const auto& dom_lo = lbound(domain);
     const auto& dom_hi = ubound(domain);
  
     Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time);
  
     const BCRec* bc_ptr = domain_bcs_type_d.data();
  
     // xlo: ori = 0
     // ylo: ori = 1
     // zlo: ori = 2
     // xhi: ori = 3
     // yhi: ori = 4
     // zhi: ori = 5
     const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
     const auto& bdatylo = (*bndry_data[1])[lev].const_array();
     const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
     const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
  
     int bccomp;
  
     for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
     {
         MultiFab& mf = *mfs[var_idx];
         const int icomp = 0;
         const int ncomp = mf.nComp();
  
         if (var_idx == Vars::xvel) {
            bccomp = BCVars::xvel_bc;
         } else if (var_idx == Vars::yvel) {
            bccomp = BCVars::yvel_bc;
         } else if (var_idx == Vars::zvel) {
            bccomp = BCVars::zvel_bc;
         } else  if (var_idx == Vars::cons) {
            bccomp = BCVars::cons_bc;
         }
  
 #ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
         for (MFIter mfi(mf); mfi.isValid(); ++mfi)
         {
             const Array4<Real>& dest_arr = mf.array(mfi);
             Box bx = mfi.growntilebox();
  
             // x-faces
             {
             Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
             if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
  
             Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
             if (var_idx == Vars::xvel) bx_xhi.setSmall(0,dom_hi.x);
  
             ParallelFor(
                 bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                     if (bc_ptr[icomp+n].lo(0) == ERFBCType::ext_dir_ingested) {
                         int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                         int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                         dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
                     }
                 },
                 bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                     if (bc_ptr[icomp+n].hi(0) == ERFBCType::ext_dir_ingested) {
                         int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                         int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                         dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
                     }
                 }
             );
             } // x-faces
  
             // y-faces
             {
             Box bx_ylo(bx); bx_ylo.setBig  (1,dom_lo.y-1);
             if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
  
             Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
             if (var_idx == Vars::yvel) bx_yhi.setSmall(1,dom_hi.y);
  
             ParallelFor(
                bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                     if (bc_ptr[icomp+n].lo(1) == ERFBCType::ext_dir_ingested) {
                         int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                         int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                         dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
                     }
                 },
                 bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                     if (bc_ptr[icomp+n].hi(1) == ERFBCType::ext_dir_ingested) {
                         int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                         int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                         dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
                     }
                 }
             );
             } // y-faces
         } // mf
     } // var_idx
 }

◆ fill_rhs()

void ERF::fill_rhs	(	amrex::MultiFab &	rhs_mf,
		const amrex::MultiFab &	state_mf,
		amrex::Real	time,
		const amrex::Geometry &	geom
	)

private

◆ FillBdyCCVels()

void ERF::FillBdyCCVels ( amrex::Vector< amrex::MultiFab > & mf_cc_vel )

 {
     // Impose bc's at domain boundaries
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         Box domain(Geom(lev).Domain());
  
         int ihi = domain.bigEnd(0);
         int jhi = domain.bigEnd(1);
         int khi = domain.bigEnd(2);
  
         // Impose periodicity first
         mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
  
         for (MFIter mfi(mf_cc_vel[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             // Note that we don't fill corners here -- only the cells that share a face
             //      with interior cells -- this is all that is needed to calculate vorticity
             const Box& bx = mfi.tilebox();
             const Array4<Real>& vel_arr = mf_cc_vel[lev].array(mfi);
  
             if (!Geom(lev).isPeriodic(0)) {
                 // Low-x side
                 if (bx.smallEnd(0) <= domain.smallEnd(0)) {
                     Real mult = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     ParallelFor(makeSlab(bx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                     {
                         vel_arr(-1,j,k,1) = mult*vel_arr(0,j,k,1); // v
                         vel_arr(-1,j,k,2) = mult*vel_arr(0,j,k,2); // w
                     });
                 }
  
                 // High-x side
                 if (bx.bigEnd(0) >= domain.bigEnd(0)) {
                     Real mult = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     ParallelFor(makeSlab(bx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                     {
                         vel_arr(ihi+1,j,k,1) = mult*vel_arr(ihi,j,k,1); // v
                         vel_arr(ihi+1,j,k,2) = mult*vel_arr(ihi,j,k,2); // w
                     });
                 }
             } // !periodic
  
             if (!Geom(lev).isPeriodic(1)) {
                 // Low-y side
                 if (bx.smallEnd(1) <= domain.smallEnd(1)) {
                     Real mult = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     ParallelFor(makeSlab(bx,1,0), [=] AMREX_GPU_DEVICE(int i, int  , int k) noexcept
                     {
                         vel_arr(i,-1,k,0) = mult*vel_arr(i,0,k,0); // u
                         vel_arr(i,-1,k,2) = mult*vel_arr(i,0,k,2); // w
                     });
                 }
  
                 // High-y side
                 if (bx.bigEnd(1) >= domain.bigEnd(1)) {
                     Real mult = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     ParallelFor(makeSlab(bx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
                     {
                         vel_arr(i,jhi+1,k,0) = mult*vel_arr(i,jhi,k,0); // u
                         vel_arr(i,jhi+1,k,2) = mult*-vel_arr(i,jhi,k,2); // w
                     });
                 }
             } // !periodic
  
             if (!Geom(lev).isPeriodic(2)) {
                 // Low-z side
                 if (bx.smallEnd(2) <= domain.smallEnd(2)) {
                     Real mult = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     ParallelFor(makeSlab(bx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                     {
                         vel_arr(i,j,-1,0) = mult*vel_arr(i,j,0,0); // u
                         vel_arr(i,j,-1,1) = mult*vel_arr(i,j,0,1); // v
                     });
                 }
  
                 // High-z side
                 if (bx.bigEnd(2) >= domain.bigEnd(2)) {
                     Real mult = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     ParallelFor(makeSlab(bx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                     {
                         vel_arr(i,j,khi+1,0) = mult*vel_arr(i,j,khi,0); // u
                         vel_arr(i,j,khi+1,1) = mult*vel_arr(i,j,khi,1); // v
                     });
                 }
             } // !periodic
         } // MFIter
  
     } // lev
 }

◆ FillCoarsePatch()

void ERF::FillCoarsePatch	(	int	lev,
		amrex::Real	time
	)

private

 {
     BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
     AMREX_ASSERT(lev > 0);
  
     //
     // First fill density at the COARSE level so we can convert velocity to momenta at the COARSE level
     //
     bool cons_only = true;
     FillPatch(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
                             &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
                            {&vars_new[lev-1][Vars::cons],
                             &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
                             false, cons_only);
  
     //
     // Convert velocity to momentum at the COARSE level
     //
     IntVect ngu = IntVect(0,0,0);
     IntVect ngv = IntVect(0,0,0);
     IntVect ngw = IntVect(0,0,0);
  
     VelocityToMomentum(vars_new[lev-1][Vars::xvel], ngu,
                        vars_new[lev-1][Vars::yvel], ngv,
                        vars_new[lev-1][Vars::zvel], ngw,
                        vars_new[lev-1][Vars::cons],
                          rU_new[lev-1],
                          rV_new[lev-1],
                          rW_new[lev-1],
                        Geom(lev).Domain(),
                        domain_bcs_type);
  
  
     //
     // Interpolate all cell-centered variables from coarse to fine level
     //
     Interpolater* mapper_c = &cell_cons_interp;
     Interpolater* mapper_f = &face_cons_linear_interp;
  
     int icomp = 0;
     int bccomp = BCVars::cons_bc;
     InterpFromCoarseLevel(vars_new[lev][Vars::cons], time, vars_new[lev-1][Vars::cons],
                           icomp, icomp, vars_new[lev][Vars::cons].nComp(),
                           geom[lev-1], geom[lev],
                           *physbcs_cons[lev-1], BCVars::cons_bc,
                           *physbcs_cons[lev  ], BCVars::cons_bc,
                           refRatio(lev-1), mapper_c, domain_bcs_type, bccomp);
  
     //
     // Interpolate x-momentum from coarse to fine level
     //
     InterpFromCoarseLevel(rU_new[lev], time, rU_new[lev-1],
                           0, 0, 1, geom[lev-1], geom[lev],
                           *physbcs_u[lev-1], BCVars::xvel_bc,
                           *physbcs_u[lev  ], BCVars::xvel_bc,
                           refRatio(lev-1), mapper_f,
                           domain_bcs_type, BCVars::xvel_bc);
  
     //
     // Interpolate y-momentum from coarse to fine level
     //
     bccomp = BCVars::yvel_bc;
     InterpFromCoarseLevel(rV_new[lev], time, rV_new[lev-1],
                           0, 0, 1, geom[lev-1], geom[lev],
                           *physbcs_v[lev-1], BCVars::yvel_bc,
                           *physbcs_v[lev  ], BCVars::yvel_bc,
                           refRatio(lev-1), mapper_f,
                           domain_bcs_type, BCVars::yvel_bc);
  
     //
     // Interpolate z-momentum from coarse to fine level
     //
  
     InterpFromCoarseLevel(rW_new[lev], time, rW_new[lev-1],
                           0, 0, 1, geom[lev-1], geom[lev],
                           *physbcs_w_no_terrain[lev-1], BCVars::zvel_bc,
                           *physbcs_w_no_terrain[lev  ], BCVars::zvel_bc,
                           refRatio(lev-1), mapper_f,
                           domain_bcs_type, BCVars::zvel_bc);
  
    for (int which_lev = lev-1; which_lev <= lev; which_lev++)
    {
         MomentumToVelocity(vars_new[which_lev][Vars::xvel],
                            vars_new[which_lev][Vars::yvel],
                            vars_new[which_lev][Vars::zvel],
                            vars_new[which_lev][Vars::cons],
                              rU_new[which_lev],
                              rV_new[which_lev],
                              rW_new[which_lev],
                            Geom(lev).Domain(),
                            domain_bcs_type);
     }
  
     vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
     vars_new[lev][Vars::xvel].FillBoundary(geom[lev].periodicity());
     vars_new[lev][Vars::yvel].FillBoundary(geom[lev].periodicity());
     vars_new[lev][Vars::zvel].FillBoundary(geom[lev].periodicity());
  
     // ***************************************************************************
     // Physical bc's at domain boundary
     // ***************************************************************************
     IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
     IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
  
     int ncomp_cons = vars_new[lev][Vars::cons].nComp();
  
     (*physbcs_cons[lev])(vars_new[lev][Vars::cons],0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc);
     (   *physbcs_u[lev])(vars_new[lev][Vars::xvel],0,1         ,ngvect_vels,time,BCVars::xvel_bc);
     (   *physbcs_v[lev])(vars_new[lev][Vars::yvel],0,1         ,ngvect_vels,time,BCVars::yvel_bc);
     (   *physbcs_w[lev])(vars_new[lev][Vars::zvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                          ngvect_vels,time,BCVars::xvel_bc,BCVars::yvel_bc,BCVars::zvel_bc);
  
     // ***************************************************************************
     // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
     // ***************************************************************************
 }

Here is the call graph for this function:

◆ FillIntermediatePatch()

void ERF::FillIntermediatePatch	(	int	lev,
		amrex::Real	time,
		const amrex::Vector< amrex::MultiFab * > &	mfs_vel,
		const amrex::Vector< amrex::MultiFab * > &	mfs_mom,
		int	ng_cons,
		int	ng_vel,
		bool	cons_only,
		int	icomp_cons,
		int	ncomp_cons,
		bool	allow_most_bcs = `true`
	)

private

 {
     BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
     int bccomp;
     Interpolater* mapper;
  
     //
     // ***************************************************************************
     // The first thing we do is interpolate the momenta on the "valid" faces of
     // the fine grids (where the interface is coarse/fine not fine/fine) -- this
     // will not be over-written by interpolation below because the FillPatch
     // operators see these as valid faces.  But we must have these interpolated
     // values in the fine data before we call FillPatchTwoLevels.
     //
     // Also -- note that we might be filling values by interpolation at physical boundaries
     //         here but that's ok because we will overwrite those values when we impose
     //         the physical bc's below
     // ***************************************************************************
     if (lev>0) {
         if (cf_set_width > 0) {
             // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
             FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
         }
         if ( !cons_only && (cf_set_width >= 0) ) {
             FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
             FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
             FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
         }
     }
  
     AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
     AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
  
     // Enforce no penetration for thin immersed body
     if (xflux_imask[lev]) {
         ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
     }
     if (yflux_imask[lev]) {
         ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
     }
     if (zflux_imask[lev]) {
         ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
     }
  
     // We always come in to this call with updated momenta but we need to create updated velocity
     //    in order to impose the rest of the bc's
     if (!cons_only) {
         // This only fills VALID region of velocity
         MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                            *mfs_vel[Vars::cons],
                            *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                             Geom(lev).Domain(), domain_bcs_type);
     }
  
     // We now start working on VELOCITY
     for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
     {
         if (cons_only && var_idx != Vars::cons) continue;
  
         MultiFab& mf = *mfs_vel[var_idx];
  
         IntVect ngvect;
         int icomp, ncomp;
         if (var_idx == Vars::cons)
         {
             bccomp = icomp_cons;
             mapper = &cell_cons_interp;
             ngvect = IntVect(ng_cons,ng_cons,ng_cons);
             icomp  = icomp_cons;
             ncomp  = ncomp_cons;
         }
         else if (var_idx == IntVars::xmom)
         {
             bccomp = BCVars::xvel_bc;
             mapper = &face_cons_linear_interp;
             ngvect = IntVect(ng_vel,ng_vel,ng_vel);
             icomp  = 0;
             ncomp  = 1;
         }
         else if (var_idx == IntVars::ymom)
         {
             bccomp = BCVars::yvel_bc;
             mapper = &face_cons_linear_interp;
             ngvect = IntVect(ng_vel,ng_vel,ng_vel);
             icomp  = 0;
             ncomp  = 1;
         }
         else if (var_idx == IntVars::zmom)
         {
             bccomp = BCVars::zvel_bc;
             mapper = &face_cons_linear_interp;
             ngvect = IntVect(ng_vel,ng_vel,0);
             icomp  = 0;
             ncomp  = 1;
         }
  
         if (lev == 0)
         {
             // This fills fine-fine ghost values of VELOCITY
             mf.FillBoundary(icomp,ncomp,ngvect,geom[lev].periodicity());
         }
         else
         {
             //
             // NOTE: All interpolation here happens on velocities not momenta;
             //       note we only do the interpolation and FillBoundary here,
             //       physical bc's are imposed later
             //
             // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
             //       unlike the FillSet calls above which filled momenta on the coarse/fine bdy
             //
             Vector<MultiFab*> fmf = {&mf};
             Vector<MultiFab*> cmf = {&vars_old[lev-1][var_idx], &vars_new[lev-1][var_idx]};
             Vector<Real> ctime    = {t_old[lev-1], t_new[lev-1]};
  
             FillPatchTwoLevels(mf, time, cmf, ctime, fmf, {time},
                                icomp, icomp, ncomp, geom[lev-1], geom[lev],
                                null_bc, 0, null_bc, 0, refRatio(lev-1),
                                mapper, domain_bcs_type, bccomp);
         } // lev > 0
     } // var_idx
  
     // ***************************************************************************
     // Physical bc's at domain boundary
     // ***************************************************************************
     IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
     IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
  
 #ifdef ERF_USE_NETCDF
     // We call this here because it is an ERF routine
     if (use_real_bcs && (lev==0)) {
         fill_from_realbdy(mfs_vel,time,false,0,ncomp_cons);
     }
 #endif
  
     if (m_r2d) fill_from_bndryregs(mfs_vel,time);
  
     // We call this even if init_type == real because this routine will fill the vertical bcs
     (*physbcs_cons[lev])(*mfs_vel[Vars::cons],icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc);
     if (!cons_only) {
         (*physbcs_u[lev])(*mfs_vel[Vars::xvel],0,1,ngvect_vels,time,BCVars::xvel_bc);
         (*physbcs_v[lev])(*mfs_vel[Vars::yvel],0,1,ngvect_vels,time,BCVars::yvel_bc);
         (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                            ngvect_vels,time,BCVars::xvel_bc,BCVars::yvel_bc,BCVars::zvel_bc);
     }
     // ***************************************************************************
  
     // MOST boundary conditions
     if (!(cons_only && ncomp_cons == 1) && m_most && allow_most_bcs) {
         m_most->impose_most_bcs(lev,mfs_vel,
 #ifdef ERF_EXPLICIT_MOST_STRESS
                                 Tau13_lev[lev].get(), Tau31_lev[lev].get(),
                                 Tau23_lev[lev].get(), Tau32_lev[lev].get(),
                                 SFS_hfx3_lev[lev].get(),
 #else
                                 eddyDiffs_lev[lev].get(),
 #endif
                                 z_phys_nd[lev].get());
     }
  
     // We always come in to this call with momenta so we need to leave with momenta!
     // We need to make sure we convert back on all ghost cells/faces because this is
     // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
     if (!cons_only) {
         IntVect ngu = mfs_vel[Vars::xvel]->nGrowVect();
         IntVect ngv = mfs_vel[Vars::yvel]->nGrowVect();
         IntVect ngw = mfs_vel[Vars::zvel]->nGrowVect();
  
         if (!solverChoice.use_NumDiff) {
             ngu = IntVect(1,1,1);
             ngv = IntVect(1,1,1);
             ngw = IntVect(1,1,0);
         }
         VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
                            *mfs_vel[Vars::yvel], ngv,
                            *mfs_vel[Vars::zvel], ngw,
                            *mfs_vel[Vars::cons],
                            *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                            Geom(lev).Domain(),
                            domain_bcs_type);
     }
 }

Here is the call graph for this function:

◆ FillPatch()

void ERF::FillPatch	(	int	lev,
		amrex::Real	time,
		const amrex::Vector< amrex::MultiFab * > &	mfs_vel,
		const amrex::Vector< amrex::MultiFab * > &	mfs_mom,
		bool	fillset = `true`,
		bool	cons_only = `false`
	)

private

 {
     BL_PROFILE_VAR("ERF::FillPatch()",ERF_FillPatch);
     Interpolater* mapper = nullptr;
  
     //
     // ***************************************************************************
     // The first thing we do is interpolate the momenta on the "valid" faces of
     // the fine grids (where the interface is coarse/fine not fine/fine) -- this
     // will not be over-written below because the FillPatch operators see these as
     // valid faces.
     //
     // Note that we interpolate momentum not velocity, but all the other boundary
     // conditions are imposed on velocity, so we convert to momentum here then
     // convert back.
     // ***************************************************************************
     if (lev>0 && fillset) {
         if (cf_set_width > 0) {
             FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
         }
         if (cf_set_width >= 0 && !cons_only) {
             //
             // This is an optimization since we won't need more than one ghost
             // cell of momentum in the integrator if not using NumDiff
             //
             //IntVect ngu = (solverChoice.use_NumDiff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
             //IntVect ngv = (solverChoice.use_NumDiff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
             //IntVect ngw = (solverChoice.use_NumDiff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
             IntVect ngu = IntVect::TheZeroVector();
             IntVect ngv = IntVect::TheZeroVector();
             IntVect ngw = IntVect::TheZeroVector();
  
             VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
                                *mfs_vel[Vars::yvel], ngv,
                                *mfs_vel[Vars::zvel], ngw,
                                *mfs_vel[Vars::cons],
                                *mfs_mom[IntVars::xmom],
                                *mfs_mom[IntVars::ymom],
                                *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(),
                                domain_bcs_type);
  
             FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
             FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
             FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
  
             MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                                *mfs_vel[Vars::cons],
                                *mfs_mom[IntVars::xmom],
                                *mfs_mom[IntVars::ymom],
                                *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(),
                                domain_bcs_type);
         }
     }
  
     IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
     IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
  
     if (lev == 0)
     {
         const int icomp = 0;
  
         Vector<Real> ftime    = {t_old[lev], t_new[lev]};
  
         Vector<MultiFab*> fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
         const int  ncomp = mfs_vel[Vars::cons]->nComp();
         FillPatchSingleLevel(*mfs_vel[Vars::cons], time, fmf, ftime, icomp, icomp, ncomp,
                              geom[lev], *physbcs_cons[lev], BCVars::cons_bc+icomp);
  
         if (!cons_only) {
             fmf = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
             const int ncomp_u = 1;
             FillPatchSingleLevel(*mfs_vel[Vars::xvel], time, fmf, ftime, icomp, icomp, ncomp_u,
                                  geom[lev], *physbcs_u[lev], BCVars::xvel_bc);
  
             fmf = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
             const int ncomp_v = 1;
             FillPatchSingleLevel(*mfs_vel[Vars::yvel], time, fmf, ftime, icomp, icomp, ncomp_v,
                                  geom[lev], *physbcs_v[lev], BCVars::yvel_bc);
  
             fmf = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
             const int ncomp_w = 1;
             FillPatchSingleLevel(*mfs_vel[Vars::zvel], time, fmf, ftime, icomp, icomp, ncomp_w,
                                  geom[lev], *physbcs_w_no_terrain[lev], BCVars::zvel_bc);
             (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                                ngvect_vels,time,BCVars::xvel_bc,BCVars::yvel_bc,BCVars::zvel_bc);
         } // !cons_only
  
     } else {
  
         Vector<Real> ftime    = {t_old[lev], t_new[lev]};
         Vector<Real> ctime    = {t_old[lev-1], t_new[lev-1]};
  
         Vector<MultiFab*> fmf = {&vars_old[lev  ][Vars::cons], &vars_new[lev  ][Vars::cons]};
         Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
         MultiFab& mf_c = *mfs_vel[Vars::cons];
         mapper = &cell_cons_interp;
         FillPatchTwoLevels(mf_c, time, cmf, ctime, fmf, ftime,
                            0, 0, mf_c.nComp(), geom[lev-1], geom[lev],
                            null_bc, BCVars::cons_bc, null_bc, BCVars::cons_bc, refRatio(lev-1),
                            mapper, domain_bcs_type, BCVars::cons_bc);
  
         if (!cons_only) {
             mapper = &face_cons_linear_interp;
  
             MultiFab& mf_u = *mfs_vel[Vars::xvel];
             fmf = {&vars_old[lev  ][Vars::xvel], &vars_new[lev  ][Vars::xvel]};
             cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
             FillPatchTwoLevels(mf_u, time, cmf, ctime, fmf, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                null_bc, BCVars::xvel_bc, null_bc, BCVars::xvel_bc, refRatio(lev-1),
                                mapper, domain_bcs_type, BCVars::xvel_bc);
  
             MultiFab& mf_v = *mfs_vel[Vars::yvel];
             fmf = {&vars_old[lev  ][Vars::yvel], &vars_new[lev  ][Vars::yvel]};
             cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
             FillPatchTwoLevels(mf_v, time, cmf, ctime, fmf, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                null_bc, BCVars::yvel_bc, null_bc, BCVars::yvel_bc, refRatio(lev-1),
                                mapper, domain_bcs_type, BCVars::yvel_bc);
  
             MultiFab& mf_w = *mfs_vel[Vars::zvel];
             fmf = {&vars_old[lev  ][Vars::zvel], &vars_new[lev  ][Vars::zvel]};
             cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
             FillPatchTwoLevels(mf_w, time, cmf, ctime, fmf, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                null_bc, BCVars::zvel_bc, null_bc, BCVars::zvel_bc, refRatio(lev-1),
                                mapper, domain_bcs_type, BCVars::zvel_bc);
         } // !cons_only
     } // lev > 0
  
     // ***************************************************************************
     // Physical bc's at domain boundary
     // ***************************************************************************
     int icomp_cons = 0;
     int ncomp_cons = mfs_vel[Vars::cons]->nComp();
  
 #ifdef ERF_USE_NETCDF
     // We call this here because it is an ERF routine
     if (use_real_bcs && (lev==0)) {
         fill_from_realbdy(mfs_vel,time,false,0,ncomp_cons);
     }
 #endif
  
     if (m_r2d) fill_from_bndryregs(mfs_vel,time);
  
     // We call these even if init_type == real because these will fill the vertical bcs
     (*physbcs_cons[lev])(*mfs_vel[Vars::cons],icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc);
     if (!cons_only) {
         (*physbcs_u[lev])(*mfs_vel[Vars::xvel],0,1,ngvect_vels,time,BCVars::xvel_bc);
         (*physbcs_v[lev])(*mfs_vel[Vars::yvel],0,1,ngvect_vels,time,BCVars::yvel_bc);
         (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                            ngvect_vels,time,BCVars::xvel_bc,BCVars::yvel_bc,BCVars::zvel_bc);
     }
 }

Here is the call graph for this function:

◆ FillPatchMoistVars()

void ERF::FillPatchMoistVars	(	int	lev,
		amrex::MultiFab &	mf
	)

private

 {
     BL_PROFILE_VAR("ERF::FillPatchMoistVars()",ERF_FillPatchMoistVars);
     // ***************************************************************************
     // Physical bc's at domain boundary
     // ***************************************************************************
     int icomp_cons = 0;
     int ncomp_cons = 1; // We only fill qv, the first component
  
     // Note that we are filling qv, stored in qmoist[lev], with the input data (if there is any), stored
     // in RhoQ1_comp.
  
     if (!use_real_bcs) {
         Real time = Real(0.0);
         IntVect ngvect_cons = mf.nGrowVect();
         int bccomp_cons = BCVars::RhoQ1_bc_comp;
  
         (*physbcs_cons[lev])(mf,icomp_cons,ncomp_cons,ngvect_cons,time,bccomp_cons);
     }
  
     mf.FillBoundary(geom[lev].periodicity());
 }

◆ getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int lev )

inlineprivate

     {
         return advflux_reg[lev];
     }

◆ getCPUTime()

static amrex::Real ERF::getCPUTime ( )

inlinestaticprivate

     {
       int numCores = amrex::ParallelDescriptor::NProcs();
 #ifdef _OPENMP
       numCores = numCores * omp_get_max_threads();
 #endif
  
       amrex::Real T =
         numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
         previousCPUTimeUsed;
  
       return T;
     }

◆ GotoNextLine()

void ERF::GotoNextLine ( std::istream & is )

staticprivate

Utility to skip to next line in Header file input stream.

 {
     constexpr std::streamsize bl_ignore_max { 100000 };
     is.ignore(bl_ignore_max, '\n');
 }

◆ init1DArrays()

void ERF::init1DArrays ( )

private

◆ init_bcs()

void ERF::init_bcs ( )

private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

 {
     auto f = [this] (std::string const& bcid, Orientation ori)
     {
         // These are simply defaults for Dirichlet faces -- they should be over-written below
         m_bc_extdir_vals[BCVars::Rho_bc_comp][ori]       =  1.0;
         m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori]  = -1.0; // It is important to set this negative
                                                                  // because the sign is tested on below
         m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori]     = 0.0;
         m_bc_extdir_vals[BCVars::RhoQKE_bc_comp][ori]    = 0.0;
         m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = 0.0;
         m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori]     = 0.0;
         m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori]     = 0.0;
  
         m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.0; // default
         m_bc_extdir_vals[BCVars::yvel_bc][ori] = 0.0;
         m_bc_extdir_vals[BCVars::zvel_bc][ori] = 0.0;
  
         // These are simply defaults for Neumann gradients -- they should be over-written below
         m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]       = 0.0;
         m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]  = 0.0;
         m_bc_neumann_vals[BCVars::RhoKE_bc_comp][ori]     = 0.0;
         m_bc_neumann_vals[BCVars::RhoQKE_bc_comp][ori]    = 0.0;
         m_bc_neumann_vals[BCVars::RhoScalar_bc_comp][ori] = 0.0;
         m_bc_neumann_vals[BCVars::RhoQ1_bc_comp][ori]     = 0.0;
         m_bc_neumann_vals[BCVars::RhoQ2_bc_comp][ori]     = 0.0;
         m_bc_neumann_vals[BCVars::xvel_bc][ori] = 0.0;
         m_bc_neumann_vals[BCVars::yvel_bc][ori] = 0.0;
         m_bc_neumann_vals[BCVars::zvel_bc][ori] = 0.0;
  
         std::string pp_text;
         if (pp_prefix == "erf") {
           pp_text = bcid;
         } else {
           pp_text = pp_prefix + "." + bcid;
         }
         ParmParse pp(pp_text);
         std::string bc_type_in = "null";
         pp.query("type", bc_type_in);
         //if (pp.query("type", bc_type_in))
         //   Print() << "INPUT BC TYPE " << bcid << " " << bc_type_in << std::endl;
         std::string bc_type = amrex::toLower(bc_type_in);
  
         if (bc_type == "symmetry")
         {
             // Print() << bcid << " set to symmetry.\n";
               phys_bc_type[ori] = ERF_BC::symmetry;
             domain_bc_type[ori] = "Symmetry";
         }
         else if (bc_type == "outflow")
         {
             // Print() << bcid << " set to outflow.\n";
               phys_bc_type[ori] = ERF_BC::outflow;
             domain_bc_type[ori] = "Outflow";
         }
         else if (bc_type == "open")
         {
             // Print() << bcid << " set to open.\n";
             AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
               phys_bc_type[ori] = ERF_BC::open;
             domain_bc_type[ori] = "Open";
         }
         else if (bc_type == "inflow")
         {
             // Print() << bcid << " set to inflow.\n";
               phys_bc_type[ori] = ERF_BC::inflow;
             domain_bc_type[ori] = "Inflow";
  
             std::vector<Real> v;
             if (input_bndry_planes && m_r2d->ingested_velocity()) {
                 m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.;
                 m_bc_extdir_vals[BCVars::yvel_bc][ori] = 0.;
                 m_bc_extdir_vals[BCVars::zvel_bc][ori] = 0.;
             } else {
                 pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
                 m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                 m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                 m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
             }
  
             Real rho_in;
             if (input_bndry_planes && m_r2d->ingested_density()) {
                 m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = 0.;
             } else {
                 pp.get("density", rho_in);
                 m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
             }
             Real theta_in;
             if (input_bndry_planes && m_r2d->ingested_theta()) {
                 m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = 0.;
             } else {
                 pp.get("theta", theta_in);
                 m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
             }
             Real scalar_in = 0.;
             if (input_bndry_planes && m_r2d->ingested_scalar()) {
                 m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = 0.;
             } else {
                 if (pp.query("scalar", scalar_in))
                 m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
             }
  
             if (solverChoice.moisture_type != MoistureType::None) {
                 Real qv_in = 0.;
                 if (input_bndry_planes && m_r2d->ingested_q1()) {
                     m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = 0.;
                 } else {
                     if (pp.query("qv", qv_in))
                     m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
                 }
                 Real qc_in = 0.;
                 if (input_bndry_planes && m_r2d->ingested_q2()) {
                     m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = 0.;
                 } else {
                     if (pp.query("qc", qc_in))
                     m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
                 }
             }
  
             Real KE_in = 0.;
             if (input_bndry_planes && m_r2d->ingested_KE()) {
                 m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = 0.;
             } else {
                 if (pp.query("KE", KE_in))
                 m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
             }
             Real QKE_in = 0.;
             if (input_bndry_planes && m_r2d->ingested_QKE()) {
                 m_bc_extdir_vals[BCVars::RhoQKE_bc_comp][ori] = 0.;
             } else {
                 if (pp.query("QKE", QKE_in))
                 m_bc_extdir_vals[BCVars::RhoQKE_bc_comp][ori] = rho_in*QKE_in;
             }
  
         }
         else if (bc_type == "noslipwall")
         {
             // Print() << bcid <<" set to no-slip wall.\n";
               phys_bc_type[ori] = ERF_BC::no_slip_wall;
             domain_bc_type[ori] = "NoSlipWall";
  
             std::vector<Real> v;
  
             // The values of m_bc_extdir_vals default to 0.
             // But if we find "velocity" in the inputs file, use those values instead.
             if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
             {
                 v[ori.coordDir()] = 0.0;
                 m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                 m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                 m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
             }
  
             Real rho_in;
             if (pp.query("density", rho_in))
             {
                 m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
             }
  
             Real theta_in;
             if (pp.query("theta", theta_in))
             {
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
             }
  
             Real theta_grad_in;
             if (pp.query("theta_grad", theta_grad_in))
             {
                 m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
             }
         }
         else if (bc_type == "slipwall")
         {
             // Print() << bcid <<" set to slip wall.\n";
  
               phys_bc_type[ori] = ERF_BC::slip_wall;
             domain_bc_type[ori] = "SlipWall";
  
             Real rho_in;
             if (pp.query("density", rho_in))
             {
                 m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
             }
  
             Real theta_in;
             if (pp.query("theta", theta_in))
             {
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
             }
  
             Real rho_grad_in;
             if (pp.query("density_grad", rho_grad_in))
             {
                m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = rho_grad_in;
             }
  
             Real theta_grad_in;
             if (pp.query("theta_grad", theta_grad_in))
             {
                m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
             }
         }
         else if (bc_type == "most")
         {
               phys_bc_type[ori] = ERF_BC::MOST;
             domain_bc_type[ori] = "MOST";
         }
         else
         {
             phys_bc_type[ori] = ERF_BC::undefined;
         }
  
         if (geom[0].isPeriodic(ori.coordDir())) {
             domain_bc_type[ori] = "Periodic";
             if (phys_bc_type[ori] == ERF_BC::undefined)
             {
                 phys_bc_type[ori] = ERF_BC::periodic;
             } else {
                 Abort("Wrong BC type for periodic boundary");
             }
         }
  
         if (phys_bc_type[ori] == ERF_BC::undefined)
         {
              Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
              Abort("This BC type is unknown");
         }
     };
  
     f("xlo", Orientation(Direction::x,Orientation::low));
     f("xhi", Orientation(Direction::x,Orientation::high));
     f("ylo", Orientation(Direction::y,Orientation::low));
     f("yhi", Orientation(Direction::y,Orientation::high));
     f("zlo", Orientation(Direction::z,Orientation::low));
     f("zhi", Orientation(Direction::z,Orientation::high));
  
     // *****************************************************************************
     //
     // Here we translate the physical boundary conditions -- one type per face --
     //     into logical boundary conditions for each velocity component
     //
     // *****************************************************************************
     {
         domain_bcs_type.resize(AMREX_SPACEDIM+NVAR_max);
         domain_bcs_type_d.resize(AMREX_SPACEDIM+NVAR_max);
  
         for (OrientationIter oit; oit; ++oit) {
             Orientation ori = oit();
             int dir = ori.coordDir();
             Orientation::Side side = ori.faceDir();
             auto const bct = phys_bc_type[ori];
             if ( bct == ERF_BC::symmetry )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::reflect_even);
                     }
                     domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::reflect_odd);
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::reflect_even);
                     }
                     domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::reflect_odd);
                 }
             }
             else if (bct == ERF_BC::outflow)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                     }
                     domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::neumann_int);
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                     }
                     domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::neumann_int);
                 }
             }
             else if (bct == ERF_BC::open)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++)
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::open);
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++)
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::open);
                 }
             }
             else if (bct == ERF_BC::inflow)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                         if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                             domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                         }
                     }
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                         if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                             domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                         }
                     }
                 }
             }
             else if (bct == ERF_BC::no_slip_wall)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                     }
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                     }
                 }
             }
             else if (bct == ERF_BC::slip_wall)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                     }
                     // Only normal direction has ext_dir
                     domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::ext_dir);
  
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                     }
                     // Only normal direction has ext_dir
                     domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::ext_dir);
                 }
             }
             else if (bct == ERF_BC::periodic)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::int_dir);
                     }
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::int_dir);
                     }
                 }
             }
             else if ( bct == ERF_BC::MOST )
             {
                 AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
                 domain_bcs_type[BCVars::xvel_bc+0].setLo(dir, ERFBCType::reflect_odd);
                 domain_bcs_type[BCVars::xvel_bc+1].setLo(dir, ERFBCType::reflect_odd);
                 domain_bcs_type[BCVars::xvel_bc+2].setLo(dir, ERFBCType::ext_dir);
             }
         }
     }
  
     // *****************************************************************************
     //
     // Here we translate the physical boundary conditions -- one type per face --
     //     into logical boundary conditions for each cell-centered variable
     //
     // *****************************************************************************
     {
         for (OrientationIter oit; oit; ++oit) {
             Orientation ori = oit();
             int dir = ori.coordDir();
             Orientation::Side side = ori.faceDir();
             auto const bct = phys_bc_type[ori];
             if ( bct == ERF_BC::symmetry )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::reflect_even);
                     }
                 } else {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::reflect_even);
                     }
                 }
             }
             else if ( bct == ERF_BC::outflow )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                     }
                 } else {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                     }
                 }
             }
             else if ( bct == ERF_BC::open )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NVAR_max; i++)
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::open);
                 } else {
                     for (int i = 0; i < NVAR_max; i++)
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::open);
                 }
             }
             else if ( bct == ERF_BC::no_slip_wall)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                     }
                     if (m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::ext_dir);
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
                     }
                 } else {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                     }
                     if (m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::ext_dir);
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
                     }
                 }
             }
             else if (bct == ERF_BC::slip_wall)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                     }
                     if (m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::ext_dir);
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::Rho_bc_comp].setLo(dir, ERFBCType::neumann);
                     }
                 } else {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                     }
                     if (m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::ext_dir);
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::Rho_bc_comp].setHi(dir, ERFBCType::neumann);
                     }
                 }
             }
             else if (bct == ERF_BC::inflow)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                         if (input_bndry_planes && dir < 2 && (
                            ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQKE_bc_comp)    && m_r2d->ingested_QKE()    ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )) ) {
                             domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                            }
                     }
                 } else {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                         if (input_bndry_planes && dir < 2 && (
                            ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQKE_bc_comp)    && m_r2d->ingested_QKE()    ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )
                            ) ) {
                             domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                            }
                     }
                 }
             }
             else if (bct == ERF_BC::periodic)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::int_dir);
                     }
                 } else {
                     for (int i = 0; i < NVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::int_dir);
                     }
                 }
             }
             else if ( bct == ERF_BC::MOST )
             {
                 AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
                 for (int i = 0; i < NVAR_max; i++) {
                     domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                 }
             }
         }
     }
  
     // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
     Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
 }

Here is the call graph for this function:

◆ init_custom()

void ERF::init_custom ( int lev )

private

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters

lev	Integer specifying the current level

 {
     auto& lev_new = vars_new[lev];
  
     MultiFab r_hse(base_state[lev], make_alias, 0, 1); // r_0 is first  component
     MultiFab p_hse(base_state[lev], make_alias, 1, 1); // p_0 is second component
  
     MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
                        lev_new[Vars::cons].nComp()   , lev_new[Vars::cons].nGrow());
     MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
     MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
     MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
  
     // Default all perturbations to zero
     cons_pert.setVal(0.);
     xvel_pert.setVal(0.);
     yvel_pert.setVal(0.);
     zvel_pert.setVal(0.);
  
     int fix_random_seed = 0;
     ParmParse pp("erf"); pp.query("fix_random_seed", fix_random_seed);
     // Note that the value of 1024UL is not significant -- the point here is just to set the
     //     same seed for all MPI processes for the purpose of regression testing
     if (fix_random_seed) {
         Print() << "Fixing the random seed" << std::endl;
         InitRandom(1024UL);
     }
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
     {
         const Box &bx  = mfi.tilebox();
         const Box &xbx = mfi.tilebox(IntVect(1,0,0));
         const Box &ybx = mfi.tilebox(IntVect(0,1,0));
         const Box &zbx = mfi.tilebox(IntVect(0,0,1));
  
  
         const auto &cons_pert_arr = cons_pert.array(mfi);
         const auto &xvel_pert_arr = xvel_pert.array(mfi);
         const auto &yvel_pert_arr = yvel_pert.array(mfi);
         const auto &zvel_pert_arr = zvel_pert.array(mfi);
  
         Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
         Array4<Real const> z_nd_arr = (solverChoice.use_terrain) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
         Array4<Real const> z_cc_arr = (solverChoice.use_terrain) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
  
         Array4<Real const> mf_m     = mapfac_m[lev]->array(mfi);
         Array4<Real const> mf_u     = mapfac_m[lev]->array(mfi);
         Array4<Real const> mf_v     = mapfac_m[lev]->array(mfi);
  
         Array4<Real> r_hse_arr = r_hse.array(mfi);
         Array4<Real> p_hse_arr = p_hse.array(mfi);
  
         prob->init_custom_pert(bx, xbx, ybx, zbx, cons_arr, cons_pert_arr,
                                xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
                                r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
                                geom[lev].data(), mf_m, mf_u, mf_v,
                                solverChoice);
     } //mfi
  
     // Add problem-specific perturbation to background flow
     MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp,      Rho_comp,      1, cons_pert.nGrow());
     MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp, 1, cons_pert.nGrow());
     MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,1, cons_pert.nGrow());
  
     // RhoKE is only relevant if using Deardorff with LES
     if (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) {
         MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp,    RhoKE_comp,    1, cons_pert.nGrow());
     }
  
     // RhoQKE is only relevant if using MYNN2.5 or YSU
     if (solverChoice.turbChoice[lev].pbl_type == PBLType::None) {
         lev_new[Vars::cons].setVal(0.0,RhoQKE_comp,1);
     } else {
         MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoQKE_comp,   RhoQKE_comp,   1, cons_pert.nGrow());
     }
  
     if (solverChoice.moisture_type != MoistureType::None) {
         int qstate_size = micro->Get_Qstate_Size();
         MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoQ1_comp,    RhoQ1_comp,    1, cons_pert.nGrow());
         MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoQ2_comp,    RhoQ2_comp,    1, cons_pert.nGrow());
         for (int q_offset(2); q_offset<qstate_size; ++q_offset) {
             int q_idx = RhoQ1_comp+q_offset;
             MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
         }
     }
  
     MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0,             0,             1, xvel_pert.nGrowVect());
     MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0,             0,             1, yvel_pert.nGrowVect());
     MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0,             0,             1, zvel_pert.nGrowVect());
 }

Here is the call graph for this function:

◆ init_from_hse()

void ERF::init_from_hse ( int lev )

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

call prob->erf_init_dens_hse(...)
- call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
- save r_hse
call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation

Parameters

lev	Integer specifying the current level

 {
     auto& lev_new = vars_new[lev];
  
     MultiFab r_hse(base_state[lev], make_alias, 0, 1); // r_0 is first  component
     MultiFab p_hse(base_state[lev], make_alias, 1, 1); // p_0 is second component
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
     {
         const Box &gbx = mfi.growntilebox(1);
         const Array4<Real      >& cons_arr  = lev_new[Vars::cons].array(mfi);
         const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
         const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
  
         ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             cons_arr(i,j,k,Rho_comp)      = r_hse_arr(i,j,k);
             cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
         });
     } //mfi
 }

Here is the call graph for this function:

◆ init_from_input_sounding()

void ERF::init_from_input_sounding ( int lev )

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters

lev	Integer specifying the current level

 {
     // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
     if (lev == 0) {
         if (input_sounding_file.empty())
             Error("input_sounding file name must be provided via input");
  
         // this will interpolate the input profiles to the nominal height levels
         // (ranging from 0 to the domain top)
         input_sounding_data.read_from_file(input_sounding_file, geom[lev], zlevels_stag);
  
         // this will calculate the hydrostatically balanced density and pressure
         // profiles following WRF ideal.exe
         if (init_sounding_ideal) input_sounding_data.calc_rho_p();
     }
  
     auto& lev_new = vars_new[lev];
  
     // update if init_sounding_ideal == true
     MultiFab r_hse (base_state[lev], make_alias, 0, 1); // r_0  is first  component
     MultiFab p_hse (base_state[lev], make_alias, 1, 1); // p_0  is second component
     MultiFab pi_hse(base_state[lev], make_alias, 2, 1); // pi_0 is third  component
  
     const Real l_gravity = solverChoice.gravity;
     const Real l_rdOcp   = solverChoice.rdOcp;
     const bool l_moist   = (solverChoice.moisture_type != MoistureType::None);
  
 #ifdef _OPENMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box &bx = mfi.tilebox();
         const auto &cons_arr = lev_new[Vars::cons].array(mfi);
         const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
         const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
         const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
         Array4<Real> r_hse_arr = r_hse.array(mfi);
         Array4<Real> p_hse_arr = p_hse.array(mfi);
         Array4<Real> pi_hse_arr = pi_hse.array(mfi);
  
         Array4<Real const> z_cc_arr = (solverChoice.use_terrain) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
         Array4<Real const> z_nd_arr = (solverChoice.use_terrain) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
  
         if (init_sounding_ideal)
         {
             // HSE will be initialized here, interpolated from values previously
             // calculated by calc_rho_p()
             init_bx_scalars_from_input_sounding_hse(
                 bx, cons_arr,
                 r_hse_arr, p_hse_arr, pi_hse_arr,
                 geom[lev].data(), z_cc_arr,
                 l_gravity, l_rdOcp, l_moist, input_sounding_data);
         }
         else
         {
             AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!solverChoice.use_terrain,
                 "Terrain is not supported without init_sounding_ideal option.");
  
             // HSE will be calculated later with call to initHSE
             init_bx_scalars_from_input_sounding(
                 bx, cons_arr,
                 geom[lev].data(), z_cc_arr,
                 l_moist, input_sounding_data);
         }
  
         init_bx_velocities_from_input_sounding(
             bx, xvel_arr, yvel_arr, zvel_arr,
             geom[lev].data(), z_nd_arr,
             input_sounding_data);
  
     } //mfi
 }

Here is the call graph for this function:

◆ init_only()

void ERF::init_only	(	int	lev,
		amrex::Real	time
	)

 {
     t_new[lev] = time;
     t_old[lev] = time - 1.e200;
  
     auto& lev_new = vars_new[lev];
     auto& lev_old = vars_old[lev];
  
     // Loop over grids at this level to initialize our grid data
     lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
     lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
     lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
     lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
  
     // Initialize background flow (optional)
     if (init_type == "input_sounding") {
         // The base state is initialized by integrating vertically through the
         // input sounding, if the init_sounding_ideal flag is set; otherwise
         // it is set by initHSE()
         init_from_input_sounding(lev);
         if (init_sounding_ideal) {
             AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
                 "Gravity should be on to be consistent with sounding initialization.");
         } else {
             initHSE();
         }
  
 #ifdef ERF_USE_NETCDF
     } else if (init_type == "ideal" || init_type == "real") {
         // The base state is initialized from WRF wrfinput data, output by
         // ideal.exe or real.exe
         init_from_wrfinput(lev);
         if (init_type == "ideal") initHSE();
  
     } else if (init_type == "metgrid") {
         // The base state is initialized from data output by WPS metgrid;
         // we will rebalance after interpolation
         init_from_metgrid(lev);
 #endif
     } else if (init_type == "uniform") {
         // Initialize a uniform background field and base state based on the
         // problem-specified reference density and temperature
         init_uniform(lev);
         initHSE(lev);
     } else {
         // No background flow initialization specified, initialize the
         // background field to be equal to the base state, calculated from the
         // problem-specific erf_init_dens_hse
         initHSE(lev); // need to call this first
         init_from_hse(lev);
     }
  
     // Add problem-specific flow features
     //
     // Notes:
     // - This calls init_custom_pert that is defined for each problem
     // - This may modify the base state
     // - The fields set by init_custom_pert are **perturbations** to the
     //   background flow set based on init_type
     init_custom(lev);
  
     // Ensure that the face-based data are the same on both sides of a periodic domain.
     // The data associated with the lower grid ID is considered the correct value.
     lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
     lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
     lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
  
     // Initialize wind farm
  
 #ifdef ERF_USE_WINDFARM
     init_windfarm(lev);
 #endif
  
    if(solverChoice.spongeChoice.sponge_type == "input_sponge"){
         input_sponge(lev);
    }
  
 }

◆ init_stuff()

void ERF::init_stuff	(	int	lev,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm,
		amrex::Vector< amrex::MultiFab > &	lev_new,
		amrex::Vector< amrex::MultiFab > &	lev_old
	)

private

 {
     if (lev == 0) {
         min_k_at_level[lev] = 0;
         max_k_at_level[lev] = geom[lev].Domain().bigEnd(2);
     } else {
         // Start with unreasonable values so we compute the right min/max
         min_k_at_level[lev] = geom[lev].Domain().bigEnd(2);
         max_k_at_level[lev] = geom[lev].Domain().smallEnd(2);
         for (int n = 0; n < ba.size(); n++)
         {
             min_k_at_level[lev] = std::min(min_k_at_level[lev], ba[n].smallEnd(2));
             max_k_at_level[lev] = std::max(max_k_at_level[lev], ba[n].bigEnd(2));
         }
     }
  
     // ********************************************************************************************
     // Base state holds r_0, pres_0, pi_0 (in that order)
     // ********************************************************************************************
     base_state[lev].define(ba,dm,3,1);
     base_state[lev].setVal(0.);
  
     if (solverChoice.use_terrain && solverChoice.terrain_type != TerrainType::Static) {
         base_state_new[lev].define(ba,dm,3,1);
         base_state_new[lev].setVal(0.);
     }
  
     // ********************************************************************************************
     // Allocate terrain arrays
     // ********************************************************************************************
     if (solverChoice.use_terrain) {
         z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
  
         if (solverChoice.terrain_type != TerrainType::Static)
         {
             detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
             detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
  
             ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
             ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
             az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
  
             ax_new[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
             ay_new[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
             az_new[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
  
             z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
         }
  
         BoxArray ba_nd(ba);
         ba_nd.surroundingNodes();
  
         // We need this to be one greater than the ghost cells to handle levels > 0
         int ngrow = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff) + 2;
         z_phys_nd[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,1));
         if (solverChoice.terrain_type != TerrainType::Static) {
             z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,1));
             z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,1));
         }
  
     } else {
             z_phys_nd[lev] = nullptr;
             z_phys_cc[lev] = nullptr;
  
         z_phys_nd_new[lev] = nullptr;
           detJ_cc_new[lev] = nullptr;
  
         z_phys_nd_src[lev] = nullptr;
           detJ_cc_src[lev] = nullptr;
  
                z_t_rk[lev] = nullptr;
     }
  
    // We use these area arrays regardless of terrain, EB or none of the above
    detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
         ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
         ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
         az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
  
    detJ_cc[lev]->setVal(1.0);
         ax[lev]->setVal(1.0);
         ay[lev]->setVal(1.0);
         az[lev]->setVal(1.0);
  
     // ********************************************************************************************
     // These are the persistent containers for the old and new data
     // ********************************************************************************************
     int ncomp;
     if (lev > 0) {
         ncomp = vars_new[lev-1][Vars::cons].nComp();
     } else {
         int n_qstate   = micro->Get_Qstate_Size();
         ncomp = NVAR_max - (NMOIST_max - n_qstate);
     }
  
     // ********************************************************************************************
     // The number of ghost cells for density must be 1 greater than that for velocity
     //     so that we can go back in forth betwen velocity and momentum on all faces
     // ********************************************************************************************
     int ngrow_state = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff) + 1;
     int ngrow_vels  = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff);
  
     // ********************************************************************************************
     // New solution data containers
     // ********************************************************************************************
     lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
     lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
  
     lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
     lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
  
     lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
     lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
  
     lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect(ngrow_vels,ngrow_vels,0));
     lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect(ngrow_vels,ngrow_vels,0));
  
     // ********************************************************************************************
     // These are just used for scratch in the time integrator but we might as well define them here
     // ********************************************************************************************
     rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
     rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
  
     rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
     rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
  
     rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
     rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
  
     // We do this here just so they won't be undefined in the initial FillPatch
     rU_new[lev].setVal(1.2e21);
     rV_new[lev].setVal(3.4e22);
     rW_new[lev].setVal(5.6e23);
  
     // ********************************************************************************************
     // These are just time averaged fields for diagnostics
     // ********************************************************************************************
  
     // NOTE: We are not completing a fillpach call on the time averaged data;
     //       which would copy on intersection and interpolate from coarse.
     //       Therefore, we are restarting the averaging when the ba changes,
     //       this may give poor statistics for dynamic mesh refinment.
     vel_t_avg[lev] = nullptr;
     if (solverChoice.time_avg_vel) {
         vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
         vel_t_avg[lev]->setVal(0.0);
         t_avg_cnt[lev] = 0.0;
     }
  
     // ********************************************************************************************
     // Initialize flux registers whenever we create/re-create a level
     // ********************************************************************************************
     if (solverChoice.coupling_type == CouplingType::TwoWay) {
         if (lev == 0) {
             advflux_reg[0] = nullptr;
         } else {
             int ncomp_reflux = vars_new[0][Vars::cons].nComp();
             advflux_reg[lev] = new YAFluxRegister(ba       , grids[lev-1],
                                                   dm       ,  dmap[lev-1],
                                                   geom[lev],  geom[lev-1],
                                                   ref_ratio[lev-1], lev, ncomp_reflux);
         }
     }
  
     // ********************************************************************************************
     // Define Theta_prim storage if using MOST BC
     // ********************************************************************************************
     if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::MOST) {
         Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,0));
         if (solverChoice.moisture_type != MoistureType::None) {
             Qv_prim[lev]    = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,0));
         } else {
             Qv_prim[lev]    = nullptr;
         }
     } else {
         Theta_prim[lev] = nullptr;
         Qv_prim[lev]    = nullptr;
     }
  
     // ********************************************************************************************
     // Map factors
     // ********************************************************************************************
     BoxList bl2d = ba.boxList();
     for (auto& b : bl2d) {
         b.setRange(2,0);
     }
     BoxArray ba2d(std::move(bl2d));
  
     mapfac_m[lev] = std::make_unique<MultiFab>(ba2d,dm,1,3);
     mapfac_u[lev] = std::make_unique<MultiFab>(convert(ba2d,IntVect(1,0,0)),dm,1,3);
     mapfac_v[lev] = std::make_unique<MultiFab>(convert(ba2d,IntVect(0,1,0)),dm,1,3);
     if (solverChoice.test_mapfactor) {
         mapfac_m[lev]->setVal(0.5);
         mapfac_u[lev]->setVal(0.5);
         mapfac_v[lev]->setVal(0.5);
     }
     else {
         mapfac_m[lev]->setVal(1.);
         mapfac_u[lev]->setVal(1.);
         mapfac_v[lev]->setVal(1.);
     }
  
 #if defined(ERF_USE_WINDFARM)
     //*********************************************************
     // Variables for Ftich model for windfarm parametrization
     //*********************************************************
     if (solverChoice.windfarm_type == WindFarmType::Fitch){
         int ngrow_state = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff) + 1;
         vars_fitch[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
         Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
     }
     if (solverChoice.windfarm_type == WindFarmType::EWP){
         int ngrow_state = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff) + 1;
         vars_ewp[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
         Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
     }
 #endif
  
 #if defined(ERF_USE_RRTMGP)
     //*********************************************************
     // Radiation heating source terms
     //*********************************************************
      qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, ngrow_state);
      qheating_rates[lev]->setVal(0.);
 #endif
 }

◆ init_uniform()

void ERF::init_uniform ( int lev )

private

Use problem-specific reference density and temperature to set the background state to a uniform value.

Parameters

lev	Integer specifying the current level

 {
     auto& lev_new = vars_new[lev];
     for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box &gbx = mfi.growntilebox(1);
         const auto &cons_arr = lev_new[Vars::cons].array(mfi);
         prob->init_uniform(gbx, cons_arr);
     }
 }

◆ InitData()

void ERF::InitData ( )

 {
     BL_PROFILE_VAR("ERF::InitData()", InitData);
  
     // Initialize the start time for our CPU-time tracker
     startCPUTime = ParallelDescriptor::second();
  
     // Create the ReadBndryPlanes object so we can read boundary plane data
     // m_r2d is used by init_bcs so we must instantiate this class before
     if (input_bndry_planes) {
         Print() << "Defining r2d for the first time " << std::endl;
         m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
     }
  
     // Map the words in the inputs file to BC types, then translate
     //     those types into what they mean for each variable
     init_bcs();
  
     // Verify BCs are compatible with solver choice
     for (int lev(0); lev <= max_level; ++lev) {
         if ( ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
                (solverChoice.turbChoice[lev].pbl_type == PBLType::YSU)       ) &&
             phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::MOST ) {
             Abort("MYNN2.5/YSU PBL Model requires MOST at lower boundary");
         }
     }
  
     if (!solverChoice.use_terrain && solverChoice.terrain_type != TerrainType::Static) {
         Abort("We do not allow non-static terrain_type with use_terrain = false");
     }
  
     last_plot_file_step_1 = -1;
     last_plot_file_step_2 = -1;
     last_check_file_step  = -1;
  
     if (restart_chkfile.empty()) {
         // start simulation from the beginning
  
         const Real time = start_time;
         InitFromScratch(time);
  
 #ifdef ERF_USE_MULTIBLOCK
 #ifndef ERF_MB_EXTERN       // enter only if multiblock does not involve an external class
         // Multiblock: hook to set BL & comms once ba/dm are known
         if(domain_p[0].bigEnd(0) < 500 ) {
             m_mbc->SetBoxLists();
             m_mbc->SetBlockCommMetaData();
         }
 #endif
 #endif
  
         if (solverChoice.use_terrain) {
             if (init_type == "ideal") {
                 Abort("We do not currently support init_type = ideal with terrain");
             }
         }
  
         //
         // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
         //
         if (solverChoice.use_terrain != 0) {
             for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
                 average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
                 average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
             }
         }
  
         // If using the Deardoff LES model,
         // we initialize rho_KE to be nonzero (and positive) so that we end up
         // with reasonable values for the eddy diffusivity and the MOST fluxes
         // (~ 1/diffusivity) do not blow up
         Real RhoKE_0;
         ParmParse pp(pp_prefix);
         if (pp.query("RhoKE_0", RhoKE_0)) {
             // Uniform initial rho*e field
             for (int lev = 0; lev <= finest_level; lev++) {
                 if (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) {
                     Print() << "Initializing uniform rhoKE=" << RhoKE_0
                         << " on level " << lev
                         << std::endl;
                     vars_new[lev][Vars::cons].setVal(RhoKE_0,RhoKE_comp,1,0);
                 } else {
                     vars_new[lev][Vars::cons].setVal(0.0,RhoKE_comp,1,0);
                 }
             }
         }
  
         Real KE_0;
         if (pp.query("KE_0", KE_0)) {
             // Uniform initial e field
             for (int lev = 0; lev <= finest_level; lev++) {
                 auto& lev_new = vars_new[lev];
                 if (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) {
                     Print() << "Initializing uniform KE=" << KE_0
                         << " on level " << lev
                         << std::endl;
                     for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                         const Box &bx = mfi.tilebox();
                         const auto &cons_arr = lev_new[Vars::cons].array(mfi);
                         // We want to set the lateral BC values, too
                         Box gbx = bx; // Copy constructor
                         gbx.grow(0,1); gbx.grow(1,1); // Grow by one in the lateral directions
                         ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                             cons_arr(i,j,k,RhoKE_comp) = cons_arr(i,j,k,Rho_comp) * KE_0;
                         });
                     } // mfi
                 } else {
                     lev_new[Vars::cons].setVal(0.0,RhoKE_comp,1,0);
                 }
             } // lev
         }
  
         Real QKE_0;
         if (pp.query("QKE_0", QKE_0)) {
             Print() << "Initializing uniform QKE=" << QKE_0 << std::endl;
             for (int lev = 0; lev <= finest_level; lev++) {
                 auto& lev_new = vars_new[lev];
                 for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                     const Box &bx = mfi.tilebox();
                     const auto &cons_arr = lev_new[Vars::cons].array(mfi);
                     // We want to set the lateral BC values, too
                     Box gbx = bx; // Copy constructor
                     gbx.grow(0,1); gbx.grow(1,1); // Grow by one in the lateral directions
                     ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                         cons_arr(i,j,k,RhoQKE_comp) = cons_arr(i,j,k,Rho_comp) * QKE_0;
                     });
                 } // mfi
             }
         }
  
         if (solverChoice.coupling_type == CouplingType::TwoWay) {
             AverageDown();
         }
  
         if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
             (solverChoice.advChoice.zero_yflux.size() > 0) ||
             (solverChoice.advChoice.zero_zflux.size() > 0))
         {
             AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
                 "Thin immersed body with refinement not currently supported.");
             if (solverChoice.use_terrain == 1) {
                 amrex::Print() << "NOTE: Thin immersed body with terrain has not been tested." << std::endl;
             }
         }
  
     } else { // Restart from a checkpoint
  
         restart();
  
  
         // TODO: Check if this is needed. I don't think it is since we now
         //       advect all the scalars...
  
         // Need to fill ghost cells here since we will use this qmoist in advance
         if (solverChoice.moisture_type != MoistureType::None)
         {
             for (int lev = 0; lev <= finest_level; lev++) {
                 if (qmoist[lev].size() > 0) FillPatchMoistVars(lev, *(qmoist[lev][0])); // qv component
             }
         }
     }
  
 #ifdef ERF_USE_PARTICLES
     /* If using a Lagrangian microphysics model, its particle container has now been
        constructed and initialized (calls to micro->Init). So, add its pointer to
        ERF::particleData and remove its name from list of unallocated particle containers. */
     if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
         const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
         const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
         particleData.pushBack(pc_name, pc_ptr);
         particleData.getNamesUnalloc().remove(pc_name);
     }
 #endif
  
     if (input_bndry_planes) {
         // Read the "time.dat" file to know what data is available
         m_r2d->read_time_file();
  
         // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
         Real dt_dummy = 0.0;
         m_r2d->read_input_files(t_new[0],dt_dummy,m_bc_extdir_vals);
     }
  
     if (solverChoice.custom_rhotheta_forcing)
     {
         h_rhotheta_src.resize(max_level+1, Vector<Real>(0));
         d_rhotheta_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
         for (int lev = 0; lev <= finest_level; lev++) {
             const int domlen = geom[lev].Domain().length(2);
             h_rhotheta_src[lev].resize(domlen, 0.0_rt);
             d_rhotheta_src[lev].resize(domlen, 0.0_rt);
             prob->update_rhotheta_sources(t_new[0],
                                           h_rhotheta_src[lev], d_rhotheta_src[lev],
                                           geom[lev], z_phys_cc[lev]);
         }
     }
  
     if (solverChoice.custom_geostrophic_profile)
     {
         h_u_geos.resize(max_level+1, Vector<Real>(0));
         d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
         h_v_geos.resize(max_level+1, Vector<Real>(0));
         d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
         for (int lev = 0; lev <= finest_level; lev++) {
             const int domlen = geom[lev].Domain().length(2);
             h_u_geos[lev].resize(domlen, 0.0_rt);
             d_u_geos[lev].resize(domlen, 0.0_rt);
             h_v_geos[lev].resize(domlen, 0.0_rt);
             d_v_geos[lev].resize(domlen, 0.0_rt);
             prob->update_geostrophic_profile(t_new[0],
                                           h_u_geos[lev], d_u_geos[lev],
                                           h_v_geos[lev], d_v_geos[lev],
                                           geom[lev], z_phys_cc[lev]);
         }
     }
  
     if (solverChoice.custom_moisture_forcing)
     {
         h_rhoqt_src.resize(max_level+1, Vector<Real>(0));
         d_rhoqt_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
         for (int lev = 0; lev <= finest_level; lev++) {
             const int domlen = geom[lev].Domain().length(2);
             h_rhoqt_src[lev].resize(domlen, 0.0_rt);
             d_rhoqt_src[lev].resize(domlen, 0.0_rt);
             prob->update_rhoqt_sources(t_new[0],
                                        h_rhoqt_src[lev], d_rhoqt_src[lev],
                                        geom[lev], z_phys_cc[lev]);
         }
     }
  
     if (solverChoice.custom_w_subsidence)
     {
         h_w_subsid.resize(max_level+1, Vector<Real>(0));
         d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
         for (int lev = 0; lev <= finest_level; lev++) {
             const int domlen = geom[lev].Domain().length(2);
             h_w_subsid[lev].resize(domlen, 0.0_rt);
             d_w_subsid[lev].resize(domlen, 0.0_rt);
             prob->update_w_subsidence(t_new[0],
                                       h_w_subsid[lev], d_w_subsid[lev],
                                       geom[lev], z_phys_cc[lev]);
         }
     }
  
     if (solverChoice.rayleigh_damp_U ||solverChoice.rayleigh_damp_V ||
         solverChoice.rayleigh_damp_W ||solverChoice.rayleigh_damp_T)
     {
         initRayleigh();
         if (init_type == "input_sounding")
         {
             // Overwrite ubar, vbar, and thetabar with input profiles;
             // wbar is assumed to be 0. Note: the tau coefficient set by
             // prob->erf_init_rayleigh() is still used
             bool restarting = (!restart_chkfile.empty());
             setRayleighRefFromSounding(restarting);
         }
     }
  
     // Read in sponge data from input file
     if(solverChoice.spongeChoice.sponge_type == "input_sponge")
     {
         initSponge();
         bool restarting = (!restart_chkfile.empty());
         setSpongeRefFromSounding(restarting);
     }
  
     if (is_it_time_for_action(istep[0], t_new[0], dt[0], sum_interval, sum_per)) {
         sum_integrated_quantities(t_new[0]);
         if (cc_profiles) {
             // all variables cell-centered
             write_1D_profiles(t_new[0]);
         } else {
             // some variables staggered
             write_1D_profiles_stag(t_new[0]);
         }
     }
  
     // We only write the file at level 0 for now
     if (output_bndry_planes)
     {
         // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
         m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
  
         Real time = 0.;
         if (time >= bndry_output_planes_start_time) {
             m_w2d->write_planes(0, time, vars_new);
         }
     }
  
 #ifdef ERF_USE_POISSON_SOLVE
     if (restart_chkfile == "")
     {
         // Note -- this projection is only defined for no terrain
         if (solverChoice.project_initial_velocity) {
             AMREX_ALWAYS_ASSERT(solverChoice.use_terrain == 0);
             project_velocities(vars_new);
         }
     }
 #endif
     // Copy from new into old just in case
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         auto& lev_new = vars_new[lev];
         auto& lev_old = vars_old[lev];
  
         int ncomp = lev_new[Vars::cons].nComp();
  
         MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,ncomp,lev_new[Vars::cons].nGrowVect());
         MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,    1,lev_new[Vars::xvel].nGrowVect());
         MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,    1,lev_new[Vars::yvel].nGrowVect());
         MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,    1,lev_new[Vars::zvel].nGrowVect());
     }
  
     // Compute the minimum dz in the domain (to be used for setting the timestep)
     dz_min = geom[0].CellSize(2);
     if ( solverChoice.use_terrain ) {
         dz_min *= (*detJ_cc[0]).min(0);
     }
  
     ComputeDt();
  
     // Fill ghost cells/faces
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         if (lev > 0 && cf_width >= 0) {
             Construct_ERFFillPatchers(lev);
         }
  
         //
         // Fill boundary conditions -- not sure why we need this here
         //
         bool fillset = false;
         auto& lev_new = vars_new[lev];
         FillPatch(lev, t_new[lev],
                   {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
                   {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                   fillset);
  
         //
         // We do this here to make sure level (lev-1) boundary conditions are filled
         // before we interpolate to level (lev) ghost cells
         //
         if (lev < finest_level) {
             auto& lev_old = vars_old[lev];
             MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
             MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
             MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
             MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
         }
  
         //
         // We fill the ghost cell values of the base state in case it wasn't done in the initialization
         //
         base_state[lev].FillBoundary(geom[lev].periodicity());
  
         // For moving terrain only
         if (solverChoice.terrain_type != TerrainType::Static) {
             MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,3,1);
             base_state_new[lev].FillBoundary(geom[lev].periodicity());
         }
     }
  
     // Configure ABLMost params if used MostWall boundary condition
     // NOTE: we must set up the MOST routine after calling FillPatch
     //       in order to have lateral ghost cells filled (MOST + terrain interp).
     //       FillPatch does not call MOST, FillIntermediatePatch does.
     if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::MOST)
     {
 #ifdef ERF_EXPLICIT_MOST_STRESS
         Print() << "Using MOST with explicitly included surface stresses" << std::endl;
 #endif
  
         m_most = std::make_unique<ABLMost>(geom, vars_old, Theta_prim, Qv_prim, z_phys_nd,
                                            sst_lev, lmask_lev, lsm_data, lsm_flux
 #ifdef ERF_USE_NETCDF
                                            ,start_bdy_time, bdy_time_interval
 #endif
                                            );
  
         // We now configure ABLMost params here so that we can print the averages at t=0
         // Note we don't fill ghost cells here because this is just for diagnostics
         for (int lev = 0; lev <= finest_level; ++lev)
         {
             Real time  = t_new[lev];
             IntVect ng = Theta_prim[lev]->nGrowVect();
             MultiFab S(vars_new[lev][Vars::cons],make_alias,0,RhoTheta_comp+1);
             MultiFab::Copy(  *Theta_prim[lev], S, RhoTheta_comp, 0, 1, ng);
             MultiFab::Divide(*Theta_prim[lev], S, Rho_comp     , 0, 1, ng);
             if (solverChoice.moisture_type != MoistureType::None) {
                 ng = Qv_prim[lev]->nGrowVect();
                 MultiFab Sm(vars_new[lev][Vars::cons],make_alias,0,RhoQ1_comp+1);
                 MultiFab::Copy(  *Qv_prim[lev], Sm, RhoQ1_comp, 0, 1, ng);
                 MultiFab::Divide(*Qv_prim[lev], Sm, Rho_comp  , 0, 1, ng);
             }
             m_most->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim);
             m_most->update_fluxes(lev, time);
         }
     }
  
     // Update micro vars before first plot file
     if (solverChoice.moisture_type != MoistureType::None) {
         for (int lev = 0; lev <= finest_level; ++lev) micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons]);
     }
  
     // Fill time averaged velocities before first plot file
     if (solverChoice.time_avg_vel) {
         for (int lev = 0; lev <= finest_level; ++lev) {
             Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
                               vars_new[lev][Vars::xvel],
                               vars_new[lev][Vars::yvel],
                               vars_new[lev][Vars::zvel]);
         }
     }
  
     // check for additional plotting variables that are available after particle containers
     // are setup.
     const std::string& pv1 = "plot_vars_1"; appendPlotVariables(pv1,plot_var_names_1);
     const std::string& pv2 = "plot_vars_2"; appendPlotVariables(pv2,plot_var_names_2);
  
     if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > 0.) )
     {
 #ifdef ERF_USE_NETCDF
         if (check_type == "netcdf") {
            WriteNCCheckpointFile();
         }
 #endif
         if (check_type == "native") {
            WriteCheckpointFile();
         }
         last_check_file_step = 0;
     }
  
     if ( (restart_chkfile.empty()) ||
          (!restart_chkfile.empty() && plot_file_on_restart) )
     {
         if (m_plot_int_1 > 0 || m_plot_per_1 > 0.)
         {
             WritePlotFile(1,plot_var_names_1);
             last_plot_file_step_1 = istep[0];
         }
         if (m_plot_int_2 > 0 || m_plot_per_2 > 0.)
         {
             WritePlotFile(2,plot_var_names_2);
             last_plot_file_step_2 = istep[0];
         }
     }
  
     // Set these up here because we need to know which MPI rank "cell" is on...
     ParmParse pp("erf");
     if (pp.contains("data_log"))
     {
         int num_datalogs = pp.countval("data_log");
         datalog.resize(num_datalogs);
         datalogname.resize(num_datalogs);
         pp.queryarr("data_log",datalogname,0,num_datalogs);
         for (int i = 0; i < num_datalogs; i++)
             setRecordDataInfo(i,datalogname[i]);
     }
  
     if (pp.contains("sample_point_log") && pp.contains("sample_point"))
     {
         int lev = 0;
  
         int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
         if (num_samplepts > 0) {
             Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
             samplepoint.resize(num_samplepts);
  
             pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
             for (int i = 0; i < num_samplepts; i++) {
                 IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                 samplepoint[i] = iv;
             }
         }
  
         int num_sampleptlogs = pp.countval("sample_point_log");
         AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
         if (num_sampleptlogs > 0) {
             sampleptlog.resize(num_sampleptlogs);
             sampleptlogname.resize(num_sampleptlogs);
             pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
  
             for (int i = 0; i < num_sampleptlogs; i++) {
                 setRecordSamplePointInfo(i,lev,samplepoint[i],sampleptlogname[i]);
             }
         }
  
     }
  
     if (pp.contains("sample_line_log") && pp.contains("sample_line"))
     {
         int lev = 0;
  
         int num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
         if (num_samplelines > 0) {
             Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
             sampleline.resize(num_samplelines);
  
             pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
             for (int i = 0; i < num_samplelines; i++) {
                 IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                 sampleline[i] = iv;
             }
         }
  
         int num_samplelinelogs = pp.countval("sample_line_log");
         AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
         if (num_samplelinelogs > 0) {
             samplelinelog.resize(num_samplelinelogs);
             samplelinelogname.resize(num_samplelinelogs);
             pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
  
             for (int i = 0; i < num_samplelinelogs; i++) {
                 setRecordSampleLineInfo(i,lev,sampleline[i],samplelinelogname[i]);
             }
         }
  
     }
  
     BL_PROFILE_VAR_STOP(InitData);
  
 #ifdef ERF_USE_EB
     bool write_eb_surface = false;
     pp.query("write_eb_surface", write_eb_surface);
     if (write_eb_surface) WriteMyEBSurface();
 #endif
  
 }

Referenced by MultiBlockContainer::InitializeBlocks(), and main().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ initHSE() [1/2]

void ERF::initHSE ( )

private

Initialize HSE.

 {
     AMREX_ALWAYS_ASSERT(!init_sounding_ideal);
     for (int lev = 0; lev <= finest_level; lev++)
     {
         initHSE(lev);
     }
 }

◆ initHSE() [2/2]

void ERF::initHSE ( int lev )

private

Initialize density and pressure base state in hydrostatic equilibrium.

 {
     MultiFab r_hse (base_state[lev], make_alias, 0, 1); // r_0  is first  component
     MultiFab p_hse (base_state[lev], make_alias, 1, 1); // p_0  is second component
     MultiFab pi_hse(base_state[lev], make_alias, 2, 1); // pi_0 is third  component
  
     if (lev > 0) {
  
         // Interp all three components: rho, p, pi
         int  icomp = 0; int bccomp = 0; int  ncomp = 3;
  
         PhysBCFunctNoOp null_bc;
         Real time = 0.;
         Interpolater* mapper = &cell_cons_interp;
  
         InterpFromCoarseLevel(base_state[lev], time, base_state[lev-1],
                               icomp, icomp, ncomp,
                               geom[lev-1], geom[lev],
                               null_bc, 0, null_bc, 0, refRatio(lev-1),
                               mapper, domain_bcs_type, bccomp);
     }
  
     // Initial r_hse may or may not be in HSE -- defined in prob.cpp
     if (solverChoice.use_moist_background){
         prob->erf_init_dens_hse_moist(r_hse, z_phys_nd[lev], geom[lev]);
     } else {
         prob->erf_init_dens_hse(r_hse, z_phys_nd[lev], z_phys_cc[lev], geom[lev]);
     }
  
     // This integrates up through column to update p_hse, pi_hse;
     // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
     erf_enforce_hse(lev, r_hse, p_hse, pi_hse, z_phys_cc[lev]);
  
 }

◆ initialize_bcs()

void ERF::initialize_bcs ( int lev )

private

 {
     physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                                m_bc_extdir_vals, m_bc_neumann_vals,
                                                                z_phys_nd[lev], use_real_bcs);
     physbcs_u[lev]    = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                             m_bc_extdir_vals, m_bc_neumann_vals,
                                                             z_phys_nd[lev], use_real_bcs);
     physbcs_v[lev]    = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                             m_bc_extdir_vals, m_bc_neumann_vals,
                                                             z_phys_nd[lev], use_real_bcs);
     physbcs_w[lev]    = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                             m_bc_extdir_vals, m_bc_neumann_vals,
                                                             solverChoice.terrain_type, z_phys_nd[lev], use_real_bcs);
     physbcs_w_no_terrain[lev]    = std::make_unique<ERFPhysBCFunct_w_no_terrain>
                                                            (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                             m_bc_extdir_vals, m_bc_neumann_vals, use_real_bcs);
 }

◆ initialize_integrator()

void ERF::initialize_integrator	(	int	lev,
		amrex::MultiFab &	cons_mf,
		amrex::MultiFab &	vel_mf
	)

private

 {
     const BoxArray& ba(cons_mf.boxArray());
     const DistributionMapping& dm(cons_mf.DistributionMap());
  
     int ncomp_cons = cons_mf.nComp();
  
     // Initialize the integrator memory
     Vector<MultiFab> int_state; // integration state data structure example
     int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons));         // cons
     int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
     int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
     int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
  
     mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
     mri_integrator_mem[lev]->setNoSubstepping(solverChoice.no_substepping);
     mri_integrator_mem[lev]->setIncompressible(solverChoice.incompressible);
     mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
     mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
 }

◆ InitializeFromFile()

void ERF::InitializeFromFile ( )

private

◆ InitializeLevelFromData()

void ERF::InitializeLevelFromData	(	int	lev,
		const amrex::MultiFab &	initial_data
	)

private

◆ initializeMicrophysics()

void ERF::initializeMicrophysics ( const int & a_nlevsmax )

private

Parameters

a_nlevsmax number of AMR levels

 {
     if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
  
         micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
  
     } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
 #ifdef ERF_USE_PARTICLES
  
         micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
         /* Lagrangian microphysics models will have a particle container; it needs to be added
            to ERF::particleData */
         const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
         /* The particle container has not yet been constructed and initialized, so just add
            its name here for now (so that functions to set plotting variables can see it). */
         particleData.addName( pc_name );
  
 #else
         Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
 #endif
     }
  
     qmoist.resize(a_nlevsmax);
     return;
 }

Here is the call graph for this function:

◆ initRayleigh()

void ERF::initRayleigh ( )

private

Initialize Rayleigh damping profiles.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

 {
     h_rayleigh_ptrs.resize(max_level+1);
     d_rayleigh_ptrs.resize(max_level+1);
  
     for (int lev = 0; lev <= finest_level; lev++)
     {
         // These have 5 components: tau, ubar, vbar, wbar, thetabar
         h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
         d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
  
         const int zlen_rayleigh = geom[lev].Domain().length(2);
  
         // Allocate space for these 1D vectors
         for (int n = 0; n < Rayleigh::nvars; n++) {
             h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
             d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
         }
  
         // Init the host vectors
         prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_cc[lev]);
  
         // Copy from host vectors to device vectors
         for (int n = 0; n < Rayleigh::nvars; n++) {
             Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
                       d_rayleigh_ptrs[lev][n].begin());
         }
     }
 }

◆ initSponge()

void ERF::initSponge ( )

private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

 {
     h_sponge_ptrs.resize(max_level+1);
     d_sponge_ptrs.resize(max_level+1);
  
     for (int lev = 0; lev <= finest_level; lev++)
     {
         // These have 2 components: ubar, vbar
         h_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
         d_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
  
         const int zlen_sponge = geom[lev].Domain().length(2);
  
         // Allocate space for these 1D vectors
         for (int n = 0; n < Sponge::nvars_sponge; n++) {
             h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
             d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
         }
  
     }
 }

◆ input_sponge()

void ERF::input_sponge ( int lev )

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters

lev	Integer specifying the current level

 {
     // We only want to read the file once
     if (lev == 0) {
         if (input_sponge_file.empty())
             Error("input_sounding file name must be provided via input");
  
         // this will interpolate the input profiles to the nominal height levels
         // (ranging from 0 to the domain top)
         input_sponge_data.read_from_file(input_sponge_file, geom[lev], zlevels_stag);
  
     }
 }

◆ is_it_time_for_action()

bool ERF::is_it_time_for_action	(	int	nstep,
		amrex::Real	time,
		amrex::Real	dt,
		int	action_interval,
		amrex::Real	action_per
	)

static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters

nstep	Timestep number
time	Current time
dtlev	Timestep for the current level
action_interval	Interval in number of timesteps for taking action
action_per	Interval in simulation time for taking action

 {
   bool int_test = (action_interval > 0 && nstep % action_interval == 0);
  
   bool per_test = false;
   if (action_per > 0.0) {
     const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
     const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
  
     if (num_per_old != num_per_new) {
       per_test = true;
     }
   }
  
   return int_test || per_test;
 }

◆ MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage	(	amrex::Vector< amrex::Real > &	h_havg,
		amrex::MultiFab &	S,
		int	n
	)

 {
     // Get the number of cells in z at level 0
     int dir_z = AMREX_SPACEDIM-1;
     auto domain = geom[0].Domain();
     int size_z = domain.length(dir_z);
     int start_z = domain.smallEnd()[dir_z];
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
  
     // resize the level 0 horizontal average vectors
     h_havg.resize(size_z, 0.0_rt);
  
     // Get the cell centered data and construct sums
 #ifdef _OPENMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(S); mfi.isValid(); ++mfi) {
         const Box& box = mfi.validbox();
         const IntVect& se = box.smallEnd();
         const IntVect& be = box.bigEnd();
  
         auto      fab_arr = S[mfi].array();
  
         FArrayBox fab_reduce(box, 1, The_Async_Arena());
         auto arr_reduce   = fab_reduce.array();
  
         ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
             arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
         });
  
         for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
             Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
             h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
         }
     }
  
     // combine sums from different MPI ranks
     ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), h_havg.size());
  
     // divide by the total number of cells we are averaging over
     for (int k = 0; k < size_z; ++k) {
         h_havg[k]     /= area_z;
     }
 }

◆ MakeHorizontalAverages()

void ERF::MakeHorizontalAverages ( )

 {
     int lev = 0;
  
     // First, average down all levels (if doing two-way coupling)
     if (solverChoice.coupling_type == CouplingType::TwoWay) {
         AverageDown();
     }
  
     MultiFab mf(grids[lev], dmap[lev], 5, 0);
  
     int zdir = 2;
     auto domain = geom[0].Domain();
  
     bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
  
     for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.validbox();
         auto  fab_arr = mf.array(mfi);
         auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
         ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
             Real dens = cons_arr(i, j, k, Rho_comp);
             fab_arr(i, j, k, 0) = dens;
             fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
             if (!use_moisture) {
                 fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
             }
         });
     }
  
     if (use_moisture)
     {
         for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
             const Box& bx = mfi.validbox();
             auto  fab_arr = mf.array(mfi);
             auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
             auto const   qv_arr = qmoist[lev][0]->const_array(mfi);
             int ncomp = vars_new[lev][Vars::cons].nComp();
  
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                 Real dens = cons_arr(i, j, k, Rho_comp);
                 fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv_arr(i,j,k));
                 fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : 0.0);
                 fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : 0.0);
             });
         }
  
         Gpu::HostVector<Real> h_avg_qv          = sumToLine(mf,3,1,domain,zdir);
         Gpu::HostVector<Real> h_avg_qc          = sumToLine(mf,4,1,domain,zdir);
     }
  
     // Sum in the horizontal plane
     Gpu::HostVector<Real> h_avg_density     = sumToLine(mf,0,1,domain,zdir);
     Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
     Gpu::HostVector<Real> h_avg_pressure    = sumToLine(mf,2,1,domain,zdir);
  
     // Divide by the total number of cells we are averaging over
     int size_z = domain.length(zdir);
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     int klen = static_cast<int>(h_avg_density.size());
  
     for (int k = 0; k < klen; ++k) {
         h_havg_density[k]     /= area_z;
         h_havg_temperature[k] /= area_z;
         h_havg_pressure[k]    /= area_z;
         if (solverChoice.moisture_type != MoistureType::None)
         {
             h_havg_qc[k]          /= area_z;
             h_havg_qv[k]          /= area_z;
         }
     } // k
  
     // resize device vectors
     d_havg_density.resize(size_z, 0.0_rt);
     d_havg_temperature.resize(size_z, 0.0_rt);
     d_havg_pressure.resize(size_z, 0.0_rt);
  
     // copy host vectors to device vectors
     Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
     Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
     Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
  
     if (solverChoice.moisture_type != MoistureType::None)
     {
         d_havg_qv.resize(size_z, 0.0_rt);
         d_havg_qc.resize(size_z, 0.0_rt);
         Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
         Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
     }
 }

Here is the call graph for this function:

◆ MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse	(	int	lev,
		amrex::Real	time,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

override

 {
     AMREX_ALWAYS_ASSERT(lev > 0);
  
     // amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
  
     //********************************************************************************************
     // This allocates all kinds of things, including but not limited to: solution arrays,
     //      terrain arrays and metrics,a nd base state.
     // *******************************************************************************************
     init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev]);
  
     t_new[lev] = time;
     t_old[lev] = time - 1.e200;
  
     // ********************************************************************************************
     // Build the data structures for terrain-related quantities
     // ********************************************************************************************
     update_terrain_arrays(lev, time);
  
     //
     // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
     //
     if (solverChoice.use_terrain != 0) {
         for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
             average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
             average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
         }
     }
  
     //********************************************************************************************
     // Microphysics
     // *******************************************************************************************
     int q_size  = micro->Get_Qmoist_Size(lev);
     qmoist[lev].resize(q_size);
     micro->Define(lev, solverChoice);
     if (solverChoice.moisture_type != MoistureType::None)
     {
         micro->Init(lev, vars_new[lev][Vars::cons],
                     grids[lev], Geom(lev), 0.0,
                     z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
     }
     for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
         qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
     }
  
     // ********************************************************************************************
     // Update the base state at this level
     // ********************************************************************************************
     // base_state[lev].define(ba,dm,3,1);
     // base_state[lev].setVal(0.);
     initHSE(lev);
  
     // ********************************************************************************************
     // Build the data structures for calculating diffusive/turbulent terms
     // ********************************************************************************************
     update_diffusive_arrays(lev, ba, dm);
  
     // *****************************************************************************************************
     // Initialize the boundary conditions (after initializing the terrain but before calling FillCoarsePatch
     // *****************************************************************************************************
     initialize_bcs(lev);
  
     // ********************************************************************************************
     // Fill data at the new level by interpolation from the coarser level
     // Note that internal to FillCoarsePatch we will convert velocity to momentum,
     //      then interpolate momentum, then convert momentum back to velocity
     // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
     // ********************************************************************************************
     FillCoarsePatch(lev, time);
  
     // ********************************************************************************************
     // Initialize the integrator class
     // ********************************************************************************************
     dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
     initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
  
     // ********************************************************************************************
     // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
     // ********************************************************************************************
     if (lev > 0 && cf_width >= 0) {
         Construct_ERFFillPatchers(lev);
            Define_ERFFillPatchers(lev);
     }
  
 #ifdef ERF_USE_PARTICLES
     // particleData.Redistribute();
 #endif
 }

◆ MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch	(	int	lev,
		amrex::Real	time,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

override

 {
     // Set BoxArray grids and DistributionMapping dmap in AMReX_AmrMesh.H class
     SetBoxArray(lev, ba);
     SetDistributionMap(lev, dm);
  
     // amrex::Print() <<" BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
  
 #ifdef AMREX_USE_EB
     m_factory[lev] = makeEBFabFactory(geom[lev], grids[lev], dmap[lev],
                                       {nghost_eb_basic(),
                                        nghost_eb_volume(),
                                        nghost_eb_full()},
                                        EBSupport::full);
 #else
     m_factory[lev] = std::make_unique<FArrayBoxFactory>();
 #endif
  
     // The number of ghost cells for density must be 1 greater than that for velocity
     //     so that we can go back in forth betwen velocity and momentum on all faces
     // int ngrow_state = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff) + 1;
     // int ngrow_vels  = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff);
  
     auto& lev_new = vars_new[lev];
     auto& lev_old = vars_old[lev];
  
     //********************************************************************************************
     // This allocates all kinds of things, including but not limited to: solution arrays,
     //      terrain arrays and metrics,a nd base state.
     // *******************************************************************************************
     init_stuff(lev, ba, dm, lev_new, lev_old);
  
     //********************************************************************************************
     // Land Surface Model
     // *******************************************************************************************
     int lsm_size  = lsm.Get_Data_Size();
     lsm_data[lev].resize(lsm_size);
     lsm_flux[lev].resize(lsm_size);
     lsm.Define(lev, solverChoice);
     if (solverChoice.lsm_type != LandSurfaceType::None)
     {
         lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
     }
     for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
         lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
         lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
     }
  
     // ********************************************************************************************
     // Build the data structures for calculating diffusive/turbulent terms
     // ********************************************************************************************
     update_diffusive_arrays(lev, ba, dm);
  
     sst_lev[lev].resize(1);     sst_lev[lev][0] = nullptr;
     lmask_lev[lev].resize(1); lmask_lev[lev][0] = nullptr;
  
     //********************************************************************************************
     // Thin immersed body
     // *******************************************************************************************
 #if 0
     if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
         (solverChoice.advChoice.zero_yflux.size() > 0) ||
         (solverChoice.advChoice.zero_zflux.size() > 0))
     {
         overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
         overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
     }
 #endif
  
     if (solverChoice.advChoice.zero_xflux.size() > 0) {
         amrex::Print() << "Setting up thin immersed body for "
             << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
         BoxArray ba_xf(ba);
         ba_xf.surroundingNodes(0);
         thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
         thin_xforce[lev]->setVal(0.0);
         xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
         xflux_imask[lev]->setVal(1);
         for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
         {
             Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
             //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
             Box xbx = mfi.nodaltilebox(0);
             for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
                 const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
                 if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
                     (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
                     (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
                 {
                     imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
                     amrex::AllPrint() << "  mask xface at " << faceidx << std::endl;
                 }
             }
         }
     } else {
         thin_xforce[lev] = nullptr;
         xflux_imask[lev] = nullptr;
     }
  
     if (solverChoice.advChoice.zero_yflux.size() > 0) {
         amrex::Print() << "Setting up thin interface boundary for "
             << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
         BoxArray ba_yf(ba);
         ba_yf.surroundingNodes(1);
         thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
         thin_yforce[lev]->setVal(0.0);
         yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
         yflux_imask[lev]->setVal(1);
         for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
         {
             Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
             //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
             Box ybx = mfi.nodaltilebox(1);
             for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
                 const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
                 if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
                     (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
                     (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
                 {
                     imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
                     amrex::AllPrint() << "  mask yface at " << faceidx << std::endl;
                 }
             }
         }
     } else {
         thin_yforce[lev] = nullptr;
         yflux_imask[lev] = nullptr;
     }
  
     if (solverChoice.advChoice.zero_zflux.size() > 0) {
         amrex::Print() << "Setting up thin interface boundary for "
             << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
         BoxArray ba_zf(ba);
         ba_zf.surroundingNodes(2);
         thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
         thin_zforce[lev]->setVal(0.0);
         zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
         zflux_imask[lev]->setVal(1);
         for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
         {
             Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
             //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
             Box zbx = mfi.nodaltilebox(2);
             for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
                 const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
                 if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
                     (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
                     (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
                 {
                     imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
                     amrex::AllPrint() << "  mask zface at " << faceidx << std::endl;
                 }
             }
         }
     } else {
         thin_zforce[lev] = nullptr;
         zflux_imask[lev] = nullptr;
     }
  
     //********************************************************************************************
     // Microphysics
     // *******************************************************************************************
     int q_size  = micro->Get_Qmoist_Size(lev);
     qmoist[lev].resize(q_size);
     micro->Define(lev, solverChoice);
     if (solverChoice.moisture_type != MoistureType::None)
     {
         micro->Init(lev, vars_new[lev][Vars::cons],
                     grids[lev], Geom(lev), 0.0,
                     z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
     }
     for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
         qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
     }
  
     // ********************************************************************************************
     // Initialize the integrator class
     // ********************************************************************************************
     initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
  
     // ********************************************************************************************
     // Initialize the data itself
     // If (init_type == "real") then we are initializing terrain and the initial data in
     //                          the same call so we must call init_only before update_terrain_arrays
     // If (init_type != "real") then we want to initialize the terrain before the initial data
     //                          since we may need to use the grid information before constructing
     //                          initial idealized data
     // ********************************************************************************************
     if ((init_type == "real") || (init_type == "metgrid")) {
  
         // If called from restart, the data structures for terrain-related quantities
         //    are built in the ReadCheckpoint routine.  Otherwise we build them here.
         if (restart_chkfile.empty()) {
             init_only(lev, start_time);
             update_terrain_arrays(lev, time);
         }
  
     } else {
         // Build the data structures for terrain-related quantities
         update_terrain_arrays(lev, time);
  
         // Initialize the solution data itself
         if (restart_chkfile.empty()) {
             init_only(lev, start_time);
         }
     }
  
     // ********************************************************************************************
     // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
     // ********************************************************************************************
     if (lev > 0 && cf_width >= 0) {
         Construct_ERFFillPatchers(lev);
            Define_ERFFillPatchers(lev);
     }
  
     // ********************************************************************************************
     // Initialize the boundary conditions
     // ********************************************************************************************
     initialize_bcs(lev);
  
 #ifdef ERF_USE_PARTICLES
     if (restart_chkfile.empty()) {
         if (lev == 0) {
             initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd);
         } else {
             particleData.Redistribute();
         }
     }
 #endif
 }

◆ NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )

inlineprivatenoexcept

     {
         return datalog.size();
     }

◆ NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )

inlineprivatenoexcept

     {
         return samplelinelog.size();
     }

◆ NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )

inlineprivatenoexcept

     {
         return sampleline.size();
     }

◆ NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )

inlineprivatenoexcept

     {
         return sampleptlog.size();
     }

◆ NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )

inlineprivatenoexcept

     {
         return samplepoint.size();
     }

◆ operator=() [1/2]

ERF& ERF::operator= ( const ERF & other )

delete

◆ operator=() [2/2]

ERF& ERF::operator= ( ERF && other )

deletenoexcept

◆ PlotFileName()

std::string ERF::PlotFileName ( int lev ) const

private

◆ PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string > plot_var_names )

staticprivate

 {
     Vector<std::string> names;
  
     names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
  
     return names;
  
 }

◆ post_timestep()

void ERF::post_timestep	(	int	nstep,
		amrex::Real	time,
		amrex::Real	dt_lev
	)

 {
     BL_PROFILE("ERF::post_timestep()");
  
 #ifdef ERF_USE_PARTICLES
     particleData.Redistribute();
 #endif
  
     if (solverChoice.coupling_type == CouplingType::TwoWay)
     {
         bool use_terrain = solverChoice.use_terrain;
         int ncomp = vars_new[0][Vars::cons].nComp();
         for (int lev = finest_level-1; lev >= 0; lev--)
         {
             // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
             // m is the map scale factor at cell centers
             // Here we pre-divide (rho S) by m^2 before refluxing
             for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.tilebox();
                 const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
                 const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
                 if (use_terrain) {
                     const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                     ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                     {
                         cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
                     });
                 } else {
                     ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                     {
                         cons_arr(i,j,k,n) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
                     });
                 }
             } // mfi
  
             // This call refluxes from the lev/lev+1 interface onto lev
             getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 0, 0, ncomp);
  
             // Here we multiply (rho S) by m^2 after refluxing
             for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.tilebox();
                 const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
                 const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
                 if (use_terrain) {
                     const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                     ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                     {
                         cons_arr(i,j,k,n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)) / detJ_arr(i,j,k);
                     });
                 } else {
                     ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                     {
                         cons_arr(i,j,k,n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
                     });
                 }
             } // mfi
  
             // We need to do this before anything else because refluxing changes the
             // values of coarse cells underneath fine grids with the assumption they'll
             // be over-written by averaging down
             AverageDownTo(lev,0,ncomp);
         }
     }
  
     if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
         sum_integrated_quantities(time);
     }
  
     if (profile_int > 0 && (nstep+1) % profile_int == 0) {
         if (cc_profiles) {
             // all variables cell-centered
             write_1D_profiles(time);
         } else {
             // some variables staggered
             write_1D_profiles_stag(time);
         }
     }
  
     if (output_1d_column) {
 #ifdef ERF_USE_NETCDF
       if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
       {
          int lev_column = 0;
          for (int lev = finest_level; lev >= 0; lev--)
          {
             Real dx_lev = geom[lev].CellSize(0);
             Real dy_lev = geom[lev].CellSize(1);
             int i_lev = static_cast<int>(std::floor(column_loc_x / dx_lev));
             int j_lev = static_cast<int>(std::floor(column_loc_y / dy_lev));
             if (grids[lev].contains(IntVect(i_lev,j_lev,0))) lev_column = lev;
          }
          writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
       }
 #else
       Abort("To output 1D column files ERF must be compiled with NetCDF");
 #endif
     }
  
     if (output_bndry_planes)
     {
       if (is_it_time_for_action(istep[0], time, dt_lev0, bndry_output_planes_interval, bndry_output_planes_per) &&
           time >= bndry_output_planes_start_time)
       {
          m_w2d->write_planes(istep[0], time, vars_new);
       }
     }
  
     // Moving terrain
     if ( solverChoice.use_terrain &&  (solverChoice.terrain_type == TerrainType::Moving) )
     {
       for (int lev = finest_level; lev >= 0; lev--)
       {
         // Copy z_phs_nd and detJ_cc at end of timestep
         MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
         MultiFab::Copy(  *detJ_cc[lev],   *detJ_cc_new[lev], 0, 0, 1,   detJ_cc[lev]->nGrowVect());
         MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,3,1);
  
         make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
       }
     }
 } // post_timestep

Here is the call graph for this function:

◆ post_update()

void ERF::post_update	(	amrex::MultiFab &	state_mf,
		amrex::Real	time,
		const amrex::Geometry &	geom
	)

private

◆ print_banner()

void ERF::print_banner	(	MPI_Comm	comm,
		std::ostream &	out
	)

static

                      : " << msg << std::endl;
 }
  
 void ERF::print_banner (MPI_Comm comm, std::ostream& out)
 {
 #ifdef AMREX_USE_MPI
     int irank = 0;
     int num_ranks = 1;
     MPI_Comm_size(comm, &num_ranks);
     MPI_Comm_rank(comm, &irank);
  
     // Only root process does the printing
     if (irank != 0) return;
 #else
     amrex::ignore_unused(comm);
 #endif
  
     auto etime = std::chrono::system_clock::now();
     auto etimet = std::chrono::system_clock::to_time_t(etime);
 #ifndef _WIN32
     char time_buf[64];
     ctime_r(&etimet, time_buf);
     const std::string tstamp(time_buf);
 #else
     char* time_buf = new char[64];
     ctime_s(time_buf, 64, &etimet);
     const std::string tstamp(time_buf);
 #endif
  
     const char* githash1 = amrex::buildInfoGetGitHash(1);
     const char* githash2 = amrex::buildInfoGetGitHash(2);
  
     // clang-format off
     out << dbl_line
         << "                ERF (https://github.com/erf-model/ERF)"
         << std::endl << std::endl
         << "  ERF Git SHA      :: " << githash1 << std::endl
         << "  AMReX Git SHA    :: " << githash2 << std::endl
         << "  AMReX version    :: " << amrex::Version() << std::endl << std::endl
         << "  Exec. time       :: " << tstamp
         << "  Build time       :: " << amrex::buildInfoGetBuildDate() << std::endl
         << "  C++ compiler     :: " << amrex::buildInfoGetComp()
         << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
         << "  MPI              :: "
 #ifdef AMREX_USE_MPI
         << "ON    (Num. ranks = " << num_ranks << ")" << std::endl
 #else
         << "OFF " << std::endl
 #endif
         << "  GPU              :: "
 #ifdef AMREX_USE_GPU
         << "ON    "
 #if defined(AMREX_USE_CUDA)
         << "(Backend: CUDA)"
 #elif defined(AMREX_USE_HIP)
         << "(Backend: HIP)"
 #elif defined(AMREX_USE_SYCL)
         << "(Backend: SYCL)"
 #endif
         << std::endl
 #else
         << "OFF" << std::endl
 #endif
         << "  OpenMP           :: "
 #ifdef AMREX_USE_OMP
         << "ON    (Num. threads = " << omp_get_max_threads() << ")" << std::endl
 #else
         << "OFF" << std::endl
 #endif
         << std::endl;
  

Referenced by main().

Here is the caller graph for this function:

◆ print_error()

void ERF::print_error	(	MPI_Comm	comm,
		const std::string &	msg
	)

static

         :
     input_file   : Input file with simulation settings
  
 Optional:
     param=value  : Overrides for parameters during runtime
 )doc" << std::endl;
 }
  
 void ERF::print_error (MPI_Comm comm, const std::string& msg)
 {
 #ifdef AMREX_USE_MPI
     int irank = 0;
     int num_ranks = 1;
     MPI_Comm_size(comm, &num_ranks);
     MPI_Comm_rank(comm, &irank);
  

Referenced by main().

Here is the caller graph for this function:

◆ print_summary()

static void ERF::print_summary ( std::ostream & )

static

◆ print_tpls()

void ERF::print_tpls ( std::ostream & out )

static

                       ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
         << dash_line << std::endl;
     // clang-format on
 }
  
 void ERF::print_tpls (std::ostream& out)
 {
     amrex::Vector<std::string> tpls;
  
 #ifdef ERF_USE_NETCDF
     tpls.push_back(std::string("NetCDF    ") + NC_VERSION);
 #endif
 #ifdef AMREX_USE_SUNDIALS
     tpls.push_back(std::string("SUNDIALS     ") + SUNDIALS_VERSION);
 #endif
  
     if (!tpls.empty()) {
         out << "  Enabled third-party libraries: ";
         for (const auto& val : tpls) {

◆ print_usage()

void ERF::print_usage	(	MPI_Comm	comm,
		std::ostream &	out
	)

static

 {
 #ifdef AMREX_USE_MPI
     int irank = 0;
     int num_ranks = 1;
     MPI_Comm_size(comm, &num_ranks);
     MPI_Comm_rank(comm, &irank);
  
     // Only root process does the printing
     if (irank != 0) return;
 #else
     amrex::ignore_unused(comm);
 #endif
  
     out << R"doc(Usage:
     ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

◆ ReadCheckpointFile()

void ERF::ReadCheckpointFile ( )

private

ERF function for reading data from a checkpoint file during restart.

 {
     Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
  
     // Header
     std::string File(restart_chkfile + "/Header");
  
     VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
  
     Vector<char> fileCharPtr;
     ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
     std::string fileCharPtrString(fileCharPtr.dataPtr());
     std::istringstream is(fileCharPtrString, std::istringstream::in);
  
     std::string line, word;
  
     int chk_ncomp_cons, chk_ncomp;
  
     // read in title line
     std::getline(is, line);
  
     // read in finest_level
     is >> finest_level;
     GotoNextLine(is);
  
     // read the number of components
     // for each variable we store
  
     // conservative, cell-centered vars
     is >> chk_ncomp_cons;
     GotoNextLine(is);
  
     // x-velocity on faces
     is >> chk_ncomp;
     GotoNextLine(is);
     AMREX_ASSERT(chk_ncomp == 1);
  
     // y-velocity on faces
     is >> chk_ncomp;
     GotoNextLine(is);
     AMREX_ASSERT(chk_ncomp == 1);
  
     // z-velocity on faces
     is >> chk_ncomp;
     GotoNextLine(is);
     AMREX_ASSERT(chk_ncomp == 1);
  
     // read in array of istep
     std::getline(is, line);
     {
         std::istringstream lis(line);
         int i = 0;
         while (lis >> word) {
             istep[i++] = std::stoi(word);
         }
     }
  
     // read in array of dt
     std::getline(is, line);
     {
         std::istringstream lis(line);
         int i = 0;
         while (lis >> word) {
             dt[i++] = std::stod(word);
         }
     }
  
     // read in array of t_new
     std::getline(is, line);
     {
         std::istringstream lis(line);
         int i = 0;
         while (lis >> word) {
             t_new[i++] = std::stod(word);
         }
     }
  
     for (int lev = 0; lev <= finest_level; ++lev) {
         // read in level 'lev' BoxArray from Header
         BoxArray ba;
         ba.readFrom(is);
         GotoNextLine(is);
  
         // create a distribution mapping
         DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
  
         MakeNewLevelFromScratch (lev, t_new[lev], ba, dm);
     }
  
     // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
     // NOTE: Data is written over ncomp, so check that we match the header file
     int ncomp_cons = vars_new[0][Vars::cons].nComp();
     AMREX_ASSERT(chk_ncomp_cons == ncomp_cons);
  
     // read in the MultiFab data
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
         VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
         MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
         vars_new[lev][Vars::cons].setBndry(1.0e34);
  
         MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
         VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
         MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
         vars_new[lev][Vars::xvel].setBndry(1.0e34);
  
         MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
         VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
         MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
         vars_new[lev][Vars::yvel].setBndry(1.0e34);
  
         MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
         VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
         MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
         vars_new[lev][Vars::zvel].setBndry(1.0e34);
  
         // Note that we read the ghost cells of the base state (unlike above)
         IntVect ng = base_state[lev].nGrowVect();
         MultiFab base(grids[lev],dmap[lev],base_state[lev].nComp(),ng);
         VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
         MultiFab::Copy(base_state[lev],base,0,0,base.nComp(),ng);
         base_state[lev].FillBoundary(geom[lev].periodicity());
  
         if (solverChoice.use_terrain)  {
            // Note that we also read the ghost cells of z_phys_nd
            ng = z_phys_nd[lev]->nGrowVect();
            MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
            VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
            MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
            update_terrain_arrays(lev, t_new[lev]);
         }
  
         // Read in the precipitation accumulation component
         if (solverChoice.moisture_type == MoistureType::Kessler) {
             ng = qmoist[lev][4]->nGrowVect();
             int nvar = 1;
             MultiFab moist_vars(grids[lev],dmap[lev],nvar,ng);
             VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "RainAccum"));
             MultiFab::Copy(*(qmoist[lev][4]),moist_vars,0,0,nvar,ng);
         }
  
          if (solverChoice.moisture_type == MoistureType::SAM) {
             ng = qmoist[lev][8]->nGrowVect();
             int nvar = 1;
             MultiFab rain_accum(grids[lev],dmap[lev],nvar,ng);
             VisMF::Read(rain_accum, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "RainAccum"));
             MultiFab::Copy(*(qmoist[lev][8]),rain_accum,0,0,nvar,ng);
  
             ng = qmoist[lev][9]->nGrowVect();
             MultiFab snow_accum(grids[lev],dmap[lev],nvar,ng);
             VisMF::Read(snow_accum, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SnowAccum"));
             MultiFab::Copy(*(qmoist[lev][9]),snow_accum,0,0,nvar,ng);
  
             ng = qmoist[lev][10]->nGrowVect();
             MultiFab graup_accum(grids[lev],dmap[lev],nvar,ng);
             VisMF::Read(graup_accum, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "GraupAccum"));
             MultiFab::Copy(*(qmoist[lev][10]),graup_accum,0,0,nvar,ng);
  
         }
  
 #if defined(ERF_USE_WINDFARM)
         if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP){
             ng = Nturb[lev].nGrowVect();
             MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
             VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
             MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
         }
 #endif
  
         if (solverChoice.lsm_type != LandSurfaceType::None) {
             for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
                 BoxArray ba = lsm_data[lev][mvar]->boxArray();
                 DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
                 ng = lsm_data[lev][mvar]->nGrowVect();
                 int nvar = lsm_data[lev][mvar]->nComp();
                 MultiFab lsm_vars(ba,dm,nvar,ng);
                 VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmVars"));
                 MultiFab::Copy(*(lsm_data[lev][mvar]),lsm_vars,0,0,nvar,ng);
             }
         }
  
         // Note that we read the ghost cells of the mapfactors
         BoxList bl2d = grids[lev].boxList();
         for (auto& b : bl2d) {
             b.setRange(2,0);
         }
         BoxArray ba2d(std::move(bl2d));
  
         ng = mapfac_m[lev]->nGrowVect();
         MultiFab mf_m(ba2d,dmap[lev],1,ng);
         VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
         MultiFab::Copy(*mapfac_m[lev],mf_m,0,0,1,ng);
  
         ng = mapfac_u[lev]->nGrowVect();
         MultiFab mf_u(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,ng);
         VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
         MultiFab::Copy(*mapfac_u[lev],mf_u,0,0,1,ng);
  
         ng = mapfac_v[lev]->nGrowVect();
         MultiFab mf_v(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,ng);
         VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
         MultiFab::Copy(*mapfac_v[lev],mf_v,0,0,1,ng);
     }
  
 #ifdef ERF_USE_PARTICLES
    particleData.Restart((ParGDBBase*)GetParGDB(),restart_chkfile);
 #endif
  
 #ifdef ERF_USE_NETCDF
     // Read bdy_data files
     if ((init_type=="real") || (init_type=="metgrid")) {
         int ioproc = ParallelDescriptor::IOProcessorNumber();  // I/O rank
         int num_time;
         int num_var;
         Vector<Box> bx_v;
         if (ParallelDescriptor::IOProcessor()) {
             // Open header file and read from it
             std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
             bdy_h_file >> num_time;
             bdy_h_file >> num_var;
             bdy_h_file >> start_bdy_time;
             bdy_h_file >> bdy_time_interval;
             bdy_h_file >> real_width;
             bx_v.resize(4*num_var);
             for (int ivar(0); ivar<num_var; ++ivar) {
                 bdy_h_file >> bx_v[4*ivar  ];
                 bdy_h_file >> bx_v[4*ivar+1];
                 bdy_h_file >> bx_v[4*ivar+2];
                 bdy_h_file >> bx_v[4*ivar+3];
             }
  
             // IO size the FABs
             bdy_data_xlo.resize(num_time);
             bdy_data_xhi.resize(num_time);
             bdy_data_ylo.resize(num_time);
             bdy_data_yhi.resize(num_time);
             for (int itime(0); itime<num_time; ++itime) {
                 bdy_data_xlo[itime].resize(num_var);
                 bdy_data_xhi[itime].resize(num_var);
                 bdy_data_ylo[itime].resize(num_var);
                 bdy_data_yhi[itime].resize(num_var);
                 for (int ivar(0); ivar<num_var; ++ivar) {
                     bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
                     bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
                     bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
                     bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
                 }
             }
  
             // Open data file and read from it
             std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
             for (int itime(0); itime<num_time; ++itime) {
                 for (int ivar(0); ivar<num_var; ++ivar) {
                     bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
                     bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
                     bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
                     bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
                 }
             }
         } // IO
  
         // Broadcast the data
         ParallelDescriptor::Barrier();
         ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
         ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
         ParallelDescriptor::Bcast(&real_width,1,ioproc);
         ParallelDescriptor::Bcast(&num_time,1,ioproc);
         ParallelDescriptor::Bcast(&num_var,1,ioproc);
  
         // Everyone size their boxes
         bx_v.resize(4*num_var);
  
         ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
  
         // Everyone but IO size their FABs
         if (!ParallelDescriptor::IOProcessor()) {
           bdy_data_xlo.resize(num_time);
           bdy_data_xhi.resize(num_time);
           bdy_data_ylo.resize(num_time);
           bdy_data_yhi.resize(num_time);
           for (int itime(0); itime<num_time; ++itime) {
             bdy_data_xlo[itime].resize(num_var);
             bdy_data_xhi[itime].resize(num_var);
             bdy_data_ylo[itime].resize(num_var);
             bdy_data_yhi[itime].resize(num_var);
             for (int ivar(0); ivar<num_var; ++ivar) {
               bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
               bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
               bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
               bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
             }
           }
         }
  
         for (int itime(0); itime<num_time; ++itime) {
             for (int ivar(0); ivar<num_var; ++ivar) {
                 ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
                 ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
                 ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
                 ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
             }
         }
     } // init real
 #endif
 }

◆ ReadParameters()

void ERF::ReadParameters ( )

private

 {
     {
         ParmParse pp;  // Traditionally, max_step and stop_time do not have prefix.
         pp.query("max_step", max_step);
         pp.query("stop_time", stop_time);
  
         pp.query("start_time", start_time); // This is optional, it defaults to 0
     }
  
     ParmParse pp(pp_prefix);
     ParmParse pp_amr("amr");
     {
         // The type of the file we restart from
         pp.query("restart_type", restart_type);
  
         pp.query("regrid_int", regrid_int);
         pp.query("check_file", check_file);
         pp.query("check_type", check_type);
  
         // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
         //    for those or "erf.restart" / "erf.m_check_int" with the former taking
         //    precedence if both are specified
         pp.query("check_int", m_check_int);
         pp.query("check_per", m_check_per);
         pp_amr.query("check_int", m_check_int);
         pp_amr.query("check_per", m_check_per);
  
         pp.query("restart", restart_chkfile);
         pp_amr.query("restart", restart_chkfile);
  
         // Verbosity
         pp.query("v", verbose);
  
         // Frequency of diagnostic output
         pp.query("sum_interval", sum_interval);
         pp.query("sum_period"  , sum_per);
  
         // Time step controls
         pp.query("cfl", cfl);
         pp.query("init_shrink", init_shrink);
         pp.query("change_max", change_max);
  
         pp.query("fixed_dt", fixed_dt);
         pp.query("fixed_fast_dt", fixed_fast_dt);
         pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
  
         // If this is set, it must be even
         if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
         {
             Abort("If you specify fixed_mri_dt_ratio, it must be even");
         }
  
         // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
         if (fixed_dt > 0. && fixed_fast_dt > 0. && fixed_mri_dt_ratio <= 0)
         {
             Real eps = 1.e-12;
             int ratio = static_cast<int>( ( (1.0+eps) * fixed_dt ) / fixed_fast_dt );
             if (fixed_dt / fixed_fast_dt != ratio)
             {
                 Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
             }
         }
  
         // If all three are specified, they must be consistent
         if (fixed_dt > 0. && fixed_fast_dt > 0. &&  fixed_mri_dt_ratio > 0)
         {
             if (fixed_dt / fixed_fast_dt != fixed_mri_dt_ratio)
             {
                 Abort("Dt is over-specfied");
             }
         }
  
         AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt > 0.);
  
         // How to initialize
         pp.query("init_type",init_type);
         if (!init_type.empty() &&
             init_type != "uniform" &&
             init_type != "ideal" &&
             init_type != "real" &&
             init_type != "metgrid" &&
             init_type != "input_sounding")
         {
             Error("if specified, init_type must be uniform, ideal, real, metgrid or input_sounding");
         }
  
         // Should we use the bcs we've read in from wrfbdy or metgrid files?
         // We default to yes if we have them, but the user can override that option
         use_real_bcs = ( (init_type == "real") || (init_type == "metgrid") );
         pp.query("use_real_bcs",use_real_bcs);
  
         // We don't allow use_real_bcs to be true if init_type is not either real or metgrid
         AMREX_ALWAYS_ASSERT(!use_real_bcs || ((init_type == "real") || (init_type == "metgrid")) );
  
         // No moving terrain with init real
         if (init_type == "real" && solverChoice.terrain_type != TerrainType::Static) {
             Abort("Moving terrain is not supported with init real");
         }
  
         // We use this to keep track of how many boxes we read in from WRF initialization
         num_files_at_level.resize(max_level+1,0);
  
         // We use this to keep track of how many boxes are specified thru the refinement indicators
         num_boxes_at_level.resize(max_level+1,0);
             boxes_at_level.resize(max_level+1);
  
         // These hold the minimum and maximum value of k in the boxes *at each level*
         min_k_at_level.resize(max_level+1,0);
         max_k_at_level.resize(max_level+1,0);
  
         // We always have exactly one file at level 0
         num_boxes_at_level[0] = 1;
         boxes_at_level[0].resize(1);
         boxes_at_level[0][0] = geom[0].Domain();
  
 #ifdef ERF_USE_NETCDF
         nc_init_file.resize(max_level+1);
  
         // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
         //        but we always have exactly one file at level 0
         for (int lev = 0; lev <= max_level; lev++)
         {
             const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
             if (pp.contains(nc_file_names.c_str()))
             {
                 int num_files = pp.countval(nc_file_names.c_str());
                 num_files_at_level[lev] = num_files;
                 nc_init_file[lev].resize(num_files);
                 pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
                 for (int j = 0; j < num_files; j++)
                     Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
             }
         }
  
         // NetCDF wrfbdy lateral boundary file
         pp.query("nc_bdy_file", nc_bdy_file);
 #endif
  
         // Text input_sounding file
         pp.query("input_sounding_file", input_sounding_file);
  
         // Text input_sounding file
         pp.query("input_sponge_file", input_sponge_file);
  
         // Flag to trigger initialization from input_sounding like WRF's ideal.exe
         pp.query("init_sounding_ideal", init_sounding_ideal);
  
         // Output format
         pp.query("plotfile_type", plotfile_type);
         if (plotfile_type != "amrex" &&
             plotfile_type != "netcdf" && plotfile_type != "NetCDF" &&
             plotfile_type != "hdf5"   && plotfile_type != "HDF5" )
         {
             Print() << "User selected plotfile_type = " << plotfile_type << std::endl;
             Abort("Dont know this plotfile_type");
         }
         pp.query("plot_file_1",   plot_file_1);
         pp.query("plot_file_2",   plot_file_2);
         pp.query("plot_int_1" , m_plot_int_1);
         pp.query("plot_int_2" , m_plot_int_2);
         pp.query("plot_per_1",  m_plot_per_1);
         pp.query("plot_per_2",  m_plot_per_2);
  
         if ( (m_plot_int_1 > 0 && m_plot_per_1 > 0) ||
              (m_plot_int_2 > 0 && m_plot_per_2 > 0.) ) {
             Abort("Must choose only one of plot_int or plot_per");
         }
  
         pp.query("profile_int", profile_int);
         pp.query("interp_profiles_to_cc", cc_profiles);
  
         pp.query("plot_lsm", plot_lsm);
  
         pp.query("output_1d_column", output_1d_column);
         pp.query("column_per", column_per);
         pp.query("column_interval", column_interval);
         pp.query("column_loc_x", column_loc_x);
         pp.query("column_loc_y", column_loc_y);
         pp.query("column_file_name", column_file_name);
  
         // Specify information about outputting planes of data
         pp.query("output_bndry_planes", output_bndry_planes);
         pp.query("bndry_output_planes_interval", bndry_output_planes_interval);
         pp.query("bndry_output_planes_per", bndry_output_planes_per);
         pp.query("bndry_output_start_time", bndry_output_planes_start_time);
  
         // Specify whether ingest boundary planes of data
         pp.query("input_bndry_planes", input_bndry_planes);
  
         // Query the set and total widths for wrfbdy interior ghost cells
         pp.query("real_width", real_width);
         pp.query("real_set_width", real_set_width);
         AMREX_ALWAYS_ASSERT(real_width >= 0);
         AMREX_ALWAYS_ASSERT(real_set_width >= 0);
         AMREX_ALWAYS_ASSERT(real_width >= real_set_width);
  
         // Query the set and total widths for crse-fine interior ghost cells
         pp.query("cf_width", cf_width);
         pp.query("cf_set_width", cf_set_width);
         if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
             Abort("You must set cf_width >= cf_set_width >= 0");
         }
         if (max_level > 0 && cf_set_width > 0) {
             for (int lev = 1; lev <= max_level; lev++) {
                 if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
                     cf_set_width%ref_ratio[lev-1][1] != 0 ||
                     cf_set_width%ref_ratio[lev-1][2] != 0 ) {
                     Abort("You must set cf_width to be a multiple of ref_ratio");
                 }
             }
         }
  
         // AmrMesh iterate on grids?
         bool iterate(true);
         pp_amr.query("iterate_grids",iterate);
         if (!iterate) SetIterateToFalse();
     }
  
 #ifdef ERF_USE_PARTICLES
     readTracersParams();
 #endif
  
 #ifdef ERF_USE_MULTIBLOCK
     solverChoice.pp_prefix = pp_prefix;
 #endif
  
     solverChoice.init_params(max_level);
  
     // What type of land surface model to use
     // NOTE: Must be checked after init_params
     if (solverChoice.lsm_type == LandSurfaceType::SLM) {
         lsm.SetModel<SLM>();
         Print() << "SLM land surface model!\n";
     } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
         lsm.SetModel<MM5>();
         Print() << "MM5 land surface model!\n";
     } else if (solverChoice.lsm_type == LandSurfaceType::None) {
         lsm.SetModel<NullSurf>();
         Print() << "Null land surface model!\n";
     } else {
         Abort("Dont know this moisture_type!") ;
     }
  
     if (verbose > 0) {
         solverChoice.display();
     }
  
     if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
         Abort("For two-way coupling you must set cf_width = 0");
     }
 }

Here is the call graph for this function:

◆ refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )

private

Function to define the refinement criteria based on user input

 {
     if (max_level > 0)
     {
         ParmParse pp(pp_prefix);
         Vector<std::string> refinement_indicators;
         pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
  
         for (int i=0; i<refinement_indicators.size(); ++i)
         {
             std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
  
             ParmParse ppr(ref_prefix);
             RealBox realbox;
             int lev_for_box;
             if (ppr.countval("in_box_lo")) {
                 std::vector<Real> box_lo(3), box_hi(3);
                 ppr.get("max_level",lev_for_box);
                 if (lev_for_box <= max_level)
                 {
                     ppr.getarr("in_box_lo",box_lo,0,AMREX_SPACEDIM);
                     ppr.getarr("in_box_hi",box_hi,0,AMREX_SPACEDIM);
                     realbox = RealBox(&(box_lo[0]),&(box_hi[0]));
  
                     Print() << "Reading " << realbox << " at level " << lev_for_box << std::endl;
                     num_boxes_at_level[lev_for_box] += 1;
  
                     const auto* dx  = geom[lev_for_box].CellSize();
                     const Real* plo = geom[lev_for_box].ProbLo();
                     int ilo = static_cast<int>((box_lo[0] - plo[0])/dx[0]);
                     int jlo = static_cast<int>((box_lo[1] - plo[1])/dx[1]);
                     int klo = static_cast<int>((box_lo[2] - plo[2])/dx[2]);
                     int ihi = static_cast<int>((box_hi[0] - plo[0])/dx[0]-1);
                     int jhi = static_cast<int>((box_hi[1] - plo[1])/dx[1]-1);
                     int khi = static_cast<int>((box_hi[2] - plo[2])/dx[2]-1);
                     Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
                     if ( (ilo%ref_ratio[lev_for_box-1][0] != 0) || ((ihi+1)%ref_ratio[lev_for_box-1][0] != 0) ||
                          (jlo%ref_ratio[lev_for_box-1][1] != 0) || ((jhi+1)%ref_ratio[lev_for_box-1][1] != 0) )
                          amrex::Error("Fine box is not legit with this ref_ratio");
                     boxes_at_level[lev_for_box].push_back(bx);
                     Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                 } // lev
                 if (init_type == "real" || init_type == "metgrid") {
                     if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                         amrex::Error("Number of boxes doesnt match number of input files");
  
                     }
                 }
             }
  
             AMRErrorTagInfo info;
  
             if (realbox.ok()) {
                 info.SetRealBox(realbox);
             }
             if (ppr.countval("start_time") > 0) {
                 Real ref_min_time; ppr.get("start_time",ref_min_time);
                 info.SetMinTime(ref_min_time);
             }
             if (ppr.countval("end_time") > 0) {
                 Real ref_max_time; ppr.get("end_time",ref_max_time);
                 info.SetMaxTime(ref_max_time);
             }
             if (ppr.countval("max_level") > 0) {
                 int ref_max_level; ppr.get("max_level",ref_max_level);
                 info.SetMaxLevel(ref_max_level);
             }
  
             if (ppr.countval("value_greater")) {
                 int num_val = ppr.countval("value_greater");
                 Vector<Real> value(num_val);
                 ppr.getarr("value_greater",value,0,num_val);
                 std::string field; ppr.get("field_name",field);
                 ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
             }
             else if (ppr.countval("value_less")) {
                 int num_val = ppr.countval("value_less");
                 Vector<Real> value(num_val);
                 ppr.getarr("value_less",value,0,num_val);
                 std::string field; ppr.get("field_name",field);
                 ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
             }
             else if (ppr.countval("adjacent_difference_greater")) {
                 int num_val = ppr.countval("adjacent_difference_greater");
                 Vector<Real> value(num_val);
                 ppr.getarr("adjacent_difference_greater",value,0,num_val);
                 std::string field; ppr.get("field_name",field);
                 ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
             }
             else if (realbox.ok())
             {
                 ref_tags.push_back(AMRErrorTag(info));
             } else {
                 Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
             }
         } // loop over criteria
     } // if max_level > 0
 }

Here is the call graph for this function:

◆ RemakeLevel()

void ERF::RemakeLevel	(	int	lev,
		amrex::Real	time,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

override

 {
     // amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
  
     BoxArray            ba_old(vars_new[lev][Vars::cons].boxArray());
     DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
  
     int     ncomp_cons  = vars_new[lev][Vars::cons].nComp();
     IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
  
     // int ngrow_state = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff) + 1;
     int ngrow_vels  = ComputeGhostCells(solverChoice.advChoice, solverChoice.use_NumDiff);
  
     Vector<MultiFab> temp_lev_new(Vars::NumTypes);
     Vector<MultiFab> temp_lev_old(Vars::NumTypes);
  
     //********************************************************************************************
     // This allocates all kinds of things, including but not limited to: solution arrays,
     //      terrain arrays and metrics, and base state.
     // *******************************************************************************************
     init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old);
  
     // *****************************************************************************************************
     // Initialize the boundary conditions (after initializing the terrain but before calling FillCoarsePatch
     // *****************************************************************************************************
     initialize_bcs(lev);
  
     // ********************************************************************************************
     // This will fill the temporary MultiFabs with data from vars_new
     // ********************************************************************************************
     FillPatch(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
                           &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
                          {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                           false);
  
     // ********************************************************************************************
     // Copy from new into old just in case
     // ********************************************************************************************
     MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
     MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0,    1,ngrow_vels);
     MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0,    1,ngrow_vels);
     MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0,    1,IntVect(ngrow_vels,ngrow_vels,0));
  
     // ********************************************************************************************
     // Now swap the pointers
     // ********************************************************************************************
     for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
         std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
         std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
     }
  
     t_new[lev] = time;
     t_old[lev] = time - 1.e200;
  
     // ********************************************************************************************
     // Build the data structures for calculating diffusive/turbulent terms
     // ********************************************************************************************
     update_diffusive_arrays(lev, ba, dm);
  
     // ********************************************************************************************
     // Build the data structures for terrain-related quantities
     // ********************************************************************************************
     update_terrain_arrays(lev, time);
  
     //
     // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
     //
     if (solverChoice.use_terrain != 0) {
         for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
             average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
             average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
         }
     }
  
     //********************************************************************************************
     // Microphysics
     // *******************************************************************************************
     int q_size = micro->Get_Qmoist_Size(lev);
     qmoist[lev].resize(q_size);
     micro->Define(lev, solverChoice);
     if (solverChoice.moisture_type != MoistureType::None)
     {
         micro->Init(lev, vars_new[lev][Vars::cons],
                     grids[lev], Geom(lev), 0.0,
                     z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
     }
     for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
         qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
     }
  
     // ********************************************************************************************
     // Update the base state at this level
     // ********************************************************************************************
     // base_state[lev].define(ba,dm,3,1);
     // base_state[lev].setVal(0.);
     initHSE(lev);
  
     // ********************************************************************************************
     // Initialize the integrator class
     // ********************************************************************************************
     initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
  
     // We need to re-define the FillPatcher if the grids have changed
     if (lev > 0 && cf_width >= 0) {
         bool ba_changed = (ba != ba_old);
         bool dm_changed = (dm != dm_old);
         if (ba_changed || dm_changed) {
           Define_ERFFillPatchers(lev);
         }
     }
  
 #ifdef ERF_USE_PARTICLES
     particleData.Redistribute();
 #endif
 }

◆ restart()

void ERF::restart ( )

 {
     // TODO: This could be deleted since ba/dm are not created yet?
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         auto& lev_new = vars_new[lev];
         auto& lev_old = vars_old[lev];
         lev_new[Vars::cons].setVal(0.); lev_old[Vars::cons].setVal(0.);
         lev_new[Vars::xvel].setVal(0.); lev_old[Vars::xvel].setVal(0.);
         lev_new[Vars::yvel].setVal(0.); lev_old[Vars::yvel].setVal(0.);
         lev_new[Vars::zvel].setVal(0.); lev_old[Vars::zvel].setVal(0.);
     }
  
 #ifdef ERF_USE_NETCDF
     if (restart_type == "netcdf") {
        ReadNCCheckpointFile();
     }
 #endif
     if (restart_type == "native") {
        ReadCheckpointFile();
     }
  
     // We set this here so that we don't over-write the checkpoint file we just started from
     last_check_file_step = istep[0];
 }

◆ sample_lines()

void ERF::sample_lines	(	int	lev,
		amrex::Real	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters

lev	Current level
time	Current time
cell	IntVect containing the x,y-dimension indices to sample along z
mf	MultiFab from which we sample the data

 {
     int datwidth = 14;
     int datprecision = 6;
  
     int ifile = 0;
  
     const int ncomp = mf.nComp(); // cell-centered state vars
  
     MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
     average_face_to_cellcenter(mf_vels, 0,
                                Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
  
     //
     // Sample the data at a line (in direction "dir") in space
     // In this case we sample in the vertical direction so dir = 2
     // The "k" value of "cell" is ignored
     //
     int dir = 2;
     MultiFab my_line       = get_line_data(mf,              dir, cell);
     MultiFab my_line_vels  = get_line_data(mf_vels,         dir, cell);
     MultiFab my_line_tau11 = get_line_data(*Tau11_lev[lev], dir, cell);
     MultiFab my_line_tau12 = get_line_data(*Tau12_lev[lev], dir, cell);
     MultiFab my_line_tau13 = get_line_data(*Tau13_lev[lev], dir, cell);
     MultiFab my_line_tau22 = get_line_data(*Tau22_lev[lev], dir, cell);
     MultiFab my_line_tau23 = get_line_data(*Tau23_lev[lev], dir, cell);
     MultiFab my_line_tau33 = get_line_data(*Tau33_lev[lev], dir, cell);
  
     for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
     {
         // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
  
         std::ostream& sample_log = SampleLineLog(ifile);
         if (sample_log.good()) {
           sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
           const auto& my_line_arr = my_line[0].const_array();
           const auto& my_line_vels_arr = my_line_vels[0].const_array();
           const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
           const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
           const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
           const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
           const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
           const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
           const Box&  my_box = my_line[0].box();
           const int klo = my_box.smallEnd(2);
           const int khi = my_box.bigEnd(2);
           int i = cell[0];
           int j = cell[1];
           for (int n = 0; n < ncomp; n++) {
               for (int k = klo; k <= khi; k++) {
                   sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
               }
           }
           for (int n = 0; n < AMREX_SPACEDIM; n++) {
               for (int k = klo; k <= khi; k++) {
                   sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
               }
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
           }
           sample_log << std::endl;
         } // if good
     } // mfi
 }

◆ sample_points()

void ERF::sample_points	(	int	lev,
		amrex::Real	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling MultiFab data at a specified cell index.

Parameters

lev	Level for the associated MultiFab data
time	Current time
cell	IntVect containing the indexes for the cell where we want to sample
mf	MultiFab from which we wish to sample data

 {
     int datwidth = 14;
  
     int ifile = 0;
  
     //
     // Sample the data at a single point in space
     //
     int ncomp = mf.nComp();
     Vector<Real> my_point = get_cell_data(mf, cell);
  
     if (!my_point.empty()) {
  
         // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
  
         std::ostream& sample_log = SamplePointLog(ifile);
         if (sample_log.good()) {
           sample_log << std::setw(datwidth) << time;
           for (int i = 0; i < ncomp; ++i)
           {
               sample_log << std::setw(datwidth) << my_point[i];
           }
           sample_log << std::endl;
         } // if good
     } // only write from processor that holds the cell
 }

◆ SampleLine()

amrex::IntVect& ERF::SampleLine ( int i )

inlineprivate

     {
         return sampleline[i];
     }

◆ SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int i )

inlineprivate

     {
         return *samplelinelog[i];
     }

◆ SampleLineLogName()

std::string ERF::SampleLineLogName ( int i ) const

inlineprivatenoexcept

The filename of the ith samplelinelog file.

1149 { return samplelinelogname[i]; }

◆ SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int i )

inlineprivate

     {
         return samplepoint[i];
     }

◆ SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int i )

inlineprivate

     {
         return *sampleptlog[i];
     }

◆ SamplePointLogName()

std::string ERF::SamplePointLogName ( int i ) const

inlineprivatenoexcept

The filename of the ith sampleptlog file.

1146 { return sampleptlogname[i]; }

◆ setPlotVariables()

void ERF::setPlotVariables	(	const std::string &	pp_plot_var_names,
		amrex::Vector< std::string > &	plot_var_names
	)

private

 {
     ParmParse pp(pp_prefix);
  
     if (pp.contains(pp_plot_var_names.c_str()))
     {
         std::string nm;
  
         int nPltVars = pp.countval(pp_plot_var_names.c_str());
  
         for (int i = 0; i < nPltVars; i++)
         {
             pp.get(pp_plot_var_names.c_str(), nm, i);
  
             // Add the named variable to our list of plot variables
             // if it is not already in the list
             if (!containerHasElement(plot_var_names, nm)) {
                 plot_var_names.push_back(nm);
             }
         }
     } else {
         //
         // The default is to add none of the variables to the list
         //
         plot_var_names.clear();
     }
  
     // Get state variables in the same order as we define them,
     // since they may be in any order in the input list
     Vector<std::string> tmp_plot_names;
  
     int n_qstate   = micro->Get_Qstate_Size();
     int ncomp_cons = NVAR_max - (NMOIST_max - n_qstate);
  
     for (int i = 0; i < ncomp_cons; ++i) {
         if ( containerHasElement(plot_var_names, cons_names[i]) ) {
             tmp_plot_names.push_back(cons_names[i]);
         }
     }
     // check for velocity since it's not in cons_names
     // if we are asked for any velocity component, we will need them all
     if (containerHasElement(plot_var_names, "x_velocity") ||
         containerHasElement(plot_var_names, "y_velocity") ||
         containerHasElement(plot_var_names, "z_velocity")) {
         tmp_plot_names.push_back("x_velocity");
         tmp_plot_names.push_back("y_velocity");
         tmp_plot_names.push_back("z_velocity");
     }
     for (int i = 0; i < derived_names.size(); ++i) {
         if ( containerHasElement(plot_var_names, derived_names[i]) ) {
             if (solverChoice.use_terrain || (derived_names[i] != "z_phys" && derived_names[i] != "detJ") ) {
                tmp_plot_names.push_back(derived_names[i]);
             }
         }
     }
 #ifdef ERF_USE_PARTICLES
     const auto& particles_namelist( particleData.getNamesUnalloc() );
     for (auto it = particles_namelist.cbegin(); it != particles_namelist.cend(); ++it) {
         std::string tmp( (*it)+"_count" );
         if (containerHasElement(plot_var_names, tmp) ) {
             tmp_plot_names.push_back(tmp);
         }
     }
 #endif
  
 #ifdef ERF_USE_EB
     if (containerHasElement(plot_var_names, "volfrac")) {
         tmp_plot_names.push_back("volfrac");
     }
 #endif
  
     plot_var_names = tmp_plot_names;
  
     for (int i=0; i<plot_var_names.size(); i++) {
     }
 }

Here is the call graph for this function:

◆ setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool restarting )

private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters

[in] restarting Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

 {
     // If we are restarting then we haven't read the input_sounding file yet
     //    so we need to read it here
     // TODO: should we store this information in the checkpoint file instead?
     if (restarting) {
         input_sounding_data.read_from_file(input_sounding_file, geom[0], zlevels_stag);
     }
  
     const Real* z_inp_sound     = input_sounding_data.z_inp_sound.dataPtr();
     const Real* U_inp_sound     = input_sounding_data.U_inp_sound.dataPtr();
     const Real* V_inp_sound     = input_sounding_data.V_inp_sound.dataPtr();
     const Real* theta_inp_sound = input_sounding_data.theta_inp_sound.dataPtr();
     const int   inp_sound_size  = input_sounding_data.size();
  
     for (int lev = 0; lev <= finest_level; lev++)
     {
         const int khi = geom[lev].Domain().bigEnd()[2];
         Vector<Real> zcc(khi+1);
  
         if (z_phys_cc[lev]) {
             // use_terrain=1
             // calculate the damping strength based on the max height at each k
             reduce_to_max_per_level(zcc, z_phys_cc[lev]);
         } else {
             const auto *const prob_lo = geom[lev].ProbLo();
             const auto *const dx = geom[lev].CellSize();
             for (int k = 0; k <= khi; k++)
             {
                 zcc[k] = prob_lo[2] + (k+0.5) * dx[2];
             }
         }
  
         for (int k = 0; k <= khi; k++)
         {
             h_rayleigh_ptrs[lev][Rayleigh::ubar][k]         = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
             h_rayleigh_ptrs[lev][Rayleigh::vbar][k]         = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
             h_rayleigh_ptrs[lev][Rayleigh::wbar][k]         = Real(0.0);
             h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
             if (h_rayleigh_ptrs[lev][Rayleigh::tau][k] > 0) {
                                                   Print() << zcc[k] << ":" << " tau=" << h_rayleigh_ptrs[lev][Rayleigh::tau][k];
                 if (solverChoice.rayleigh_damp_U) Print() << " ubar    = " << h_rayleigh_ptrs[lev][Rayleigh::ubar][k];
                 if (solverChoice.rayleigh_damp_V) Print() << " vbar    = " << h_rayleigh_ptrs[lev][Rayleigh::vbar][k];
                 if (solverChoice.rayleigh_damp_W) Print() << " wbar    = " << h_rayleigh_ptrs[lev][Rayleigh::wbar][k];
                 if (solverChoice.rayleigh_damp_T) Print() << " thetabar= " << h_rayleigh_ptrs[lev][Rayleigh::thetabar][k];
                 Print() << std::endl;
             }
         }
  
         // Copy from host version to device version
         Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
                          d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
         Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
                          d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
         Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
                          d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
         Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
                          d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
     }
 }

Here is the call graph for this function:

◆ setRecordDataInfo()

void ERF::setRecordDataInfo	(	int	i,
		const std::string &	filename
	)

inlineprivate

     {
         if (amrex::ParallelDescriptor::IOProcessor())
         {
             datalog[i] = std::make_unique<std::fstream>();
             datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
             if (!datalog[i]->good()) {
                 amrex::FileOpenFailed(filename);
             }
         }
         amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
     }

◆ setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo	(	int	i,
		int	lev,
		amrex::IntVect &	cell,
		const std::string &	filename
	)

inlineprivate

     {
         amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
         for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
         {
             const amrex::Box& bx = mfi.validbox();
             if (bx.contains(cell)) {
                 samplelinelog[i] = std::make_unique<std::fstream>();
                 samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                 if (!samplelinelog[i]->good()) {
                     amrex::FileOpenFailed(filename);
                 }
             }
         }
         amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
     }

◆ setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo	(	int	i,
		int	lev,
		amrex::IntVect &	cell,
		const std::string &	filename
	)

inlineprivate

     {
         amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
         for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
         {
             const amrex::Box& bx = mfi.validbox();
             if (bx.contains(cell)) {
                 sampleptlog[i] = std::make_unique<std::fstream>();
                 sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                 if (!sampleptlog[i]->good()) {
                     amrex::FileOpenFailed(filename);
                 }
             }
         }
         amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
     }

◆ setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool restarting )

private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters

[in] restarting Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

 {
  
     // If we are restarting then we haven't read the input_sponge file yet
     //    so we need to read it here
     // TODO: should we store this information in the checkpoint file instead?
     if (restarting) {
         input_sponge_data.read_from_file(input_sponge_file, geom[0], zlevels_stag);
     }
  
     const Real* z_inp_sponge     = input_sponge_data.z_inp_sponge.dataPtr();
     const Real* U_inp_sponge     = input_sponge_data.U_inp_sponge.dataPtr();
     const Real* V_inp_sponge     = input_sponge_data.V_inp_sponge.dataPtr();
     const int   inp_sponge_size  = input_sponge_data.size();
  
     for (int lev = 0; lev <= finest_level; lev++)
     {
         const int khi = geom[lev].Domain().bigEnd()[2];
         Vector<Real> zcc(khi+1);
  
         if (z_phys_cc[lev]) {
             // use_terrain=1
             // calculate the damping strength based on the max height at each k
             reduce_to_max_per_level(zcc, z_phys_cc[lev]);
         } else {
             const auto *const prob_lo = geom[lev].ProbLo();
             const auto *const dx = geom[lev].CellSize();
             for (int k = 0; k <= khi; k++)
             {
                 zcc[k] = prob_lo[2] + (k+0.5) * dx[2];
             }
         }
  
         for (int k = 0; k <= khi; k++)
         {
             h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
             h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
         }
  
         // Copy from host version to device version
         Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
                          d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
         Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
                          d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
     }
 }

Here is the call graph for this function:

◆ sum_integrated_quantities()

void ERF::sum_integrated_quantities ( amrex::Real time )

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters

time	Current time

 {
     BL_PROFILE("ERF::sum_integrated_quantities()");
  
     if (verbose <= 0)
       return;
  
     int datwidth = 14;
     int datprecision = 6;
  
     // Single level sum
     Real mass_sl;
  
     // Multilevel sums
     Real mass_ml = 0.0;
     Real rhth_ml = 0.0;
     Real scal_ml = 0.0;
  
 #if 1
     mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,*mapfac_m[0],true,false);
     for (int lev = 0; lev <= finest_level; lev++) {
         mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,*mapfac_m[lev],true,true);
     }
 #else
     for (int lev = 0; lev <= finest_level; lev++) {
         MultiFab pert_dens(vars_new[lev][Vars::cons].boxArray(),
                            vars_new[lev][Vars::cons].DistributionMap(),
                            1,0);
         MultiFab r_hse (base_state[lev], make_alias, 0, 1); // r_0 is first  component
         for ( MFIter mfi(pert_dens,TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             const Box& bx = mfi.tilebox();
             const Array4<Real      >& pert_dens_arr = pert_dens.array(mfi);
             const Array4<Real const>&         S_arr = vars_new[lev][Vars::cons].const_array(mfi);
             const Array4<Real const>&        r0_arr = r_hse.const_array(mfi);
             ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                 pert_dens_arr(i, j, k, 0) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
             });
         }
         if (lev == 0) {
             mass_sl = volWgtSumMF(0,pert_dens,0,*mapfac_m[0],true,false);
         }
         mass_ml += volWgtSumMF(lev,pert_dens,0,*mapfac_m[lev],true,true);
     } // lev
 #endif
  
     Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,*mapfac_m[0],true,false);
     Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,*mapfac_m[0],true,false);
  
     for (int lev = 0; lev <= finest_level; lev++) {
         rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,*mapfac_m[lev],true,true);
         scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,*mapfac_m[lev],true,true);
     }
  
     Gpu::HostVector<Real> h_avg_ustar; h_avg_ustar.resize(1);
     Gpu::HostVector<Real> h_avg_tstar; h_avg_tstar.resize(1);
     Gpu::HostVector<Real> h_avg_olen; h_avg_olen.resize(1);
     if ((m_most != nullptr) && (NumDataLogs() > 0)) {
         Box domain = geom[0].Domain();
         int zdir = 2;
         h_avg_ustar = sumToLine(*m_most->get_u_star(0),0,1,domain,zdir);
         h_avg_tstar = sumToLine(*m_most->get_t_star(0),0,1,domain,zdir);
         h_avg_olen  = sumToLine(*m_most->get_olen(0),0,1,domain,zdir);
  
         // Divide by the total number of cells we are averaging over
         Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
         h_avg_ustar[0] /= area_z;
         h_avg_tstar[0] /= area_z;
         h_avg_olen[0]  /= area_z;
  
     } else {
         h_avg_ustar[0] = 0.;
         h_avg_tstar[0] = 0.;
         h_avg_olen[0]  = 0.;
     }
  
     const int nfoo = 6;
     Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mass_ml,rhth_ml,scal_ml};
 #ifdef AMREX_LAZY
     Lazy::QueueReduction([=]() mutable {
 #endif
     ParallelDescriptor::ReduceRealSum(
         foo, nfoo, ParallelDescriptor::IOProcessorNumber());
  
       if (ParallelDescriptor::IOProcessor()) {
         int i = 0;
         mass_sl = foo[i++];
         rhth_sl = foo[i++];
         scal_sl = foo[i++];
         mass_ml = foo[i++];
         rhth_ml = foo[i++];
         scal_ml = foo[i++];
  
         Print() << '\n';
         if (finest_level ==  0) {
 #if 1
            Print() << "TIME= " << time << "     MASS          = " << mass_sl << '\n';
 #else
            Print() << "TIME= " << time << " PERT MASS         = " << mass_sl << '\n';
 #endif
            Print() << "TIME= " << time << " RHO THETA         = " << rhth_sl << '\n';
            Print() << "TIME= " << time << " RHO SCALAR        = " << scal_sl << '\n';
         } else {
 #if 1
            Print() << "TIME= " << time << "      MASS   SL/ML = " << mass_sl << " " << mass_ml << '\n';
 #else
            Print() << "TIME= " << time << " PERT MASS   SL/ML = " << mass_sl << " " << mass_ml << '\n';
 #endif
            Print() << "TIME= " << time << " RHO THETA   SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
            Print() << "TIME= " << time << " RHO SCALAR  SL/ML = " << scal_sl << " " << scal_ml << '\n';
         }
  
         // The first data log only holds scalars
         if (NumDataLogs() > 0)
         {
             int nd = 0;
             std::ostream& data_log1 = DataLog(nd);
             if (data_log1.good()) {
                 if (time == 0.0) {
                     data_log1 << std::setw(datwidth) << "          time";
                     data_log1 << std::setw(datwidth) << "          u_star";
                     data_log1 << std::setw(datwidth) << "          t_star";
                     data_log1 << std::setw(datwidth) << "          olen";
                     data_log1 << std::endl;
                 } // time = 0
  
               // Write the quantities at this time
               data_log1 << std::setw(datwidth) << time;
               data_log1 << std::setw(datwidth) << std::setprecision(datprecision)
                         << h_avg_ustar[0];
               data_log1 << std::setw(datwidth) << std::setprecision(datprecision)
                         << h_avg_tstar[0];
               data_log1 << std::setw(datwidth) << std::setprecision(datprecision)
                         << h_avg_olen[0];
               data_log1 << std::endl;
             } // if good
         } // loop over i
       } // if IOProcessor
 #ifdef AMREX_LAZY
     });
 #endif
  
     // This is just an alias for convenience
     int lev = 0;
     if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
         for (int i = 0; i < NumSamplePoints(); ++i)
         {
             sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
         }
     }
     if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
         for (int i = 0; i < NumSampleLines(); ++i)
         {
             sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
         }
     }
 }

◆ timeStep()

void ERF::timeStep	(	int	lev,
		amrex::Real	time,
		int	iteration
	)

private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	iteration	time step counter

 {
     if (regrid_int > 0)  // We may need to regrid
     {
         // help keep track of whether a level was already regridded
         // from a coarser level call to regrid
         static Vector<int> last_regrid_step(max_level+1, 0);
  
         // regrid changes level "lev+1" so we don't regrid on max_level
         // also make sure we don't regrid fine levels again if
         // it was taken care of during a coarser regrid
         if (lev < max_level)
         {
             if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
             {
                 // regrid could add newly refine levels (if finest_level < max_level)
                 // so we save the previous finest level index
                 int old_finest = finest_level;
  
                 regrid(lev, time);
  
 #ifdef ERF_USE_PARTICLES
                 if (finest_level != old_finest) {
                     particleData.Redistribute();
                 }
 #endif
  
                 // mark that we have regridded this level already
                 for (int k = lev; k <= finest_level; ++k) {
                     last_regrid_step[k] = istep[k];
                 }
  
                 // if there are newly created levels, set the time step
                 for (int k = old_finest+1; k <= finest_level; ++k) {
                     dt[k] = dt[k-1] / MaxRefRatio(k-1);
                 }
             } // if
         } // lev
     }
  
     // Update what we call "old" and "new" time
     t_old[lev] = t_new[lev];
     t_new[lev] += dt[lev];
  
     if (Verbose()) {
         amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
         amrex::Print() << "ADVANCE from time = " << t_old[lev] << " to " << t_new[lev]
                        << " with dt = " << dt[lev] << std::endl;
     }
  
     // Advance a single level for a single time step
     Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
  
     ++istep[lev];
  
     if (Verbose()) {
         amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
         amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
     }
  
     if (lev < finest_level)
     {
         // recursive call for next-finer level
         for (int i = 1; i <= nsubsteps[lev+1]; ++i)
         {
             Real strt_time_for_fine = time + (i-1)*dt[lev+1];
             timeStep(lev+1, strt_time_for_fine, i);
         }
     }
 }

◆ update_diffusive_arrays()

void ERF::update_diffusive_arrays	(	int	lev,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

private

 {
     // ********************************************************************************************
     // Diffusive terms
     // ********************************************************************************************
     bool l_use_terrain = solverChoice.use_terrain;
     bool l_use_diff    = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
                            (solverChoice.turbChoice[lev].les_type        !=       LESType::None) ||
                            (solverChoice.turbChoice[lev].pbl_type        !=       PBLType::None) );
     bool l_use_kturb   = ( (solverChoice.turbChoice[lev].les_type        != LESType::None)   ||
                            (solverChoice.turbChoice[lev].pbl_type        != PBLType::None) );
     bool l_use_ddorf   = (  solverChoice.turbChoice[lev].les_type        == LESType::Deardorff);
  
     BoxArray ba12 = convert(ba, IntVect(1,1,0));
     BoxArray ba13 = convert(ba, IntVect(1,0,1));
     BoxArray ba23 = convert(ba, IntVect(0,1,1));
  
     if (l_use_diff) {
         Tau11_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,0) );
         Tau22_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,0) );
         Tau33_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,0) );
         Tau12_lev[lev] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,0) );
         Tau13_lev[lev] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,0) );
         Tau23_lev[lev] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,0) );
         if (l_use_terrain) {
             Tau21_lev[lev] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,0) );
             Tau31_lev[lev] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,0) );
             Tau32_lev[lev] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,0) );
         } else {
             Tau21_lev[lev] = nullptr;
             Tau31_lev[lev] = nullptr;
             Tau32_lev[lev] = nullptr;
         }
         SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,0) );
         SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,0) );
         SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
         SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,0) );
         SFS_hfx1_lev[lev]->setVal(0.);
         SFS_hfx2_lev[lev]->setVal(0.);
         SFS_hfx3_lev[lev]->setVal(0.);
         SFS_diss_lev[lev]->setVal(0.);
     } else {
         Tau11_lev[lev] = nullptr; Tau22_lev[lev] = nullptr; Tau33_lev[lev] = nullptr;
         Tau12_lev[lev] = nullptr; Tau21_lev[lev] = nullptr;
         Tau13_lev[lev] = nullptr; Tau31_lev[lev] = nullptr;
         Tau23_lev[lev] = nullptr; Tau32_lev[lev] = nullptr;
         SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
         SFS_diss_lev[lev] = nullptr;
     }
  
     if (l_use_kturb) {
       eddyDiffs_lev[lev] = std::make_unique<MultiFab>( ba, dm, EddyDiff::NumDiffs, 1 );
         eddyDiffs_lev[lev]->setVal(0.0);
         if(l_use_ddorf) {
             SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
         } else {
             SmnSmn_lev[lev] = nullptr;
         }
     } else {
         eddyDiffs_lev[lev] = nullptr;
         SmnSmn_lev[lev]    = nullptr;
     }
 }

◆ update_terrain_arrays()

void ERF::update_terrain_arrays	(	int	lev,
		amrex::Real	time
	)

private

 {
     if (solverChoice.use_terrain) {
  
         if (lev == 0 && (init_type != "real" && init_type != "metgrid") ) {
             prob->init_custom_terrain(geom[lev],*z_phys_nd[lev],time);
             init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag);
         }
  
         if (lev > 0) {
             Vector<MultiFab*> fmf = {z_phys_nd[lev].get(), z_phys_nd[lev].get()};
             Vector<Real> ftime    = {t_old[lev], t_new[lev]};
             Vector<MultiFab*> cmf = {z_phys_nd[lev-1].get(), z_phys_nd[lev-1].get()};
             Vector<Real> ctime    = {t_old[lev-1], t_new[lev-1]};
  
             //
             // First we fill z_phys_nd at lev>0 through interpolation
             //
             Interpolater* mapper = &node_bilinear_interp;
             PhysBCFunctNoOp null_bc;
             InterpFromCoarseLevel(*z_phys_nd[lev], time, *z_phys_nd[lev-1],
                                   0, 0, 1,
                                   geom[lev-1], geom[lev],
                                   null_bc, 0, null_bc, 0, refRatio(lev-1),
                                   mapper, domain_bcs_type, 0);
  
             //
             // Then if not using real/metgrid data, we
             // 1) redefine the terrain at k=0 for every fine box which includes k=0
             // 2) recreate z_phys_nd at every fine node using
             // the data at the bottom of each fine grid
             // which has been either been interpolated from the coarse grid (k>0)
             // or set in init_custom_terrain (k=0)
             //
             prob->init_custom_terrain(geom[lev],*z_phys_nd[lev],time);
             if (init_type != "real" && init_type != "metgrid") {
                 init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag);
             }
         }
  
         make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
         make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
  
         make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
     }
 }

Here is the call graph for this function:

◆ volWgtSumMF()

Real ERF::volWgtSumMF	(	int	lev,
		const amrex::MultiFab &	mf,
		int	comp,
		const amrex::MultiFab &	mapfac,
		bool	local,
		bool	finemask
	)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters

lev	Current level
mf	MultiFab on which we do the volume weighted sum
comp	Index of the component we want to sum
local	Boolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)
finemask	If a finer level is available, determines whether we mask fine data

 {
     BL_PROFILE("ERF::volWgtSumMF()");
  
     Real sum = 0.0;
     MultiFab tmp(grids[lev], dmap[lev], 1, 0);
     MultiFab::Copy(tmp, mf, comp, 0, 1, 0);
  
     // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
     // m is the map scale factor at cell centers
     for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.tilebox();
         const Array4<      Real>    tmp_arr =    tmp.array(mfi);
         const Array4<const Real> mapfac_arr = mapfac.const_array(mfi);
         ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
         {
             tmp_arr(i,j,k) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
         });
     } // mfi
  
     if (lev < finest_level && finemask) {
         const MultiFab& mask = build_fine_mask(lev+1);
         MultiFab::Multiply(tmp, mask, 0, 0, 1, 0);
     }
  
     MultiFab volume(grids[lev], dmap[lev], 1, 0);
     auto const& dx = geom[lev].CellSizeArray();
     Real cell_vol = dx[0]*dx[1]*dx[2];
     volume.setVal(cell_vol);
     if (solverChoice.use_terrain)
         MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
     sum = MultiFab::Dot(tmp, 0, volume, 0, 1, 0, local);
  
     if (!local)
       ParallelDescriptor::ReduceRealSum(sum);
  
     return sum;
 }

◆ write_1D_profiles()

void ERF::write_1D_profiles ( amrex::Real time )

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters

time	Current time

 {
     BL_PROFILE("ERF::write_1D_profiles()");
  
     if (verbose <= 0)
       return;
  
     int datwidth = 14;
     int datprecision = 6;
     int timeprecision = 13; // e.g., 1-yr LES: 31,536,000 s with dt ~ 0.01 ==> min prec = 10
  
     if (verbose > 0 && NumDataLogs() > 1)
     {
         // Define the 1d arrays we will need
         Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
         Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs;
         Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr;
         Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
         Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
         Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
         Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
         Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
         Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
  
         if (NumDataLogs() > 1) {
             derive_diag_profiles(h_avg_u, h_avg_v, h_avg_w,
                                  h_avg_rho, h_avg_th, h_avg_ksgs,
                                  h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr,
                                  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw);
         }
  
         if (NumDataLogs() > 3) {
             derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                    h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                    h_avg_sgshfx,
                                    h_avg_sgsdiss);
         }
  
         int hu_size =  h_avg_u.size();
  
         auto const& dx = geom[0].CellSizeArray();
         if (ParallelDescriptor::IOProcessor()) {
             if (NumDataLogs() > 1) {
                 std::ostream& data_log1 = DataLog(1);
                 if (data_log1.good()) {
                   // Write the quantities at this time
                   for (int k = 0; k < hu_size; k++) {
                       Real z = (k + 0.5)* dx[2];
                       data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_u[k]  << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
                                 << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
                                 << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k]
                                 << std::endl;
                   } // loop over z
                 } // if good
             } // NumDataLogs
  
             if (NumDataLogs() > 2) {
                 std::ostream& data_log2 = DataLog(2);
                 if (data_log2.good()) {
                   // Write the perturbational quantities at this time
                   for (int k = 0; k < hu_size; k++) {
                       Real z = (k + 0.5)* dx[2];
                       data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]  << " "
                                 << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]  << " "
                                 << h_avg_uw[k]   - h_avg_u[k]*h_avg_w[k]  << " "
                                 << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]  << " "
                                 << h_avg_vw[k]   - h_avg_v[k]*h_avg_w[k]  << " "
                                 << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]  << " "
                                 << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k] << " "
                                 << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k] << " "
                                 << h_avg_wth[k]  - h_avg_w[k]*h_avg_th[k] << " "
                                 << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
                                 // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                 //                        -   <u_i u_i> * <u_j>
                                 //                        - 2*<u_i> * <u_i u_j>
                                 //                        + 2*<u_i>*<u_i> * <u_j>
                                 << h_avg_uiuiu[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
                                  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
                                    << " " // (u'_i u'_i)u'
                                 << h_avg_uiuiv[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
                                  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
                                    << " " // (u'_i u'_i)v'
                                 << h_avg_uiuiw[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
                                  - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                    << " " // (u'_i u'_i)w'
                                 << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k] << " "
                                 << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k] << " "
                                 << h_avg_pw[k]   - h_avg_p[k]*h_avg_w[k] << " "
                                 << h_avg_wqv[k]   - h_avg_qv[k]*h_avg_w[k] << " "
                                 << h_avg_wqc[k]   - h_avg_qc[k]*h_avg_w[k] << " "
                                 << h_avg_wqr[k]   - h_avg_qr[k]*h_avg_w[k]
                                 << std::endl;
                   } // loop over z
                 } // if good
             } // NumDataLogs
  
             if (NumDataLogs() > 3) {
                 std::ostream& data_log3 = DataLog(3);
                 if (data_log3.good()) {
                   // Write the average stresses
                   for (int k = 0; k < hu_size; k++) {
                       Real z = (k + 0.5)* dx[2];
                       data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                 << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                 << h_avg_sgshfx[k] << " " << h_avg_sgsdiss[k]
                                 << std::endl;
                   } // loop over z
                 } // if good
             } // NumDataLogs
         } // if IOProcessor
     } // if verbose
 }

◆ write_1D_profiles_stag()

void ERF::write_1D_profiles_stag ( amrex::Real time )

Writes 1-dimensional averaged quantities as profiles to output log files at the given time. Quantities are output at their native grid locations, therefore W and associated quantities <(*)'w'>, tau13, and tau23 (where * includes u, v, p, theta, and k) will be output at staggered heights (i.e., z faces) rather than cell-center heights to avoid performing additional averaging. The unstaggered quantities are associated with the left cell face, i.e., they share the same k index. These quantities will have a zero value at the max height of Nz+1.

Parameters

time	Current time

 {
     BL_PROFILE("ERF::write_1D_profiles()");
  
     if (verbose <= 0)
       return;
  
     int datwidth = 14;
     int datprecision = 6;
     int timeprecision = 13; // e.g., 1-yr LES: 31,536,000 s with dt ~ 0.01 ==> min prec = 10
  
     if (verbose > 0 && NumDataLogs() > 1)
     {
         // Define the 1d arrays we will need
         Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
         Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs;
         Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
         Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
         Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
         Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
         Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
         Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
  
         if (NumDataLogs() > 1) {
             derive_diag_profiles_stag(h_avg_u, h_avg_v, h_avg_w,
                                       h_avg_rho, h_avg_th, h_avg_ksgs,
                                       h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                       h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                       h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                       h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw);
         }
  
         if (NumDataLogs() > 3) {
             derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                         h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                         h_avg_sgshfx,
                                         h_avg_sgsdiss);
         }
  
         int unstag_size =  h_avg_w.size() - 1; // _un_staggered heights
  
         auto const& dx = geom[0].CellSizeArray();
         if (ParallelDescriptor::IOProcessor()) {
             if (NumDataLogs() > 1) {
                 std::ostream& data_log1 = DataLog(1);
                 if (data_log1.good()) {
                   // Write the quantities at this time
                   for (int k = 0; k < unstag_size; k++) {
                       Real z = k * dx[2];
                       data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_u[k]  << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
                                 << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k]
                                 << std::endl;
                   } // loop over z
                   // Write top face values
                   data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                             << std::setw(datwidth) << std::setprecision(datprecision) << unstag_size * dx[2] << " "
                             << 0  << " " << 0 << " " << h_avg_w[unstag_size+1] << " "
                             << 0 << " " << 0 << " " << 0
                             << std::endl;
                 } // if good
             } // NumDataLogs
  
             if (NumDataLogs() > 2) {
                 std::ostream& data_log2 = DataLog(2);
                 if (data_log2.good()) {
                   // Write the perturbational quantities at this time
                   // For surface values (k=0), assume w = uw = vw = ww = 0
                   Real w_cc  = h_avg_w[1] / 2;  // w at first cell center
                   Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
                   Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
                   Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
                   data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                             << std::setw(datwidth) << std::setprecision(datprecision) << 0 << " "
                             << h_avg_uu[0]   - h_avg_u[0]*h_avg_u[0]   << " " // u'u'
                             << h_avg_uv[0]   - h_avg_u[0]*h_avg_v[0]   << " " // u'v'
                             << 0                                       << " " // u'w'
                             << h_avg_vv[0]   - h_avg_v[0]*h_avg_v[0]   << " " // v'v'
                             << 0                                       << " " // v'w'
                             << 0                                       << " " // w'w'
                             << h_avg_uth[0]  - h_avg_u[0]*h_avg_th[0]  << " " // u'th'
                             << h_avg_vth[0]  - h_avg_v[0]*h_avg_th[0]  << " " // v'th'
                             << 0                                       << " " // w'th'
                             << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
                             << h_avg_uiuiu[0]
                              - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
                              - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
                              + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
                                << " " // (u'_i u'_i)u'
                             << h_avg_uiuiv[0]
                              - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
                              - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
                              + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
                                << " " // (u'_i u'_i)v'
                             << 0 << " " // (u'_i u'_i)w'
                             << h_avg_pu[0]   - h_avg_p[0]*h_avg_u[0]   << " " // pu'
                             << h_avg_pv[0]   - h_avg_p[0]*h_avg_v[0]   << " " // pv'
                             << 0                                              // pw'
                             << std::endl;
  
                   // For internal values, interpolate scalar quantities to faces
                   for (int k = 1; k < unstag_size; k++) {
                       Real z = k * dx[2];
                       Real uface  = 0.5*(h_avg_u[k]  + h_avg_u[k-1]);
                       Real vface  = 0.5*(h_avg_v[k]  + h_avg_v[k-1]);
                       Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]);
                       Real pface  = 0.5*(h_avg_p[k]  + h_avg_p[k-1]);
                       Real uuface = 0.5*(h_avg_uu[k] + h_avg_uu[k-1]);
                       Real uvface = 0.5*(h_avg_uv[k] + h_avg_uv[k-1]);
                       Real vvface = 0.5*(h_avg_vv[k] + h_avg_vv[k-1]);
                       w_cc   = 0.5*(h_avg_w[k-1]  + h_avg_w[k]);
                       uw_cc  = 0.5*(h_avg_uw[k-1] + h_avg_uw[k]);
                       vw_cc  = 0.5*(h_avg_vw[k-1] + h_avg_vw[k]);
                       ww_cc  = 0.5*(h_avg_ww[k-1] + h_avg_ww[k]);
                       amrex::ignore_unused(uvface);
                       data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]   << " " // u'u'
                                 << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]   << " " // u'v'
                                 << h_avg_uw[k]   -      uface*h_avg_w[k]   << " " // u'w'
                                 << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]   << " " // v'v'
                                 << h_avg_vw[k]   -      vface*h_avg_w[k]   << " " // v'w'
                                 << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]   << " " // w'w'
                                 << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k]  << " " // u'th'
                                 << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k]  << " " // v'th'
                                 << h_avg_wth[k]  - h_avg_w[k]*thface       << " " // w'th'
                                 << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
                                 // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                 //                        -   <u_i u_i> * <u_j>
                                 //                        - 2*<u_i> * <u_i u_j>
                                 //                        + 2*<u_i>*<u_i> * <u_j>
                                 << h_avg_uiuiu[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
                                  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
                                    << " " // cell-centered (u'_i u'_i)u'
                                 << h_avg_uiuiv[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
                                  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
                                    << " " // cell-centered (u'_i u'_i)v'
                                 << h_avg_uiuiw[k]
                                  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                                  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                    << " " // face-centered (u'_i u'_i)w'
                                 << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k]   << " " // pu'
                                 << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k]   << " " // pv'
                                 << h_avg_pw[k]   -      pface*h_avg_w[k]          // pw'
                                 << std::endl;
                   } // loop over z
  
                   // Write top face values, extrapolating scalar quantities
                   const int k = unstag_size;
                   Real uface  = 1.5*h_avg_u[k-1]  - 0.5*h_avg_u[k-2];
                   Real vface  = 1.5*h_avg_v[k-1]  - 0.5*h_avg_v[k-2];
                   Real thface = 1.5*h_avg_th[k-1] - 0.5*h_avg_th[k-2];
                   Real pface  = 1.5*h_avg_p[k-1]  - 0.5*h_avg_p[k-2];
                   Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2];
                   Real uvface = 1.5*h_avg_uv[k-1] - 0.5*h_avg_uv[k-2];
                   Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2];
                   amrex::ignore_unused(uvface);
                   data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                             << std::setw(datwidth) << std::setprecision(datprecision) << k * dx[2] << " "
                             << 0                                     << " " // u'u'
                             << 0                                     << " " // u'v'
                             << h_avg_uw[k]   -      uface*h_avg_w[k] << " " // u'w'
                             << 0                                     << " " // v'v'
                             << h_avg_vw[k]   -      vface*h_avg_w[k] << " " // v'w'
                             << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k] << " " // w'w'
                             << 0                                     << " " // u'th'
                             << 0                                     << " " // v'th'
                             << h_avg_wth[k]  -     thface*h_avg_w[k] << " " // w'th'
                             << 0                                     << " " // th'th'
                             << 0                                     << " " // (u'_i u'_i)u'
                             << 0                                     << " " // (u'_i u'_i)v'
                             << h_avg_uiuiw[k]
                              - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                              - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                              + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                << " " // (u'_i u'_i)w'
                             << 0                                     << " " // pu'
                             << 0                                     << " " // pv'
                             << h_avg_pw[k]   -      pface*h_avg_w[k]        // pw'
                             << std::endl;
                 } // if good
             } // NumDataLogs
  
             if (NumDataLogs() > 3) {
                 std::ostream& data_log3 = DataLog(3);
                 if (data_log3.good()) {
                   // Write the average stresses
                   for (int k = 0; k < unstag_size; k++) {
                       Real z = k * dx[2];
                       data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                 << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                 << h_avg_sgshfx[k] << " " << h_avg_sgsdiss[k]
                                 << std::endl;
                   } // loop over z
                   // Write top face values
                   data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                             << std::setw(datwidth) << std::setprecision(datprecision) << unstag_size * dx[2] << " "
                             << 0 << " " << 0 << " " << h_avg_tau13[unstag_size] << " "
                             << 0 << " " << h_avg_tau23[unstag_size] << " " << 0 << " "
                             << 0 << " " << 0
                             << std::endl;
                 } // if good
             } // NumDataLogs
         } // if IOProcessor
     } // if verbose
 }

◆ writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream & os )

static

 {
     std::string PrettyLine = std::string(78, '=') + "\n";
     std::string OtherLine = std::string(78, '-') + "\n";
     std::string SkipSpace = std::string(8, ' ');
  
     // build information
     os << PrettyLine;
     os << " ERF Build Information\n";
     os << PrettyLine;
  
     os << "build date:    " << buildInfoGetBuildDate() << "\n";
     os << "build machine: " << buildInfoGetBuildMachine() << "\n";
     os << "build dir:     " << buildInfoGetBuildDir() << "\n";
     os << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
  
     os << "\n";
  
     os << "COMP:          " << buildInfoGetComp() << "\n";
     os << "COMP version:  " << buildInfoGetCompVersion() << "\n";
  
     os << "C++ compiler:  " << buildInfoGetCXXName() << "\n";
     os << "C++ flags:     " << buildInfoGetCXXFlags() << "\n";
  
     os << "\n";
  
     os << "Link flags:    " << buildInfoGetLinkFlags() << "\n";
     os << "Libraries:     " << buildInfoGetLibraries() << "\n";
  
     os << "\n";
  
     for (int n = 1; n <= buildInfoGetNumModules(); n++) {
       os << buildInfoGetModuleName(n) << ": "
          << buildInfoGetModuleVal(n) << "\n";
     }
  
     os << "\n";
     const char* githash1 = buildInfoGetGitHash(1);
     const char* githash2 = buildInfoGetGitHash(2);
     if (strlen(githash1) > 0) {
       os << "ERF       git hash: " << githash1 << "\n";
     }
     if (strlen(githash2) > 0) {
       os << "AMReX       git hash: " << githash2 << "\n";
     }
  
     const char* buildgithash = buildInfoGetBuildGitHash();
     const char* buildgitname = buildInfoGetBuildGitName();
     if (strlen(buildgithash) > 0) {
       os << buildgitname << " git hash: " << buildgithash << "\n";
     }
  
     os << "\n";
     os << " ERF Compile time variables: \n";
  
     os << "\n";
     os << " ERF Defines: \n";
 #ifdef _OPENMP
     os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
        << std::endl;
 #else
     os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
        << std::endl;
 #endif
  
 #ifdef MPI_VERSION
     os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
        << MPI_VERSION << std::endl;
 #else
     os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
        << "UNDEFINED" << std::endl;
 #endif
  
 #ifdef MPI_SUBVERSION
     os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
        << MPI_SUBVERSION << std::endl;
 #else
     os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
        << "UNDEFINED" << std::endl;
 #endif
  
     os << "\n\n";
 }

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ WriteCheckpointFile()

void ERF::WriteCheckpointFile ( ) const

private

ERF function for writing a checkpoint file.

 {
     // chk00010            write a checkpoint file with this root directory
     // chk00010/Header     this contains information you need to save (e.g., finest_level, t_new, etc.) and also
     //                     the BoxArrays at each level
     // chk00010/Level_0/
     // chk00010/Level_1/
     // etc.                these subdirectories will hold the MultiFab data at each level of refinement
  
     // checkpoint file name, e.g., chk00010
     const std::string& checkpointname = Concatenate(check_file,istep[0],5);
  
     Print() << "Writing native checkpoint " << checkpointname << "\n";
  
     const int nlevels = finest_level+1;
  
     // ---- prebuild a hierarchy of directories
     // ---- dirName is built first.  if dirName exists, it is renamed.  then build
     // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
     // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
     // ---- after all directories are built
     // ---- ParallelDescriptor::IOProcessor() creates the directories
     PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
  
     int ncomp_cons = vars_new[0][Vars::cons].nComp();
  
     // write Header file
     if (ParallelDescriptor::IOProcessor()) {
  
        std::string HeaderFileName(checkpointname + "/Header");
        VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
        std::ofstream HeaderFile;
        HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
        HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                                std::ofstream::trunc |
                                                std::ofstream::binary);
        if( ! HeaderFile.good()) {
            FileOpenFailed(HeaderFileName);
        }
  
        HeaderFile.precision(17);
  
        // write out title line
        HeaderFile << "Checkpoint file for ERF\n";
  
        // write out finest_level
        HeaderFile << finest_level << "\n";
  
        // write the number of components
        // for each variable we store
  
        // conservative, cell-centered vars
        HeaderFile << ncomp_cons << "\n";
  
        // x-velocity on faces
        HeaderFile << 1 << "\n";
  
        // y-velocity on faces
        HeaderFile << 1 << "\n";
  
        // z-velocity on faces
        HeaderFile << 1 << "\n";
  
        // write out array of istep
        for (int i = 0; i < istep.size(); ++i) {
            HeaderFile << istep[i] << " ";
        }
        HeaderFile << "\n";
  
        // write out array of dt
        for (int i = 0; i < dt.size(); ++i) {
            HeaderFile << dt[i] << " ";
        }
        HeaderFile << "\n";
  
        // write out array of t_new
        for (int i = 0; i < t_new.size(); ++i) {
            HeaderFile << t_new[i] << " ";
        }
        HeaderFile << "\n";
  
        // write the BoxArray at each level
        for (int lev = 0; lev <= finest_level; ++lev) {
            boxArray(lev).writeOn(HeaderFile);
            HeaderFile << '\n';
        }
    }
  
     // write the MultiFab data to, e.g., chk00010/Level_0/
     // Here we make copies of the MultiFab with no ghost cells
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
         MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
         VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
  
         MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
         MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
         VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
  
         MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
         MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
         VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
  
         MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
         MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
         VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
  
         // Note that we write the ghost cells of the base state (unlike above)
         IntVect ng = base_state[lev].nGrowVect();
         MultiFab base(grids[lev],dmap[lev],base_state[lev].nComp(),ng);
         MultiFab::Copy(base,base_state[lev],0,0,base.nComp(),ng);
         VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
  
         if (solverChoice.use_terrain)  {
             // Note that we also write the ghost cells of z_phys_nd
             ng = z_phys_nd[lev]->nGrowVect();
             MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
             MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
             VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
         }
  
          // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
          // Write the precipitation accumulation component only
         if (solverChoice.moisture_type == MoistureType::Kessler) {
             ng = qmoist[lev][4]->nGrowVect();
             int nvar = 1;
             MultiFab moist_vars(grids[lev],dmap[lev],nvar,ng);
             MultiFab::Copy(moist_vars,*(qmoist[lev][4]),0,0,nvar,ng);
             VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "RainAccum"));
         }
  
         if(solverChoice.moisture_type == MoistureType::SAM){
             ng = qmoist[lev][8]->nGrowVect();
             int nvar = 1;
             MultiFab rain_accum(grids[lev],dmap[lev],nvar,ng);
             MultiFab::Copy(rain_accum,*(qmoist[lev][8]),0,0,nvar,ng);
             VisMF::Write(rain_accum, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "RainAccum"));
  
             ng = qmoist[lev][9]->nGrowVect();
             MultiFab snow_accum(grids[lev],dmap[lev],nvar,ng);
             MultiFab::Copy(snow_accum,*(qmoist[lev][9]),0,0,nvar,ng);
             VisMF::Write(snow_accum, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SnowAccum"));
  
             ng = qmoist[lev][10]->nGrowVect();
             MultiFab graup_accum(grids[lev],dmap[lev],nvar,ng);
             MultiFab::Copy(graup_accum,*(qmoist[lev][10]),0,0,nvar,ng);
             VisMF::Write(graup_accum, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "GraupAccum"));
         }
  
  
 #if defined(ERF_USE_WINDFARM)
         if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP){
             ng = Nturb[lev].nGrowVect();
             MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
             MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng);
             VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
         }
 #endif
  
  
         if (solverChoice.lsm_type != LandSurfaceType::None) {
             for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
                 BoxArray ba = lsm_data[lev][mvar]->boxArray();
                 DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
                 ng = lsm_data[lev][mvar]->nGrowVect();
                 int nvar = lsm_data[lev][mvar]->nComp();
                 MultiFab lsm_vars(ba,dm,nvar,ng);
                 MultiFab::Copy(lsm_vars,*(lsm_data[lev][mvar]),0,0,nvar,ng);
                 VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmVars"));
             }
         }
  
         // Note that we also write the ghost cells of the mapfactors (2D)
         BoxList bl2d = grids[lev].boxList();
         for (auto& b : bl2d) {
             b.setRange(2,0);
         }
         BoxArray ba2d(std::move(bl2d));
  
         ng = mapfac_m[lev]->nGrowVect();
         MultiFab mf_m(ba2d,dmap[lev],1,ng);
         MultiFab::Copy(mf_m,*mapfac_m[lev],0,0,1,ng);
         VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_m"));
  
         ng = mapfac_u[lev]->nGrowVect();
         MultiFab mf_u(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,ng);
         MultiFab::Copy(mf_u,*mapfac_u[lev],0,0,1,ng);
         VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_u"));
  
         ng = mapfac_v[lev]->nGrowVect();
         MultiFab mf_v(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,ng);
         MultiFab::Copy(mf_v,*mapfac_v[lev],0,0,1,ng);
         VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_v"));
     }
  
 #ifdef ERF_USE_PARTICLES
    particleData.Checkpoint(checkpointname);
 #endif
  
 #ifdef ERF_USE_NETCDF
    // Write bdy_data files
    if (ParallelDescriptor::IOProcessor() && ((init_type=="real") || (init_type=="metgrid"))) {
  
      // Vector dimensions
      int num_time = bdy_data_xlo.size();
      int num_var  = bdy_data_xlo[0].size();
  
      // Open header file and write to it
      std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
      bdy_h_file << std::setprecision(1) << std::fixed;
      bdy_h_file << num_time << "\n";
      bdy_h_file << num_var  << "\n";
      bdy_h_file << start_bdy_time << "\n";
      bdy_h_file << bdy_time_interval << "\n";
      bdy_h_file << real_width << "\n";
      for (int ivar(0); ivar<num_var; ++ivar) {
        bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
        bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
        bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
        bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
      }
  
      // Open data file and write to it
      std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
      for (int itime(0); itime<num_time; ++itime) {
        for (int ivar(0); ivar<num_var; ++ivar) {
          bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
          bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
          bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
          bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
        }
      }
    }
 #endif
  
 }

◆ WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain	(	std::ostream &	HeaderFile,
		int	nlevels,
		const amrex::Vector< amrex::BoxArray > &	bArray,
		const amrex::Vector< std::string > &	varnames,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const std::string &	versionName,
		const std::string &	levelPrefix,
		const std::string &	mfPrefix
	)		const

 {
     AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
     AMREX_ALWAYS_ASSERT(nlevels <= ref_ratio.size()+1);
     AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
  
     HeaderFile.precision(17);
  
     // ---- this is the generic plot file type name
     HeaderFile << versionName << '\n';
  
     HeaderFile << varnames.size() << '\n';
  
     for (int ivar = 0; ivar < varnames.size(); ++ivar) {
         HeaderFile << varnames[ivar] << "\n";
     }
     HeaderFile << AMREX_SPACEDIM << '\n';
     HeaderFile << time << '\n';
     HeaderFile << finest_level << '\n';
     for (int i = 0; i < AMREX_SPACEDIM; ++i) {
         HeaderFile << geom[0].ProbLo(i) << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i < AMREX_SPACEDIM; ++i) {
         HeaderFile << geom[0].ProbHi(i) << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i < finest_level; ++i) {
         HeaderFile << ref_ratio[i][0] << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i <= finest_level; ++i) {
         HeaderFile << geom[i].Domain() << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i <= finest_level; ++i) {
         HeaderFile << level_steps[i] << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i <= finest_level; ++i) {
         for (int k = 0; k < AMREX_SPACEDIM; ++k) {
             HeaderFile << geom[i].CellSize()[k] << ' ';
         }
         HeaderFile << '\n';
     }
     HeaderFile << (int) geom[0].Coord() << '\n';
     HeaderFile << "0\n";
  
     for (int level = 0; level <= finest_level; ++level) {
         HeaderFile << level << ' ' << bArray[level].size() << ' ' << time << '\n';
         HeaderFile << level_steps[level] << '\n';
  
         const IntVect& domain_lo = geom[level].Domain().smallEnd();
         for (int i = 0; i < bArray[level].size(); ++i)
         {
             // Need to shift because the RealBox ctor we call takes the
             // physical location of index (0,0,0).  This does not affect
             // the usual cases where the domain index starts with 0.
             const Box& b = shift(bArray[level][i], -domain_lo);
             RealBox loc = RealBox(b, geom[level].CellSize(), geom[level].ProbLo());
             for (int n = 0; n < AMREX_SPACEDIM; ++n) {
                 HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
             }
         }
  
         HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
     }
     HeaderFile << "1" << "\n";
     HeaderFile << "3" << "\n";
     HeaderFile << "amrexvec_nu_x" << "\n";
     HeaderFile << "amrexvec_nu_y" << "\n";
     HeaderFile << "amrexvec_nu_z" << "\n";
     std::string mf_nodal_prefix = "Nu_nd";
     for (int level = 0; level <= finest_level; ++level) {
         HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
     }
 }

◆ writeJobInfo()

void ERF::writeJobInfo ( const std::string & dir ) const

 {
     // job_info file with details about the run
     std::ofstream jobInfoFile;
     std::string FullPathJobInfoFile = dir;
     FullPathJobInfoFile += "/job_info";
     jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out);
  
     std::string PrettyLine = "==================================================="
       "============================\n";
     std::string OtherLine = "----------------------------------------------------"
       "----------------------------\n";
     std::string SkipSpace = "        ";
  
     // job information
     jobInfoFile << PrettyLine;
     jobInfoFile << " ERF Job Information\n";
     jobInfoFile << PrettyLine;
  
     jobInfoFile << "inputs file: " << inputs_name << "\n\n";
  
     jobInfoFile << "number of MPI processes: "
                 << ParallelDescriptor::NProcs() << "\n";
 #ifdef _OPENMP
     jobInfoFile << "number of threads:       " << omp_get_max_threads() << "\n";
 #endif
  
     jobInfoFile << "\n";
     jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
                 << getCPUTime() / 3600.0;
  
     jobInfoFile << "\n\n";
  
     // plotfile information
     jobInfoFile << PrettyLine;
     jobInfoFile << " Plotfile Information\n";
     jobInfoFile << PrettyLine;
  
     time_t now = time(nullptr);
  
     // Convert now to tm struct for local timezone
     tm* localtm = localtime(&now);
     jobInfoFile << "output data / time: " << asctime(localtm);
  
     std::string currentDir = FileSystem::CurrentPath();
     jobInfoFile << "output dir:         " << currentDir << "\n";
  
     jobInfoFile << "\n\n";
  
     // build information
     jobInfoFile << PrettyLine;
     jobInfoFile << " Build Information\n";
     jobInfoFile << PrettyLine;
  
     jobInfoFile << "build date:    " << buildInfoGetBuildDate() << "\n";
     jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
     jobInfoFile << "build dir:     " << buildInfoGetBuildDir() << "\n";
     jobInfoFile << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
  
     jobInfoFile << "\n";
  
     jobInfoFile << "COMP:          " << buildInfoGetComp() << "\n";
     jobInfoFile << "COMP version:  " << buildInfoGetCompVersion() << "\n";
  
     jobInfoFile << "\n";
  
     for (int n = 1; n <= buildInfoGetNumModules(); n++) {
       jobInfoFile << buildInfoGetModuleName(n) << ": "
                   << buildInfoGetModuleVal(n) << "\n";
     }
  
     jobInfoFile << "\n";
  
     const char* githash1 = buildInfoGetGitHash(1);
     const char* githash2 = buildInfoGetGitHash(2);
     if (strlen(githash1) > 0) {
       jobInfoFile << "ERF       git hash: " << githash1 << "\n";
     }
     if (strlen(githash2) > 0) {
       jobInfoFile << "AMReX       git hash: " << githash2 << "\n";
     }
  
     const char* buildgithash = buildInfoGetBuildGitHash();
     const char* buildgitname = buildInfoGetBuildGitName();
     if (strlen(buildgithash) > 0) {
       jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
     }
  
     jobInfoFile << "\n\n";
  
     // grid information
     jobInfoFile << PrettyLine;
     jobInfoFile << " Grid Information\n";
     jobInfoFile << PrettyLine;
  
     int f_lev = finest_level;
  
     for (int i = 0; i <= f_lev; i++) {
       jobInfoFile << " level: " << i << "\n";
       jobInfoFile << "   number of boxes = " << grids[i].size() << "\n";
       jobInfoFile << "   maximum zones   = ";
       for (int n = 0; n < AMREX_SPACEDIM; n++) {
         jobInfoFile << geom[i].Domain().length(n) << " ";
       }
       jobInfoFile << "\n\n";
     }
  
     jobInfoFile << " Boundary conditions\n";
  
     jobInfoFile << "   -x: " << domain_bc_type[0] << "\n";
     jobInfoFile << "   +x: " << domain_bc_type[3] << "\n";
     jobInfoFile << "   -y: " << domain_bc_type[1] << "\n";
     jobInfoFile << "   +y: " << domain_bc_type[4] << "\n";
     jobInfoFile << "   -z: " << domain_bc_type[2] << "\n";
     jobInfoFile << "   +z: " << domain_bc_type[5] << "\n";
  
     jobInfoFile << "\n\n";
  
     // runtime parameters
     jobInfoFile << PrettyLine;
     jobInfoFile << " Inputs File Parameters\n";
     jobInfoFile << PrettyLine;
  
     ParmParse::dumpTable(jobInfoFile, true);
     jobInfoFile.close();
 }

Here is the call graph for this function:

◆ WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain	(	const std::string &	plotfilename,
		int	nlevels,
		const amrex::Vector< const amrex::MultiFab * > &	mf,
		const amrex::Vector< const amrex::MultiFab * > &	mf_nd,
		const amrex::Vector< std::string > &	varnames,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const std::string &	versionName = `"HyperCLaw-V1.1"`,
		const std::string &	levelPrefix = `"Level_"`,
		const std::string &	mfPrefix = `"Cell"`,
		const amrex::Vector< std::string > &	extra_dirs = `amrex::Vector<std::string>()`
	)		const

 {
     BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
  
     AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
     AMREX_ALWAYS_ASSERT(nlevels <= ref_ratio.size()+1);
     AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
     AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
  
     bool callBarrier(false);
     PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
     if (!extra_dirs.empty()) {
         for (const auto& d : extra_dirs) {
             const std::string ed = plotfilename+"/"+d;
             PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
         }
     }
     ParallelDescriptor::Barrier();
  
     if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
         Vector<BoxArray> boxArrays(nlevels);
         for(int level(0); level < boxArrays.size(); ++level) {
             boxArrays[level] = mf[level]->boxArray();
         }
  
         auto f = [=]() {
             VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
             std::string HeaderFileName(plotfilename + "/Header");
             std::ofstream HeaderFile;
             HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
             HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                                     std::ofstream::trunc |
                                                     std::ofstream::binary);
             if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
             WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
                                                   time, level_steps, versionName,
                                                   levelPrefix, mfPrefix);
         };
  
         if (AsyncOut::UseAsyncOut()) {
             AsyncOut::Submit(std::move(f));
         } else {
             f();
         }
     }
  
     std::string mf_nodal_prefix = "Nu_nd";
     for (int level = 0; level <= finest_level; ++level)
     {
         if (AsyncOut::UseAsyncOut()) {
             VisMF::AsyncWrite(*mf[level],
                               MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
                               true);
             VisMF::AsyncWrite(*mf_nd[level],
                               MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
                               true);
         } else {
             const MultiFab* data;
             std::unique_ptr<MultiFab> mf_tmp;
             if (mf[level]->nGrowVect() != 0) {
                 mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
                                                     mf[level]->DistributionMap(),
                                                     mf[level]->nComp(), 0, MFInfo(),
                                                     mf[level]->Factory());
                 MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
                 data = mf_tmp.get();
             } else {
                 data = mf[level];
             }
             VisMF::Write(*data        , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
             VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
         }
     }
 }

◆ writeNow()

bool ERF::writeNow	(	const amrex::Real	cur_time,
		const amrex::Real	dt,
		const int	nstep,
		const int	plot_int,
		const amrex::Real	plot_per
	)

 {
     bool write_now = false;
  
     if ( plot_int > 0 && (nstep % plot_int == 0) ) {
         write_now = true;
  
     } else if (plot_per > 0.0) {
  
         // Check to see if we've crossed a plot_per interval by comparing
         // the number of intervals that have elapsed for both the current
         // time and the time at the beginning of this timestep.
  
         const Real eps = std::numeric_limits<Real>::epsilon() * Real(10.0) * std::abs(cur_time);
  
         int num_per_old = static_cast<int>(std::round((cur_time-eps-dt_lev) / plot_per));
         int num_per_new = static_cast<int>(std::round((cur_time-eps       ) / plot_per));
  
         // Before using these, however, we must test for the case where we're
         // within machine epsilon of the next interval. In that case, increment
         // the counter, because we have indeed reached the next plot_per interval
         // at this point.
  
         const Real next_plot_time = (num_per_old + 1) * plot_per;
  
         if ((num_per_new == num_per_old) && std::abs(cur_time - next_plot_time) <= eps)
         {
             num_per_new += 1;
         }
  
         // Similarly, we have to account for the case where the old time is within
         // machine epsilon of the beginning of this interval, so that we don't double
         // count that time threshold -- we already plotted at that time on the last timestep.
  
         if ((num_per_new != num_per_old) && std::abs((cur_time - dt_lev) - next_plot_time) <= eps)
             num_per_old += 1;
  
         if (num_per_old != num_per_new)
             write_now = true;
     }
     return write_now;
 }

◆ WritePlotFile()

void ERF::WritePlotFile	(	int	which,
		amrex::Vector< std::string >	plot_var_names
	)

 {
     const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
     const int ncomp_mf = varnames.size();
  
     int ncomp_cons = vars_new[0][Vars::cons].nComp();
  
     if (ncomp_mf == 0) return;
  
     // We Fillpatch here because some of the derived quantities require derivatives
     //     which require ghost cells to be filled.  We do not need to call FillPatcher
     //     because we don't need to set interior fine points.
     for (int lev = 0; lev <= finest_level; ++lev) {
         bool fillset = false;
         FillPatch(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
                                     &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
                                    {&vars_new[lev][Vars::cons], &rU_new[lev],
                                     &rV_new[lev], &rW_new[lev]}, fillset);
     }
  
     // Get qmoist pointers if using moisture
     bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
     for (int lev = 0; lev <= finest_level; ++lev) {
         for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
             qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
         }
     }
  
     // Vector of MultiFabs for cell-centered data
     Vector<MultiFab> mf(finest_level+1);
     for (int lev = 0; lev <= finest_level; ++lev) {
         mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
     }
  
     // Vector of MultiFabs for nodal data
     Vector<MultiFab> mf_nd(finest_level+1);
     if (solverChoice.use_terrain) {
         for (int lev = 0; lev <= finest_level; ++lev) {
             BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
             mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
             mf_nd[lev].setVal(0.);
         }
     }
  
     // Array of MultiFabs for cell-centered velocity
     Vector<MultiFab> mf_cc_vel(finest_level+1);
  
     if (containerHasElement(plot_var_names, "x_velocity" ) ||
         containerHasElement(plot_var_names, "y_velocity" ) ||
         containerHasElement(plot_var_names, "z_velocity" ) ||
         containerHasElement(plot_var_names, "magvel"     ) ||
         containerHasElement(plot_var_names, "vorticity_x") ||
         containerHasElement(plot_var_names, "vorticity_y") ||
         containerHasElement(plot_var_names, "vorticity_z") ) {
  
         for (int lev = 0; lev <= finest_level; ++lev) {
             mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
             average_face_to_cellcenter(mf_cc_vel[lev],0,
                                        Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                                 &vars_new[lev][Vars::yvel],
                                                                 &vars_new[lev][Vars::zvel]});
         } // lev
     } // if (vel or vort)
  
     // We need ghost cells if computing vorticity
     if ( containerHasElement(plot_var_names, "vorticity_x")||
          containerHasElement(plot_var_names, "vorticity_y") ||
          containerHasElement(plot_var_names, "vorticity_z") )
     {
         amrex::Interpolater* mapper = &cell_cons_interp;
         for (int lev = 1; lev <= finest_level; ++lev)
         {
             Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
             Vector<Real> ftime    = {t_new[lev], t_new[lev]};
             Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
             Vector<Real> ctime    = {t_new[lev], t_new[lev]};
  
             amrex::FillPatchTwoLevels(mf_cc_vel[lev], t_new[lev], cmf, ctime, fmf, ftime,
                                       0, 0, AMREX_SPACEDIM, geom[lev-1], geom[lev],
                                       null_bc_for_fill, 0, null_bc_for_fill, 0, refRatio(lev-1),
                                       mapper, domain_bcs_type, 0);
         } // lev
  
         // Impose bc's at domain boundaries at all levels
         FillBdyCCVels(mf_cc_vel);
     } // if (vort)
  
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         int mf_comp = 0;
  
         // First, copy any of the conserved state variables into the output plotfile
         AMREX_ALWAYS_ASSERT(cons_names.size() >= ncomp_cons);
         for (int i = 0; i < ncomp_cons; ++i) {
             if (containerHasElement(plot_var_names, cons_names[i])) {
                 MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,0);
                 mf_comp++;
             }
         }
  
         // Next, check for velocities
         if (containerHasElement(plot_var_names, "x_velocity")) {
             MultiFab::Copy(mf[lev], mf_cc_vel[lev], 0, mf_comp, 1, 0);
             mf_comp += 1;
         }
         if (containerHasElement(plot_var_names, "y_velocity")) {
             MultiFab::Copy(mf[lev], mf_cc_vel[lev], 1, mf_comp, 1, 0);
             mf_comp += 1;
         }
         if (containerHasElement(plot_var_names, "z_velocity")) {
             MultiFab::Copy(mf[lev], mf_cc_vel[lev], 2, mf_comp, 1, 0);
             mf_comp += 1;
         }
  
         // Finally, check for any derived quantities and compute them, inserting
         // them into our output multifab
         auto calculate_derived = [&](const std::string& der_name,
                                      MultiFab& src_mf,
                                      decltype(derived::erf_dernull)& der_function)
         {
             if (containerHasElement(plot_var_names, der_name)) {
                 MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     auto& dfab = dmf[mfi];
                     auto& sfab = src_mf[mfi];
                     der_function(bx, dfab, 0, 1, sfab, Geom(lev), t_new[0], nullptr, lev);
                 }
  
                 mf_comp++;
             }
         };
  
         // Note: All derived variables must be computed in order of "derived_names" defined in ERF.H
         calculate_derived("soundspeed",  vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
         calculate_derived("temp",        vars_new[lev][Vars::cons], derived::erf_dertemp);
         calculate_derived("theta",       vars_new[lev][Vars::cons], derived::erf_dertheta);
         calculate_derived("KE",          vars_new[lev][Vars::cons], derived::erf_derKE);
         calculate_derived("QKE",         vars_new[lev][Vars::cons], derived::erf_derQKE);
         calculate_derived("scalar",      vars_new[lev][Vars::cons], derived::erf_derscalar);
         calculate_derived("vorticity_x", mf_cc_vel[lev]           , derived::erf_dervortx);
         calculate_derived("vorticity_y", mf_cc_vel[lev]           , derived::erf_dervorty);
         calculate_derived("vorticity_z", mf_cc_vel[lev]           , derived::erf_dervortz);
         calculate_derived("magvel"     , mf_cc_vel[lev]           , derived::erf_dermagvel);
  
         if (containerHasElement(plot_var_names, "divU"))
         {
             MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
             Array<MultiFab const*, AMREX_SPACEDIM> u;
             u[0] = &(vars_new[lev][Vars::xvel]);
             u[1] = &(vars_new[lev][Vars::yvel]);
             u[2] = &(vars_new[lev][Vars::zvel]);
             computeDivergence(dmf, u, geom[lev]);
             mf_comp += 1;
         }
  
         MultiFab r_hse(base_state[lev], make_alias, 0, 1); // r_0 is first  component
         MultiFab p_hse(base_state[lev], make_alias, 1, 1); // p_0 is second component
         if (containerHasElement(plot_var_names, "pres_hse"))
         {
             // p_0 is second component of base_state
             MultiFab::Copy(mf[lev],p_hse,0,mf_comp,1,0);
             mf_comp += 1;
         }
         if (containerHasElement(plot_var_names, "dens_hse"))
         {
             // r_0 is first component of base_state
             MultiFab::Copy(mf[lev],r_hse,0,mf_comp,1,0);
             mf_comp += 1;
         }
  
         if (containerHasElement(plot_var_names, "pressure"))
         {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real      >& derdat = mf[lev].array(mfi);
                 const Array4<Real const>&  S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 const int ncomp = vars_new[lev][Vars::cons].nComp();
  
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                 {
                     Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
                     const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                     derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p);
                 });
             }
             mf_comp += 1;
         }
         if (containerHasElement(plot_var_names, "pert_pres"))
         {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 const int ncomp = vars_new[lev][Vars::cons].nComp();
  
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                 {
                     Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
                     const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                     derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p) - p0_arr(i,j,k);
                 });
             }
             mf_comp += 1;
         }
         if (containerHasElement(plot_var_names, "pert_dens"))
         {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat  = mf[lev].array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
                 });
             }
             mf_comp ++;
         }
  
         if (containerHasElement(plot_var_names, "eq_pot_temp"))
         {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat  = mf[lev].array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 const int ncomp = vars_new[lev][Vars::cons].nComp();
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
                     Real qc = (use_moisture && (ncomp > RhoQ2_comp)) ? S_arr(i,j,k,RhoQ2_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
                     Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                     Real pressure = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp), qv);
                     Real fac = Cp_d + Cp_l*(qv + qc);
                     Real pv = erf_esatw(T)*100.0;
  
                     derdat(i, j, k, mf_comp) = T*std::pow((pressure - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
                 });
             }
             mf_comp ++;
         }
  
 #ifdef ERF_USE_WINDFARM
         if (containerHasElement(plot_var_names, "num_turb"))
         {
         std::cout << "Plotting num_turb" << "\n";
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat  = mf[lev].array(mfi);
                 const Array4<Real const>& Nturb_array = Nturb[lev].const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     derdat(i, j, k, mf_comp) = Nturb_array(i,j,k,0);
                 });
             }
             mf_comp ++;
         }
 #endif
  
         int klo = geom[lev].Domain().smallEnd(2);
         int khi = geom[lev].Domain().bigEnd(2);
  
         if (containerHasElement(plot_var_names, "dpdx"))
         {
             auto dxInv = geom[lev].InvCellSizeArray();
             MultiFab pres(vars_new[lev][Vars::cons].boxArray(), vars_new[lev][Vars::cons].DistributionMap(), 1, 1);
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 // First define pressure on grown box
                 const Box& gbx = mfi.growntilebox(1);
                 const Array4<Real> & p_arr  = pres.array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                     p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp));
                 });
             }
             pres.FillBoundary(geom[lev].periodicity());
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 // Now compute pressure gradient on valid box
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<Real> & p_arr  = pres.array(mfi);
  
                 if (solverChoice.use_terrain) {
                     const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
  
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
  
                         // Pgrad at lower I face
                         Real met_h_xi_lo   = Compute_h_xi_AtIface  (i, j, k, dxInv, z_nd);
                         Real met_h_zeta_lo = Compute_h_zeta_AtIface(i, j, k, dxInv, z_nd);
                         Real gp_xi_lo = dxInv[0] * (p_arr(i,j,k) - p_arr(i-1,j,k));
                         Real gp_zeta_on_iface_lo;
                         if(k == klo) {
                             gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
                                 p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
                               - p_arr(i-1,j,k  ) - p_arr(i,j,k  ) );
                         } else if (k == khi) {
                             gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
                                 p_arr(i-1,j,k  ) + p_arr(i,j,k  )
                               - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
                         } else {
                             gp_zeta_on_iface_lo = 0.25 * dxInv[2] * (
                                 p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
                               - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
                         }
                         Real gpx_lo = gp_xi_lo - (met_h_xi_lo/ met_h_zeta_lo) * gp_zeta_on_iface_lo;
  
                         // Pgrad at higher I face
                         Real met_h_xi_hi   = Compute_h_xi_AtIface  (i+1, j, k, dxInv, z_nd);
                         Real met_h_zeta_hi = Compute_h_zeta_AtIface(i+1, j, k, dxInv, z_nd);
                         Real gp_xi_hi = dxInv[0] * (p_arr(i+1,j,k) - p_arr(i,j,k));
                         Real gp_zeta_on_iface_hi;
                         if(k == klo) {
                             gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
                                 p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
                                 - p_arr(i+1,j,k  ) - p_arr(i,j,k  ) );
                         } else if (k == khi) {
                             gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
                                 p_arr(i+1,j,k  ) + p_arr(i,j,k  )
                                 - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
                         } else {
                             gp_zeta_on_iface_hi = 0.25 * dxInv[2] * (
                                 p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
                                 - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
                         }
                         Real gpx_hi = gp_xi_hi - (met_h_xi_hi/ met_h_zeta_hi) * gp_zeta_on_iface_hi;
  
                         // Average P grad to CC
                         derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_lo + gpx_hi);
                     });
                 } else {
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                         derdat(i ,j ,k, mf_comp) = 0.5 * (p_arr(i+1,j,k) - p_arr(i-1,j,k)) * dxInv[0];
                     });
                 }
             } // mfi
             mf_comp ++;
         } // dpdx
  
         if (containerHasElement(plot_var_names, "dpdy"))
         {
             auto dxInv = geom[lev].InvCellSizeArray();
  
             MultiFab pres(vars_new[lev][Vars::cons].boxArray(), vars_new[lev][Vars::cons].DistributionMap(), 1, 1);
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 // First define pressure on grown box
                 const Box& gbx = mfi.growntilebox(1);
                 const Array4<Real> & p_arr  = pres.array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                     p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp));
                 });
             }
             pres.FillBoundary(geom[lev].periodicity());
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 // Now compute pressure gradient on valid box
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<Real> & p_arr  = pres.array(mfi);
  
                 if (solverChoice.use_terrain) {
                     const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
  
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
  
                         Real met_h_eta_lo  = Compute_h_eta_AtJface (i, j, k, dxInv, z_nd);
                         Real met_h_zeta_lo = Compute_h_zeta_AtJface(i, j, k, dxInv, z_nd);
                         Real gp_eta_lo = dxInv[1] * (p_arr(i,j,k) - p_arr(i,j-1,k));
                         Real gp_zeta_on_jface_lo;
                         if (k == klo) {
                             gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
                                 p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
                                 - p_arr(i,j,k  ) - p_arr(i,j-1,k  ) );
                         } else if (k == khi) {
                             gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
                                 p_arr(i,j,k  ) + p_arr(i,j-1,k  )
                                 - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
                         } else {
                             gp_zeta_on_jface_lo = 0.25 * dxInv[2] * (
                                 p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
                                 - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
                         }
                         Real gpy_lo = gp_eta_lo - (met_h_eta_lo / met_h_zeta_lo) * gp_zeta_on_jface_lo;
  
                         Real met_h_eta_hi  = Compute_h_eta_AtJface (i, j+1, k, dxInv, z_nd);
                         Real met_h_zeta_hi = Compute_h_zeta_AtJface(i, j+1, k, dxInv, z_nd);
                         Real gp_eta_hi = dxInv[1] * (p_arr(i,j+1,k) - p_arr(i,j,k));
                         Real gp_zeta_on_jface_hi;
                         if (k == klo) {
                             gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
                                 p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
                                 - p_arr(i,j+1,k  ) - p_arr(i,j,k  ) );
                         } else if (k == khi) {
                             gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
                                 p_arr(i,j+1,k  ) + p_arr(i,j,k  )
                                 - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
                         } else {
                             gp_zeta_on_jface_hi = 0.25 * dxInv[2] * (
                                 p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
                                 - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
                         }
                         Real gpy_hi = gp_eta_hi - (met_h_eta_hi / met_h_zeta_hi) * gp_zeta_on_jface_hi;
  
                         derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_lo + gpy_hi);
                     });
                 } else {
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                         derdat(i ,j ,k, mf_comp) = 0.5 * (p_arr(i,j+1,k) - p_arr(i,j-1,k)) * dxInv[1];
                     });
                 }
             } // mf
             mf_comp ++;
         } // dpdy
  
         if (containerHasElement(plot_var_names, "pres_hse_x"))
         {
             auto dxInv = geom[lev].InvCellSizeArray();
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real      >&  derdat = mf[lev].array(mfi);
                 const Array4<Real const>&   p_arr = p_hse.const_array(mfi);
  
                 //USE_TERRAIN POSSIBLE ISSUE HERE
                 const Array4<Real const>& z_nd  = z_phys_nd[lev]->const_array(mfi);
  
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     Real met_h_xi_lo   = Compute_h_xi_AtIface  (i, j, k, dxInv, z_nd);
                     Real met_h_zeta_lo = Compute_h_zeta_AtIface(i, j, k, dxInv, z_nd);
                     Real gp_xi_lo = dxInv[0] * (p_arr(i,j,k) - p_arr(i-1,j,k));
                     Real gp_zeta_on_iface_lo;
                     if (k == klo) {
                       gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
                                                               p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
                                                             - p_arr(i-1,j,k  ) - p_arr(i,j,k  ) );
                     } else if (k == khi) {
                       gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
                                                               p_arr(i-1,j,k  ) + p_arr(i,j,k  )
                                                             - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
                     } else {
                       gp_zeta_on_iface_lo = 0.25 * dxInv[2] * (
                                                                p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
                                                              - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
                     }
                     Real gpx_lo = gp_xi_lo - (met_h_xi_lo/ met_h_zeta_lo) * gp_zeta_on_iface_lo;
  
                     Real met_h_xi_hi   = Compute_h_xi_AtIface  (i+1, j, k, dxInv, z_nd);
                     Real met_h_zeta_hi = Compute_h_zeta_AtIface(i+1, j, k, dxInv, z_nd);
                     Real gp_xi_hi = dxInv[0] * (p_arr(i+1,j,k) - p_arr(i,j,k));
                     Real gp_zeta_on_iface_hi;
                     if (k == klo) {
                       gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
                                                               p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
                                                             - p_arr(i+1,j,k  ) - p_arr(i,j,k  ) );
                     } else if (k == khi) {
                       gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
                                                               p_arr(i+1,j,k  ) + p_arr(i,j,k  )
                                                             - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
                     } else {
                       gp_zeta_on_iface_hi = 0.25 * dxInv[2] * (
                                                                p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
                                                              - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
                     }
                     Real gpx_hi = gp_xi_hi - (met_h_xi_hi/ met_h_zeta_hi) * gp_zeta_on_iface_hi;
  
                     derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_lo + gpx_hi);
                 });
             }
             mf_comp += 1;
         } // pres_hse_x
  
         if (containerHasElement(plot_var_names, "pres_hse_y"))
         {
             auto dxInv = geom[lev].InvCellSizeArray();
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real      >& derdat = mf[lev].array(mfi);
                 const Array4<Real const>&   p_arr = p_hse.const_array(mfi);
                 const Array4<Real const>& z_nd    = z_phys_nd[lev]->const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     Real met_h_eta_lo  = Compute_h_eta_AtJface (i, j, k, dxInv, z_nd);
                     Real met_h_zeta_lo = Compute_h_zeta_AtJface(i, j, k, dxInv, z_nd);
                     Real gp_eta_lo = dxInv[1] * (p_arr(i,j,k) - p_arr(i,j-1,k));
                     Real gp_zeta_on_jface_lo;
                     if (k == klo) {
                       gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
                                                               p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
                                                             - p_arr(i,j,k  ) - p_arr(i,j-1,k  ) );
                     } else if (k == khi) {
                       gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
                                                               p_arr(i,j,k  ) + p_arr(i,j-1,k  )
                                                             - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
                     } else {
                       gp_zeta_on_jface_lo = 0.25 * dxInv[2] * (
                                                                p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
                                                              - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
                     }
                     Real gpy_lo = gp_eta_lo - (met_h_eta_lo / met_h_zeta_lo) * gp_zeta_on_jface_lo;
  
                     Real met_h_eta_hi  = Compute_h_eta_AtJface (i, j+1, k, dxInv, z_nd);
                     Real met_h_zeta_hi = Compute_h_zeta_AtJface(i, j+1, k, dxInv, z_nd);
                     Real gp_eta_hi = dxInv[1] * (p_arr(i,j+1,k) - p_arr(i,j,k));
                     Real gp_zeta_on_jface_hi;
                     if (k == klo) {
                       gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
                                                               p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
                                                             - p_arr(i,j+1,k  ) - p_arr(i,j,k  ) );
                     } else if (k == khi) {
                       gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
                                                               p_arr(i,j+1,k  ) + p_arr(i,j,k  )
                                                             - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
                     } else {
                       gp_zeta_on_jface_hi = 0.25 * dxInv[2] * (
                                                                p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
                                                              - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
                     }
                     Real gpy_hi = gp_eta_hi - (met_h_eta_hi / met_h_zeta_hi) * gp_zeta_on_jface_hi;
  
                     derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_lo + gpy_hi);
                 });
             }
             mf_comp += 1;
         } // pres_hse_y
  
         if (solverChoice.use_terrain) {
             if (containerHasElement(plot_var_names, "z_phys"))
             {
                 MultiFab::Copy(mf[lev],*z_phys_cc[lev],0,mf_comp,1,0);
                 mf_comp ++;
             }
  
             if (containerHasElement(plot_var_names, "detJ"))
             {
                 MultiFab::Copy(mf[lev],*detJ_cc[lev],0,mf_comp,1,0);
                 mf_comp ++;
             }
         } // use_terrain
  
         if (containerHasElement(plot_var_names, "mapfac")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<Real>& mf_m   = mapfac_m[lev]->array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
                 });
             }
             mf_comp ++;
         }
  
 #ifdef ERF_USE_NETCDF
         if (use_real_bcs) {
             if (containerHasElement(plot_var_names, "lat_m")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = lat_m[lev]->array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = data(i,j,0);
                     });
                 }
                 mf_comp ++;
             } // lat_m
             if (containerHasElement(plot_var_names, "lon_m")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = lon_m[lev]->array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = data(i,j,0);
                     });
                 }
                 mf_comp ++;
             } // lon_m
         } // use_real_bcs
 #endif
  
  
         if (solverChoice.time_avg_vel) {
             if (containerHasElement(plot_var_names, "u_t_avg")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                     const Real norm = t_avg_cnt[lev];
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         derdat(i ,j ,k, mf_comp) = data(i,j,k,0) / norm;
                     });
                 }
                 mf_comp ++;
             }
  
             if (containerHasElement(plot_var_names, "v_t_avg")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                     const Real norm = t_avg_cnt[lev];
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         derdat(i ,j ,k, mf_comp) = data(i,j,k,1) / norm;
                     });
                 }
                 mf_comp ++;
             }
  
             if (containerHasElement(plot_var_names, "w_t_avg")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                     const Real norm = t_avg_cnt[lev];
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         derdat(i ,j ,k, mf_comp) = data(i,j,k,2) / norm;
                     });
                 }
                 mf_comp ++;
             }
  
             if (containerHasElement(plot_var_names, "umag_t_avg")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                     const Real norm = t_avg_cnt[lev];
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         derdat(i ,j ,k, mf_comp) = data(i,j,k,3) / norm;
                     });
                 }
                 mf_comp ++;
             }
         }
  
         if (containerHasElement(plot_var_names, "Kmv")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_v,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "Kmh")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "Khv")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_v,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "Khh")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "Lpbl")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::PBL_lengthscale,mf_comp,1,0);
             mf_comp ++;
         }
  
         // TODO: The size of the q variables can vary with different
         //       moisture models. Therefore, certain components may
         //       reside at different indices. For example, Kessler is
         //       warm but precipitating. This puts qp at index 3.
         //       However, SAM is cold and precipitating so qp is index 4.
         //       Need to built an external enum struct or a better pathway.
  
         // NOTE: Protect against accessing non-existent data
         if (use_moisture) {
             int n_qstate   = micro->Get_Qstate_Size();
  
             // Non-precipitating components
             //--------------------------------------------------------------------------
             if(containerHasElement(plot_var_names, "qt"))
             {
                 int n_start = RhoQ1_comp;
                 int n_end   = RhoQ2_comp;
                 if (n_qstate > 3) n_end = RhoQ3_comp;
                 MultiFab Sm(vars_new[lev][Vars::cons],make_alias,0,ncomp_cons);
                 MultiFab::Copy(  mf[lev], Sm, n_start, mf_comp, 1, 0);
                 for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
                     MultiFab::Add(  mf[lev], Sm, n_comp, mf_comp, 1, 0);
                 }
                 MultiFab::Divide(mf[lev], Sm, Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qv") && (n_qstate >= 1))
             {
                 MultiFab Sm(vars_new[lev][Vars::cons],make_alias,0,RhoQ1_comp+1);
                 MultiFab::Copy(  mf[lev], Sm, RhoQ1_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], Sm, Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qc") && (n_qstate >= 2))
             {
                 MultiFab Sm(vars_new[lev][Vars::cons],make_alias,0,RhoQ2_comp+1);
                 MultiFab::Copy(  mf[lev], Sm, RhoQ2_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], Sm, Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qi") && (n_qstate >= 4))
             {
                 MultiFab Sm(vars_new[lev][Vars::cons],make_alias,0,RhoQ3_comp+1);
                 MultiFab::Copy(  mf[lev], Sm, RhoQ3_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], Sm, Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             // Precipitating components
             //--------------------------------------------------------------------------
             if(containerHasElement(plot_var_names, "qp"))
             {
                 int n_start = RhoQ3_comp;
                 int n_end   = ncomp_cons - 1;
                 if (n_qstate > 3) n_start = RhoQ4_comp;
                 MultiFab Sm(vars_new[lev][Vars::cons],make_alias,0,ncomp_cons);
                 MultiFab::Copy(  mf[lev], Sm, n_start, mf_comp, 1, 0);
                 for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
                     MultiFab::Add(  mf[lev], Sm, n_comp, mf_comp, 1, 0);
                 }
                 MultiFab::Divide(mf[lev], Sm, Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qrain") && (n_qstate >= 3))
             {
                 int n_start = RhoQ3_comp;
                 if (n_qstate > 3) n_start = RhoQ4_comp;
                 MultiFab Sm(vars_new[lev][Vars::cons],make_alias,0,ncomp_cons);
                 MultiFab::Copy(  mf[lev], Sm, n_start , mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], Sm, Rho_comp, mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qsnow") && (n_qstate >= 5))
             {
                 MultiFab Sm(vars_new[lev][Vars::cons],make_alias,0,RhoQ5_comp+1);
                 MultiFab::Copy( mf[lev], Sm, RhoQ5_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], Sm, Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qgraup") && (n_qstate >= 6))
             {
                 MultiFab Sm(vars_new[lev][Vars::cons],make_alias,0,RhoQ6_comp+1);
                 MultiFab::Copy(  mf[lev], Sm, RhoQ6_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], Sm, Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
         if(solverChoice.moisture_type == MoistureType::Kessler){
             if (containerHasElement(plot_var_names, "rain_accum"))
             {
                 MultiFab rain_accum_mf(*(qmoist[lev][4]), make_alias, 0, 1);
                 MultiFab::Copy(mf[lev],rain_accum_mf,0,mf_comp,1,0);
                 mf_comp += 1;
             }
         }
         else if(solverChoice.moisture_type == MoistureType::SAM)
         {
             if (containerHasElement(plot_var_names, "rain_accum"))
             {
                 MultiFab rain_accum_mf(*(qmoist[lev][8]), make_alias, 0, 1);
                 MultiFab::Copy(mf[lev],rain_accum_mf,0,mf_comp,1,0);
                 mf_comp += 1;
             }
             if (containerHasElement(plot_var_names, "snow_accum"))
             {
                 MultiFab snow_accum_mf(*(qmoist[lev][9]), make_alias, 0, 1);
                 MultiFab::Copy(mf[lev],snow_accum_mf,0,mf_comp,1,0);
                 mf_comp += 1;
             }
             if (containerHasElement(plot_var_names, "graup_accum"))
             {
                 MultiFab graup_accum_mf(*(qmoist[lev][10]), make_alias, 0, 1);
                 MultiFab::Copy(mf[lev],graup_accum_mf,0,mf_comp,1,0);
                 mf_comp += 1;
             }
         }
         }
  
 #ifdef ERF_USE_PARTICLES
         const auto& particles_namelist( particleData.getNames() );
         for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++) {
             if (containerHasElement(plot_var_names, std::string(particles_namelist[i]+"_count"))) {
                 MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
                 temp_dat.setVal(0);
                 particleData[particles_namelist[i]]->Increment(temp_dat, lev);
                 MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                 mf_comp += 1;
             }
         }
  
         Vector<std::string> particle_mesh_plot_names(0);
         particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
         for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
             std::string plot_var_name(particle_mesh_plot_names[i]);
             if (containerHasElement(plot_var_names, plot_var_name) ) {
                 MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
                 temp_dat.setVal(0);
                 particleData.GetMeshPlotVar(plot_var_name, temp_dat, lev);
                 MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                 mf_comp += 1;
             }
         }
 #endif
  
 #ifdef ERF_USE_EB
         if (containerHasElement(plot_var_names, "volfrac")) {
             MultiFab::Copy(mf[lev], EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
             mf_comp += 1;
         }
 #endif
  
 #ifdef ERF_COMPUTE_ERROR
         // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
         if (containerHasElement(plot_var_names, "xvel_err") ||
             containerHasElement(plot_var_names, "yvel_err") ||
             containerHasElement(plot_var_names, "zvel_err"))
         {
             //
             // Moving terrain ANALYTICAL
             //
             Real H           = geom[lev].ProbHi()[2];
             Real Ampl        = 0.16;
             Real wavelength  = 100.;
             Real kp          = 2. * PI / wavelength;
             Real g           = CONST_GRAV;
             Real omega       = std::sqrt(g * kp);
             Real omega_t     = omega * t_new[lev];
  
             const auto dx = geom[lev].CellSizeArray();
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.validbox();
                 Box xbx(bx); xbx.surroundingNodes(0);
                 const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
                 const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
  
                 const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
  
                 ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     Real x = i * dx[0];
                     Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
  
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
                     z -= z_base;
  
                     Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
  
                     xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
                 });
  
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     Real x   = (i + 0.5) * dx[0];
                     Real z   = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
  
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
                     z -= z_base;
  
                     Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
  
                     zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
                 });
             }
  
             MultiFab temp_mf(mf[lev].boxArray(), mf[lev].DistributionMap(), AMREX_SPACEDIM, 0);
             average_face_to_cellcenter(temp_mf,0,
                 Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
  
             if (containerHasElement(plot_var_names, "xvel_err")) {
                 MultiFab::Copy(mf[lev],temp_mf,0,mf_comp,1,0);
                 mf_comp += 1;
             }
             if (containerHasElement(plot_var_names, "yvel_err")) {
                 MultiFab::Copy(mf[lev],temp_mf,1,mf_comp,1,0);
                 mf_comp += 1;
             }
             if (containerHasElement(plot_var_names, "zvel_err")) {
                 MultiFab::Copy(mf[lev],temp_mf,2,mf_comp,1,0);
                 mf_comp += 1;
             }
  
             // Now restore the velocities to what they were
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.validbox();
                 Box xbx(bx); xbx.surroundingNodes(0);
  
                 const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
                 const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
  
                 const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
  
                 ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     Real x = i * dx[0];
                     Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
  
                     z -= z_base;
  
                     Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                     xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
                 });
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     Real x   = (i + 0.5) * dx[0];
                     Real z   = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
  
                     z -= z_base;
                     Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
  
                     zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
                 });
             }
         } // end xvel_err, yvel_err, zvel_err
  
         if (containerHasElement(plot_var_names, "pp_err"))
         {
             // Moving terrain ANALYTICAL
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
  
                 const auto dx = geom[lev].CellSizeArray();
                 const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
                 const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
  
                 Real H           = geom[lev].ProbHi()[2];
                 Real Ampl        = 0.16;
                 Real wavelength  = 100.;
                 Real kp          = 2. * PI / wavelength;
                 Real g           = CONST_GRAV;
                 Real omega       = std::sqrt(g * kp);
                 Real omega_t     = omega * t_new[lev];
  
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                 {
                     const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                     derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta) - p0_arr(i,j,k);
  
                     Real rho_hse     = r0_arr(i,j,k);
  
                     Real x   = (i + 0.5) * dx[0];
                     Real z   = 0.125 * ( z_nd(i,j,k  ) + z_nd(i+1,j,k  ) + z_nd(i,j+1,k  ) + z_nd(i+1,j+1,k  )
                                         +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
  
                     z -= z_base;
                     Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                     Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
                                               std::sin(kp * x - omega_t) * r0_arr(i,j,k);
  
                     derdat(i,j,k,mf_comp) -= pprime_exact;
                 });
             }
             mf_comp += 1;
         }
 #endif
     }
  
 #ifdef EB_USE_EB
     for (int lev = 0; lev <= finest_level; ++lev) {
         EB_set_covered(mf[lev], 0.0);
     }
 #endif
  
     // Fill terrain distortion MF
     if (solverChoice.use_terrain) {
         for (int lev(0); lev <= finest_level; ++lev) {
             MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
             Real dz = Geom()[lev].CellSizeArray()[2];
             for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.tilebox();
                 Array4<      Real> mf_arr = mf_nd[lev].array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                     mf_arr(i,j,k,2) -= k * dz;
                 });
             }
         }
     }
  
     std::string plotfilename;
     if (which == 1)
        plotfilename = Concatenate(plot_file_1, istep[0], 5);
     else if (which == 2)
        plotfilename = Concatenate(plot_file_2, istep[0], 5);
  
     // LSM writes it's own data
     if (which==1 && plot_lsm) {
         lsm.Plot_Lsm_Data(t_new[0], istep, refRatio());
     }
  
     if (finest_level == 0)
     {
         if (plotfile_type == "amrex") {
             Print() << "Writing native plotfile " << plotfilename << "\n";
             if (solverChoice.use_terrain) {
                 WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                    GetVecOfConstPtrs(mf),
                                                    GetVecOfConstPtrs(mf_nd),
                                                    varnames,
                                                    t_new[0], istep);
             } else {
                 WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                         GetVecOfConstPtrs(mf),
                                         varnames,
                                         Geom(), t_new[0], istep, refRatio());
             }
             writeJobInfo(plotfilename);
  
 #ifdef ERF_USE_PARTICLES
             particleData.Checkpoint(plotfilename);
 #endif
 #ifdef ERF_USE_HDF5
         } else if (plotfile_type == "hdf5" || plotfile_type == "HDF5") {
             Print() << "Writing plotfile " << plotfilename+"d01.h5" << "\n";
             WriteMultiLevelPlotfileHDF5(plotfilename, finest_level+1,
                                         GetVecOfConstPtrs(mf),
                                         varnames,
                                         Geom(), t_new[0], istep, refRatio());
 #endif
 #ifdef ERF_USE_NETCDF
         } else if (plotfile_type == "netcdf" || plotfile_type == "NetCDF") {
              int lev   = 0;
              int l_which = 0;
              writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep, t_new[0]);
 #endif
         } else {
             Print() << "User specified plot_filetype = " << plotfile_type << std::endl;
             Abort("Dont know this plot_filetype");
         }
  
     } else { // multilevel
  
         Vector<IntVect>   r2(finest_level);
         Vector<Geometry>  g2(finest_level+1);
         Vector<MultiFab> mf2(finest_level+1);
  
         mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
  
         // Copy level 0 as is
         MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
  
         // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
         Array<int,AMREX_SPACEDIM> periodicity =
                      {Geom()[0].isPeriodic(0),Geom()[0].isPeriodic(1),Geom()[0].isPeriodic(2)};
         g2[0].define(Geom()[0].Domain(),&(Geom()[0].ProbDomain()),0,periodicity.data());
  
         if (plotfile_type == "amrex") {
             r2[0] = IntVect(1,1,ref_ratio[0][0]);
             for (int lev = 1; lev <= finest_level; ++lev) {
                 if (lev > 1) {
                     r2[lev-1][0] = 1;
                     r2[lev-1][1] = 1;
                     r2[lev-1][2] = r2[lev-2][2] * ref_ratio[lev-1][0];
                 }
  
                 mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
  
                 // Set the new problem domain
                 Box d2(Geom()[lev].Domain());
                 d2.refine(r2[lev-1]);
  
                 g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
             }
  
             // Do piecewise interpolation of mf into mf2
             for (int lev = 1; lev <= finest_level; ++lev) {
                 Interpolater* mapper_c = &pc_interp;
                 InterpFromCoarseLevel(mf2[lev], t_new[lev], mf[lev],
                                       0, 0, ncomp_mf,
                                       geom[lev], g2[lev],
                                       null_bc_for_fill, 0, null_bc_for_fill, 0,
                                       r2[lev-1], mapper_c, domain_bcs_type, 0);
             }
  
             // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
             Vector<IntVect> rr(finest_level);
             for (int lev = 0; lev < finest_level; ++lev) {
                 rr[lev] = IntVect(ref_ratio[lev][0],ref_ratio[lev][1],ref_ratio[lev][0]);
             }
  
             Print() << "Writing plotfile " << plotfilename << "\n";
             if (solverChoice.use_terrain) {
                 WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                    GetVecOfConstPtrs(mf),
                                                    GetVecOfConstPtrs(mf_nd),
                                                    varnames,
                                                    t_new[0], istep);
             } else {
                 WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                         GetVecOfConstPtrs(mf2), varnames,
                                         g2, t_new[0], istep, rr);
             }
  
             writeJobInfo(plotfilename);
  
 #ifdef ERF_USE_PARTICLES
             particleData.Checkpoint(plotfilename);
 #endif
 #ifdef ERF_USE_NETCDF
         } else if (plotfile_type == "netcdf" || plotfile_type == "NetCDF") {
              for (int lev = 0; lev <= finest_level; ++lev) {
                  for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
                      writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep, t_new[0]);
                  }
              }
 #endif
         }
     } // end multi-level
 }

Here is the call graph for this function:

Member Data Documentation

◆ advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg

private

Referenced by getAdvFluxReg().

◆ ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax

private

◆ ax_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_new

private

◆ ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src

private

◆ ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay

private

◆ ay_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_new

private

◆ ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src

private

◆ az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az

private

◆ az_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_new

private

◆ az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src

private

◆ base_state

amrex::Vector<amrex::MultiFab> ERF::base_state

private

◆ base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new

private

◆ bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1

staticprivate

◆ bndry_output_planes_per

Real ERF::bndry_output_planes_per = -1.0

staticprivate

◆ bndry_output_planes_start_time

Real ERF::bndry_output_planes_start_time = 0.0

staticprivate

◆ boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level

private

◆ cc_profiles

bool ERF::cc_profiles = true

private

◆ cf_set_width

int ERF::cf_set_width {0}

private

◆ cf_width

int ERF::cf_width {0}

private

◆ cfl

Real ERF::cfl = 0.8

staticprivate

◆ change_max

Real ERF::change_max = 1.1

staticprivate

◆ check_file

std::string ERF::check_file {"chk"}

private

◆ check_type

std::string ERF::check_type {"native"}

private

◆ column_file_name

std::string ERF::column_file_name = "column_data.nc"

staticprivate

◆ column_interval

int ERF::column_interval = -1

staticprivate

◆ column_loc_x

Real ERF::column_loc_x = 0.0

staticprivate

◆ column_loc_y

Real ERF::column_loc_y = 0.0

staticprivate

◆ column_per

Real ERF::column_per = -1.0

staticprivate

◆ cons_names

const amrex::Vector<std::string> ERF::cons_names

private

Initial value:

{"density", "rhotheta", "rhoKE", "rhoQKE", "rhoadv_0",
                                                     "rhoQ1", "rhoQ2", "rhoQ3",
                                                     "rhoQ4", "rhoQ5", "rhoQ6"}

◆ d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density

private

◆ d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure

private

◆ d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc

private

◆ d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv

private

◆ d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature

private

◆ d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs

private

◆ d_rhoqt_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhoqt_src

private

◆ d_rhotheta_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhotheta_src

private

◆ d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs

private

◆ d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos

private

◆ d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos

private

◆ d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid

private

◆ datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog

private

Referenced by DataLog(), NumDataLogs(), and setRecordDataInfo().

◆ datalogname

amrex::Vector<std::string> ERF::datalogname

private

Referenced by DataLogName().

◆ derived_names

const amrex::Vector<std::string> ERF::derived_names

private

Initial value:

{"soundspeed", "temp", "theta", "KE", "QKE", "scalar",
                                                    "vorticity_x","vorticity_y","vorticity_z",
                                                    "magvel", "divU",
                                                    "pres_hse", "dens_hse", "pressure", "pert_pres", "pert_dens", "eq_pot_temp", "num_turb",
                                                    "dpdx", "dpdy", "pres_hse_x", "pres_hse_y",
                                                    "z_phys", "detJ" , "mapfac", "lat_m", "lon_m",
                                                    
                                                    "u_t_avg", "v_t_avg", "w_t_avg", "umag_t_avg",
                                                    
                                                    "Kmv","Kmh",
                                                    
                                                    "Khv","Khh",
                                                    
                                                    "Lpbl",
                                                    
                                                    "qt", "qv", "qc", "qi", "qp", "qrain", "qsnow", "qgraup", "rain_accum", "snow_accum", "graup_accum"
 
 
 
                                                   }

◆ detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc

private

◆ detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new

private

◆ detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src

private

◆ domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type

private

◆ domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type

private

◆ domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d

private

◆ dt

amrex::Vector<amrex::Real> ERF::dt

private

◆ dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio

private

◆ dz_min

amrex::Real ERF::dz_min

private

◆ eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev

private

◆ fine_mask

amrex::MultiFab ERF::fine_mask

private

◆ fixed_dt

Real ERF::fixed_dt = -1.0

staticprivate

◆ fixed_fast_dt

Real ERF::fixed_fast_dt = -1.0

staticprivate

◆ fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0

staticprivate

◆ FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c

private

◆ FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u

private

◆ FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v

private

◆ FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w

private

◆ h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density

private

◆ h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure

private

◆ h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc

private

◆ h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv

private

◆ h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature

private

◆ h_rayleigh_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs

private

◆ h_rhoqt_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhoqt_src

private

◆ h_rhotheta_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhotheta_src

private

◆ h_sponge_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs

private

◆ h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos

private

◆ h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos

private

◆ h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid

private

◆ hub_height

amrex::Real ERF::hub_height

private

◆ init_shrink

Real ERF::init_shrink = 1.0

staticprivate

◆ init_sounding_ideal

bool ERF::init_sounding_ideal = false

staticprivate

◆ init_type

std::string ERF::init_type

staticprivate

◆ input_bndry_planes

int ERF::input_bndry_planes = 0

staticprivate

◆ input_sounding_data

InputSoundingData ERF::input_sounding_data

private

◆ input_sounding_file

std::string ERF::input_sounding_file = "input_sounding"

staticprivate

◆ input_sponge_data

InputSpongeData ERF::input_sponge_data

private

◆ input_sponge_file

std::string ERF::input_sponge_file = "input_sponge_file.txt"

staticprivate

◆ istep

amrex::Vector<int> ERF::istep

private

◆ last_check_file_step

int ERF::last_check_file_step

private

◆ last_plot_file_step_1

int ERF::last_plot_file_step_1

private

◆ last_plot_file_step_2

int ERF::last_plot_file_step_2

private

◆ lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev

private

◆ lsm

LandSurface ERF::lsm

private

◆ lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data

private

◆ lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux

private

◆ m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NVAR_max> ERF::m_bc_extdir_vals

private

◆ m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NVAR_max> ERF::m_bc_neumann_vals

private

◆ m_check_int

int ERF::m_check_int = -1

private

◆ m_check_per

amrex::Real ERF::m_check_per = -1.0

private

◆ m_factory

amrex::Vector<std::unique_ptr<amrex::FabFactory<amrex::FArrayBox> > > ERF::m_factory

private

Referenced by Factory().

◆ m_most

std::unique_ptr<ABLMost> ERF::m_most = nullptr

private

◆ m_plot_int_1

int ERF::m_plot_int_1 = -1

private

◆ m_plot_int_2

int ERF::m_plot_int_2 = -1

private

◆ m_plot_per_1

amrex::Real ERF::m_plot_per_1 = -1.0

private

◆ m_plot_per_2

amrex::Real ERF::m_plot_per_2 = -1.0

private

◆ m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr

private

◆ m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr

private

◆ mapfac_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mapfac_m

private

◆ mapfac_u

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mapfac_u

private

◆ mapfac_v

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mapfac_v

private

◆ max_k_at_level

amrex::Vector<int> ERF::max_k_at_level

private

◆ max_step

int ERF::max_step = std::numeric_limits<int>::max()

private

◆ micro

std::unique_ptr<Microphysics> ERF::micro

private

◆ min_k_at_level

amrex::Vector<int> ERF::min_k_at_level

private

◆ mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem

private

◆ nc_bdy_file

std::string ERF::nc_bdy_file

staticprivate

◆ nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}

staticprivate

◆ ng_dens_hse

int ERF::ng_dens_hse

staticprivate

◆ ng_pres_hse

int ERF::ng_pres_hse

staticprivate

◆ nominal_power

amrex::Real ERF::nominal_power

private

◆ nsubsteps

amrex::Vector<int> ERF::nsubsteps

private

◆ Nturb

amrex::Vector<amrex::MultiFab> ERF::Nturb

private

◆ num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level

private

◆ num_files_at_level

amrex::Vector<int> ERF::num_files_at_level

private

◆ output_1d_column

int ERF::output_1d_column = 0

staticprivate

◆ output_bndry_planes

int ERF::output_bndry_planes = 0

staticprivate

◆ phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type

private

◆ physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons

private

◆ physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u

private

◆ physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v

private

◆ physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w

private

◆ physbcs_w_no_terrain

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w_no_terrain> > ERF::physbcs_w_no_terrain

private

◆ plot_file_1

std::string ERF::plot_file_1 {"plt_1_"}

private

◆ plot_file_2

std::string ERF::plot_file_2 {"plt_2_"}

private

◆ plot_file_on_restart

int ERF::plot_file_on_restart = 1

private

◆ plot_lsm

bool ERF::plot_lsm = false

private

◆ plot_var_names_1

amrex::Vector<std::string> ERF::plot_var_names_1

private

◆ plot_var_names_2

amrex::Vector<std::string> ERF::plot_var_names_2

private

◆ plotfile_type

std::string ERF::plotfile_type = "amrex"

staticprivate

◆ power

amrex::Vector<amrex::Real> ERF::power

private

◆ pp_prefix

std::string ERF::pp_prefix {"erf"}

◆ previousCPUTimeUsed

Real ERF::previousCPUTimeUsed = 0.0

staticprivate

Referenced by getCPUTime().

◆ prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr

private

◆ profile_int

int ERF::profile_int = -1

private

◆ qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist

private

◆ Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim

private

◆ real_set_width

int ERF::real_set_width {0}

private

◆ real_width

int ERF::real_width {0}

private

◆ ref_tags

Vector< AMRErrorTag > ERF::ref_tags

staticprivate

◆ regrid_int

int ERF::regrid_int = -1

private

◆ restart_chkfile

std::string ERF::restart_chkfile = ""

private

◆ restart_type

std::string ERF::restart_type {"native"}

private

◆ rotor_dia

amrex::Real ERF::rotor_dia

private

◆ rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new

private

◆ rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old

private

◆ rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new

private

◆ rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old

private

◆ rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new

private

◆ rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old

private

◆ sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline

private

Referenced by NumSampleLines(), and SampleLine().

◆ samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog

private

Referenced by NumSampleLineLogs(), SampleLineLog(), and setRecordSampleLineInfo().

◆ samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname

private

Referenced by SampleLineLogName().

◆ samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint

private

Referenced by NumSamplePoints(), and SamplePoint().

◆ sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog

private

Referenced by NumSamplePointLogs(), SamplePointLog(), and setRecordSamplePointInfo().

◆ sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname

private

Referenced by SamplePointLogName().

◆ SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev

private

◆ SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev

private

◆ SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev

private

◆ SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev

private

◆ SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev

private

◆ solverChoice

SolverChoice ERF::solverChoice

staticprivate

◆ sponge_type

std::string ERF::sponge_type

staticprivate

◆ sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev

private

◆ start_time

amrex::Real ERF::start_time = 0.0

private

◆ startCPUTime

Real ERF::startCPUTime = 0.0

staticprivate

Referenced by getCPUTime().

◆ stop_time

amrex::Real ERF::stop_time = std::numeric_limits<amrex::Real>::max()

private

◆ sum_interval

int ERF::sum_interval = -1

staticprivate

◆ sum_per

Real ERF::sum_per = -1.0

staticprivate

◆ t_avg_cnt

amrex::Vector<amrex::Real> ERF::t_avg_cnt

private

◆ t_new

amrex::Vector<amrex::Real> ERF::t_new

private

◆ t_old

amrex::Vector<amrex::Real> ERF::t_old

private

◆ Tau11_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau11_lev

private

◆ Tau12_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau12_lev

private

◆ Tau13_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau13_lev

private

◆ Tau21_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau21_lev

private

◆ Tau22_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau22_lev

private

◆ Tau23_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau23_lev

private

◆ Tau31_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau31_lev

private

◆ Tau32_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau32_lev

private

◆ Tau33_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Tau33_lev

private

◆ Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim

private

◆ thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce

private

◆ thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce

private

◆ thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce

private

◆ thrust_coeff

amrex::Vector<amrex::Real> ERF::thrust_coeff

private

◆ thrust_coeff_standing

amrex::Real ERF::thrust_coeff_standing

private

◆ use_real_bcs

bool ERF::use_real_bcs

staticprivate

◆ vars_ewp

amrex::Vector<amrex::MultiFab> ERF::vars_ewp

private

◆ vars_fitch

amrex::Vector<amrex::MultiFab> ERF::vars_fitch

private

◆ vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new

private

Referenced by MultiBlockContainer::FillPatchBlocks(), MultiBlockContainer::SetBlockCommMetaData(), and MultiBlockContainer::SetBoxLists().

◆ vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old

private

◆ vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg

private

◆ verbose

int ERF::verbose = 0

staticprivate

◆ wind_speed

amrex::Vector<amrex::Real> ERF::wind_speed

private

◆ xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask

private

◆ yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask

private

◆ z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc

private

◆ z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd

private

◆ z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new

private

◆ z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src

private

◆ z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk

private

◆ zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask

private

◆ zlevels_stag

amrex::Vector<amrex::Real> ERF::zlevels_stag

private

The documentation for this class was generated from the following files:

Source/ERF.H
Source/BoundaryConditions/BoundaryConditions_bndryreg.cpp
Source/BoundaryConditions/ERF_FillPatch.cpp
Source/ERF.cpp
Source/ERF_make_new_arrays.cpp
Source/ERF_make_new_level.cpp
Source/ERF_Tagging.cpp
Source/Initialization/ERF_init1d.cpp
Source/Initialization/ERF_init_bcs.cpp
Source/Initialization/ERF_init_custom.cpp
Source/Initialization/ERF_init_from_hse.cpp
Source/Initialization/ERF_init_from_input_sounding.cpp
Source/Initialization/ERF_init_uniform.cpp
Source/Initialization/ERF_input_sponge.cpp
Source/IO/Checkpoint.cpp
Source/IO/console_io.cpp
Source/IO/ERF_Write1DProfiles.cpp
Source/IO/ERF_Write1DProfiles_stag.cpp
Source/IO/ERF_WriteScalarProfiles.cpp
Source/IO/Plotfile.cpp
Source/IO/writeJobInfo.cpp
Source/TimeIntegration/ERF_Advance.cpp
Source/TimeIntegration/ERF_advance_dycore.cpp
Source/TimeIntegration/ERF_advance_lsm.cpp
Source/TimeIntegration/ERF_advance_microphysics.cpp
Source/TimeIntegration/ERF_ComputeTimestep.cpp
Source/TimeIntegration/ERF_TimeStep.cpp

Public Member Functions

Static Public Member Functions

Public Attributes

Private Member Functions

Static Private Member Functions

Private Attributes

Static Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ ERF() [1/3]

◆ ~ERF()

◆ ERF() [2/3]

◆ ERF() [3/3]

Member Function Documentation

◆ Advance()

◆ advance_dycore()

◆ advance_lsm()

◆ advance_microphysics()

◆ appendPlotVariables()

◆ AverageDown()

◆ AverageDownTo()

◆ build_fine_mask()

◆ ClearLevel()

◆ ComputeDt()

◆ ComputeGhostCells()

◆ Construct_ERFFillPatchers()

◆ DataLog()

◆ DataLogName()

◆ Define_ERFFillPatchers()

◆ derive_diag_profiles()

◆ derive_diag_profiles_stag()

◆ derive_stress_profiles()

◆ derive_stress_profiles_stag()

◆ derive_upwp()

◆ erf_enforce_hse()

◆ ErrorEst()

◆ estTimeStep()

◆ Evolve()

◆ Factory()

◆ fill_from_bndryregs()

◆ fill_rhs()

◆ FillBdyCCVels()

◆ FillCoarsePatch()

◆ FillIntermediatePatch()

◆ FillPatch()

◆ FillPatchMoistVars()

◆ getAdvFluxReg()

◆ getCPUTime()

◆ GotoNextLine()

◆ init1DArrays()

◆ init_bcs()

◆ init_custom()

◆ init_from_hse()

◆ init_from_input_sounding()

◆ init_only()

◆ init_stuff()

◆ init_uniform()

◆ InitData()

◆ initHSE() [1/2]

◆ initHSE() [2/2]

◆ initialize_bcs()

◆ initialize_integrator()

◆ InitializeFromFile()

◆ InitializeLevelFromData()

◆ initializeMicrophysics()

◆ initRayleigh()

◆ initSponge()

◆ input_sponge()

◆ is_it_time_for_action()

◆ MakeDiagnosticAverage()

◆ MakeHorizontalAverages()

◆ MakeNewLevelFromCoarse()

◆ MakeNewLevelFromScratch()

◆ NumDataLogs()

◆ NumSampleLineLogs()

◆ NumSampleLines()

◆ NumSamplePointLogs()

◆ NumSamplePoints()

◆ operator=() [1/2]

◆ operator=() [2/2]