ERF Class Reference

#include <ERF.H>

Inheritance diagram for ERF:

Collaboration diagram for ERF:

Public Member Functions
	ERF ()
	~ERF () override
void	ERF_shared ()
	ERF (ERF &&) noexcept=delete
ERF &	operator= (ERF &&other) noexcept=delete
	ERF (const ERF &other)=delete
ERF &	operator= (const ERF &other)=delete
void	Evolve ()
void	ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override
void	HurricaneTracker (int lev, amrex::Real time, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::TagBoxArray *tags=nullptr)
bool	FindInitialEye (int lev, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::Real &eye_x, amrex::Real &eye_y)
std::string	MakeVTKFilename (int nstep)
std::string	MakeVTKFilename_TrackerCircle (int nstep)
std::string	MakeVTKFilename_EyeTracker_xy (int nstep)
std::string	MakeFilename_EyeTracker_latlon (int nstep)
std::string	MakeFilename_EyeTracker_maxvel (int nstep)
std::string	MakeFilename_EyeTracker_minpressure (int nstep)
void	WriteVTKPolyline (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
void	WriteLinePlot (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
void	InitData ()
void	InitData_pre ()
void	InitData_post ()
void	Interp2DArrays (int lev, const amrex::BoxArray &my_ba2d, const amrex::DistributionMapping &my_dm)
void	WriteMyEBSurface ()
void	compute_divergence (int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
void	project_initial_velocity (int lev, amrex::Real time, amrex::Real dt)
void	project_momenta (int lev, amrex::Real l_time, amrex::Real l_dt, amrex::Vector< amrex::MultiFab > &vars)
void	project_velocity_tb (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
void	poisson_wall_dist (int lev)
void	make_subdomains (const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
void	solve_with_gmres (int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &az_sub, amrex::MultiFab &, amrex::MultiFab &znd_sub)
void	ImposeBCsOnPhi (int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
void	init_only (int lev, amrex::Real time)
void	restart ()
void	check_state_for_nans (amrex::MultiFab const &S)
void	check_vels_for_nans (amrex::MultiFab const &xvel, amrex::MultiFab const &yvel, amrex::MultiFab const &zvel)
void	check_for_negative_theta (amrex::MultiFab &S)
void	check_for_low_temp (amrex::MultiFab &S)
bool	writeNow (const amrex::Real cur_time, const int nstep, const int plot_int, const amrex::Real plot_per, const amrex::Real dt_0, amrex::Real &last_file_time)
void	post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev)
void	sum_integrated_quantities (amrex::Real time)
void	sum_derived_quantities (amrex::Real time)
void	sum_energy_quantities (amrex::Real time)
void	write_1D_profiles (amrex::Real time)
void	write_1D_profiles_stag (amrex::Real time)
amrex::Real	cloud_fraction (amrex::Real time)
void	FillBdyCCVels (amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)
void	sample_points (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
void	sample_lines (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
void	derive_diag_profiles (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
void	derive_diag_profiles_stag (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
void	derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
void	derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
amrex::Real	volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, const amrex::MultiFab &dJ, const amrex::MultiFab &mfx, const amrex::MultiFab &mfy, bool finemask, bool local=true)
void	volWgtColumnSum (int lev, const amrex::MultiFab &mf, int comp, amrex::MultiFab &mf_2d, const amrex::MultiFab &dJ)
void	MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
void	RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
void	ClearLevel (int lev) override
void	MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
amrex::Real	estTimeStep (int lev, long &dt_fast_ratio) const
void	advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
void	advance_microphysics (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
void	advance_lsm (int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)
void	advance_radiation (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance)
void	build_fine_mask (int lev, amrex::MultiFab &fine_mask)
void	MakeHorizontalAverages ()
void	MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)
void	derive_upwp (amrex::Vector< amrex::Real > &h_havg)
void	Write3DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
void	Write2DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
void	WriteSubvolume (int isub, amrex::Vector< std::string > subvol_var_names)
void	WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
void	WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
void	erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
void	init_from_input_sounding (int lev)
void	init_immersed_forcing (int lev)
void	input_sponge (int lev)
void	init_from_hse (int lev)
void	init_thin_body (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
void	FillForecastStateMultiFabs (const int lev, const std::string &filename, const std::unique_ptr< amrex::MultiFab > &z_phys_nd, amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
void	FillSurfaceStateMultiFabs (const int lev, const std::string &filename, amrex::Vector< amrex::MultiFab > &surface_state)
void	WeatherDataInterpolation (const int nlevs, const amrex::Real time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)
void	SurfaceDataInterpolation (const int nlevs, const amrex::Real time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)
void	create_random_perturbations (const int lev, amrex::MultiFab &cons_pert, amrex::MultiFab &xvel_pert, amrex::MultiFab &yvel_pert, amrex::MultiFab &zvel_pert)
void	apply_gaussian_smoothing_to_perturbations (const int lev, amrex::MultiFab &cons_pert, amrex::MultiFab &xvel_pert, amrex::MultiFab &yvel_pert, amrex::MultiFab &zvel_pert)
void	fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
void	MakeEBGeometry ()
void	make_eb_box ()
void	make_eb_regular ()
void	AverageDownTo (int crse_lev, int scomp, int ncomp)
void	WriteCheckpointFile () const
void	ReadCheckpointFile ()
void	ReadCheckpointFileSurfaceLayer ()
void	init_zphys (int lev, amrex::Real time)
void	remake_zphys (int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
void	update_terrain_arrays (int lev)
void	writeJobInfo (const std::string &dir) const

Static Public Member Functions
static bool	is_it_time_for_action (int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
static void	writeBuildInfo (std::ostream &os)
static void	print_banner (MPI_Comm, std::ostream &)
static void	print_usage (MPI_Comm, std::ostream &)
static void	print_error (MPI_Comm, const std::string &msg)
static void	print_summary (std::ostream &)
static void	print_tpls (std::ostream &)

Public Attributes
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_track_xy
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_eye_track_xy
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_eye_track_latlon
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_maxvel_vs_time
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_minpressure_vs_time
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_tracker_circle
amrex::Vector< amrex::MultiFab >	weather_forecast_data_1
amrex::Vector< amrex::MultiFab >	weather_forecast_data_2
amrex::Vector< amrex::Vector< amrex::MultiFab > >	forecast_state_1
amrex::Vector< amrex::Vector< amrex::MultiFab > >	forecast_state_2
amrex::Vector< amrex::Vector< amrex::MultiFab > >	forecast_state_interp
amrex::Vector< amrex::MultiFab >	surface_state_1
amrex::Vector< amrex::MultiFab >	surface_state_2
amrex::Vector< amrex::MultiFab >	surface_state_interp
std::string	pp_prefix {"erf"}

Private Member Functions
void	ReadParameters ()
void	ParameterSanityChecks ()
void	AverageDown ()
void	update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
void	Construct_ERFFillPatchers (int lev)
void	Define_ERFFillPatchers (int lev)
void	init1DArrays ()
void	init_bcs ()
void	init_phys_bcs (bool &rho_read, bool &read_prim_theta)
void	init_custom (int lev)
void	init_uniform (int lev)
void	init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
void	turbPert_update (const int lev, const amrex::Real dt)
void	turbPert_amplitude (const int lev)
void	initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
void	make_physbcs (int lev)
void	initializeMicrophysics (const int &)
void	FillPatchCrseLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
void	FillPatchFineLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
void	FillIntermediatePatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
void	FillCoarsePatch (int lev, amrex::Real time)
void	timeStep (int lev, amrex::Real time, int iteration)
void	Advance (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
void	initHSE ()
	Initialize HSE. More...
void	initHSE (int lev)
void	initRayleigh ()
	Initialize Rayleigh damping profiles. More...
void	initSponge ()
	Initialize sponge profiles. More...
void	setRayleighRefFromSounding (bool restarting)
	Set Rayleigh mean profiles from input sounding. More...
void	setSpongeRefFromSounding (bool restarting)
	Set sponge mean profiles from input sounding. More...
void	ComputeDt (int step=-1)
std::string	PlotFileName (int lev) const
void	setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
void	setPlotVariables2D (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
void	appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
void	setSubVolVariables (const std::string &pp_subvol_var_names, amrex::Vector< std::string > &subvol_var_names)
void	init_Dirichlet_bc_data (const std::string input_file)
void	InitializeFromFile ()
void	InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)
void	post_update (amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
void	fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
void	init_geo_wind_profile (const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
void	refinement_criteria_setup ()
AMREX_FORCE_INLINE amrex::YAFluxRegister *	getAdvFluxReg (int lev)
AMREX_FORCE_INLINE std::ostream &	DataLog (int i)
AMREX_FORCE_INLINE std::ostream &	DerDataLog (int i)
AMREX_FORCE_INLINE int	NumDataLogs () noexcept
AMREX_FORCE_INLINE int	NumDerDataLogs () noexcept
AMREX_FORCE_INLINE std::ostream &	SamplePointLog (int i)
AMREX_FORCE_INLINE int	NumSamplePointLogs () noexcept
AMREX_FORCE_INLINE std::ostream &	SampleLineLog (int i)
AMREX_FORCE_INLINE int	NumSampleLineLogs () noexcept
amrex::IntVect &	SamplePoint (int i)
AMREX_FORCE_INLINE int	NumSamplePoints () noexcept
amrex::IntVect &	SampleLine (int i)
AMREX_FORCE_INLINE int	NumSampleLines () noexcept
void	setRecordDataInfo (int i, const std::string &filename)
void	setRecordDerDataInfo (int i, const std::string &filename)
void	setRecordEnergyDataInfo (int i, const std::string &filename)
void	setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
void	setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
std::string	DataLogName (int i) const noexcept
	The filename of the ith datalog file. More...
std::string	DerDataLogName (int i) const noexcept
std::string	SamplePointLogName (int i) const noexcept
	The filename of the ith sampleptlog file. More...
std::string	SampleLineLogName (int i) const noexcept
	The filename of the ith samplelinelog file. More...
eb_ const &	get_eb (int lev) const noexcept
amrex::EBFArrayBoxFactory const &	EBFactory (int lev) const noexcept

Static Private Member Functions
static amrex::Vector< std::string >	PlotFileVarNames (amrex::Vector< std::string > plot_var_names)
static void	GotoNextLine (std::istream &is)
static AMREX_FORCE_INLINE int	ComputeGhostCells (const SolverChoice &sc)
static amrex::Real	getCPUTime ()
static int	nghost_eb_basic ()
static int	nghost_eb_volume ()
static int	nghost_eb_full ()

Private Attributes
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	lat_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	lon_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	sinPhi_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	cosPhi_m
InputSoundingData	input_sounding_data
InputSpongeData	input_sponge_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	xvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	yvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	zvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	th_bc_data
std::unique_ptr< ProblemBase >	prob = nullptr
amrex::Vector< int >	num_boxes_at_level
amrex::Vector< int >	num_files_at_level
amrex::Vector< amrex::Vector< amrex::Box > >	boxes_at_level
amrex::Vector< int >	istep
amrex::Vector< int >	nsubsteps
amrex::Vector< amrex::Real >	t_new
amrex::Vector< amrex::Real >	t_old
amrex::Vector< amrex::Real >	dt
amrex::Vector< long >	dt_mri_ratio
amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_new
amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_old
amrex::Vector< amrex::Vector< amrex::MultiFab > >	gradp
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vel_t_avg
amrex::Vector< amrex::Real >	t_avg_cnt
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > >	mri_integrator_mem
amrex::Vector< amrex::MultiFab >	pp_inc
amrex::Vector< amrex::MultiFab >	lagged_delta_rt
amrex::Vector< amrex::MultiFab >	avg_xmom
amrex::Vector< amrex::MultiFab >	avg_ymom
amrex::Vector< amrex::MultiFab >	avg_zmom
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > >	physbcs_cons
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > >	physbcs_u
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > >	physbcs_v
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > >	physbcs_w
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > >	physbcs_base
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Theta_prim
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Qv_prim
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Qr_prim
amrex::Vector< amrex::MultiFab >	rU_old
amrex::Vector< amrex::MultiFab >	rU_new
amrex::Vector< amrex::MultiFab >	rV_old
amrex::Vector< amrex::MultiFab >	rV_new
amrex::Vector< amrex::MultiFab >	rW_old
amrex::Vector< amrex::MultiFab >	rW_new
amrex::Vector< amrex::MultiFab >	zmom_crse_rhs
std::unique_ptr< Microphysics >	micro
amrex::Vector< amrex::Vector< amrex::MultiFab * > >	qmoist
LandSurface	lsm
amrex::Vector< std::string >	lsm_data_name
amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_data
amrex::Vector< std::string >	lsm_flux_name
amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_flux
amrex::Vector< std::unique_ptr< IRadiation > >	rad
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	qheating_rates
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	rad_fluxes
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	sw_lw_fluxes
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	solar_zenith
bool	plot_rad = false
int	rad_datalog_int = -1
int	cf_width {0}
int	cf_set_width {0}
amrex::Vector< ERFFillPatcher >	FPr_c
amrex::Vector< ERFFillPatcher >	FPr_u
amrex::Vector< ERFFillPatcher >	FPr_v
amrex::Vector< ERFFillPatcher >	FPr_w
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	Tau
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	Tau_corr
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	eddyDiffs_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SmnSmn_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	sst_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	tsk_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	lmask_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	land_type_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	soil_type_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	urb_frac_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx1_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx2_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx3_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_diss_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx1_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx2_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx3_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q2fx3_lev
amrex::Vector< amrex::Vector< amrex::Real > >	zlevels_stag
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_cc
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_cc_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_t_rk
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	terrain_blanking
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	walldist
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	mapfac
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	fine_mask
amrex::Vector< amrex::Vector< amrex::Real > >	stretched_dz_h
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	stretched_dz_d
amrex::Vector< amrex::MultiFab >	base_state
amrex::Vector< amrex::MultiFab >	base_state_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Hwave
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Lwave
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Hwave_onegrid
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Lwave_onegrid
bool	finished_wave = false
amrex::Vector< amrex::YAFluxRegister * >	advflux_reg
amrex::Vector< amrex::BCRec >	domain_bcs_type
amrex::Gpu::DeviceVector< amrex::BCRec >	domain_bcs_type_d
amrex::Array< std::string, 2 *AMREX_SPACEDIM >	domain_bc_type
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max >	m_bc_extdir_vals
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max >	m_bc_neumann_vals
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 >	phys_bc_type
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	xflux_imask
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	yflux_imask
amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	zflux_imask
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_xforce
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_yforce
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_zforce
amrex::Vector< int >	last_subvol_step
amrex::Vector< amrex::Real >	last_subvol_time
const int	datwidth = 14
const int	datprecision = 6
const int	timeprecision = 13
int	max_step = -1
bool	use_datetime = false
const std::string	datetime_format = "%Y-%m-%d %H:%M:%S"
std::string	restart_chkfile = ""
amrex::Vector< amrex::Real >	fixed_dt
amrex::Vector< amrex::Real >	fixed_fast_dt
int	regrid_int = -1
bool	regrid_level_0_on_restart = false
std::string	plot3d_file_1 {"plt_1_"}
std::string	plot3d_file_2 {"plt_2_"}
std::string	plot2d_file_1 {"plt2d_1_"}
std::string	plot2d_file_2 {"plt2d_2_"}
std::string	subvol_file {"subvol"}
bool	m_expand_plotvars_to_unif_rr = false
int	m_plot3d_int_1 = -1
int	m_plot3d_int_2 = -1
int	m_plot2d_int_1 = -1
int	m_plot2d_int_2 = -1
amrex::Vector< int >	m_subvol_int
amrex::Vector< amrex::Real >	m_subvol_per
amrex::Real	m_plot3d_per_1 = -1.0
amrex::Real	m_plot3d_per_2 = -1.0
amrex::Real	m_plot2d_per_1 = -1.0
amrex::Real	m_plot2d_per_2 = -1.0
bool	m_plot_face_vels = false
bool	plot_lsm = false
int	profile_int = -1
bool	destag_profiles = true
std::string	check_file {"chk"}
int	m_check_int = -1
amrex::Real	m_check_per = -1.0
amrex::Vector< std::string >	subvol3d_var_names
amrex::Vector< std::string >	plot3d_var_names_1
amrex::Vector< std::string >	plot3d_var_names_2
amrex::Vector< std::string >	plot2d_var_names_1
amrex::Vector< std::string >	plot2d_var_names_2
const amrex::Vector< std::string >	cons_names
const amrex::Vector< std::string >	derived_names
const amrex::Vector< std::string >	derived_names_2d
const amrex::Vector< std::string >	derived_subvol_names {"soundspeed", "temp", "theta", "KE", "scalar"}
TurbulentPerturbation	turbPert
int	file_name_digits = 5
bool	use_real_time_in_pltname = false
int	real_width {0}
int	real_set_width {0}
bool	real_extrap_w {true}
bool	metgrid_debug_quiescent {false}
bool	metgrid_debug_isothermal {false}
bool	metgrid_debug_dry {false}
bool	metgrid_debug_psfc {false}
bool	metgrid_debug_msf {false}
bool	metgrid_interp_theta {false}
bool	metgrid_basic_linear {false}
bool	metgrid_use_below_sfc {true}
bool	metgrid_use_sfc {true}
bool	metgrid_retain_sfc {false}
amrex::Real	metgrid_proximity {500.0}
int	metgrid_order {2}
int	metgrid_force_sfc_k {6}
amrex::Vector< amrex::BoxArray >	ba1d
amrex::Vector< amrex::BoxArray >	ba2d
std::unique_ptr< amrex::MultiFab >	mf_C1H
std::unique_ptr< amrex::MultiFab >	mf_C2H
std::unique_ptr< amrex::MultiFab >	mf_MUB
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mf_PSFC
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	rhotheta_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	rhoqt_src
amrex::Vector< amrex::Vector< amrex::Real > >	h_w_subsid
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_w_subsid
amrex::Vector< amrex::Vector< amrex::Real > >	h_u_geos
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_u_geos
amrex::Vector< amrex::Vector< amrex::Real > >	h_v_geos
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_v_geos
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_rayleigh_ptrs
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_sponge_ptrs
amrex::Vector< amrex::Vector< amrex::Real > >	h_sinesq_ptrs
amrex::Vector< amrex::Vector< amrex::Real > >	h_sinesq_stag_ptrs
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_rayleigh_ptrs
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_sponge_ptrs
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_sinesq_ptrs
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_sinesq_stag_ptrs
amrex::Vector< amrex::Real >	h_havg_density
amrex::Vector< amrex::Real >	h_havg_temperature
amrex::Vector< amrex::Real >	h_havg_pressure
amrex::Vector< amrex::Real >	h_havg_qv
amrex::Vector< amrex::Real >	h_havg_qc
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_density
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_temperature
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_pressure
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qv
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qc
std::unique_ptr< WriteBndryPlanes >	m_w2d = nullptr
std::unique_ptr< ReadBndryPlanes >	m_r2d = nullptr
std::unique_ptr< SurfaceLayer >	m_SurfaceLayer = nullptr
amrex::Vector< std::unique_ptr< ForestDrag > >	m_forest_drag
amrex::Vector< amrex::Vector< amrex::BoxArray > >	subdomains
amrex::Vector< amrex::Real >	dz_min
int	line_sampling_interval = -1
int	plane_sampling_interval = -1
amrex::Real	line_sampling_per = -1.0
amrex::Real	plane_sampling_per = -1.0
std::unique_ptr< LineSampler >	line_sampler = nullptr
std::unique_ptr< PlaneSampler >	plane_sampler = nullptr
amrex::Vector< std::unique_ptr< std::fstream > >	datalog
amrex::Vector< std::unique_ptr< std::fstream > >	der_datalog
amrex::Vector< std::unique_ptr< std::fstream > >	tot_e_datalog
amrex::Vector< std::string >	datalogname
amrex::Vector< std::string >	der_datalogname
amrex::Vector< std::string >	tot_e_datalogname
amrex::Vector< std::unique_ptr< std::fstream > >	sampleptlog
amrex::Vector< std::string >	sampleptlogname
amrex::Vector< amrex::IntVect >	samplepoint
amrex::Vector< std::unique_ptr< std::fstream > >	samplelinelog
amrex::Vector< std::string >	samplelinelogname
amrex::Vector< amrex::IntVect >	sampleline
amrex::Vector< std::unique_ptr< eb_ > >	eb

Static Private Attributes
static int	last_plot3d_file_step_1 = -1
static int	last_plot3d_file_step_2 = -1
static int	last_plot2d_file_step_1 = -1
static int	last_plot2d_file_step_2 = -1
static int	last_check_file_step = -1
static amrex::Real	last_plot3d_file_time_1 = 0.0
static amrex::Real	last_plot3d_file_time_2 = 0.0
static amrex::Real	last_plot2d_file_time_1 = 0.0
static amrex::Real	last_plot2d_file_time_2 = 0.0
static amrex::Real	last_check_file_time = 0.0
static bool	plot_file_on_restart = true
static amrex::Real	start_time = 0.0
static amrex::Real	stop_time = std::numeric_limits<amrex::Real>::max()
static amrex::Real	cfl = 0.8
static amrex::Real	sub_cfl = 1.0
static amrex::Real	init_shrink = 1.0
static amrex::Real	change_max = 1.1
static amrex::Real	dt_max_initial = 2.0e100
static amrex::Real	dt_max = 1.0e9
static int	fixed_mri_dt_ratio = 0
static SolverChoice	solverChoice
static int	verbose = 0
static int	mg_verbose = 0
static bool	use_fft = false
static int	check_for_nans = 0
static int	sum_interval = -1
static int	pert_interval = -1
static amrex::Real	sum_per = -1.0
static PlotFileType	plotfile3d_type_1 = PlotFileType::None
static PlotFileType	plotfile3d_type_2 = PlotFileType::None
static PlotFileType	plotfile2d_type_1 = PlotFileType::None
static PlotFileType	plotfile2d_type_2 = PlotFileType::None
static StateInterpType	interpolation_type
static std::string	sponge_type
static amrex::Vector< amrex::Vector< std::string > >	nc_init_file = {{""}}
static amrex::Vector< amrex::Vector< int > >	have_read_nc_init_file = {{0}}
static std::string	nc_bdy_file
static std::string	nc_low_file
static int	output_1d_column = 0
static int	column_interval = -1
static amrex::Real	column_per = -1.0
static amrex::Real	column_loc_x = 0.0
static amrex::Real	column_loc_y = 0.0
static std::string	column_file_name = "column_data.nc"
static int	output_bndry_planes = 0
static int	bndry_output_planes_interval = -1
static amrex::Real	bndry_output_planes_per = -1.0
static amrex::Real	bndry_output_planes_start_time = 0.0
static int	input_bndry_planes = 0
static int	ng_dens_hse
static int	ng_pres_hse
static amrex::Vector< amrex::AMRErrorTag >	ref_tags
static amrex::Real	startCPUTime = 0.0
static amrex::Real	previousCPUTimeUsed = 0.0

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

ERF() [1/3]

ERF::ERF

(

)

{
    int fix_random_seed = 0;
    ParmParse pp("erf"); pp.query("fix_random_seed", fix_random_seed);
    // Note that the value of 1024UL is not significant -- the point here is just to set the
    // same seed for all MPI processes for the purpose of regression testing
    if (fix_random_seed) {
        Print() << "Fixing the random seed" << std::endl;
        InitRandom(1024UL, ParallelDescriptor::NProcs(), 1024UL);
    }
 
    ERF_shared();
}

Here is the call graph for this function:

~ERF()

ERF::~ERF ( )

overridedefault

ERF() [2/3]

ERF::ERF ( ERF &&  )

deletenoexcept

ERF() [3/3]

ERF::ERF ( const ERF &  other )

delete

Member Function Documentation

Advance()

void ERF::Advance ( int  lev, amrex::Real  time, amrex::Real  dt_lev, int  iteration, int  ncycle  )

private

Function that advances the solution at one level for a single time step – this does some preliminaries then calls erf_advance

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	dt_lev	time step for this time advance

{
    BL_PROFILE("ERF::Advance()");
 
    // We must swap the pointers so the previous step's "new" is now this step's "old"
    std::swap(vars_old[lev], vars_new[lev]);
 
    MultiFab& S_old = vars_old[lev][Vars::cons];
    MultiFab& S_new = vars_new[lev][Vars::cons];
 
    MultiFab& U_old = vars_old[lev][Vars::xvel];
    MultiFab& V_old = vars_old[lev][Vars::yvel];
    MultiFab& W_old = vars_old[lev][Vars::zvel];
 
    MultiFab& U_new = vars_new[lev][Vars::xvel];
    MultiFab& V_new = vars_new[lev][Vars::yvel];
    MultiFab& W_new = vars_new[lev][Vars::zvel];
 
    // We need to set these because otherwise in the first call to erf_advance we may
    //    read uninitialized data on ghost values in setting the bc's on the velocities
    U_new.setVal(1.e34,U_new.nGrowVect());
    V_new.setVal(1.e34,V_new.nGrowVect());
    W_new.setVal(1.e34,W_new.nGrowVect());
 
    //
    // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
    // If lev == 0 we have already FillPatched this in ERF::TimeStep
    //
    if (lev > 0) {
        FillPatchFineLevel(lev, time, {&S_old, &U_old, &V_old, &W_old},
                           {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]},
                           base_state[lev], base_state[lev]);
    }
 
    //
    // So we must convert the fillpatched to momenta, including the ghost values
    //
    const MultiFab* c_vfrac = nullptr;
    if (solverChoice.terrain_type == TerrainType::EB) {
        c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
    }
 
    VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
                       V_old, rV_old[lev].nGrowVect(),
                       W_old, rW_old[lev].nGrowVect(),
                       S_old, rU_old[lev], rV_old[lev], rW_old[lev],
                       Geom(lev).Domain(),
                       domain_bcs_type, c_vfrac);
 
    // Update the inflow perturbation update time and amplitude
    if (solverChoice.pert_type == PerturbationType::Source ||
        solverChoice.pert_type == PerturbationType::Direct ||
        solverChoice.pert_type == PerturbationType::CPM)
    {
        turbPert.calc_tpi_update(lev, dt_lev, U_old, V_old, S_old);
    }
 
    // If PerturbationType::Direct or CPM is selected, directly add the computed perturbation
    // on the conserved field
    if (solverChoice.pert_type == PerturbationType::Direct ||
        solverChoice.pert_type == PerturbationType::CPM)
    {
        auto m_ixtype = S_old.boxArray().ixType(); // Conserved term
        for (MFIter mfi(S_old,TileNoZ()); mfi.isValid(); ++mfi) {
            Box bx  = mfi.tilebox();
            const Array4<Real> &cell_data  = S_old.array(mfi);
            const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
            turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cell_data, pert_cell);
        }
    }
 
    // configure SurfaceLayer params if needed
    if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
        if (m_SurfaceLayer) {
            IntVect ng = Theta_prim[lev]->nGrowVect();
            MultiFab::Copy(  *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
            MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp     , 0, 1, ng);
            if (solverChoice.moisture_type != MoistureType::None) {
                ng = Qv_prim[lev]->nGrowVect();
 
                MultiFab::Copy(  *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
                MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp  , 0, 1, ng);
 
                if (solverChoice.moisture_indices.qr > -1) {
                    MultiFab::Copy(  *Qr_prim[lev], S_old, solverChoice.moisture_indices.qr, 0, 1, ng);
                    MultiFab::Divide(*Qr_prim[lev], S_old, Rho_comp  , 0, 1, ng);
                } else {
                    Qr_prim[lev]->setVal(0.0);
                }
            }
            // NOTE: std::swap above causes the field ptrs to be out of date.
            //       Reassign the field ptrs for MAC avg computation.
            m_SurfaceLayer->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim, Qr_prim);
            m_SurfaceLayer->update_pblh(lev, vars_old, z_phys_cc[lev].get(),
                                        solverChoice.moisture_indices);
            m_SurfaceLayer->update_fluxes(lev, time, S_old, z_phys_nd[lev], walldist[lev]);
        }
    }
 
#if defined(ERF_USE_WINDFARM)
    // **************************************************************************************
    // Update the windfarm sources
    // **************************************************************************************
    if (solverChoice.windfarm_type != WindFarmType::None) {
        advance_windfarm(Geom(lev), dt_lev, S_old,
                         U_old, V_old, W_old, vars_windfarm[lev],
                         Nturb[lev], SMark[lev], time);
    }
 
#endif
 
    // **************************************************************************************
    // Update the radiation sources with the "old" state
    // **************************************************************************************
    advance_radiation(lev, S_old, dt_lev);
 
#ifdef ERF_USE_SHOC
    // **************************************************************************************
    // Update the "old" state using SHOC
    // **************************************************************************************
    if (solverChoice.use_shoc) {
        // Get SFC fluxes from SurfaceLayer
        if (m_SurfaceLayer) {
            Vector<const MultiFab*> mfs = {&S_old, &U_old, &V_old, &W_old};
            m_SurfaceLayer->impose_SurfaceLayer_bcs(lev, mfs, Tau[lev],
                                                    SFS_hfx1_lev[lev].get() , SFS_hfx2_lev[lev].get() , SFS_hfx3_lev[lev].get(),
                                                    SFS_q1fx1_lev[lev].get(), SFS_q1fx2_lev[lev].get(), SFS_q1fx3_lev[lev].get(),
                                                    z_phys_nd[lev].get());
        }
 
        // Get Shoc tendencies and update the state
        Real* w_sub = (solverChoice.custom_w_subsidence) ? d_w_subsid[lev].data() : nullptr;
        compute_shoc_tendencies(lev, &S_old, &U_old, &V_old, &W_old, w_sub,
                                Tau[lev][TauType::tau13].get(), Tau[lev][TauType::tau23].get(),
                                SFS_hfx3_lev[lev].get()       , SFS_q1fx3_lev[lev].get()      ,
                                eddyDiffs_lev[lev].get()      , z_phys_nd[lev].get()          ,
                                dt_lev);
    }
#endif
 
    const BoxArray&            ba = S_old.boxArray();
    const DistributionMapping& dm = S_old.DistributionMap();
 
    int nvars = S_old.nComp();
 
    // Source array for conserved cell-centered quantities -- this will be filled
    //     in the call to make_sources in ERF_TI_slow_rhs_pre.H
    MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0.0);
 
    // Source arrays for momenta -- these will be filled
    //     in the call to make_mom_sources in ERF_TI_slow_rhs_pre.H
    BoxArray ba_x(ba); ba_x.surroundingNodes(0);
    MultiFab xmom_source(ba_x,dm,1,1); xmom_source.setVal(0.0);
 
    BoxArray ba_y(ba); ba_y.surroundingNodes(1);
    MultiFab ymom_source(ba_y,dm,1,1); ymom_source.setVal(0.0);
 
    BoxArray ba_z(ba); ba_z.surroundingNodes(2);
    MultiFab zmom_source(ba_z,dm,1,1); zmom_source.setVal(0.0);
    MultiFab    buoyancy(ba_z,dm,1,1); buoyancy.setVal(0.0);
 
    amrex::Vector<MultiFab> state_old;
    amrex::Vector<MultiFab> state_new;
 
    // **************************************************************************************
    // Here we define state_old and state_new which are to be advanced
    // **************************************************************************************
    // Initial solution
    // Note that "old" and "new" here are relative to each RK stage.
    state_old.push_back(MultiFab(S_old      , amrex::make_alias, 0, nvars)); // cons
    state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0,     1)); // xmom
    state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0,     1)); // ymom
    state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0,     1)); // zmom
 
    // Final solution
    // state_new at the end of the last RK stage holds the t^{n+1} data
    state_new.push_back(MultiFab(S_new      , amrex::make_alias, 0, nvars)); // cons
    state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0,     1)); // xmom
    state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0,     1)); // ymom
    state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0,     1)); // zmom
 
    // **************************************************************************************
    // Tests on the reasonableness of the solution before the dycore
    // **************************************************************************************
    // Test for NaNs after dycore
    if (check_for_nans > 1) {
        if (verbose > 1) {
            amrex::Print() << "Testing old state and vels for NaNs before dycore" << std::endl;
        }
        check_state_for_nans(S_old);
        check_vels_for_nans(rU_old[lev],rV_old[lev],rW_old[lev]);
    }
 
    // We only test on low temp if we have a moisture model because we are protecting against
    //    the test on low temp inside the moisture models
    if (solverChoice.moisture_type != MoistureType::None) {
        if (verbose > 1) {
            amrex::Print() << "Testing on low temperature before dycore" << std::endl;
        }
        check_for_low_temp(S_old);
    } else {
        if (verbose > 1) {
            amrex::Print() << "Testing on negative temperature before dycore" << std::endl;
        }
        check_for_negative_theta(S_old);
    }
 
    // **************************************************************************************
    // Update the dycore
    // **************************************************************************************
    advance_dycore(lev, state_old, state_new,
                   U_old, V_old, W_old,
                   U_new, V_new, W_new,
                   cc_source, xmom_source, ymom_source, zmom_source, buoyancy,
                   Geom(lev), dt_lev, time);
 
    // **************************************************************************************
    // Tests on the reasonableness of the solution after the dycore
    // **************************************************************************************
    // Test for NaNs after dycore
    if (check_for_nans > 0) {
        if (verbose > 1) {
            amrex::Print() << "Testing new state and vels for NaNs after dycore" << std::endl;
        }
        check_state_for_nans(S_new);
        check_vels_for_nans(rU_new[lev],rV_new[lev],rW_new[lev]);
    }
 
    // We only test on low temp if we have a moisture model because we are protecting against
    //    the test on low temp inside the moisture models
    if (solverChoice.moisture_type != MoistureType::None) {
        if (verbose > 1) {
            amrex::Print() << "Testing on low temperature after dycore" << std::endl;
        }
        check_for_low_temp(S_new);
    } else {
        // Otherwise we will test on negative (rhotheta) coming out of the dycore
        if (verbose > 1) {
            amrex::Print() << "Testing on negative temperature after dycore" << std::endl;
        }
        check_for_negative_theta(S_new);
    }
 
    // **************************************************************************************
    // Update the microphysics (moisture)
    // **************************************************************************************
    if (!solverChoice.moisture_tight_coupling)
    {
        advance_microphysics(lev, S_new, dt_lev, iteration, time);
 
        // Test for NaNs after microphysics
        if (check_for_nans > 0) {
            amrex::Print() << "Testing new state for NaNs after advance_microphysics" << std::endl;
            check_state_for_nans(S_new);
        }
    }
 
    // **************************************************************************************
    // Update the land surface model
    // **************************************************************************************
    advance_lsm(lev, S_new, U_new, V_new, dt_lev);
 
#ifdef ERF_USE_PARTICLES
    // **************************************************************************************
    // Update the particle positions
    // **************************************************************************************
   evolveTracers( lev, dt_lev, vars_new, z_phys_nd );
#endif
 
    // ***********************************************************************************************
    // Impose domain boundary conditions here so that in FillPatching the fine data we won't
    // need to re-fill these
    // ***********************************************************************************************
    if (lev < finest_level) {
         IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
         (*physbcs_cons[lev])(vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                              0,vars_new[lev][Vars::cons].nComp(),
                              vars_new[lev][Vars::cons].nGrowVect(),time,BCVars::cons_bc,true);
            (*physbcs_u[lev])(vars_new[lev][Vars::xvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                              ngvect_vels,time,BCVars::xvel_bc,true);
            (*physbcs_v[lev])(vars_new[lev][Vars::yvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                              ngvect_vels,time,BCVars::yvel_bc,true);
            (*physbcs_w[lev])(vars_new[lev][Vars::zvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                              ngvect_vels,time,BCVars::zvel_bc,true);
    }
 
    // **************************************************************************************
    // Register old and new coarse data if we are at a level less than the finest level
    // **************************************************************************************
    if (lev < finest_level) {
        if (cf_width > 0) {
            // We must fill the ghost cells of these so that the parallel copy works correctly
            state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
            FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
                                          {time, time+dt_lev});
        }
 
        if (cf_width >= 0) {
            // We must fill the ghost cells of these so that the parallel copy works correctly
            state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
            FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
                                          {time, time+dt_lev});
 
            state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
            FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
                                          {time, time+dt_lev});
 
            state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
            state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
            FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
                                          {time, time+dt_lev});
        }
 
            //
            // Now create a MultiFab that holds (S_new - S_old) / dt from the coarse level interpolated
            //     on to the coarse/fine boundary at the fine resolution
            //
            Interpolater* mapper_f = &face_cons_linear_interp;
 
            // PhysBCFunctNoOp null_bc;
            // MultiFab tempx(vars_new[lev+1][Vars::xvel].boxArray(),vars_new[lev+1][Vars::xvel].DistributionMap(),1,0);
            // tempx.setVal(0.0);
            // xmom_crse_rhs[lev+1].setVal(0.0);
            // FPr_u[lev].FillSet(tempx               , time       , null_bc, domain_bcs_type);
            // FPr_u[lev].FillSet(xmom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
            // MultiFab::Subtract(xmom_crse_rhs[lev+1],tempx,0,0,1,IntVect{0});
            // xmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
 
            // MultiFab tempy(vars_new[lev+1][Vars::yvel].boxArray(),vars_new[lev+1][Vars::yvel].DistributionMap(),1,0);
            // tempy.setVal(0.0);
            // ymom_crse_rhs[lev+1].setVal(0.0);
            // FPr_v[lev].FillSet(tempy               , time       , null_bc, domain_bcs_type);
            // FPr_v[lev].FillSet(ymom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
            // MultiFab::Subtract(ymom_crse_rhs[lev+1],tempy,0,0,1,IntVect{0});
            // ymom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
 
            MultiFab temp_state(zmom_crse_rhs[lev+1].boxArray(),zmom_crse_rhs[lev+1].DistributionMap(),1,0);
            InterpFromCoarseLevel(temp_state,            IntVect{0}, IntVect{0}, state_old[IntVars::zmom], 0, 0, 1,
                                  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
            InterpFromCoarseLevel(zmom_crse_rhs[lev+1],  IntVect{0}, IntVect{0}, state_new[IntVars::zmom], 0, 0, 1,
                                  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
            MultiFab::Subtract(zmom_crse_rhs[lev+1],temp_state,0,0,1,IntVect{0});
            zmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
    }
 
    // ***********************************************************************************************
    // Update the time averaged velocities if they are requested
    // ***********************************************************************************************
    if (solverChoice.time_avg_vel) {
        Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
    }
}

Here is the call graph for this function:

advance_dycore()

void ERF::advance_dycore	(	int	level,
		amrex::Vector< amrex::MultiFab > &	state_old,
		amrex::Vector< amrex::MultiFab > &	state_new,
		amrex::MultiFab &	xvel_old,
		amrex::MultiFab &	yvel_old,
		amrex::MultiFab &	zvel_old,
		amrex::MultiFab &	xvel_new,
		amrex::MultiFab &	yvel_new,
		amrex::MultiFab &	zvel_new,
		amrex::MultiFab &	source,
		amrex::MultiFab &	xmom_src,
		amrex::MultiFab &	ymom_src,
		amrex::MultiFab &	zmom_src,
		amrex::MultiFab &	buoyancy,
		amrex::Geometry	fine_geom,
		amrex::Real	dt,
		amrex::Real	time
	)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters

[in]	level	level of refinement (coarsest level is 0)
[in]	state_old	old-time conserved variables
[in]	state_new	new-time conserved variables
[in]	xvel_old	old-time x-component of velocity
[in]	yvel_old	old-time y-component of velocity
[in]	zvel_old	old-time z-component of velocity
[in]	xvel_new	new-time x-component of velocity
[in]	yvel_new	new-time y-component of velocity
[in]	zvel_new	new-time z-component of velocity
[in]	cc_src	source term for conserved variables
[in]	xmom_src	source term for x-momenta
[in]	ymom_src	source term for y-momenta
[in]	zmom_src	source term for z-momenta
[in]	fine_geom	container for geometry information at current level
[in]	dt_advance	time step for this time advance
[in]	old_time	old time for this time advance

{
    BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
 
    const Box& domain = fine_geom.Domain();
 
    DiffChoice dc    = solverChoice.diffChoice;
    TurbChoice tc    = solverChoice.turbChoice[level];
    SpongeChoice sc  = solverChoice.spongeChoice;
 
    MultiFab r_hse (base_state[level], make_alias, BaseState::r0_comp , 1);
    MultiFab p_hse (base_state[level], make_alias, BaseState::p0_comp , 1);
    MultiFab pi_hse(base_state[level], make_alias, BaseState::pi0_comp, 1);
 
    // These pointers are used in the MRI utility functions
    MultiFab* r0  = &r_hse;
    MultiFab* p0  = &p_hse;
    MultiFab* pi0 = &pi_hse;
 
    MultiFab* rhotheta_src_ptr = solverChoice.custom_rhotheta_forcing ? rhotheta_src[level].get() : nullptr;
    MultiFab* rhoqt_src_ptr    = solverChoice.custom_moisture_forcing ? rhoqt_src[level].get()   : nullptr;
    Real* dptr_wbar_sub        = solverChoice.custom_w_subsidence     ? d_w_subsid[level].data()     : nullptr;
 
    // Turbulent Perturbation Pointer
    //Real* dptr_rhotheta_src = solverChoice.pert_type ? d_rhotheta_src[level].data() : nullptr;
 
    Vector<Real*> d_rayleigh_ptrs_at_lev;
    d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
    d_rayleigh_ptrs_at_lev[Rayleigh::ubar]     = solverChoice.dampingChoice.rayleigh_damp_U   ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
    d_rayleigh_ptrs_at_lev[Rayleigh::vbar]     = solverChoice.dampingChoice.rayleigh_damp_V   ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
    d_rayleigh_ptrs_at_lev[Rayleigh::wbar]     = solverChoice.dampingChoice.rayleigh_damp_W   ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
    d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.dampingChoice.rayleigh_damp_T   ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
 
    bool use_rayleigh =
       (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
        solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T);
    Real* d_sinesq_at_lev      = (use_rayleigh)  ? d_sinesq_ptrs[level].data() : nullptr;
    Real* d_sinesq_stag_at_lev = (use_rayleigh)  ? d_sinesq_stag_ptrs[level].data() : nullptr;
 
    Vector<Real*> d_sponge_ptrs_at_lev;
    if(sc.sponge_type=="input_sponge")
    {
        d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
        d_sponge_ptrs_at_lev[Sponge::ubar_sponge]  =  d_sponge_ptrs[level][Sponge::ubar_sponge].data();
        d_sponge_ptrs_at_lev[Sponge::vbar_sponge]  =  d_sponge_ptrs[level][Sponge::vbar_sponge].data();
    }
 
    bool l_use_terrain_fitted_coords = (solverChoice.mesh_type != MeshType::ConstantDz);
    bool l_use_kturb   = tc.use_kturb;
    bool l_use_diff    = ( (dc.molec_diff_type != MolecDiffType::None) ||
                           l_use_kturb );
 
    const bool use_SurfLayer = (m_SurfaceLayer != nullptr);
    const MultiFab* z_0     = (use_SurfLayer) ? m_SurfaceLayer->get_z0(level) : nullptr;
 
    const BoxArray& ba            = state_old[IntVars::cons].boxArray();
    const BoxArray& ba_z          = zvel_old.boxArray();
    const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
 
    int num_prim = state_old[IntVars::cons].nComp() - 1;
 
    MultiFab    S_prim  (ba  , dm, num_prim,          state_old[IntVars::cons].nGrowVect());
    MultiFab  pi_stage  (ba  , dm,        1,          1);
    MultiFab fast_coeffs(ba_z, dm,        5,          0);
 
    MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
    MultiFab* SmnSmn    = SmnSmn_lev[level].get();
 
    // **************************************************************************************
    // Compute strain for use in slow RHS and Smagorinsky model
    // **************************************************************************************
    {
    BL_PROFILE("erf_advance_strain");
    if (l_use_diff) {
 
        const BCRec* bc_ptr_h = domain_bcs_type.data();
        const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
 
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
        {
            Box bxcc  = mfi.growntilebox(IntVect(1,1,0));
            Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
            Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
            Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
 
            if (bxcc.smallEnd(2) != domain.smallEnd(2)) {
                 bxcc.growLo(2,1);
                tbxxy.growLo(2,1);
                tbxxz.growLo(2,1);
                tbxyz.growLo(2,1);
            }
 
            if (bxcc.bigEnd(2) != domain.bigEnd(2)) {
                 bxcc.growHi(2,1);
                tbxxy.growHi(2,1);
                tbxxz.growHi(2,1);
                tbxyz.growHi(2,1);
            }
 
            const Array4<const Real> & u = xvel_old.array(mfi);
            const Array4<const Real> & v = yvel_old.array(mfi);
            const Array4<const Real> & w = zvel_old.array(mfi);
 
            Array4<Real> tau11 = Tau[level][TauType::tau11].get()->array(mfi);
            Array4<Real> tau22 = Tau[level][TauType::tau22].get()->array(mfi);
            Array4<Real> tau33 = Tau[level][TauType::tau33].get()->array(mfi);
            Array4<Real> tau12 = Tau[level][TauType::tau12].get()->array(mfi);
            Array4<Real> tau13 = Tau[level][TauType::tau13].get()->array(mfi);
            Array4<Real> tau23 = Tau[level][TauType::tau23].get()->array(mfi);
 
            Array4<Real> tau21 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau21].get()->array(mfi) : Array4<Real>{};
            Array4<Real> tau31 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau31].get()->array(mfi) : Array4<Real>{};
            Array4<Real> tau32 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau32].get()->array(mfi) : Array4<Real>{};
            const Array4<const Real>& z_nd = z_phys_nd[level]->const_array(mfi);
 
            const Array4<const Real> mf_mx = mapfac[level][MapFacType::m_x]->const_array(mfi);
            const Array4<const Real> mf_ux = mapfac[level][MapFacType::u_x]->const_array(mfi);
            const Array4<const Real> mf_vx = mapfac[level][MapFacType::v_x]->const_array(mfi);
            const Array4<const Real> mf_my = mapfac[level][MapFacType::m_y]->const_array(mfi);
            const Array4<const Real> mf_uy = mapfac[level][MapFacType::u_y]->const_array(mfi);
            const Array4<const Real> mf_vy = mapfac[level][MapFacType::v_y]->const_array(mfi);
 
            // We update Tau_corr[level] in erf_make_tau_terms, not here
            Array4<Real> no_tau_corr_update_here{};
 
            if (solverChoice.mesh_type == MeshType::StretchedDz) {
                ComputeStrain_S(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                u, v, w,
                                tau11, tau22, tau33,
                                tau12, tau21,
                                tau13, tau31,
                                tau23, tau32,
                                stretched_dz_d[level], dxInv,
                                mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
                                no_tau_corr_update_here, no_tau_corr_update_here);
            } else if (l_use_terrain_fitted_coords) {
                ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                u, v, w,
                                tau11, tau22, tau33,
                                tau12, tau21,
                                tau13, tau31,
                                tau23, tau32,
                                z_nd, detJ_cc[level]->const_array(mfi), dxInv,
                                mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
                                no_tau_corr_update_here, no_tau_corr_update_here);
            } else {
                ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                u, v, w,
                                tau11, tau22, tau33,
                                tau12, tau13, tau23,
                                dxInv,
                                mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
                                no_tau_corr_update_here, no_tau_corr_update_here);
            }
        } // mfi
    } // l_use_diff
    } // profile
 
#include "ERF_TI_utils.H"
 
    // Additional SFS quantities, calculated once per timestep
    MultiFab* Hfx1  = SFS_hfx1_lev[level].get();
    MultiFab* Hfx2  = SFS_hfx2_lev[level].get();
    MultiFab* Hfx3  = SFS_hfx3_lev[level].get();
    MultiFab* Q1fx1 = SFS_q1fx1_lev[level].get();
    MultiFab* Q1fx2 = SFS_q1fx2_lev[level].get();
    MultiFab* Q1fx3 = SFS_q1fx3_lev[level].get();
    MultiFab* Q2fx3 = SFS_q2fx3_lev[level].get();
    MultiFab* Diss  = SFS_diss_lev[level].get();
 
    // *************************************************************************
    // Calculate cell-centered eddy viscosity & diffusivities
    //
    // Notes -- we fill all the data in ghost cells before calling this so
    //    that we can fill the eddy viscosity in the ghost regions and
    //    not have to call a boundary filler on this data itself
    //
    // LES - updates both horizontal and vertical eddy viscosity components
    // PBL - only updates vertical eddy viscosity components so horizontal
    //       components come from the LES model or are left as zero.
    // *************************************************************************
    if (l_use_kturb)
    {
        // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
        bool l_use_moisture = ( solverChoice.moisture_type != MoistureType::None );
        const BCRec* bc_ptr_h = domain_bcs_type.data();
        ComputeTurbulentViscosity(dt_advance, xvel_old, yvel_old,Tau[level],
                                  state_old[IntVars::cons],
                                  *walldist[level].get(),
                                  *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
                                  fine_geom, mapfac[level],
                                  z_phys_nd[level], solverChoice,
                                  m_SurfaceLayer, z_0, l_use_terrain_fitted_coords,
                                  l_use_moisture, level,
                                  bc_ptr_h);
    }
 
    // ***********************************************************************************************
    // Update user-defined source terms -- these are defined once per time step (not per RK stage)
    // ***********************************************************************************************
    if (solverChoice.custom_rhotheta_forcing) {
        prob->update_rhotheta_sources(old_time,
                                      rhotheta_src_ptr,
                                      fine_geom, z_phys_cc[level]);
    }
 
    if (solverChoice.custom_moisture_forcing) {
        prob->update_rhoqt_sources(old_time,
                                   rhoqt_src_ptr,
                                   fine_geom, z_phys_cc[level]);
    }
 
    if (solverChoice.custom_geostrophic_profile) {
        prob->update_geostrophic_profile(old_time,
                                   h_u_geos[level], d_u_geos[level],
                                   h_v_geos[level], d_v_geos[level],
                                   fine_geom, z_phys_cc[level]);
    }
 
    if (solverChoice.custom_w_subsidence) {
        prob->update_w_subsidence(old_time,
                                  h_w_subsid[level], d_w_subsid[level],base_state[level],
                                  fine_geom, z_phys_nd[level]);
    }
 
    // ***********************************************************************************************
    // Convert old velocity available on faces to old momentum on faces to be used in time integration
    // ***********************************************************************************************
    MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
 
    //
    // This is an optimization since we won't need more than one ghost
    // cell of momentum in the integrator if not using numerical diffusion
    //
    IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
    IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
    IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
 
    const MultiFab* c_vfrac = nullptr;
    if (solverChoice.terrain_type == TerrainType::EB) {
        c_vfrac = &((get_eb(level).get_const_factory())->getVolFrac());
    }
 
    VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
                       state_old[IntVars::xmom],
                       state_old[IntVars::ymom],
                       state_old[IntVars::zmom],
                       domain, domain_bcs_type, c_vfrac);
 
    MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
    MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
    MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
 
    bool fast_only = false;
    bool vel_and_mom_synced = true;
 
    apply_bcs(state_old, old_time,
              state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
              fast_only, vel_and_mom_synced);
    cons_to_prim(state_old[IntVars::cons], state_old[IntVars::cons].nGrow());
 
    // ***********************************************************************************************
    // Define a new MultiFab that holds q_total and fill it by summing the moisture components --
    //      to be used in buoyancy calculation and as part of the inertial weighting in the
    // ***********************************************************************************************
 
    const bool l_eb_terrain = (solverChoice.terrain_type == TerrainType::EB);
    MultiFab qt(grids[level], dmap[level], 1, (l_eb_terrain) ? 2 : 1);
    qt.setVal(0.0);
 
#include "ERF_TI_no_substep_fun.H"
#include "ERF_TI_substep_fun.H"
#include "ERF_TI_slow_rhs_pre.H"
#include "ERF_TI_slow_rhs_post.H"
 
    // ***************************************************************************************
    // Setup the integrator and integrate for a single timestep
    // **************************************************************************************
    MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
 
    // Define rhs and 'post update' utility function that is called after calculating
    // any state data (e.g. at RK stages or at the end of a timestep)
    mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
    mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
 
    mri_integrator.set_acoustic_substepping(acoustic_substepping_fun);
    mri_integrator.set_slow_fast_timestep_ratio(fixed_mri_dt_ratio > 0 ? fixed_mri_dt_ratio : dt_mri_ratio[level]);
    mri_integrator.set_no_substep(no_substep_fun);
 
    mri_integrator.advance(state_old, state_new, old_time, dt_advance);
 
    if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
}

Here is the call graph for this function:

advance_lsm()

void ERF::advance_lsm	(	int	lev,
		amrex::MultiFab &	cons_in,
		amrex::MultiFab &	xvel_in,
		amrex::MultiFab &	yvel_in,
		const amrex::Real &	dt_advance
	)

{
    if (solverChoice.lsm_type != LandSurfaceType::None) {
        if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
            lsm.Advance(lev, cons_in, xvel_in, yvel_in, SFS_hfx3_lev[lev].get(), SFS_q1fx3_lev[lev].get(), dt_advance, istep[0]);
        } else {
            lsm.Advance(lev, dt_advance);
        }
    }
}

advance_microphysics()

void ERF::advance_microphysics	(	int	lev,
		amrex::MultiFab &	cons_in,
		const amrex::Real &	dt_advance,
		const int &	iteration,
		const amrex::Real &	time
	)

{
    if (solverChoice.moisture_type != MoistureType::None) {
        micro->Set_RealWidth(lev, real_width);
        micro->Update_Micro_Vars_Lev(lev, cons);
        micro->Advance(lev, dt_advance, iteration, time, solverChoice, vars_new, z_phys_nd, phys_bc_type);
        micro->Update_State_Vars_Lev(lev, cons);
    }
}

advance_radiation()

void ERF::advance_radiation	(	int	lev,
		amrex::MultiFab &	cons_in,
		const amrex::Real &	dt_advance
	)

{
    if (solverChoice.rad_type != RadiationType::None) {
#ifdef ERF_USE_NETCDF
        MultiFab *lat_ptr = lat_m[lev].get();
        MultiFab *lon_ptr = lon_m[lev].get();
#else
        MultiFab *lat_ptr = nullptr;
        MultiFab *lon_ptr = nullptr;
#endif
        // T surf from SurfaceLayer if we have it
        MultiFab* t_surf = (m_SurfaceLayer) ? m_SurfaceLayer->get_t_surf(lev) : nullptr;
 
        // RRTMGP inputs names and pointers
        Vector<std::string> lsm_input_names = rad[lev]->get_lsm_input_varnames();
        Vector<MultiFab*> lsm_input_ptrs(lsm_input_names.size(),nullptr);
        for (int i(0); i<lsm_input_ptrs.size(); ++i) {
            int varIdx = lsm.Get_DataIdx(lev,lsm_input_names[i]);
            lsm_input_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
        }
 
        // RRTMGP output names and pointers
        Vector<std::string> lsm_output_names = rad[lev]->get_lsm_output_varnames();
        Vector<MultiFab*> lsm_output_ptrs(lsm_output_names.size(),nullptr);
        for (int i(0); i<lsm_output_ptrs.size(); ++i) {
            int varIdx = lsm.Get_DataIdx(lev,lsm_output_names[i]);
            lsm_output_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
        }
 
        // Enter radiation class driver
        amrex::Real time_for_rad = t_new[lev] + start_time;
        rad[lev]->Run(lev, istep[lev], time_for_rad, dt_advance,
                      cons.boxArray(), geom[lev], &(cons),
                      lmask_lev[lev][0].get(), t_surf,
                      sw_lw_fluxes[lev].get(), solar_zenith[lev].get(),
                      lsm_input_ptrs, lsm_output_ptrs,
                      qheating_rates[lev].get(), rad_fluxes[lev].get(),
                      z_phys_nd[lev].get()     , lat_ptr, lon_ptr);
    }
}

appendPlotVariables()

void ERF::appendPlotVariables ( const std::string &  pp_plot_var_names, amrex::Vector< std::string > &  plot_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    Vector<std::string> plot_var_names(0);
    if (pp.contains(pp_plot_var_names.c_str())) {
        std::string nm;
        int nPltVars = pp.countval(pp_plot_var_names.c_str());
        for (int i = 0; i < nPltVars; i++) {
            pp.get(pp_plot_var_names.c_str(), nm, i);
            // Add the named variable to our list of plot variables
            // if it is not already in the list
            if (!containerHasElement(plot_var_names, nm)) {
                plot_var_names.push_back(nm);
            }
        }
    }
 
    Vector<std::string> tmp_plot_names(0);
#ifdef ERF_USE_PARTICLES
    Vector<std::string> particle_mesh_plot_names;
    particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
    if (particle_mesh_plot_names.size() > 0) {
        static bool first_call = true;
        if (first_call) {
            Print() << "ParticleData: the following additional Eulerian variables are available to plot:\n";
            for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
                Print() << "    " << particle_mesh_plot_names[i] << "\n";
            }
            first_call = false;
        }
        for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
            std::string tmp(particle_mesh_plot_names[i]);
            if (containerHasElement(plot_var_names, tmp) ) {
                tmp_plot_names.push_back(tmp);
            }
        }
    }
#endif
 
    {
        Vector<std::string> microphysics_plot_names;
        micro->GetPlotVarNames(microphysics_plot_names);
        if (microphysics_plot_names.size() > 0) {
            static bool first_call = true;
            if (first_call) {
                Print() << getEnumNameString(solverChoice.moisture_type)
                        << ": the following additional variables are available to plot:\n";
                for (int i = 0; i < microphysics_plot_names.size(); i++) {
                    Print() << "    " << microphysics_plot_names[i] << "\n";
                }
                first_call = false;
            }
            for (auto& plot_name : microphysics_plot_names) {
                if (containerHasElement(plot_var_names, plot_name)) {
                    tmp_plot_names.push_back(plot_name);
                }
            }
        }
    }
 
    for (int i = 0; i < tmp_plot_names.size(); i++) {
        a_plot_var_names.push_back( tmp_plot_names[i] );
    }
 
    // Finally, check to see if we found all the requested variables
    for (const auto& plot_name : plot_var_names) {
        if (!containerHasElement(a_plot_var_names, plot_name)) {
             if (amrex::ParallelDescriptor::IOProcessor()) {
                 Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
             }
        }
    }
}

Here is the call graph for this function:

apply_gaussian_smoothing_to_perturbations()

void ERF::apply_gaussian_smoothing_to_perturbations	(	const int	lev,
		amrex::MultiFab &	cons_pert,
		amrex::MultiFab &	xvel_pert,
		amrex::MultiFab &	yvel_pert,
		amrex::MultiFab &	zvel_pert
	)

{
    ignore_unused(cons_pert);
    ignore_unused(yvel_pert);
    ignore_unused(zvel_pert);
 
    const Geometry& gm = geom[lev];
    const Real dx = gm.CellSize(0);
    const Real dy = gm.CellSize(1);
 
    const Real dmesh = std::min(dx, dy);
    // ---- User choices ----
    const Real sigma = solverChoice.pert_correlated_radius; // e.g. 2 km correlation length
    const int  r     = static_cast<int>(3.0 * sigma / dmesh);  // stencil radius
 
    // ---- Precompute Gaussian weights on host ----
    const int wsize = 2*r + 1;
    Vector<Real> w_host(wsize * wsize);
 
    Real Z = 0.0;
    for (int m = -r; m <= r; ++m) {
        for (int n = -r; n <= r; ++n) {
            Real val = std::exp(-(m*m*dx*dx + n*n*dy*dy)/(2.0*sigma*sigma));
            w_host[(m+r)*wsize + (n+r)] = val;
            Z += val;
        }
    }
    for (auto& v : w_host) {
        v = v/Z;
    }
 
    Gpu::DeviceVector<Real> w_dev;
    w_dev.resize(w_host.size());
    Gpu::copy(Gpu::hostToDevice, w_host.begin(), w_host.end(), w_dev.begin());
 
    Real const* w = w_dev.data();
 
    // 1. Define ngrow_big using the actual dimension macro
    IntVect ngrow_big(AMREX_D_DECL(r, r, 0));
 
    // 2. Create the copy
    MultiFab xvel_pert_copy(xvel_pert.boxArray(),
                        xvel_pert.DistributionMap(),
                        1, ngrow_big);
    //MultiFab::Copy(xvel_pert_copy, xvel_pert, 0, 0, 1, 0);
 
    // 3. Use the built-in copy that includes ghost cell logic
    // Copy(dst, src, src_comp, dst_comp, num_comp, ngrow)
    // Setting ngrow to 0 ensures we only take valid data from the original
    xvel_pert_copy.ParallelCopy(xvel_pert, 0, 0, 1, IntVect(0), ngrow_big, gm.periodicity());
 
    for (MFIter mfi(xvel_pert, TileNoZ()); mfi.isValid(); ++mfi)
    {
        const Box& tbx = mfi.tilebox();
 
        auto const& in  = xvel_pert_copy.array(mfi);
        auto const& out = xvel_pert.array(mfi);
 
        ParallelFor(tbx,
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            Real sum = 0.0;
            for (int m = -r; m <= r; ++m) {
                for (int n = -r; n <= r; ++n) {
                    Real wij = w[(m+r)*wsize + (n+r)];
                    sum += wij * in(i+m, j+n, k);
                }
            }
            out(i,j,k) = sum;
        });
    }
}

Here is the call graph for this function:

AverageDown()

void ERF::AverageDown ( )

private

{
    AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
 
    int src_comp, num_comp;
    for (int lev = finest_level-1; lev >= 0; --lev)
    {
        // If anelastic we don't average down rho because rho == rho0.
        if (solverChoice.anelastic[lev]) {
            src_comp = 1;
        } else {
            src_comp = 0;
        }
        num_comp = vars_new[0][Vars::cons].nComp() - src_comp;
        AverageDownTo(lev,src_comp,num_comp);
    }
}

AverageDownTo()

void ERF::AverageDownTo	(	int	crse_lev,
		int	scomp,
		int	ncomp
	)

{
    if (solverChoice.anelastic[crse_lev]) {
        AMREX_ALWAYS_ASSERT(scomp == 1);
    } else {
        AMREX_ALWAYS_ASSERT(scomp == 0);
    }
 
    AMREX_ALWAYS_ASSERT(ncomp == vars_new[crse_lev][Vars::cons].nComp() - scomp);
    AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
 
    // ******************************************************************************************
    // First do cell-centered quantities
    // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
    // m is the map scale factor at cell centers
    // Here we multiply (rho S) by detJ and divide (rho S) by m^2 before average down
    // ******************************************************************************************
    for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
      for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
        const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
            });
        } else {
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
            });
        }
      } // mfi
    } // lev
 
    int fine_lev = crse_lev+1;
 
    if (interpolation_type == StateInterpType::Perturbational) {
        // Make the fine rho and (rho theta) be perturbational
        MultiFab::Divide(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
                         Rho_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                           BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                           BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
 
        // Make the crse rho and (rho theta) be perturbational
        MultiFab::Divide(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                         Rho_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                           BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                           BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
    }
 
    if (SolverChoice::terrain_type != TerrainType::EB) {
        average_down(vars_new[crse_lev+1][Vars::cons],vars_new[crse_lev  ][Vars::cons],
                    scomp, ncomp, refRatio(crse_lev));
    } else {
        // const auto dx = geom[fine_lev].CellSize();
        // Setting cell_vol to the exact value may cause round-off errors in volume average.
        // const Real cell_vol = dx[0]*dx[1]*dx[2];
        constexpr Real cell_vol = 1.0;
        const BoxArray& ba = vars_new[fine_lev][IntVars::cons].boxArray();
        const DistributionMapping& dm = vars_new[fine_lev][IntVars::cons].DistributionMap();
        MultiFab vol_fine(ba, dm, 1, 0);
        vol_fine.setVal(cell_vol);
        EB_average_down(vars_new[fine_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                    vol_fine, *detJ_cc[fine_lev],
                    scomp, ncomp, refRatio(crse_lev));
    }
 
    if (interpolation_type == StateInterpType::Perturbational) {
        // Restore the fine data to what it was
        MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                      BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                      BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Multiply(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
                           Rho_comp,RhoTheta_comp,1,IntVect{0});
 
        // Make the crse data be full values not perturbational
        MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                      BaseState::r0_comp,Rho_comp,1,IntVect{0});
        MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                      BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        MultiFab::Multiply(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                           Rho_comp,RhoTheta_comp,1,IntVect{0});
    }
 
    vars_new[crse_lev][Vars::cons].FillBoundary(geom[crse_lev].periodicity());
 
    // ******************************************************************************************
    // Here we multiply (rho S) by m^2 and divide by detJ after average down
    // ******************************************************************************************
    for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
      for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.tilebox();
        const Array4<      Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
        const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
            });
        } else { // MeshType::ConstantDz
            ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
            {
                cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
            });
        }
      } // mfi
    } // lev
 
    // Fill EB covered cells by old values
    // (This won't be needed because EB_average_down copyies the covered value.)
    if (SolverChoice::terrain_type == TerrainType::EB) {
        for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
            for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<      Real> cons_new = vars_new[lev][Vars::cons].array(mfi);
                const Array4<const Real> cons_old = vars_old[lev][Vars::cons].array(mfi);
                const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
                ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                {
                    if (detJ_arr(i,j,k) == 0.0) {
                        cons_new(i,j,k,scomp+n) = cons_old(i,j,k,scomp+n);
                    }
                });
            } // mfi
        } // lev
    }
 
    // ******************************************************************************************
    // Now average down momenta.
    // Note that vars_new holds velocities not momenta, but we want to do conservative
    //    averaging so we first convert to momentum, then average down, then convert
    //    back to velocities -- only on the valid region
    // ******************************************************************************************
    for (int lev = crse_lev; lev <= crse_lev+1; lev++)
    {
        // FillBoundary for density so we can go back and forth between velocity and momentum
        vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
 
        const MultiFab* c_vfrac = nullptr;
        if (SolverChoice::terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
        }
 
        VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
                        vars_new[lev][Vars::yvel], IntVect(0,0,0),
                        vars_new[lev][Vars::zvel], IntVect(0,0,0),
                        vars_new[lev][Vars::cons],
                            rU_new[lev],
                            rV_new[lev],
                            rW_new[lev],
                        Geom(lev).Domain(),
                        domain_bcs_type,
                        c_vfrac);
    }
 
    if (SolverChoice::terrain_type != TerrainType::EB) {
        average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
        average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
        average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
    } else {
        EB_average_down_faces({&rU_new[crse_lev+1], &rV_new[crse_lev+1], &rW_new[crse_lev+1]},
                            {&rU_new[crse_lev], &rV_new[crse_lev], &rW_new[crse_lev]},
                            refRatio(crse_lev), 0);
    }
 
    for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
 
        const MultiFab* c_vfrac = nullptr;
        if (SolverChoice::terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
        }
 
        MomentumToVelocity(vars_new[lev][Vars::xvel],
                        vars_new[lev][Vars::yvel],
                        vars_new[lev][Vars::zvel],
                        vars_new[lev][Vars::cons],
                            rU_new[lev],
                            rV_new[lev],
                            rW_new[lev],
                        Geom(lev).Domain(),
                        domain_bcs_type,
                        c_vfrac);
    }
}

Here is the call graph for this function:

build_fine_mask()

void ERF::build_fine_mask	(	int	lev,
		amrex::MultiFab &	fine_mask
	)

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters

level

Fine level index which masks underlying coarser data

{
    // Mask for zeroing covered cells
    AMREX_ASSERT(level > 0);
 
    BoxArray cba            = grids[level-1];
    DistributionMapping cdm = dmap[level-1];
 
    BoxArray fba            = fine_mask_lev.boxArray();
 
    iMultiFab ifine_mask_lev = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
 
    const auto  fma =  fine_mask_lev.arrays();
    const auto ifma = ifine_mask_lev.arrays();
    ParallelFor(fine_mask_lev, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
    {
       fma[bno](i,j,k) = ifma[bno](i,j,k);
    });
}

check_for_low_temp()

void ERF::check_for_low_temp

(

amrex::MultiFab &

S

)

{
    // *****************************************************************************
    // Test for low temp (low is defined as beyond the microphysics range of validity)
    // *****************************************************************************
    //
    // This value is defined in erf_dtesati in Source/Utils/ERF_MicrophysicsUtils.H
    Real t_low = 273.16 - 85.;
    //
    for (MFIter mfi(S); mfi.isValid(); ++mfi)
    {
        Box bx = mfi.tilebox();
        const Array4<Real> &s_arr  = S.array(mfi);
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            const Real rho      = s_arr(i, j, k, Rho_comp);
            const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
            const Real qv       = s_arr(i, j, k, RhoQ1_comp);
 
            Real temp = getTgivenRandRTh(rho, rhotheta, qv);
 
            if (temp < t_low) {
#ifdef AMREX_USE_GPU
                AMREX_DEVICE_PRINTF("Temperature too low in cell: %d %d %d %e \n", i,j,k,temp);
#else
                printf("Temperature too low in cell: %d %d %d \n", i,j,k);
                printf("Based on temp / rhotheta / rho %e %e %e \n", temp,rhotheta,rho);
#endif
                Abort();
            }
        });
    }
}

Here is the call graph for this function:

check_for_negative_theta()

void ERF::check_for_negative_theta

(

amrex::MultiFab &

S

)

{
    // *****************************************************************************
    // Test for negative (rho theta)
    // *****************************************************************************
    for (MFIter mfi(S); mfi.isValid(); ++mfi)
    {
        Box bx = mfi.tilebox();
        const Array4<Real> &s_arr  = S.array(mfi);
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
            if (rhotheta <= 0.) {
#ifdef AMREX_USE_GPU
                AMREX_DEVICE_PRINTF("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
#else
                printf("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
                Abort("Bad theta in check_for_negative_theta");
#endif
            }
            });
    } // mfi
}

check_state_for_nans()

void ERF::check_state_for_nans

(

amrex::MultiFab const &

S

)

{
    int ncomp = S.nComp();
    for (int lev = 0; lev <= finest_level; lev++)
    {
        //
        // Test at the end of every full timestep whether the solution data contains NaNs
        //
        bool any_have_nans = false;
        for (int i = 0; i < ncomp; i++) {
            if (S.contains_nan(i,1,0))
            {
                amrex::Print() << "Component " << i << " of conserved variables contains NaNs" << '\n';
                any_have_nans = true;
            }
        }
        if (any_have_nans) {
            exit(0);
        }
    }
}

check_vels_for_nans()

void ERF::check_vels_for_nans	(	amrex::MultiFab const &	xvel,
		amrex::MultiFab const &	yvel,
		amrex::MultiFab const &	zvel
	)

{
    //
    // Test at the end of every full timestep whether the solution data contains NaNs
    //
    bool any_have_nans = false;
    if (xvel.contains_nan(0,1,0))
    {
        amrex::Print() << "x-velocity contains NaNs " << '\n';
        any_have_nans = true;
    }
    if (yvel.contains_nan(0,1,0))
    {
        amrex::Print() << "y-velocity contains NaNs" << '\n';
        any_have_nans = true;
    }
    if (zvel.contains_nan(0,1,0))
    {
        amrex::Print() << "z-velocity contains NaNs" << '\n';
        any_have_nans = true;
    }
    if (any_have_nans) {
        exit(0);
    }
}

ClearLevel()

void ERF::ClearLevel ( int  lev )

override

{
    for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
        vars_new[lev][var_idx].clear();
        vars_old[lev][var_idx].clear();
    }
 
    base_state[lev].clear();
 
    rU_new[lev].clear();
    rU_old[lev].clear();
    rV_new[lev].clear();
    rV_old[lev].clear();
    rW_new[lev].clear();
    rW_old[lev].clear();
 
    if (lev > 0) {
        zmom_crse_rhs[lev].clear();
    }
 
    if ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) {
        pp_inc[lev].clear();
    }
    if (solverChoice.anelastic[lev] == 0) {
        lagged_delta_rt[lev].clear();
    }
    avg_xmom[lev].clear();
    avg_ymom[lev].clear();
    avg_zmom[lev].clear();
 
    // Clears the integrator memory
    mri_integrator_mem[lev].reset();
 
    // Clears the physical boundary condition routines
    physbcs_cons[lev].reset();
    physbcs_u[lev].reset();
    physbcs_v[lev].reset();
    physbcs_w[lev].reset();
    physbcs_base[lev].reset();
 
    // Clears the flux register array
    advflux_reg[lev]->reset();
 
    // Clears the 2D arrays
    if (sst_lev[lev][0]) {
        for (int n = 0; n < sst_lev[lev].size(); n++) {
            sst_lev[lev][n].reset();
        }
    }
    if (tsk_lev[lev][0]) {
        for (int n = 0; n < tsk_lev[lev].size(); n++) {
            tsk_lev[lev][n].reset();
        }
    }
    if (lat_m[lev]) {
        lat_m[lev].reset();
    }
    if (lon_m[lev]) {
        lon_m[lev].reset();
    }
    if (sinPhi_m[lev]) {
        sinPhi_m[lev].reset();
    }
    if (cosPhi_m[lev]) {
        cosPhi_m[lev].reset();
    }
}

cloud_fraction()

Real ERF::cloud_fraction

(

amrex::Real

time

)

{
    BL_PROFILE("ERF::cloud_fraction()");
 
    int lev = 0;
    // This holds all of qc
    MultiFab qc(vars_new[lev][Vars::cons],make_alias,RhoQ2_comp,1);
 
    int direction = 2; // z-direction
    Box const& domain = geom[lev].Domain();
 
    auto const& qc_arr = qc.const_arrays();
 
    // qc_2d is an BaseFab<int> holding the max value over the column
    auto qc_2d = ReduceToPlane<ReduceOpMax,int>(direction, domain, qc,
         [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) -> int
         {
             if (qc_arr[box_no](i,j,k) > 0) {
                 return 1;
             } else {
                 return 0;
             }
         });
 
    auto* p = qc_2d.dataPtr();
 
    Long numpts = qc_2d.numPts();
 
    AMREX_ASSERT(numpts < Long(std::numeric_limits<int>::max));
 
#if 1
    if (ParallelDescriptor::UseGpuAwareMpi()) {
        ParallelDescriptor::ReduceIntMax(p,static_cast<int>(numpts));
    } else {
        Gpu::PinnedVector<int> hv(numpts);
        Gpu::copyAsync(Gpu::deviceToHost, p, p+numpts, hv.data());
        Gpu::streamSynchronize();
        ParallelDescriptor::ReduceIntMax(hv.data(),static_cast<int>(numpts));
        Gpu::copyAsync(Gpu::hostToDevice, hv.data(), hv.data()+numpts, p);
    }
 
    // Sum over component 0
    Long num_cloudy = qc_2d.template sum<RunOn::Device>(0);
 
#else
    //
    // We need this if we allow domain decomposition in the vertical
    //    but for now we leave it commented out
    //
    Long num_cloudy = Reduce::Sum<Long>(numpts,
         [=] AMREX_GPU_DEVICE (Long i) -> Long {
             if (p[i] == 1) {
                 return 1;
             } else {
                 return 0;
             }
         });
    ParallelDescriptor::ReduceLongSum(num_cloudy);
#endif
 
    Real num_total = qc_2d.box().d_numPts();
 
    Real cloud_frac = num_cloudy / num_total;
 
    return cloud_frac;
}

compute_divergence()

void ERF::compute_divergence	(	int	lev,
		amrex::MultiFab &	rhs,
		amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM >	rho0_u_const,
		amrex::Geometry const &	geom_at_lev
	)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level 0.

{
    BL_PROFILE("ERF::compute_divergence()");
 
    auto dxInv = geom_at_lev.InvCellSizeArray();
 
    // ****************************************************************************
    // Compute divergence which will form RHS
    // Note that we replace "rho0w" with the contravariant momentum, Omega
    // ****************************************************************************
    if (solverChoice.terrain_type == TerrainType::EB)
    {
        bool already_on_centroids = true;
        EB_computeDivergence(rhs, rho0_u_const, geom_at_lev, already_on_centroids);
    }
    else if (SolverChoice::mesh_type == MeshType::ConstantDz)
    {
        computeDivergence(rhs, rho0_u_const, geom_at_lev);
    }
    else
    {
        for ( MFIter mfi(rhs,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            Box bx = mfi.tilebox();
            const Array4<Real const>& rho0u_arr = rho0_u_const[0]->const_array(mfi);
            const Array4<Real const>& rho0v_arr = rho0_u_const[1]->const_array(mfi);
            const Array4<Real const>& rho0w_arr = rho0_u_const[2]->const_array(mfi);
            const Array4<Real      >&   rhs_arr = rhs.array(mfi);
 
            const Array4<Real const>&      mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
            const Array4<Real const>&      mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
            const Array4<Real const>&      mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
            const Array4<Real const>&      mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
 
            if (SolverChoice::mesh_type == MeshType::StretchedDz)
            {
                Real* stretched_dz_d_ptr = stretched_dz_d[lev].data();
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    Real inv_dz = 1.0/stretched_dz_d_ptr[k];
                    Real mfsq   = mf_mx(i,j,0) * mf_my(i,j,0);
                    rhs_arr(i,j,k) = (  (rho0u_arr(i+1,j  ,k  )/mf_uy(i+1,j,0) - rho0u_arr(i,j,k)/mf_uy(i,j,0)) * dxInv[0]
                                       +(rho0v_arr(i  ,j+1,k  )/mf_vx(i,j+1,0) - rho0v_arr(i,j,k)/mf_vx(i,j,0)) * dxInv[1]
                                       +(rho0w_arr(i  ,j  ,k+1)/mfsq           - rho0w_arr(i,j,k)/mfsq        ) * inv_dz  ) * mfsq;
                });
            }
            else
            {
                //
                // Note we compute the divergence using "rho0w" == Omega
                //
                const Array4<Real const>& ax_arr = ax[lev]->const_array(mfi);
                const Array4<Real const>& ay_arr = ay[lev]->const_array(mfi);
                const Array4<Real const>& dJ_arr = detJ_cc[lev]->const_array(mfi);
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
                    rhs_arr(i,j,k) = ( ( ax_arr(i+1,j,k)*rho0u_arr(i+1,j,k)/mf_uy(i+1,j,0)
                                        -ax_arr(i  ,j,k)*rho0u_arr(i  ,j,k)/mf_uy(i  ,j,0)  ) * dxInv[0]
                                     + ( ay_arr(i,j+1,k)*rho0v_arr(i,j+1,k)/mf_vx(i,j+1,0)
                                        -ay_arr(i,j  ,k)*rho0v_arr(i,j  ,k)/mf_vx(i,j  ,0)  ) * dxInv[1]
                                      +(                 rho0w_arr(i,j,k+1)/mfsq
                                        -                rho0w_arr(i,j,k  )/mfsq            ) * dxInv[2] ) * mfsq / dJ_arr(i,j,k);
                });
            }
        } // mfi
    }
}

ComputeDt()

void ERF::ComputeDt ( int  step = -1 )

private

Function that calls estTimeStep for each level

{
    Vector<Real> dt_tmp(finest_level+1);
 
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
    }
 
    ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
 
    Real dt_0 = dt_tmp[0];
    int n_factor = 1;
    for (int lev = 0; lev <= finest_level; ++lev) {
        dt_tmp[lev] = amrex::min(dt_tmp[lev], change_max*dt[lev]);
        n_factor *= nsubsteps[lev];
        dt_0 = amrex::min(dt_0, n_factor*dt_tmp[lev]);
        if (step == 0){
            dt_0 *= init_shrink;
            if (verbose && init_shrink != 1.0) {
                Print() << "Timestep 0: shrink initial dt at level " << lev << " by " << init_shrink << std::endl;
            }
        }
    }
    // Limit dt's by the value of stop_time.
    const Real eps = 1.e-3*dt_0;
    if (t_new[0] + dt_0 > stop_time - eps) {
        dt_0 = stop_time - t_new[0];
    }
 
    dt[0] = dt_0;
    for (int lev = 1; lev <= finest_level; ++lev) {
        dt[lev] = dt[lev-1] / nsubsteps[lev];
    }
}

ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells ( const SolverChoice &  sc )

inlinestaticprivate

    {
        int ngrow = 0;
 
        if (sc.use_num_diff)
        {
            ngrow = 3;
        } else {
            if (
                 (sc.advChoice.dycore_horiz_adv_type    == AdvType::Centered_6th)
              || (sc.advChoice.dycore_vert_adv_type     == AdvType::Centered_6th)
              || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
              || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
              || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Centered_6th)
              || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Centered_6th) )
            { ngrow = 3; }
            else if (
                 (sc.advChoice.dycore_horiz_adv_type    == AdvType::Upwind_5th)
              || (sc.advChoice.dycore_vert_adv_type     == AdvType::Upwind_5th)
              || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
              || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
              || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Upwind_5th)
              || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Upwind_5th) )
            { ngrow = 3; }
            else if (
                  (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5)
               || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_5)
               || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_5)
               || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_5)
               || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_5Z)
               || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_5Z)
               || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5Z)
               || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_5Z) )
            { ngrow = 3; }
            else if (
                  (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_7)
               || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_7)
               || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_7)
               || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_7)
               || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_7Z)
               || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_7Z)
               || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_7Z)
               || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_7Z) )
            { ngrow = 4; }
            else
            {
                if (sc.terrain_type == TerrainType::EB){
                    ngrow = 4;
                } else {
                    ngrow = 2;
                }
            }
        }
 
        return ngrow;
    }

Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int  lev )

private

{
    auto& fine_new = vars_new[lev];
    auto& crse_new = vars_new[lev-1];
    auto& ba_fine  = fine_new[Vars::cons].boxArray();
    auto& ba_crse  = crse_new[Vars::cons].boxArray();
    auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
    auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
 
    int ncomp = vars_new[lev][Vars::cons].nComp();
 
    FPr_c.emplace_back(ba_fine, dm_fine, geom[lev]  ,
                       ba_crse, dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
    FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                       convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                       convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                       convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                       -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
}

create_random_perturbations()

void ERF::create_random_perturbations	(	const int	lev,
		amrex::MultiFab &	cons_pert,
		amrex::MultiFab &	xvel_pert,
		amrex::MultiFab &	yvel_pert,
		amrex::MultiFab &	zvel_pert
	)

{
    ignore_unused(cons_pert);
    ignore_unused(yvel_pert);
    ignore_unused(zvel_pert);
 
    auto& lev_new = vars_new[lev];
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
        const auto &xvel_pert_arr = xvel_pert.array(mfi);
        const Box &xbx = mfi.tilebox(IntVect(1,0,0));
        ParallelForRNG(xbx, [=] AMREX_GPU_DEVICE(int i, int j, int k, const amrex::RandomEngine& engine) noexcept
        {
            xvel_pert_arr(i, j, k) = amrex::Random(engine);
        });
    }
}

Here is the call graph for this function:

DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int  i )

inlineprivate

    {
        return *datalog[i];
    }

DataLogName()

std::string ERF::DataLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith datalog file.

{ return datalogname[i]; }

Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int  lev )

private

{
    auto& fine_new = vars_new[lev];
    auto& crse_new = vars_new[lev-1];
    auto& ba_fine  = fine_new[Vars::cons].boxArray();
    auto& ba_crse  = crse_new[Vars::cons].boxArray();
    auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
    auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
 
    int ncomp = fine_new[Vars::cons].nComp();
 
    FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev]  ,
                        ba_crse, dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
    FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
    FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
}

DerDataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DerDataLog ( int  i )

inlineprivate

    {
        return *der_datalog[i];
    }

DerDataLogName()

std::string ERF::DerDataLogName ( int  i ) const

inlineprivatenoexcept

{ return der_datalogname[i]; }

derive_diag_profiles()

void ERF::derive_diag_profiles	(	amrex::Real	time,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Kmv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Khv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qi,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qg,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wthv
	)

Computes the profiles for diagnostic quantities.

Parameters

h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_uiuiu	Profile for u_i*u_i*u triple product on Host
h_avg_uiuiv	Profile for u_i*u_i*v triple product on Host
h_avg_uiuiw	Profile for u_i*u_i*w triple product on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host

{
    // We assume that this is always called at level 0
    int lev = 0;
 
    bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
    bool l_use_KE    = solverChoice.turbChoice[lev].use_tke;
    // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, uw, vv, vw, ww, uth, vth, wth,
    //                  0      1     2    3    4   5   6   7   8   9  10   11   12   13
    //                thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr,
    //                  14     15     16     17 18  19  20  21  22  23  24   25   26   27
    //                  qi, qs, qg, wthv
    //                  28  29  30    31
    MultiFab mf_out(grids[lev], dmap[lev], 32, 0);
 
    MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
 
    MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
    MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
    MultiFab  w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
 
    average_face_to_cellcenter(mf_vels,0,
        Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
 
    int zdir = 2;
    auto domain = geom[0].Domain();
 
    // Sum in the horizontal plane
    h_avg_u  = sumToLine(mf_vels ,0,1,domain,zdir);
    h_avg_v  = sumToLine(mf_vels ,1,1,domain,zdir);
    h_avg_w  = sumToLine(mf_vels ,2,1,domain,zdir);
 
    int hu_size =  h_avg_u.size();
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < hu_size; ++k) {
        h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
    }
 
    Gpu::DeviceVector<Real> d_avg_u(hu_size, Real(0.0));
    Gpu::DeviceVector<Real> d_avg_v(hu_size, Real(0.0));
    Gpu::DeviceVector<Real> d_avg_w(hu_size, Real(0.0));
 
#if 0
    auto* avg_u_ptr = d_avg_u.data();
    auto* avg_v_ptr = d_avg_v.data();
    auto* avg_w_ptr = d_avg_w.data();
#endif
 
    Gpu::copy(Gpu::hostToDevice, h_avg_u.begin(), h_avg_u.end(), d_avg_u.begin());
    Gpu::copy(Gpu::hostToDevice, h_avg_v.begin(), h_avg_v.end(), d_avg_v.begin());
    Gpu::copy(Gpu::hostToDevice, h_avg_w.begin(), h_avg_w.end(), d_avg_w.begin());
 
    int nvars = vars_new[lev][Vars::cons].nComp();
    MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
 
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
 
    for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
        const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
        const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
        const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
        const Array4<Real>& cons_arr = mf_cons.array(mfi);
        const Array4<Real>&   p0_arr = p_hse.array(mfi);
        const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
                                                            Array4<const Real>{};
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 1) = theta;
            Real ksgs = 0.0;
            if (l_use_KE) {
                ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
            }
            fab_arr(i, j, k, 2) = ksgs;
#if 1
            if (l_use_kturb) {
                fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
                fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
            } else {
                fab_arr(i, j, k, 3) = 0.0;
                fab_arr(i, j, k, 4) = 0.0;
            }
#else
            // Here we hijack the "Kturb" variable name to print out the resolved kinetic energy
            Real upert = u_cc_arr(i,j,k) - avg_u_ptr[k];
            Real vpert = v_cc_arr(i,j,k) - avg_v_ptr[k];
            Real wpert = w_cc_arr(i,j,k) - avg_w_ptr[k];
            fab_arr(i, j, k, 3) = 0.5 * (upert*upert + vpert*vpert + wpert*wpert);
#endif
            fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
            fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
            fab_arr(i, j, k, 7) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // u*w
            fab_arr(i, j, k, 8) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
            fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // v*w
            fab_arr(i, j, k,10) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // w*w
            fab_arr(i, j, k,11) = u_cc_arr(i,j,k) * theta; // u*th
            fab_arr(i, j, k,12) = v_cc_arr(i,j,k) * theta; // v*th
            fab_arr(i, j, k,13) = w_cc_arr(i,j,k) * theta; // w*th
            fab_arr(i, j, k,14) = theta * theta;           // th*th
 
            // if the number of fields is changed above, then be sure to update
            // the following def!
            Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,8) + fab_arr(i,j,k,10);
            fab_arr(i, j, k,15) = uiui * u_cc_arr(i,j,k);   // (ui*ui)*u
            fab_arr(i, j, k,16) = uiui * v_cc_arr(i,j,k);   // (ui*ui)*v
            fab_arr(i, j, k,17) = uiui * w_cc_arr(i,j,k);   // (ui*ui)*w
 
            if (!use_moisture) {
                Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,18) = p;                       // p
                fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k);     // p*w
                fab_arr(i, j, k,22) = 0.;  // qv
                fab_arr(i, j, k,23) = 0.;  // qc
                fab_arr(i, j, k,24) = 0.;  // qr
                fab_arr(i, j, k,25) = 0.;  // w*qv
                fab_arr(i, j, k,26) = 0.;  // w*qc
                fab_arr(i, j, k,27) = 0.;  // w*qr
                fab_arr(i, j, k,28) = 0.;  // qi
                fab_arr(i, j, k,29) = 0.;  // qs
                fab_arr(i, j, k,30) = 0.;  // qg
                fab_arr(i, j, k,31) = 0.;  // w*thv
            }
        });
    } // mfi
 
    if (use_moisture)
    {
        int n_qstate_moist = micro->Get_Qstate_Moist_Size();
 
        for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& fab_arr  = mf_out.array(mfi);
            const Array4<Real>& cons_arr = mf_cons.array(mfi);
            const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
            const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
            const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
            const Array4<Real>&   p0_arr = p_hse.array(mfi);
 
            int rhoqr_comp = solverChoice.moisture_indices.qr;
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
                Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
                Real qr = (rhoqr_comp > -1) ?  cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
                                               Real(0.0);
                Real p  = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
 
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,18) = p;                       // p
                fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k);     // p*w
                fab_arr(i, j, k,22) = qv;  // qv
                fab_arr(i, j, k,23) = qc;  // qc
                fab_arr(i, j, k,24) = qr;  // qr
                fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * qv;  // w*qv
                fab_arr(i, j, k,26) = w_cc_arr(i,j,k) * qc;  // w*qc
                fab_arr(i, j, k,27) = w_cc_arr(i,j,k) * qr;  // w*qr
                if (n_qstate_moist > 3) {
                    fab_arr(i, j, k,28) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp);  // qi
                    fab_arr(i, j, k,29) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp);  // qs
                    fab_arr(i, j, k,30) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp);  // qg
                } else {
                    fab_arr(i, j, k,28) = 0.0;  // qi
                    fab_arr(i, j, k,29) = 0.0;  // qs
                    fab_arr(i, j, k,30) = 0.0;  // qg
                }
                Real ql    = qc + qr;
                Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
                Real thv   = theta * (1 + 0.61*qv - ql);
                fab_arr(i, j, k,31) = w_cc_arr(i,j,k) * thv; // w*thv
            });
        } // mfi
    } // use_moisture
 
    h_avg_rho   = sumToLine(mf_out, 0,1,domain,zdir);
    h_avg_th    = sumToLine(mf_out, 1,1,domain,zdir);
    h_avg_ksgs  = sumToLine(mf_out, 2,1,domain,zdir);
    h_avg_Kmv   = sumToLine(mf_out, 3,1,domain,zdir);
    h_avg_Khv   = sumToLine(mf_out, 4,1,domain,zdir);
    h_avg_uu    = sumToLine(mf_out, 5,1,domain,zdir);
    h_avg_uv    = sumToLine(mf_out, 6,1,domain,zdir);
    h_avg_uw    = sumToLine(mf_out, 7,1,domain,zdir);
    h_avg_vv    = sumToLine(mf_out, 8,1,domain,zdir);
    h_avg_vw    = sumToLine(mf_out, 9,1,domain,zdir);
    h_avg_ww    = sumToLine(mf_out,10,1,domain,zdir);
    h_avg_uth   = sumToLine(mf_out,11,1,domain,zdir);
    h_avg_vth   = sumToLine(mf_out,12,1,domain,zdir);
    h_avg_wth   = sumToLine(mf_out,13,1,domain,zdir);
    h_avg_thth  = sumToLine(mf_out,14,1,domain,zdir);
    h_avg_uiuiu = sumToLine(mf_out,15,1,domain,zdir);
    h_avg_uiuiv = sumToLine(mf_out,16,1,domain,zdir);
    h_avg_uiuiw = sumToLine(mf_out,17,1,domain,zdir);
    h_avg_p     = sumToLine(mf_out,18,1,domain,zdir);
    h_avg_pu    = sumToLine(mf_out,19,1,domain,zdir);
    h_avg_pv    = sumToLine(mf_out,20,1,domain,zdir);
    h_avg_pw    = sumToLine(mf_out,21,1,domain,zdir);
    h_avg_qv    = sumToLine(mf_out,22,1,domain,zdir);
    h_avg_qc    = sumToLine(mf_out,23,1,domain,zdir);
    h_avg_qr    = sumToLine(mf_out,24,1,domain,zdir);
    h_avg_wqv   = sumToLine(mf_out,25,1,domain,zdir);
    h_avg_wqc   = sumToLine(mf_out,26,1,domain,zdir);
    h_avg_wqr   = sumToLine(mf_out,27,1,domain,zdir);
    h_avg_qi    = sumToLine(mf_out,28,1,domain,zdir);
    h_avg_qs    = sumToLine(mf_out,29,1,domain,zdir);
    h_avg_qg    = sumToLine(mf_out,30,1,domain,zdir);
    h_avg_wthv  = sumToLine(mf_out,31,1,domain,zdir);
 
    // Divide by the total number of cells we are averaging over
    int h_avg_u_size = static_cast<int>(h_avg_u.size());
    for (int k = 0; k < h_avg_u_size; ++k) {
        h_avg_rho[k] /= area_z;
        h_avg_ksgs[k]  /= area_z;
        h_avg_Kmv[k]   /= area_z;
        h_avg_Khv[k]   /= area_z;
        h_avg_th[k]    /= area_z;
        h_avg_thth[k]  /= area_z;
        h_avg_uu[k]    /= area_z;
        h_avg_uv[k]    /= area_z;
        h_avg_uw[k]    /= area_z;
        h_avg_vv[k]    /= area_z;
        h_avg_vw[k]    /= area_z;
        h_avg_ww[k]    /= area_z;
        h_avg_uth[k]   /= area_z;
        h_avg_vth[k]   /= area_z;
        h_avg_wth[k]   /= area_z;
        h_avg_uiuiu[k] /= area_z;
        h_avg_uiuiv[k] /= area_z;
        h_avg_uiuiw[k] /= area_z;
        h_avg_p[k]     /= area_z;
        h_avg_pu[k]    /= area_z;
        h_avg_pv[k]    /= area_z;
        h_avg_pw[k]    /= area_z;
        h_avg_qv[k]    /= area_z;
        h_avg_qc[k]    /= area_z;
        h_avg_qr[k]    /= area_z;
        h_avg_wqv[k]   /= area_z;
        h_avg_wqc[k]   /= area_z;
        h_avg_wqr[k]   /= area_z;
        h_avg_qi[k]    /= area_z;
        h_avg_qs[k]    /= area_z;
        h_avg_qg[k]    /= area_z;
        h_avg_wthv[k]  /= area_z;
    }
 
#if 0
    // Here we print the integrated total kinetic energy as computed in the 1D profile above
    Real sum = 0.;
    Real dz = geom[0].ProbHi(2) / static_cast<Real>(h_avg_u_size);
    for (int k = 0; k < h_avg_u_size; ++k) {
        sum += h_avg_kturb[k] * h_avg_rho[k] * dz;
    }
    amrex::Print() << "ITKE " << time << " " << sum << " using " << h_avg_u_size << " " << dz << std::endl;
#endif
}

Here is the call graph for this function:

derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag	(	amrex::Real	time,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Kmv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Khv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qi,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qg,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wthv
	)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters

h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_uiuiu	Profile for u_i*u_i*u triple product on Host
h_avg_uiuiv	Profile for u_i*u_i*v triple product on Host
h_avg_uiuiw	Profile for u_i*u_i*w triple product on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host

{
    // We assume that this is always called at level 0
    int lev = 0;
 
    bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
    bool l_use_KE    = solverChoice.turbChoice[lev].use_tke;
    // Note: "uiui" == u_i*u_i = u*u + v*v + w*w
    // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, vv, uth, vth,
    //       indices:   0      1     2    3    4   5   6   7    8    9
    //                thth, uiuiu, uiuiv, p, pu, pv, qv, qc, qr, qi, qs, qg
    //                  10     11     12 13  14  15  16  17  18  19  20  21
    MultiFab mf_out(grids[lev], dmap[lev], 22, 0);
 
    // This will hold uw, vw, ww, wth, uiuiw, pw, wqv, wqc, wqr, wthv
    //       indices:  0   1   2    3      4   5    6    7    8     9
    MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 10, 0);
 
    // This is only used to average u and v; w is not averaged to cell centers
    MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
 
    MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
    MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
    MultiFab  w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers (staggered)
 
    int zdir = 2;
    auto domain = geom[0].Domain();
    Box stag_domain = domain;
    stag_domain.convert(IntVect(0,0,1));
 
    int nvars = vars_new[lev][Vars::cons].nComp();
    MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
 
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
 
    for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
        const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
        const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
        const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
        const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
        const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
        const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
        const Array4<Real>& cons_arr = mf_cons.array(mfi);
        const Array4<Real>&   p0_arr = p_hse.array(mfi);
        const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
                                                            Array4<const Real>{};
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            u_cc_arr(i,j,k) = 0.5 * (u_arr(i,j,k) + u_arr(i+1,j  ,k));
            v_cc_arr(i,j,k) = 0.5 * (v_arr(i,j,k) + v_arr(i  ,j+1,k));
 
            Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
            fab_arr(i, j, k, 1) = theta;
            Real ksgs = 0.0;
            if (l_use_KE) {
                ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
            }
            fab_arr(i, j, k, 2) = ksgs;
            if (l_use_kturb) {
                fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
                fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
            } else {
                fab_arr(i, j, k, 3) = 0.0;
                fab_arr(i, j, k, 4) = 0.0;
            }
            fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
            fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
            fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
            fab_arr(i, j, k, 8) = u_cc_arr(i,j,k) * theta;             // u*th
            fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * theta;             // v*th
            fab_arr(i, j, k,10) = theta * theta;                       // th*th
 
            Real wcc = 0.5 * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
 
            // if the number of fields is changed above, then be sure to update
            // the following def!
            Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,7) + wcc*wcc;
            fab_arr(i, j, k,11) = uiui * u_cc_arr(i,j,k);           // (ui*ui)*u
            fab_arr(i, j, k,12) = uiui * v_cc_arr(i,j,k);           // (ui*ui)*v
 
            if (!use_moisture) {
                Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,13) = p;                       // p
                fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,16) = 0.;  // qv
                fab_arr(i, j, k,17) = 0.;  // qc
                fab_arr(i, j, k,18) = 0.;  // qr
                fab_arr(i, j, k,19) = 0.;  // qi
                fab_arr(i, j, k,20) = 0.;  // qs
                fab_arr(i, j, k,21) = 0.;  // qg
            }
        });
 
        const Box& zbx = mfi.tilebox(IntVect(0,0,1));
        ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            // average to z faces (first to cell centers, then in z)
            Real uface = 0.25 * ( u_arr(i  ,j,k) + u_arr(i  ,j,k-1)
                                + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
            Real vface = 0.25 * ( v_arr(i,j  ,k) + v_arr(i,j  ,k-1)
                                + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
            Real theta0 = cons_arr(i,j,k  ,RhoTheta_comp) / cons_arr(i,j,k  ,Rho_comp);
            Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
            Real thface = 0.5*(theta0 + theta1);
            fab_arr_stag(i,j,k,0) =           uface * w_fc_arr(i,j,k); // u*w
            fab_arr_stag(i,j,k,1) =           vface * w_fc_arr(i,j,k); // v*w
            fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
            fab_arr_stag(i,j,k,3) =          thface * w_fc_arr(i,j,k); // th*w
            Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
            fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
            if (!use_moisture) {
                Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp)) - p0_arr(i,j,k  );
                Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
                Real pface = 0.5 * (p0 + p1);
                fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k);       // p*w
                fab_arr_stag(i,j,k,6) = 0.;  // w*qv
                fab_arr_stag(i,j,k,7) = 0.;  // w*qc
                fab_arr_stag(i,j,k,8) = 0.;  // w*qr
                fab_arr_stag(i,j,k,9) = 0.;  // w*thv
            }
        });
 
    } // mfi
 
    if (use_moisture)
    {
        int n_qstate_moist = micro->Get_Qstate_Moist_Size();
 
        for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& fab_arr  = mf_out.array(mfi);
            const Array4<Real>& fab_arr_stag  = mf_out_stag.array(mfi);
            const Array4<Real>& cons_arr = mf_cons.array(mfi);
            const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
            const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
            const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
            const Array4<Real>&   p0_arr = p_hse.array(mfi);
 
            int rhoqr_comp = solverChoice.moisture_indices.qr;
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
                Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
                Real qr = (rhoqr_comp > -1) ?  cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
                                               Real(0.0);
                Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
 
                p -= p0_arr(i,j,k);
                fab_arr(i, j, k,13) = p;                       // p
                fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k);     // p*u
                fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k);     // p*v
                fab_arr(i, j, k,16) = qv;  // qv
                fab_arr(i, j, k,17) = qc;  // qc
                fab_arr(i, j, k,18) = qr;  // qr
                if (n_qstate_moist > 3) { // SAM model
                    fab_arr(i, j, k,19) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp);  // qi
                    fab_arr(i, j, k,20) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp);  // qs
                    fab_arr(i, j, k,21) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp);  // qg
                } else {
                    fab_arr(i, j, k,19) = 0.0;  // qi
                    fab_arr(i, j, k,20) = 0.0;  // qs
                    fab_arr(i, j, k,21) = 0.0;  // qg
                }
            });
 
            const Box& zbx = mfi.tilebox(IntVect(0,0,1));
            ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
            {
                Real qv0 = cons_arr(i,j,k  ,RhoQ1_comp) / cons_arr(i,j,k  ,Rho_comp);
                Real qv1 = cons_arr(i,j,k-1,RhoQ1_comp) / cons_arr(i,j,k-1,Rho_comp);
                Real qc0 = cons_arr(i,j,k  ,RhoQ2_comp) / cons_arr(i,j,k  ,Rho_comp);
                Real qc1 = cons_arr(i,j,k-1,RhoQ2_comp) / cons_arr(i,j,k-1,Rho_comp);
                Real qr0 = (rhoqr_comp > -1) ? cons_arr(i,j,k  ,RhoQ3_comp) / cons_arr(i,j,k  ,Rho_comp) :
                                               Real(0.0);
                Real qr1 = (rhoqr_comp > -1) ? cons_arr(i,j,k-1,RhoQ3_comp) / cons_arr(i,j,k-1,Rho_comp) :
                                               Real(0.0);
                Real qvface = 0.5 * (qv0 + qv1);
                Real qcface = 0.5 * (qc0 + qc1);
                Real qrface = 0.5 * (qr0 + qr1);
 
                Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp), qv0) - p0_arr(i,j,k  );
                Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv1) - p0_arr(i,j,k-1);
                Real pface = 0.5 * (p0 + p1);
 
                Real theta0 = cons_arr(i,j,k  ,RhoTheta_comp) / cons_arr(i,j,k  ,Rho_comp);
                Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
                Real thface = 0.5*(theta0 + theta1);
                Real ql = qcface + qrface;
                Real thv = thface * (1 + 0.61*qvface - ql);
 
                fab_arr_stag(i,j,k,5) = pface  * w_fc_arr(i,j,k); // p*w
                fab_arr_stag(i,j,k,6) = qvface * w_fc_arr(i,j,k); // w*qv
                fab_arr_stag(i,j,k,7) = qcface * w_fc_arr(i,j,k); // w*qc
                fab_arr_stag(i,j,k,8) = qrface * w_fc_arr(i,j,k); // w*qr
                fab_arr_stag(i,j,k,9) = thv    * w_fc_arr(i,j,k); // w*thv
            });
        } // mfi
    } // use_moisture
 
    // Sum in the horizontal plane
    h_avg_u = sumToLine(u_cc,0,1,     domain,zdir);
    h_avg_v = sumToLine(v_cc,0,1,     domain,zdir);
    h_avg_w = sumToLine(w_fc,0,1,stag_domain,zdir);
 
    h_avg_rho   = sumToLine(mf_out, 0,1,domain,zdir);
    h_avg_th    = sumToLine(mf_out, 1,1,domain,zdir);
    h_avg_ksgs  = sumToLine(mf_out, 2,1,domain,zdir);
    h_avg_Kmv   = sumToLine(mf_out, 3,1,domain,zdir);
    h_avg_Khv   = sumToLine(mf_out, 4,1,domain,zdir);
    h_avg_uu    = sumToLine(mf_out, 5,1,domain,zdir);
    h_avg_uv    = sumToLine(mf_out, 6,1,domain,zdir);
    h_avg_vv    = sumToLine(mf_out, 7,1,domain,zdir);
    h_avg_uth   = sumToLine(mf_out, 8,1,domain,zdir);
    h_avg_vth   = sumToLine(mf_out, 9,1,domain,zdir);
    h_avg_thth  = sumToLine(mf_out,10,1,domain,zdir);
    h_avg_uiuiu = sumToLine(mf_out,11,1,domain,zdir);
    h_avg_uiuiv = sumToLine(mf_out,12,1,domain,zdir);
    h_avg_p     = sumToLine(mf_out,13,1,domain,zdir);
    h_avg_pu    = sumToLine(mf_out,14,1,domain,zdir);
    h_avg_pv    = sumToLine(mf_out,15,1,domain,zdir);
    h_avg_qv    = sumToLine(mf_out,16,1,domain,zdir);
    h_avg_qc    = sumToLine(mf_out,17,1,domain,zdir);
    h_avg_qr    = sumToLine(mf_out,18,1,domain,zdir);
    h_avg_qi    = sumToLine(mf_out,19,1,domain,zdir);
    h_avg_qs    = sumToLine(mf_out,20,1,domain,zdir);
    h_avg_qg    = sumToLine(mf_out,21,1,domain,zdir);
 
    h_avg_uw    = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
    h_avg_vw    = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
    h_avg_ww    = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
    h_avg_wth   = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
    h_avg_uiuiw = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
    h_avg_pw    = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
    h_avg_wqv   = sumToLine(mf_out_stag,6,1,stag_domain,zdir);
    h_avg_wqc   = sumToLine(mf_out_stag,7,1,stag_domain,zdir);
    h_avg_wqr   = sumToLine(mf_out_stag,8,1,stag_domain,zdir);
    h_avg_wthv  = sumToLine(mf_out_stag,9,1,stag_domain,zdir);
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
    for (int k = 0; k < unstag_size; ++k) {
        h_avg_u[k]     /= area_z;
        h_avg_v[k]     /= area_z;
        h_avg_rho[k]   /= area_z;
        h_avg_ksgs[k]  /= area_z;
        h_avg_Kmv[k]   /= area_z;
        h_avg_Khv[k]   /= area_z;
        h_avg_th[k]    /= area_z;
        h_avg_thth[k]  /= area_z;
        h_avg_uu[k]    /= area_z;
        h_avg_uv[k]    /= area_z;
        h_avg_vv[k]    /= area_z;
        h_avg_uth[k]   /= area_z;
        h_avg_vth[k]   /= area_z;
        h_avg_uiuiu[k] /= area_z;
        h_avg_uiuiv[k] /= area_z;
        h_avg_p[k]     /= area_z;
        h_avg_pu[k]    /= area_z;
        h_avg_pv[k]    /= area_z;
        h_avg_qv[k]    /= area_z;
        h_avg_qc[k]    /= area_z;
        h_avg_qr[k]    /= area_z;
        h_avg_qi[k]    /= area_z;
        h_avg_qs[k]    /= area_z;
        h_avg_qg[k]    /= area_z;
    }
 
    for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
        h_avg_w[k]     /= area_z;
        h_avg_uw[k]    /= area_z;
        h_avg_vw[k]    /= area_z;
        h_avg_ww[k]    /= area_z;
        h_avg_wth[k]   /= area_z;
        h_avg_uiuiw[k] /= area_z;
        h_avg_pw[k]    /= area_z;
        h_avg_wqv[k]   /= area_z;
        h_avg_wqc[k]   /= area_z;
        h_avg_wqr[k]   /= area_z;
        h_avg_wthv[k]  /= area_z;
    }
}

Here is the call graph for this function:

derive_stress_profiles()

void ERF::derive_stress_profiles	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q1fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q2fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

{
    int lev = 0;
 
    // This will hold the stress tensor components
    MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
 
    MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
 
    bool l_use_moist   = ( solverChoice.moisture_type != MoistureType::None );
 
    for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
 
        const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
 
        // NOTE: These are from the last RK stage...
        const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
        const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
        const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
        const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
        const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
        const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
 
        // These should be re-calculated during ERF_slow_rhs_post
        // -- just vertical SFS kinematic heat flux for now
        //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
        //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
        const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
        const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
            fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 1) = ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
                                / (   rho_arr(i,j  ,k) +   rho_arr(i+1,j  ,k)
                                  +   rho_arr(i,j+1,k) +   rho_arr(i+1,j+1,k) );
            fab_arr(i, j, k, 2) = ( tau13_arr(i,j,k  ) + tau13_arr(i+1,j,k  )
                                  + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) )
                                / (   rho_arr(i,j,k  ) +   rho_arr(i+1,j,k  )
                                  +   rho_arr(i,j,k+1) +   rho_arr(i+1,j,k+1) );
            fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 4) = ( tau23_arr(i,j,k  ) + tau23_arr(i,j+1,k  )
                                  + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) )
                                / (   rho_arr(i,j,k  ) +   rho_arr(i,j+1,k  )
                                  +   rho_arr(i,j,k+1) +   rho_arr(i,j+1,k+1) );
            fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 6) = 0.5 * ( hfx3_arr(i,j,k) + hfx3_arr(i,j,k+1) ) / rho_arr(i,j,k);
            fab_arr(i, j, k, 7) = (l_use_moist) ? 0.5 * ( q1fx3_arr(i,j,k) + q1fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
            fab_arr(i, j, k, 8) = (l_use_moist) ? 0.5 * ( q2fx3_arr(i,j,k) + q2fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
            fab_arr(i, j, k, 9) =  diss_arr(i,j,k) / rho_arr(i,j,k);
        });
    }
 
    int zdir = 2;
    auto domain = geom[0].Domain();
 
    h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
    h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
    h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
    h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
    h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
    h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
    h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
    h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
    h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
    h_avg_diss  = sumToLine(mf_out,9,1,domain,zdir);
 
    int ht_size =  h_avg_tau11.size();
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < ht_size; ++k) {
        h_avg_tau11[k] /= area_z;
        h_avg_tau12[k] /= area_z;
        h_avg_tau13[k] /= area_z;
        h_avg_tau22[k] /= area_z;
        h_avg_tau23[k] /= area_z;
        h_avg_tau33[k] /= area_z;
        h_avg_hfx3[k] /= area_z;
        h_avg_q1fx3[k] /= area_z;
        h_avg_q2fx3[k] /= area_z;
        h_avg_diss[k] /= area_z;
    }
}

derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q1fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q2fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

{
    int lev = 0;
 
    // This will hold the stress tensor components
    MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
 
    // This will hold Tau13 and Tau23
    MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 5, 0);
 
    MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
 
    bool l_use_moist   = ( solverChoice.moisture_type != MoistureType::None );
 
    for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        const Array4<Real>& fab_arr = mf_out.array(mfi);
        const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
 
        const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
 
        // NOTE: These are from the last RK stage...
        const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
        const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
        const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
        const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
        const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
        const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
 
        // These should be re-calculated during ERF_slow_rhs_post
        // -- just vertical SFS kinematic heat flux for now
        //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
        //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
        const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
        const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
                                                              Array4<const Real>{};
        const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
            fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 1) = ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
                                / (   rho_arr(i,j  ,k) +   rho_arr(i+1,j  ,k)
                                  +   rho_arr(i,j+1,k) +   rho_arr(i+1,j+1,k) );
            fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
            fab_arr(i, j, k, 9) =  diss_arr(i,j,k) / rho_arr(i,j,k);
        });
 
        const Box& zbx = mfi.tilebox(IntVect(0,0,1));
        ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            Real rho_face = 0.5 * (rho_arr(i,j,k-1) + rho_arr(i,j,k));
            // average from edge to face center
            fab_arr_stag(i,j,k,0) = 0.5*(tau13_arr(i,j,k) + tau13_arr(i+1,j  ,k)) / rho_face;
            fab_arr_stag(i,j,k,1) = 0.5*(tau23_arr(i,j,k) + tau23_arr(i  ,j+1,k)) / rho_face;
 
            fab_arr_stag(i,j,k,2) = hfx3_arr(i,j,k) / rho_face;
            fab_arr_stag(i,j,k,3) = (l_use_moist) ? q1fx3_arr(i,j,k) / rho_face : 0.0;
            fab_arr_stag(i,j,k,4) = (l_use_moist) ? q2fx3_arr(i,j,k) / rho_face : 0.0;
        });
    }
 
    int zdir = 2;
    auto domain = geom[0].Domain();
    Box stag_domain = domain;
    stag_domain.convert(IntVect(0,0,1));
 
    h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
    h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
//  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
    h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
//  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
    h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
//  h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
//  h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
//  h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
    h_avg_diss  = sumToLine(mf_out,9,1,domain,zdir);
 
    h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
    h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
    h_avg_hfx3  = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
    h_avg_q1fx3 = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
    h_avg_q2fx3 = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
 
    int ht_size =  h_avg_tau11.size(); // _un_staggered
 
    // Divide by the total number of cells we are averaging over
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    for (int k = 0; k < ht_size; ++k) {
        h_avg_tau11[k] /= area_z;
        h_avg_tau12[k] /= area_z;
        h_avg_tau13[k] /= area_z;
        h_avg_tau22[k] /= area_z;
        h_avg_tau23[k] /= area_z;
        h_avg_tau33[k] /= area_z;
        h_avg_hfx3[k] /= area_z;
        h_avg_q1fx3[k] /= area_z;
        h_avg_q2fx3[k] /= area_z;
        h_avg_diss[k] /= area_z;
    }
    // staggered heights
    h_avg_tau13[ht_size] /= area_z;
    h_avg_tau23[ht_size] /= area_z;
    h_avg_hfx3[ht_size]  /= area_z;
    h_avg_q1fx3[ht_size] /= area_z;
    h_avg_q2fx3[ht_size] /= area_z;
}

derive_upwp()

void ERF::derive_upwp

(

amrex::Vector< amrex::Real > &

h_havg

)

EBFactory()

amrex::EBFArrayBoxFactory const& ERF::EBFactory ( int  lev ) const

inlineprivatenoexcept

                                       {
        return *(eb[lev]->get_const_factory());
    }

erf_enforce_hse()

void ERF::erf_enforce_hse	(	int	lev,
		amrex::MultiFab &	dens,
		amrex::MultiFab &	pres,
		amrex::MultiFab &	pi,
		amrex::MultiFab &	th,
		amrex::MultiFab &	qv,
		std::unique_ptr< amrex::MultiFab > &	z_cc
	)

Enforces hydrostatic equilibrium when using terrain.

Parameters

[in]	lev	Integer specifying the current level
[out]	dens	MultiFab storing base state density
[out]	pres	MultiFab storing base state pressure
[out]	pi	MultiFab storing base state Exner function
[in]	z_cc	Pointer to MultiFab storing cell centered z-coordinates

{
    Real l_gravity = solverChoice.gravity;
    bool l_use_terrain = (solverChoice.mesh_type != MeshType::ConstantDz);
 
    const auto geomdata = geom[lev].data();
    const Real dz = geomdata.CellSize(2);
 
    for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
    {
        // Create a flat box with same horizontal extent but only one cell in vertical
        const Box& tbz = mfi.nodaltilebox(2);
        int klo = tbz.smallEnd(2);
        int khi = tbz.bigEnd(2);
 
        // Note we only grow by 1 because that is how big z_cc is.
        Box b2d = tbz; // Copy constructor
        b2d.grow(0,1);
        b2d.grow(1,1);
        b2d.setRange(2,0);
 
        // Intersect this box with the domain
        Box zdomain = convert(geom[lev].Domain(),tbz.ixType());
        b2d &= zdomain;
 
        // We integrate to the first cell (and below) by using rho in this cell
        // If gravity == 0 this is constant pressure
        // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
        // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
        // Note ng_pres_hse = 1
 
       // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
       // Note that gravity is positive
 
        Array4<Real>  rho_arr = dens.array(mfi);
        Array4<Real> pres_arr = pres.array(mfi);
        Array4<Real>   pi_arr =   pi.array(mfi);
        Array4<Real>   th_arr = theta.array(mfi);
        Array4<Real> zcc_arr;
        if (l_use_terrain) {
           zcc_arr = z_cc->array(mfi);
        }
 
        const Real rdOcp = solverChoice.rdOcp;
 
        ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
            // Set value at surface from Newton iteration for rho
            if (klo == 0)
            {
                // Physical height of the terrain at cell center
                Real hz;
                if (l_use_terrain) {
                    hz = zcc_arr(i,j,klo);
                } else {
                    hz = 0.5*dz;
                }
 
                pres_arr(i,j,klo) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
                  pi_arr(i,j,klo) = getExnergivenP(pres_arr(i,j,klo), rdOcp);
                  th_arr(i,j,klo) = getRhoThetagivenP(pres_arr(i,j,klo)) / rho_arr(i,j,klo);
 
                //
                // Set ghost cell with dz and rho at boundary
                // (We will set the rest of the ghost cells in the boundary condition routine)
                //
                pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
                  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
                  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
 
            } else {
 
                // If level > 0 and klo > 0, we need to use the value of pres_arr(i,j,klo-1) which was
                //    filled from FillPatch-ing it.
                Real dz_loc;
                if (l_use_terrain) {
                    dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
                } else {
                    dz_loc = dz;
                }
 
                Real dens_interp = 0.5*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
                pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
 
                pi_arr(i,j,klo  ) = getExnergivenP(pres_arr(i,j,klo  ), rdOcp);
                th_arr(i,j,klo  ) = getRhoThetagivenP(pres_arr(i,j,klo  )) / rho_arr(i,j,klo  );
 
                pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
                th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
            }
 
            Real dens_interp;
            if (l_use_terrain) {
                for (int k = klo+1; k <= khi; k++) {
                    Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
                    dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                    pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
                    pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                    th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
                }
            } else {
                for (int k = klo+1; k <= khi; k++) {
                    dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                    pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
                    pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                    th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
                }
            }
        });
 
    } // mfi
 
     dens.FillBoundary(geom[lev].periodicity());
     pres.FillBoundary(geom[lev].periodicity());
       pi.FillBoundary(geom[lev].periodicity());
    theta.FillBoundary(geom[lev].periodicity());
       qv.FillBoundary(geom[lev].periodicity());
}

Here is the call graph for this function:

ERF_shared()

void ERF::ERF_shared

(

)

{
    if (ParallelDescriptor::IOProcessor()) {
        const char* erf_hash = buildInfoGetGitHash(1);
        const char* amrex_hash = buildInfoGetGitHash(2);
        const char* buildgithash = buildInfoGetBuildGitHash();
        const char* buildgitname = buildInfoGetBuildGitName();
 
        if (strlen(erf_hash) > 0) {
          Print() << "\n"
                         << "ERF git hash: " << erf_hash << "\n";
        }
        if (strlen(amrex_hash) > 0) {
          Print() << "AMReX git hash: " << amrex_hash << "\n";
        }
        if (strlen(buildgithash) > 0) {
          Print() << buildgitname << " git hash: " << buildgithash << "\n";
        }
 
        Print() << "\n";
    }
 
    int nlevs_max = max_level + 1;
 
#ifdef ERF_USE_WINDFARM
    Nturb.resize(nlevs_max);
    vars_windfarm.resize(nlevs_max);
    SMark.resize(nlevs_max);
#endif
 
    qheating_rates.resize(nlevs_max);
    rad_fluxes.resize(nlevs_max);
    sw_lw_fluxes.resize(nlevs_max);
    solar_zenith.resize(nlevs_max);
 
    // NOTE: size lsm before readparams (chooses the model at all levels)
    lsm.ReSize(nlevs_max);
    lsm_data.resize(nlevs_max);
    lsm_flux.resize(nlevs_max);
 
    rhotheta_src.resize(nlevs_max);
    rhoqt_src.resize(nlevs_max);
 
    // NOTE: size canopy model before readparams (if file exists, we construct)
    m_forest_drag.resize(nlevs_max);
    for (int lev = 0; lev <= max_level; ++lev) { m_forest_drag[lev] = nullptr;}
 
    ReadParameters();
    initializeMicrophysics(nlevs_max);
 
#ifdef ERF_USE_WINDFARM
    initializeWindFarm(nlevs_max);
#endif
 
#ifdef ERF_USE_SHOC
    shoc_interface.resize(nlevs_max);
    if (solverChoice.use_shoc) {
        for (int lev = 0; lev <= max_level; ++lev) {
            shoc_interface[lev] = std::make_unique<SHOCInterface>(lev, solverChoice);
        }
    }
#endif
 
    rad.resize(nlevs_max);
    for (int lev = 0; lev <= max_level; ++lev) {
        if (solverChoice.rad_type == RadiationType::RRTMGP) {
#ifdef ERF_USE_RRTMGP
            rad[lev] = std::make_unique<Radiation>(lev, solverChoice);
            // pass radiation datalog frequency to model - RRTMGP needs to know when to save data for profiles
            rad[lev]->setDataLogFrequency(rad_datalog_int);
#endif
        } else if (solverChoice.rad_type != RadiationType::None) {
            Abort("Don't know this radiation model!");
        }
    }
 
    const std::string& pv3d_1 = "plot_vars_1"  ; setPlotVariables(pv3d_1,plot3d_var_names_1);
    const std::string& pv3d_2 = "plot_vars_2"  ; setPlotVariables(pv3d_2,plot3d_var_names_2);
    const std::string& pv2d_1 = "plot2d_vars_1"; setPlotVariables2D(pv2d_1,plot2d_var_names_1);
    const std::string& pv2d_2 = "plot2d_vars_2"; setPlotVariables2D(pv2d_2,plot2d_var_names_2);
 
    // This is only used when we have mesh_type == MeshType::StretchedDz
    stretched_dz_h.resize(nlevs_max);
    stretched_dz_d.resize(nlevs_max);
 
    // Initialize staggered vertical levels for grid stretching or terrain, and
    // to simplify Rayleigh damping layer calculations.
    zlevels_stag.resize(max_level+1);
    init_zlevels(zlevels_stag,
                 stretched_dz_h,
                 stretched_dz_d,
                 geom,
                 refRatio(),
                 solverChoice.grid_stretching_ratio,
                 solverChoice.zsurf,
                 solverChoice.dz0);
 
    if (SolverChoice::mesh_type == MeshType::StretchedDz ||
        SolverChoice::mesh_type == MeshType::VariableDz) {
        int nz = geom[0].Domain().length(2) + 1; // staggered
        if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
            Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
                << " does not match highest requested z level " << zlevels_stag[0][nz-1]
                << std::endl;
        }
        if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > 1.0e-4) {
            Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
                << " does not match lowest requested level " << zlevels_stag[0][0]
                << std::endl;
        }
 
        // Redefine the problem domain here?
    }
 
    // Get lo/hi indices for massflux calc
    if ((solverChoice.const_massflux_u != 0) || (solverChoice.const_massflux_v != 0)) {
        if (solverChoice.mesh_type == MeshType::ConstantDz) {
            const Real massflux_zlo = solverChoice.const_massflux_layer_lo - geom[0].ProbLo(2);
            const Real massflux_zhi = solverChoice.const_massflux_layer_hi - geom[0].ProbLo(2);
            const Real dz = geom[0].CellSize(2);
            if (massflux_zlo == -1e34) {
                solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
            } else {
                solverChoice.massflux_klo = static_cast<int>(std::ceil(massflux_zlo / dz - 0.5));
            }
            if (massflux_zhi ==  1e34) {
                solverChoice.massflux_khi = geom[0].Domain().bigEnd(2);
            } else {
                solverChoice.massflux_khi = static_cast<int>(std::floor(massflux_zhi / dz - 0.5));
            }
        } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
            const Real massflux_zlo = solverChoice.const_massflux_layer_lo;
            const Real massflux_zhi = solverChoice.const_massflux_layer_hi;
            solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
            solverChoice.massflux_khi = geom[0].Domain().bigEnd(2) + 1;
            for (int k=0; k <= geom[0].Domain().bigEnd(2)+1; ++k) {
                if (zlevels_stag[0][k] <= massflux_zlo) solverChoice.massflux_klo = k;
                if (zlevels_stag[0][k] <= massflux_zhi) solverChoice.massflux_khi = k;
            }
        } else { // solverChoice.mesh_type == MeshType::VariableDz
            Error("Const massflux with variable dz not supported -- planar averages are on k rather than constant-z planes");
        }
 
        Print() << "Constant mass flux based on k in ["
            << solverChoice.massflux_klo << ", " << solverChoice.massflux_khi << "]" << std::endl;
    }
 
    prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
 
    // Geometry on all levels has been defined already.
 
    // No valid BoxArray and DistributionMapping have been defined.
    // But the arrays for them have been resized.
 
    t_new.resize(nlevs_max, 0.0);
    t_old.resize(nlevs_max, -1.e100);
    dt.resize(nlevs_max, std::min(1.e100,dt_max_initial));
    dt_mri_ratio.resize(nlevs_max, 1);
 
    vars_new.resize(nlevs_max);
    vars_old.resize(nlevs_max);
    gradp.resize(nlevs_max);
 
    // We resize this regardless in order to pass it without error
    pp_inc.resize(nlevs_max);
 
    // Used in the fast substepping only
    lagged_delta_rt.resize(nlevs_max);
    avg_xmom.resize(nlevs_max);
    avg_ymom.resize(nlevs_max);
    avg_zmom.resize(nlevs_max);
 
    rU_new.resize(nlevs_max);
    rV_new.resize(nlevs_max);
    rW_new.resize(nlevs_max);
 
    rU_old.resize(nlevs_max);
    rV_old.resize(nlevs_max);
    rW_old.resize(nlevs_max);
 
    // xmom_crse_rhs.resize(nlevs_max);
    // ymom_crse_rhs.resize(nlevs_max);
    zmom_crse_rhs.resize(nlevs_max);
 
    for (int lev = 0; lev < nlevs_max; ++lev) {
        vars_new[lev].resize(Vars::NumTypes);
        vars_old[lev].resize(Vars::NumTypes);
        gradp[lev].resize(AMREX_SPACEDIM);
    }
 
    // Time integrator
    mri_integrator_mem.resize(nlevs_max);
 
    // Physical boundary conditions
    physbcs_cons.resize(nlevs_max);
    physbcs_u.resize(nlevs_max);
    physbcs_v.resize(nlevs_max);
    physbcs_w.resize(nlevs_max);
    physbcs_base.resize(nlevs_max);
 
    // Planes to hold Dirichlet values at boundaries
    xvel_bc_data.resize(nlevs_max);
    yvel_bc_data.resize(nlevs_max);
    zvel_bc_data.resize(nlevs_max);
    th_bc_data.resize(nlevs_max);
 
    advflux_reg.resize(nlevs_max);
 
    // Stresses
    Tau.resize(nlevs_max);
    Tau_corr.resize(nlevs_max);
    SFS_hfx1_lev.resize(nlevs_max); SFS_hfx2_lev.resize(nlevs_max); SFS_hfx3_lev.resize(nlevs_max);
    SFS_diss_lev.resize(nlevs_max);
    SFS_q1fx1_lev.resize(nlevs_max); SFS_q1fx2_lev.resize(nlevs_max); SFS_q1fx3_lev.resize(nlevs_max);
    SFS_q2fx3_lev.resize(nlevs_max);
    eddyDiffs_lev.resize(nlevs_max);
    SmnSmn_lev.resize(nlevs_max);
 
    // Sea surface temps
    sst_lev.resize(nlevs_max);
    tsk_lev.resize(nlevs_max);
    lmask_lev.resize(nlevs_max);
 
    // Land and soil grid type and urban fractions
    land_type_lev.resize(nlevs_max);
    soil_type_lev.resize(nlevs_max);
    urb_frac_lev.resize(nlevs_max);
 
    // Metric terms
    z_phys_nd.resize(nlevs_max);
    z_phys_cc.resize(nlevs_max);
    detJ_cc.resize(nlevs_max);
    ax.resize(nlevs_max);
    ay.resize(nlevs_max);
    az.resize(nlevs_max);
 
    z_phys_nd_new.resize(nlevs_max);
    detJ_cc_new.resize(nlevs_max);
 
    z_phys_nd_src.resize(nlevs_max);
    z_phys_cc_src.resize(nlevs_max);
    detJ_cc_src.resize(nlevs_max);
    ax_src.resize(nlevs_max);
    ay_src.resize(nlevs_max);
    az_src.resize(nlevs_max);
 
    z_t_rk.resize(nlevs_max);
 
    terrain_blanking.resize(nlevs_max);
 
    // Wall distance
    walldist.resize(nlevs_max);
 
    // BoxArrays to make MultiFabs needed to convert WRFBdy data
    ba1d.resize(nlevs_max);
    ba2d.resize(nlevs_max);
 
    // MultiFabs needed to convert WRFBdy data
    mf_PSFC.resize(nlevs_max);
 
    // Map factors
    mapfac.resize(nlevs_max);
 
    // Fine mask
    fine_mask.resize(nlevs_max);
 
    // Thin immersed body
    xflux_imask.resize(nlevs_max);
    yflux_imask.resize(nlevs_max);
    zflux_imask.resize(nlevs_max);
    //overset_imask.resize(nlevs_max);
    thin_xforce.resize(nlevs_max);
    thin_yforce.resize(nlevs_max);
    thin_zforce.resize(nlevs_max);
 
    // Base state
    base_state.resize(nlevs_max);
    base_state_new.resize(nlevs_max);
 
    // Wave coupling data
    Hwave.resize(nlevs_max);
    Lwave.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev)
    {
        Hwave[lev] = nullptr;
        Lwave[lev] = nullptr;
    }
    Hwave_onegrid.resize(nlevs_max);
    Lwave_onegrid.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev)
    {
        Hwave_onegrid[lev] = nullptr;
        Lwave_onegrid[lev] = nullptr;
    }
 
    // Theta prim for MOST
    Theta_prim.resize(nlevs_max);
 
    // Qv prim for MOST
    Qv_prim.resize(nlevs_max);
 
    // Qr prim for MOST
    Qr_prim.resize(nlevs_max);
 
    // Time averaged velocity field
    vel_t_avg.resize(nlevs_max);
    t_avg_cnt.resize(nlevs_max);
 
    // Size lat long arrays and default to null pointers
    lat_m.resize(nlevs_max);
    lon_m.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev) {
        lat_m[lev] = nullptr;
        lon_m[lev] = nullptr;
    }
 
    // Variable coriolis
    sinPhi_m.resize(nlevs_max);
    cosPhi_m.resize(nlevs_max);
    for (int lev = 0; lev < max_level; ++lev) {
        sinPhi_m[lev] = nullptr;
        cosPhi_m[lev] = nullptr;
    }
 
    // Initialize tagging criteria for mesh refinement
    refinement_criteria_setup();
 
    for (int lev = 0; lev < max_level; ++lev)
    {
       Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
          ref_ratio[lev][0]  << " " << ref_ratio[lev][1]  <<  " " << ref_ratio[lev][2] << std::endl;
    }
 
    // We will create each of these in MakeNewLevelFromScratch
    eb.resize(max_level+1);
    for (int lev = 0; lev < max_level + 1; lev++){
        eb[lev] = std::make_unique<eb_>();
    }
 
    //
    // Construct the EB data structures and store in a separate class
    //
    // This is needed before initializing level MultiFabs
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing)
    {
        std::string geometry ="terrain";
        ParmParse pp("eb2");
        pp.queryAdd("geometry", geometry);
 
        constexpr int ngrow_for_eb = 4;  // This is the default in amrex but we need to explicitly pass it here since
                               // we want to also pass the build_coarse_level_by_coarsening argument
        const bool build_eb_for_multigrid = (solverChoice.terrain_type == TerrainType::EB &&
                                            ((solverChoice.project_initial_velocity[0] == 1) ||
                                            solverChoice.anelastic[0] == 1));
        // Note this just needs to be an integer > number of V-cycles one might use
        const int max_coarsening_level = (build_eb_for_multigrid) ? 100 : 0;
        const bool build_coarse_level_by_coarsening(false);
 
        // Define GeometryShop using the implicit function
        if (geometry == "terrain") {
            Box terrain_bx(surroundingNodes(geom[max_level].Domain())); terrain_bx.grow(3);
            FArrayBox terrain_fab(makeSlab(terrain_bx,2,0),1);
            Real dummy_time = 0.0;
            prob->init_terrain_surface(geom[max_level], terrain_fab, dummy_time);
            TerrainIF implicit_fun(terrain_fab, geom[max_level], stretched_dz_d[max_level]);
            auto gshop = EB2::makeShop(implicit_fun);
            if (build_eb_for_multigrid) {
                EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                            ngrow_for_eb, build_coarse_level_by_coarsening);
            } else {
                EB2::Build(gshop, this->Geom(), ngrow_for_eb);
            }
        } else if (geometry == "box") {
            RealArray box_lo{0.0, 0.0, 0.0};
            RealArray box_hi{0.0, 0.0, 0.0};
            pp.query("box_lo", box_lo);
            pp.query("box_hi", box_hi);
            EB2::BoxIF implicit_fun(box_lo, box_hi, false);
            auto gshop = EB2::makeShop(implicit_fun);
            if (build_eb_for_multigrid) {
                EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                            ngrow_for_eb, build_coarse_level_by_coarsening);
            } else {
                EB2::Build(gshop, this->Geom(), ngrow_for_eb);
            }
        } else if (geometry == "sphere") {
            auto ProbLoArr = geom[max_level].ProbLoArray();
            auto ProbHiArr = geom[max_level].ProbHiArray();
            const Real xcen = 0.5 * (ProbLoArr[0] + ProbHiArr[0]);
            const Real ycen = 0.5 * (ProbLoArr[1] + ProbHiArr[1]);
            RealArray sphere_center = {xcen, ycen, 0.0};
            EB2::SphereIF implicit_fun(0.5, sphere_center, false);
            auto gshop = EB2::makeShop(implicit_fun);
            if (build_eb_for_multigrid) {
                EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                            ngrow_for_eb, build_coarse_level_by_coarsening);
            } else {
                EB2::Build(gshop, this->Geom(), ngrow_for_eb);
            }
        }
    }
 
    if ( solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
        constexpr int ngrow_for_eb = 4;
        Box buildings_bx(surroundingNodes(geom[max_level].Domain())); buildings_bx.grow(3);
        FArrayBox buildings_fab(makeSlab(buildings_bx,2,0),1);
        Real dummy_time = 0.0;
        prob->init_buildings_surface(geom[max_level], buildings_fab, dummy_time);
        TerrainIF implicit_fun(buildings_fab, geom[max_level], stretched_dz_d[max_level]);
        auto gshop = EB2::makeShop(implicit_fun);
        EB2::Build(gshop, this->Geom(), ngrow_for_eb);
    }
 
    forecast_state_1.resize(nlevs_max);
    forecast_state_2.resize(nlevs_max);
    forecast_state_interp.resize(nlevs_max);
 
    surface_state_1.resize(nlevs_max);
    surface_state_2.resize(nlevs_max);
    surface_state_interp.resize(nlevs_max);
}

Here is the call graph for this function:

ErrorEst()

void ERF::ErrorEst ( int  lev, amrex::TagBoxArray &  tags, amrex::Real  time, int  ngrow  )

override

{
    const int clearval = TagBox::CLEAR;
    const int   tagval = TagBox::SET;
 
#ifdef ERF_USE_NETCDF
    if (solverChoice.init_type == InitType::WRFInput) {
        int ratio;
        Box subdomain;
 
        if (!nc_init_file[levc+1].empty())
        {
            Real levc_start_time = read_start_time_from_wrfinput(levc  , nc_init_file[levc  ][0]);
            amrex::Print() << " WRFInput       time at level " << levc << " is " << levc_start_time << std::endl;
 
            for (int isub = 0; isub < nc_init_file[levc+1].size(); isub++) {
                if (!have_read_nc_init_file[levc+1][isub])
                {
                    Real levf_start_time = read_start_time_from_wrfinput(levc+1, nc_init_file[levc+1][isub]);
                    amrex::Print() << " WRFInput start_time at level " << levc+1 << " is " << levf_start_time << std::endl;
 
                    // We assume there is only one subdomain at levc; otherwise we don't know
                    //     which one is the parent of the fine region we are trying to create
                    AMREX_ALWAYS_ASSERT(subdomains[levc].size() == 1);
 
                    if ( (ref_ratio[levc][2]) != 1) {
                        amrex::Abort("The ref_ratio specified in the inputs file must have 1 in the z direction; please use ref_ratio_vect rather than ref_ratio");
                    }
 
                    if ( levf_start_time <= (levc_start_time + t_new[levc]) ) {
                        amrex::Print() << " WRFInput file to read: " << nc_init_file[levc+1][isub] << std::endl;
                        subdomain = read_subdomain_from_wrfinput(levc, nc_init_file[levc+1][isub], ratio);
                        amrex::Print() << " WRFInput subdomain " << isub << " at level " << levc+1 << " is " << subdomain << std::endl;
 
                        if ( (ratio != ref_ratio[levc][0]) || (ratio != ref_ratio[levc][1]) ) {
                            amrex::Print() << "File " << nc_init_file[levc+1][0] << " has refinement ratio = " << ratio << std::endl;
                            amrex::Print() << "The inputs file has refinement ratio = " << ref_ratio[levc] << std::endl;
                            amrex::Abort("These must be the same -- please edit your inputs file and try again.");
                        }
 
                        subdomain.coarsen(IntVect(ratio,ratio,1));
 
                        Box coarser_level(subdomains[levc][isub].minimalBox());
                        subdomain.shift(coarser_level.smallEnd());
 
                        if (verbose > 0) {
                            amrex::Print() << " Crse subdomain to be tagged is" << subdomain << std::endl;
                        }
 
                        Box new_fine(subdomain); new_fine.refine(IntVect(ratio,ratio,1));
                        num_boxes_at_level[levc+1] = 1;
                        boxes_at_level[levc+1].push_back(new_fine);
 
                        for (MFIter mfi(tags); mfi.isValid(); ++mfi) {
                            auto tag_arr = tags.array(mfi);  // Get device-accessible array
 
                            Box bx = mfi.validbox(); bx &= subdomain;
 
                            if (!bx.isEmpty()) {
                                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                                    tag_arr(i,j,k) = TagBox::SET;
                                });
                            }
                        }
                    } // time is right
                } else {
                    // Re-tag this region
                    for (MFIter mfi(tags); mfi.isValid(); ++mfi)
                    {
                        auto tag_arr = tags.array(mfi);  // Get device-accessible array
 
                        Box existing_bx_coarsened(boxes_at_level[levc+1][isub]);
                        existing_bx_coarsened.coarsen(ref_ratio[levc]);
 
                        Box bx = mfi.validbox(); bx &= existing_bx_coarsened;
 
                        if (!bx.isEmpty()) {
                            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                                tag_arr(i,j,k) = TagBox::SET;
                            });
                        }
                    }
                } // has file been read?
            } // isub
            return;
        } // file not empty
    }
#endif
 
    //
    // Make sure the ghost cells of the level we are tagging at are filled
    //    in case we take differences that require them
    // NOTE: We are Fillpatching only the cell-centered variables here
    //
    MultiFab& S_new = vars_new[levc][Vars::cons];
    MultiFab& U_new = vars_new[levc][Vars::xvel];
    MultiFab& V_new = vars_new[levc][Vars::yvel];
    MultiFab& W_new = vars_new[levc][Vars::zvel];
    //
    if (levc == 0) {
        FillPatchCrseLevel(levc, time, {&S_new, &U_new, &V_new, &W_new});
    } else {
        FillPatchFineLevel(levc, time, {&S_new, &U_new, &V_new, &W_new},
                           {&S_new, &rU_new[levc], &rV_new[levc], &rW_new[levc]},
                           base_state[levc], base_state[levc],
                           false, true);
    }
 
    for (int j=0; j < ref_tags.size(); ++j)
    {
        //
        // This mf must have ghost cells because we may take differences between adjacent values
        //
        std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 1);
        mf->setVal(0.0);
 
        // This allows dynamic refinement based on the value of the density
        if (ref_tags[j].Field() == "density")
        {
            MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,1);
 
        // This allows dynamic refinement based on the value of qv
        } else if ( ref_tags[j].Field() == "qv" ) {
            MultiFab::Copy(  *mf, vars_new[levc][Vars::cons], RhoQ1_comp, 0, 1, 1);
            MultiFab::Divide(*mf, vars_new[levc][Vars::cons],   Rho_comp, 0, 1, 1);
 
 
        // This allows dynamic refinement based on the value of qc
        } else if (ref_tags[j].Field() == "qc" ) {
            MultiFab::Copy(  *mf, vars_new[levc][Vars::cons], RhoQ2_comp, 0, 1, 1);
            MultiFab::Divide(*mf, vars_new[levc][Vars::cons],   Rho_comp, 0, 1, 1);
 
        // This allows dynamic refinement based on the value of the z-component of vorticity
        } else if (ref_tags[j].Field() == "vorticity" ) {
            Vector<MultiFab> mf_cc_vel(1);
            mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
            average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
 
            // Impose bc's at domain boundaries at all levels
            FillBdyCCVels(mf_cc_vel,levc);
 
            mf->setVal(0.);
 
            for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                auto& dfab = (*mf)[mfi];
                auto& sfab = mf_cc_vel[0][mfi];
                derived::erf_dervortz(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
            }
 
        // This allows dynamic refinement based on the value of the scalar/theta
        } else if ( (ref_tags[j].Field() == "scalar"  ) ||
                    (ref_tags[j].Field() == "theta"   ) )
        {
            for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.growntilebox();
                auto& dfab = (*mf)[mfi];
                auto& sfab = vars_new[levc][Vars::cons][mfi];
                if (ref_tags[j].Field() == "scalar") {
                    derived::erf_derscalar(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                } else if (ref_tags[j].Field() == "theta") {
                    derived::erf_dertheta(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                }
            } // mfi
        // This allows dynamic refinement based on the value of the density
        } else if ( (SolverChoice::terrain_type == TerrainType::ImmersedForcing) &&
                    (ref_tags[j].Field() == "terrain_blanking") )
        {
            MultiFab::Copy(*mf,*terrain_blanking[levc],0,0,1,1);
        }
        else if (ref_tags[j].Field() == "velmag")
        {
            ParmParse pp(pp_prefix);
            Vector<std::string> refinement_indicators;
            pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
            Real velmag_threshold;
            bool is_hurricane_tracker = false;
            for (int i=0; i<refinement_indicators.size(); ++i)
            {
                if (refinement_indicators[i]=="hurricane_tracker") {
                    is_hurricane_tracker = true;
                    std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
                    ParmParse ppr(ref_prefix);
                    ppr.get("value_greater", velmag_threshold);
                    break;
                }
            }
 
            Vector<MultiFab> mf_cc_vel(1);
            mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
            average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
 
            if (is_hurricane_tracker) {
                HurricaneTracker(levc, time, mf_cc_vel[0], velmag_threshold, &tags);
            } else {
                for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    auto& dfab = (*mf)[mfi];
                    auto& sfab = mf_cc_vel[0][mfi];
                    derived::erf_dermagvel(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                }
            }
 
#ifdef ERF_USE_PARTICLES
        } else {
            //
            // This allows dynamic refinement based on the number of particles per cell
            //
            // Note that we must count all the particles in levels both at and above the current,
            //      since otherwise, e.g., if the particles are all at level 1, counting particles at
            //      level 0 will not trigger refinement when regridding so level 1 will disappear,
            //      then come back at the next regridding
            //
            const auto& particles_namelist( particleData.getNames() );
            mf->setVal(0.0);
            for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
            {
                std::string tmp_string(particles_namelist[i]+"_count");
                IntVect rr = IntVect::TheUnitVector();
                if (ref_tags[j].Field() == tmp_string) {
                    for (int lev = levc; lev <= finest_level; lev++)
                    {
                        MultiFab temp_dat(grids[lev], dmap[lev], 1, 0); temp_dat.setVal(0);
                        particleData[particles_namelist[i]]->IncrementWithTotal(temp_dat, lev);
 
                        MultiFab temp_dat_crse(grids[levc], dmap[levc], 1, 0); temp_dat_crse.setVal(0);
 
                        if (lev == levc) {
                            MultiFab::Copy(*mf, temp_dat, 0, 0, 1, 0);
                        } else {
                            for (int d = 0; d < AMREX_SPACEDIM; d++) {
                                rr[d] *= ref_ratio[levc][d];
                            }
                            average_down(temp_dat, temp_dat_crse, 0, 1, rr);
                            MultiFab::Add(*mf, temp_dat_crse, 0, 0, 1, 0);
                        }
                    }
                }
            }
#endif
        }
 
        ref_tags[j](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
    } // loop over j
 
    // ********************************************************************************************
    // Refinement based on 2d distance from the "eye" which is defined here as the (x,y) location of
    //    the integrated qv
    // ********************************************************************************************
    ParmParse pp(pp_prefix);
    Vector<std::string> refinement_indicators;
    pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
    for (int i=0; i<refinement_indicators.size(); ++i)
    {
        if ( (refinement_indicators[i]=="storm_tracker") && (solverChoice.moisture_type != MoistureType::None) )
        {
            std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
            ParmParse ppr(ref_prefix);
 
            Real ref_start_time = -1.0;
            ppr.query("start_time",ref_start_time);
 
            if (time >= ref_start_time) {
 
                Real max_radius = -1.0;
                ppr.get("max_radius", max_radius);
 
                // Create the volume-weighted sum of (rho qv) in each column
                MultiFab mf_qv_int(ba2d[levc], dmap[levc], 1, 0); mf_qv_int.setVal(0.);
 
                // Define the 2D MultiFab holding the column-integrated (rho qv)
                volWgtColumnSum(levc, S_new, RhoQ1_comp, mf_qv_int, *detJ_cc[levc]);
 
                // Find the max value in the domain
                IntVect eye = mf_qv_int.maxIndex(0);
 
                const auto dx      = geom[levc].CellSizeArray();
                const auto prob_lo = geom[levc].ProbLoArray();
 
                Real eye_x = prob_lo[0] + (eye[0] + 0.5) * dx[0];
                Real eye_y = prob_lo[1] + (eye[1] + 0.5) * dx[1];
 
                tag_on_distance_from_eye(geom[levc], &tags, eye_x, eye_y, max_radius);
            }
        }
    }
}

Here is the call graph for this function:

estTimeStep()

Real ERF::estTimeStep	(	int	level,
		long &	dt_fast_ratio
	)		const

Function that calls estTimeStep for each level

Parameters

[in]	level	level of refinement (coarsest level i 0)
[out]	dt_fast_ratio	ratio of slow to fast time step

{
    BL_PROFILE("ERF::estTimeStep()");
 
    Real estdt_comp = 1.e20;
    Real estdt_lowM = 1.e20;
 
    // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation
    const int nxc = geom[0].Domain().length(0);
    const int nyc = geom[0].Domain().length(1);
 
    auto const dxinv = geom[level].InvCellSizeArray();
    auto const dzinv = 1.0 / dz_min[level];
 
    MultiFab const& S_new = vars_new[level][Vars::cons];
 
    MultiFab ccvel(grids[level],dmap[level],3,0);
 
    average_face_to_cellcenter(ccvel,0,
                               Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
                                                        &vars_new[level][Vars::yvel],
                                                        &vars_new[level][Vars::zvel]});
 
    bool l_substepping = (solverChoice.substepping_type[level] == SubsteppingType::Implicit);
    int  l_anelastic   = solverChoice.anelastic[level];
 
    bool l_comp_substepping_diag = (verbose && l_substepping && !l_anelastic && solverChoice.substepping_diag);
 
    Real estdt_comp_inv;
    Real estdt_vert_comp_inv;
    Real estdt_vert_lowM_inv;
 
    if (l_substepping && (nxc==1) && (nyc==1)) {
        // SCM -- should not depend on dx or dy; force minimum number of substeps
        estdt_comp_inv = std::numeric_limits<Real>::min();
    }
    else if (solverChoice.terrain_type == TerrainType::EB)
    {
        const eb_& eb_lev = get_eb(level);
        const MultiFab& detJ = (eb_lev.get_const_factory())->getVolFrac();
 
        estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
        [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                   Array4<Real const> const& s,
                                   Array4<Real const> const& u,
                                   Array4<Real const> const& vf) -> Real
        {
           Real new_comp_dt = -1.e100;
           amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
           {
               if (vf(i,j,k) > 0.)
               {
                   const Real rho      = s(i, j, k, Rho_comp);
                   const Real rhotheta = s(i, j, k, RhoTheta_comp);
 
                   // NOTE: even when moisture is present,
                   //       we only use the partial pressure of the dry air
                   //       to compute the soundspeed
                   Real pressure = getPgivenRTh(rhotheta);
                   Real c = std::sqrt(Gamma * pressure / rho);
 
                   // If we are doing implicit acoustic substepping, then the z-direction does not contribute
                   //    to the computation of the time step
                   if (l_substepping) {
                       if ((nxc > 1) && (nyc==1)) {
                           // 2-D in x-z
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
                       } else if ((nyc > 1) && (nxc==1)) {
                           // 2-D in y-z
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                       } else {
                           // 3-D
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                    ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                       }
 
                   // If we are not doing implicit acoustic substepping, then the z-direction contributes
                   //    to the computation of the time step
                   } else {
                       if (nxc > 1 && nyc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                    ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       } else if (nxc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       } else if (nyc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       } else {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       }
 
                   }
               }
           });
           return new_comp_dt;
       });
 
    } else {
       estdt_comp_inv = ReduceMax(S_new, ccvel, 0,
       [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                  Array4<Real const> const& s,
                                  Array4<Real const> const& u) -> Real
       {
           Real new_comp_dt = -1.e100;
           amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
           {
               {
                   const Real rho      = s(i, j, k, Rho_comp);
                   const Real rhotheta = s(i, j, k, RhoTheta_comp);
 
                   // NOTE: even when moisture is present,
                   //       we only use the partial pressure of the dry air
                   //       to compute the soundspeed
                   Real pressure = getPgivenRTh(rhotheta);
                   Real c = std::sqrt(Gamma * pressure / rho);
 
                   // If we are doing implicit acoustic substepping, then the z-direction does not contribute
                   //    to the computation of the time step
                   if (l_substepping) {
                       if ((nxc > 1) && (nyc==1)) {
                           // 2-D in x-z
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
                       } else if ((nyc > 1) && (nxc==1)) {
                           // 2-D in y-z
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                       } else {
                           // 3-D
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                    ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                       }
 
                   // If we are not doing implicit acoustic substepping, then the z-direction contributes
                   //    to the computation of the time step
                   } else {
                       if (nxc > 1 && nyc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                    ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       } else if (nxc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       } else if (nyc > 1) {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                    ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       } else {
                           new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                       }
 
                   }
               }
           });
           return new_comp_dt;
       });
    } // not EB
 
    ParallelDescriptor::ReduceRealMax(estdt_comp_inv);
    estdt_comp = cfl / estdt_comp_inv;
 
     Real estdt_lowM_inv = ReduceMax(ccvel, 0,
       [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                  Array4<Real const> const& u) -> Real
       {
           Real new_lm_dt = -1.e100;
           Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
           {
               new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
                                      ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
                                      ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
           });
           return new_lm_dt;
       });
 
     ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
     if (estdt_lowM_inv > 0.0_rt)
         estdt_lowM = cfl / estdt_lowM_inv;
 
     // Additional vertical diagnostics
     if (l_comp_substepping_diag) {
         estdt_vert_comp_inv = ReduceMax(S_new, ccvel, 0,
         [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                    Array4<Real const> const& s,
                                    Array4<Real const> const& u) -> Real
         {
             Real new_comp_dt = -1.e100;
             amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
             {
                 {
                     const Real rho      = s(i, j, k, Rho_comp);
                     const Real rhotheta = s(i, j, k, RhoTheta_comp);
 
                     // NOTE: even when moisture is present,
                     //       we only use the partial pressure of the dry air
                     //       to compute the soundspeed
                     Real pressure = getPgivenRTh(rhotheta);
                     Real c = std::sqrt(Gamma * pressure / rho);
 
                     // Look at z-direction only
                     new_comp_dt = amrex::max((amrex::Math::abs(u(i,j,k,2)) + c) * dzinv, new_comp_dt);
                 }
             });
             return new_comp_dt;
         });
 
         estdt_vert_lowM_inv = ReduceMax(ccvel, 0,
         [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                    Array4<Real const> const& u) -> Real
         {
             Real new_lowM_dt = -1.e100;
             amrex::Loop(b, [=,&new_lowM_dt] (int i, int j, int k) noexcept
             {
                 new_lowM_dt = amrex::max((amrex::Math::abs(u(i,j,k,2))) * dzinv, new_lowM_dt);
             });
             return new_lowM_dt;
         });
 
         ParallelDescriptor::ReduceRealMax(estdt_vert_comp_inv);
         ParallelDescriptor::ReduceRealMax(estdt_vert_lowM_inv);
     }
 
     if (verbose) {
         if (fixed_dt[level] <= 0.0) {
             Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " <<
               1.0/dxinv[0] << " " << 1.0/dxinv[1] << " " << dz_min[level] << std::endl;
             Print() << "Compressible dt at level " << level << ":  " << estdt_comp << std::endl;
             if (estdt_lowM_inv > 0.0_rt) {
                 Print() << "Anelastic dt at level " << level << ":  " << estdt_lowM << std::endl;
             } else {
                 Print() << "Anelastic dt at level " << level << ": undefined " << std::endl;
             }
         }
 
         if (fixed_dt[level] > 0.0) {
             Print() << "Based on cfl of 1.0 " << std::endl;
             Print() << "Compressible dt at level " << level << " would be:  " << estdt_comp/cfl << std::endl;
             if (estdt_lowM_inv > 0.0_rt) {
                 Print() << "Anelastic dt at level " << level << " would be:  " << estdt_lowM/cfl << std::endl;
             } else {
                 Print() << "Anelastic dt at level " << level << " would be undefined " << std::endl;
             }
             Print() << "Fixed dt at level " << level << "       is:  " << fixed_dt[level] << std::endl;
             if (fixed_fast_dt[level] > 0.0) {
                 Print() << "Fixed fast dt at level " << level << "       is:  " << fixed_fast_dt[level] << std::endl;
             }
         }
     }
 
     if (solverChoice.substepping_type[level] != SubsteppingType::None) {
         if (fixed_dt[level] > 0. && fixed_fast_dt[level] > 0.) {
             dt_fast_ratio = static_cast<long>( fixed_dt[level] / fixed_fast_dt[level] );
         } else if (fixed_dt[level] > 0.) {
             // Max CFL_c = 1.0 for substeps by default, but we enforce a min of 4 substeps
             auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
             dt_fast_ratio = static_cast<long>( std::max(fixed_dt[level]/dt_sub_max,4.) );
         } else {
             // auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
             // dt_fast_ratio = static_cast<long>( std::max(estdt_comp/dt_sub_max,4.) );
             dt_fast_ratio = static_cast<long>( std::max(cfl / sub_cfl, 4.) );
         }
 
         // Force time step ratio to be an even value
         if (solverChoice.force_stage1_single_substep) {
             if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
         } else {
             if ( dt_fast_ratio%6 != 0) {
                 Print() << "mri_dt_ratio = " << dt_fast_ratio
                         << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
                 dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/6.0) * 6);
             }
         }
 
         if (verbose) {
             Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
         }
     } // if substepping
 
     // Print out some extra diagnostics -- dt calcs are repeated so as to not
     // disrupt the overall code flow...
     if (l_comp_substepping_diag) {
         Real dt_diag = (fixed_dt[level] > 0.0) ? fixed_dt[level] : estdt_comp;
         int  ns      = (fixed_mri_dt_ratio > 0.0) ? fixed_mri_dt_ratio : dt_fast_ratio;
 
         // horizontal acoustic CFL must be < 1 (fully explicit)
         // vertical   acoustic CFL may  be > 1
         Print() << "effective horiz,vert acoustic CFL with " << ns << " substeps : "
            << (dt_diag / ns) * estdt_comp_inv << " "
            << (dt_diag / ns) * estdt_vert_comp_inv << std::endl;
 
         // vertical advective CFL should be < 1, otherwise w-damping may be needed
         Print() << "effective vert advective CFL : "
            << dt_diag * estdt_vert_lowM_inv << std::endl;
     }
 
     if (fixed_dt[level] > 0.0) {
         return fixed_dt[level];
     } else {
         // Anelastic (substepping is not allowed)
         if (l_anelastic) {
 
            // Make sure that timestep is less than the dt_max
            estdt_lowM = amrex::min(estdt_lowM, dt_max);
 
            // On the first timestep enforce dt_max_initial
            if (istep[level] == 0) {
                return amrex::min(dt_max_initial, estdt_lowM);
            } else {
                return estdt_lowM;
            }
 
 
         // Compressible with or without substepping
         } else {
             return estdt_comp;
         }
     }
}

Here is the call graph for this function:

Evolve()

void ERF::Evolve

(

)

{
    BL_PROFILE_VAR("ERF::Evolve()", evolve);
 
    Real cur_time = t_new[0];
 
    // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
    //      for finer levels (with or without subcycling)
    for (int step = istep[0]; step < max_step && start_time+cur_time < stop_time; ++step)
    {
        if (use_datetime) {
            Print() << "\n" << getTimestamp(start_time+cur_time, datetime_format)
                    << " (" << cur_time << " s elapsed)" << std::endl;
        }
        Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
 
        ComputeDt(step);
 
        // Make sure we have read enough of the boundary plane data to make it through this timestep
        if (input_bndry_planes)
        {
            m_r2d->read_input_files(cur_time,dt[0],m_bc_extdir_vals);
        }
 
#ifdef ERF_USE_PARTICLES
        // We call this every time step with the knowledge that the particles may be
        //    initialized at a later time than the simulation start time.
        // The ParticleContainer carries a "start time" so the initialization will happen
        //    only when a) time > start_time, and b) particles have not yet been initialized
        initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,cur_time);
#endif
 
        if(solverChoice.init_type == InitType::HindCast and
          solverChoice.hindcast_lateral_forcing) {
            for(int lev=0;lev<finest_level+1;lev++){
                WeatherDataInterpolation(lev,cur_time,z_phys_nd,false);
            }
        }
 
        if(solverChoice.init_type == InitType::HindCast and
          solverChoice.hindcast_surface_bcs) {
            for(int lev=0;lev<finest_level+1;lev++){
                SurfaceDataInterpolation(lev,cur_time,z_phys_nd,false);
            }
        }
 
        auto dEvolveTime0 = amrex::second();
 
        int iteration = 1;
        timeStep(0, cur_time, iteration);
 
        cur_time  += dt[0];
 
        Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
                << " DT = " << dt[0]  << std::endl;
 
        if (check_for_nans > 0) {
            amrex::Print() << "Testing new state and vels for NaNs at end of timestep" << std::endl;
            for (int lev = 0; lev <= finest_level; ++lev) {
                check_state_for_nans(vars_new[lev][IntVars::cons]);
                check_vels_for_nans(vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],vars_new[lev][Vars::zvel]);
            }
        }
 
        if (verbose > 0)
        {
            auto dEvolveTime = amrex::second() - dEvolveTime0;
            ParallelDescriptor::ReduceRealMax(dEvolveTime,ParallelDescriptor::IOProcessorNumber());
            amrex::Print() << "Timestep time = " << dEvolveTime << " seconds." << '\n';
        }
 
        post_timestep(step, cur_time, dt[0]);
 
        if (writeNow(cur_time, step+1, m_plot3d_int_1, m_plot3d_per_1, dt[0], last_plot3d_file_time_1)) {
            last_plot3d_file_step_1 = step+1;
            Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
            for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
            if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 += m_plot3d_per_1;}
        }
        if (writeNow(cur_time, step+1, m_plot3d_int_2, m_plot3d_per_2, dt[0], last_plot3d_file_time_2)) {
            last_plot3d_file_step_2 = step+1;
            Write3DPlotFile(2,plotfile3d_type_2,plot3d_var_names_2);
            for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
            if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 += m_plot3d_per_2;}
        }
 
        if (writeNow(cur_time, step+1, m_plot2d_int_1, m_plot2d_per_1, dt[0], last_plot2d_file_time_1)) {
            last_plot2d_file_step_1 = step+1;
            Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
            if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 += m_plot2d_per_1;}
        }
 
        if (writeNow(cur_time, step+1, m_plot2d_int_2, m_plot2d_per_2, dt[0], last_plot2d_file_time_2)) {
            last_plot2d_file_step_2 = step+1;
            Write2DPlotFile(2,plotfile2d_type_2,plot2d_var_names_2);
            if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 += m_plot2d_per_2;}
        }
 
        for (int i = 0; i < m_subvol_int.size(); i++) {
            if (writeNow(cur_time, step+1, m_subvol_int[i], m_subvol_per[i], dt[0], last_subvol_time[i])) {
                last_subvol_step[i] = step+1;
                WriteSubvolume(i,subvol3d_var_names);
                if (m_subvol_per[i] > 0.) {last_subvol_time[i] += m_subvol_per[i];}
            }
        }
 
        if (writeNow(cur_time, step+1, m_check_int, m_check_per, dt[0], last_check_file_time)) {
            last_check_file_step = step+1;
            WriteCheckpointFile();
            if (m_check_per > 0.) {last_check_file_time += m_check_per;}
        }
 
#ifdef AMREX_MEM_PROFILING
        {
            std::ostringstream ss;
            ss << "[STEP " << step+1 << "]";
            MemProfiler::report(ss.str());
        }
#endif
 
        if (cur_time >= stop_time - 1.e-6*dt[0]) break;
    }
 
    // Write plotfiles at final time
    if ( (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.) && istep[0] > last_plot3d_file_step_1 ) {
        Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
        if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 += m_plot3d_per_1;}
    }
    if ( (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.) && istep[0] > last_plot3d_file_step_2) {
        Write3DPlotFile(2,plotfile3d_type_1,plot3d_var_names_2);
        if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 += m_plot3d_per_2;}
    }
    if ( (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.) && istep[0] > last_plot2d_file_step_1 ) {
        Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
        if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 += m_plot2d_per_1;}
    }
    if ( (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.) && istep[0] > last_plot2d_file_step_2) {
        Write2DPlotFile(2,plotfile2d_type_1,plot2d_var_names_2);
        if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 += m_plot2d_per_2;}
    }
 
    for (int i = 0; i < m_subvol_int.size(); i++) {
        if ( (m_subvol_int[i] > 0 || m_subvol_per[i] > 0.) && istep[0] > last_subvol_step[i]) {
            WriteSubvolume(i,subvol3d_var_names);
            if (m_subvol_per[i] > 0.) {last_subvol_time[i] += m_subvol_per[i];}
        }
    }
 
    if ( (m_check_int > 0 || m_check_per > 0.) && istep[0] > last_check_file_step) {
        WriteCheckpointFile();
        if (m_check_per > 0.) {last_check_file_time += m_check_per;}
    }
 
    BL_PROFILE_VAR_STOP(evolve);
}

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

fill_from_bndryregs()

void ERF::fill_from_bndryregs	(	const amrex::Vector< amrex::MultiFab * > &	mfs,
		amrex::Real	time
	)

{
    //
    // We now assume that if we read in on one face, we read in on all faces
    //
    AMREX_ALWAYS_ASSERT(m_r2d);
 
    int lev = 0;
    const Box& domain = geom[lev].Domain();
 
    const auto& dom_lo = lbound(domain);
    const auto& dom_hi = ubound(domain);
 
    Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time);
 
    const BCRec* bc_ptr = domain_bcs_type_d.data();
 
    // xlo: ori = 0
    // ylo: ori = 1
    // zlo: ori = 2
    // xhi: ori = 3
    // yhi: ori = 4
    // zhi: ori = 5
    const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
    const auto& bdatylo = (*bndry_data[1])[lev].const_array();
    const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
    const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
 
    int bccomp;
 
    for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
    {
        MultiFab& mf = *mfs[var_idx];
        const int icomp = 0;
        const int ncomp = mf.nComp();
 
        if (var_idx == Vars::xvel) {
           bccomp = BCVars::xvel_bc;
        } else if (var_idx == Vars::yvel) {
           bccomp = BCVars::yvel_bc;
        } else if (var_idx == Vars::zvel) {
           bccomp = BCVars::zvel_bc;
        } else  if (var_idx == Vars::cons) {
           bccomp = BCVars::cons_bc;
        }
 
#ifdef AMREX_USE_OMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
        for (MFIter mfi(mf); mfi.isValid(); ++mfi)
        {
            const Array4<Real>& dest_arr = mf.array(mfi);
            Box bx = mfi.growntilebox();
 
            // x-faces
            {
            Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
            if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
 
            Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
            if (var_idx == Vars::xvel) bx_xhi.setSmall(0,dom_hi.x);
 
            ParallelFor(
                bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].lo(0) == ERFBCType::ext_dir_ingested) {
                        int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
                    }
                },
                bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].hi(0) == ERFBCType::ext_dir_ingested) {
                        int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
                    }
                }
            );
            } // x-faces
 
            // y-faces
            {
            Box bx_ylo(bx); bx_ylo.setBig  (1,dom_lo.y-1);
            if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
 
            Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
            if (var_idx == Vars::yvel) bx_yhi.setSmall(1,dom_hi.y);
 
            ParallelFor(
               bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].lo(1) == ERFBCType::ext_dir_ingested) {
                        int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
                    }
                },
                bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                    int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                   BCVars::RhoScalar_bc_comp : icomp+n;
                    if (bc_ptr[bc_comp].hi(1) == ERFBCType::ext_dir_ingested) {
                        int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                        int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                        dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
                    }
                }
            );
            } // y-faces
        } // mf
    } // var_idx
}

fill_rhs()

void ERF::fill_rhs ( amrex::MultiFab &  rhs_mf, const amrex::MultiFab &  state_mf, amrex::Real  time, const amrex::Geometry &  geom  )

private

FillBdyCCVels()

void ERF::FillBdyCCVels	(	amrex::Vector< amrex::MultiFab > &	mf_cc_vel,
		int	levc = 0
	)

{
    // Impose bc's at domain boundaries
    for (int ilev(0); ilev < mf_cc_vel.size(); ++ilev)
    {
        int lev = ilev + levc;
        Box domain(Geom(lev).Domain());
 
        int ihi = domain.bigEnd(0);
        int jhi = domain.bigEnd(1);
        int khi = domain.bigEnd(2);
 
        // Impose periodicity first
        mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
 
        int jper = (Geom(lev).isPeriodic(1));
        int kper = (Geom(lev).isPeriodic(2));
 
        for (MFIter mfi(mf_cc_vel[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            const Array4<Real>& vel_arr = mf_cc_vel[lev].array(mfi);
 
            if (!Geom(lev).isPeriodic(0)) {
                // Low-x side
                if (bx.smallEnd(0) <= domain.smallEnd(0)) {
                    Real multn = ( (phys_bc_type[0] == ERF_BC::slip_wall   ) ||
                                   (phys_bc_type[0] == ERF_BC::no_slip_wall) ||
                                   (phys_bc_type[0] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                    Real multt = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
                    ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                    {
                        vel_arr(-1,j,k,0) = multn*vel_arr(0,j,k,0); // u
                        vel_arr(-1,j,k,1) = multt*vel_arr(0,j,k,1); // v
                        vel_arr(-1,j,k,2) = multt*vel_arr(0,j,k,2); // w
                    });
                }
 
                // High-x side
                if (bx.bigEnd(0) >= domain.bigEnd(0)) {
                    Real multn = ( (phys_bc_type[3] == ERF_BC::slip_wall   ) ||
                                   (phys_bc_type[3] == ERF_BC::no_slip_wall) ||
                                   (phys_bc_type[3] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                    Real multt = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
                    ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                    {
                        vel_arr(ihi+1,j,k,0) = multn*vel_arr(ihi,j,k,0); // u
                        vel_arr(ihi+1,j,k,1) = multt*vel_arr(ihi,j,k,1); // v
                        vel_arr(ihi+1,j,k,2) = multt*vel_arr(ihi,j,k,2); // w
                    });
                }
            } // !periodic
 
            if (!Geom(lev).isPeriodic(1)) {
                // Low-y side
                if (bx.smallEnd(1) <= domain.smallEnd(1)) {
                    Real multn = ( (phys_bc_type[1] == ERF_BC::slip_wall   ) ||
                                   (phys_bc_type[1] == ERF_BC::no_slip_wall) ||
                                   (phys_bc_type[1] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                    Real multt = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
                    ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int  , int k) noexcept
                    {
                        vel_arr(i,-1,k,0) = multt*vel_arr(i,0,k,0); // u
                        vel_arr(i,-1,k,1) = multn*vel_arr(i,0,k,1); // u
                        vel_arr(i,-1,k,2) = multt*vel_arr(i,0,k,2); // w
                    });
                }
 
                // High-y side
                if (bx.bigEnd(1) >= domain.bigEnd(1)) {
                    Real multn = ( (phys_bc_type[4] == ERF_BC::slip_wall   ) ||
                                   (phys_bc_type[4] == ERF_BC::no_slip_wall) ||
                                   (phys_bc_type[4] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                    Real multt = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
                    ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
                    {
                        vel_arr(i,jhi+1,k,0) = multt*vel_arr(i,jhi,k,0); // u
                        vel_arr(i,jhi+1,k,1) = multn*vel_arr(i,jhi,k,1); // v
                        vel_arr(i,jhi+1,k,2) = multt*vel_arr(i,jhi,k,2); // w
                    });
                }
            } // !periodic
 
            if (!Geom(lev).isPeriodic(2)) {
                // Low-z side
                if (bx.smallEnd(2) <= domain.smallEnd(2)) {
                    Real multn = ( (phys_bc_type[2] == ERF_BC::slip_wall   ) ||
                                   (phys_bc_type[2] == ERF_BC::no_slip_wall) ||
                                   (phys_bc_type[2] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                    Real multt = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
                    ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                    {
                        vel_arr(i,j,-1,0) = multt*vel_arr(i,j,0,0); // u
                        vel_arr(i,j,-1,1) = multt*vel_arr(i,j,0,1); // v
                        vel_arr(i,j,-1,2) = multn*vel_arr(i,j,0,2); // w
                    });
                }
 
                // High-z side
                if (bx.bigEnd(2) >= domain.bigEnd(2)) {
                    Real multn = ( (phys_bc_type[5] == ERF_BC::slip_wall   ) ||
                                   (phys_bc_type[5] == ERF_BC::no_slip_wall) ||
                                   (phys_bc_type[5] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                    Real multt = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -1. : 1.;
                    Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
                    ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                    {
                        vel_arr(i,j,khi+1,0) = multt*vel_arr(i,j,khi,0); // u
                        vel_arr(i,j,khi+1,1) = multt*vel_arr(i,j,khi,1); // v
                        vel_arr(i,j,khi+1,2) = multn*vel_arr(i,j,khi,2); // w
                    });
                }
            } // !periodic
        } // MFIter
 
        // Impose periodicity again
        mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
    } // lev
}

FillCoarsePatch()

void ERF::FillCoarsePatch ( int  lev, amrex::Real  time  )

private

{
    BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
    AMREX_ASSERT(lev > 0);
 
    //
    //****************************************************************************************************************
    // First fill velocities and density at the COARSE level so we can convert velocity to momenta at the COARSE level
    //****************************************************************************************************************
    //
    bool cons_only = false;
    if (lev == 1) {
        FillPatchCrseLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
                           &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
                           cons_only);
    } else {
        FillPatchFineLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
                           &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
                          {&vars_new[lev-1][Vars::cons],
                           &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
                           base_state[lev-1], base_state[lev-1],
                           false, cons_only);
    }
 
    //
    // ************************************************
    // Convert velocity to momentum at the COARSE level
    // ************************************************
    //
    const MultiFab* c_vfrac = nullptr;
    if (solverChoice.terrain_type == TerrainType::EB) {
        c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
    }
 
    VelocityToMomentum(vars_new[lev-1][Vars::xvel], IntVect{0},
                       vars_new[lev-1][Vars::yvel], IntVect{0},
                       vars_new[lev-1][Vars::zvel], IntVect{0},
                       vars_new[lev-1][Vars::cons],
                         rU_new[lev-1],
                         rV_new[lev-1],
                         rW_new[lev-1],
                       Geom(lev).Domain(),
                       domain_bcs_type, c_vfrac);
    //
    // *****************************************************************
    // Interpolate all cell-centered variables from coarse to fine level
    // *****************************************************************
    //
    Interpolater* mapper_c = &cell_cons_interp;
    Interpolater* mapper_f = &face_cons_linear_interp;
 
    //
    //************************************************************************************************
    // Interpolate cell-centered data from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
    // ************************************************************************************************
    IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
    int      ncomp_cons = vars_new[lev][Vars::cons].nComp();
 
    InterpFromCoarseLevel(vars_new[lev  ][Vars::cons], ngvect_cons, IntVect(0,0,0),
                          vars_new[lev-1][Vars::cons], 0, 0, ncomp_cons,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_c, domain_bcs_type, BCVars::cons_bc);
 
    // ***************************************************************************
    // Physical bc's for cell centered variables at domain boundary
    // ***************************************************************************
    (*physbcs_cons[lev])(vars_new[lev][Vars::cons],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                         0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc,true);
 
    //
    //************************************************************************************************
    // Interpolate x-momentum from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
    // ************************************************************************************************
    //
    InterpFromCoarseLevel(rU_new[lev], IntVect{0}, IntVect{0}, rU_new[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::xvel_bc);
 
    //
    //************************************************************************************************
    // Interpolate y-momentum from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
    // ************************************************************************************************
    //
    InterpFromCoarseLevel(rV_new[lev], IntVect{0}, IntVect{0}, rV_new[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::yvel_bc);
 
    //************************************************************************************************
    // Interpolate z-momentum from coarse to fine level
    // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
    // ************************************************************************************************
    InterpFromCoarseLevel(rW_new[lev],  IntVect{0}, IntVect{0}, rW_new[lev-1], 0, 0, 1,
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::zvel_bc);
    //
    // *********************************************************
    // After interpolation of momentum, convert back to velocity
    // *********************************************************
    //
    for (int which_lev = lev-1; which_lev <= lev; which_lev++)
    {
        c_vfrac = nullptr;
        if (solverChoice.terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(which_lev).get_const_factory())->getVolFrac());
        }
 
        MomentumToVelocity(vars_new[which_lev][Vars::xvel],
                           vars_new[which_lev][Vars::yvel],
                           vars_new[which_lev][Vars::zvel],
                           vars_new[which_lev][Vars::cons],
                             rU_new[which_lev],
                             rV_new[which_lev],
                             rW_new[which_lev],
                           Geom(which_lev).Domain(),
                           domain_bcs_type, c_vfrac);
    }
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
 
    (*physbcs_u[lev])(vars_new[lev][Vars::xvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                      ngvect_vels,time,BCVars::xvel_bc,true);
    (*physbcs_v[lev])(vars_new[lev][Vars::yvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                      ngvect_vels,time,BCVars::yvel_bc,true);
    (*physbcs_w[lev])(vars_new[lev][Vars::zvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                      ngvect_vels,time,BCVars::zvel_bc,true);
 
    // ***************************************************************************
    // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
    // ***************************************************************************
}

Here is the call graph for this function:

FillForecastStateMultiFabs()

void ERF::FillForecastStateMultiFabs	(	const int	lev,
		const std::string &	filename,
		const std::unique_ptr< amrex::MultiFab > &	z_phys_nd,
		amrex::Vector< amrex::Vector< amrex::MultiFab >> &	forecast_state
	)

{
 
    Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
    Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
 
    ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
                       xvec_h, yvec_h, zvec_h, rho_h,
                       uvel_h, vvel_h, wvel_h,
                       theta_h, qv_h, qc_h, qr_h);
 
    Real zmax = *std::max_element(zvec_h.begin(), zvec_h.end());
 
    const auto prob_lo_erf  = geom[lev].ProbLoArray();
    const auto prob_hi_erf  = geom[lev].ProbHiArray();
    const auto dx_erf       = geom[lev].CellSizeArray();
 
    if (prob_hi_erf[2] >= zmax) {
        Abort("ERROR: the maximum z of the domain (" + std::to_string(prob_hi_erf[2]) +
        ") should be less than the maximum z in the forecast data (" + std::to_string(zmax) +
        "). Change geometry.prob_hi[2] in the inputs to be less than " + std::to_string(zmax) + "."
        );
    }
 
    if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
        amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
    }
    if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
        amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
    }
    if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
        amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
    }
    if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
        amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
    }
 
 
    int nx = xvec_h.size();
    int ny = yvec_h.size();
    int nz = zvec_h.size();
 
    amrex::Real dxvec = (xvec_h[nx-1]-xvec_h[0])/(nx-1);
    amrex::Real dyvec = (yvec_h[ny-1]-yvec_h[0])/(ny-1);
 
    amrex::Gpu::DeviceVector<Real> latvec_d(nx*ny), lonvec_d(nx*ny), zvec_d(nz);
    amrex::Gpu::DeviceVector<Real> xvec_d(nx*ny*nz), yvec_d(nx*ny*nz);
    amrex::Gpu::DeviceVector<Real> rho_d(nx*ny*nz), uvel_d(nx*ny*nz), vvel_d(nx*ny*nz), wvel_d(nx*ny*nz),
                                   theta_d(nx*ny*nz), qv_d(nx*ny*nz), qc_d(nx*ny*nz), qr_d(nx*ny*nz);
 
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, latvec_h.begin(), latvec_h.end(), latvec_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, lonvec_h.begin(), lonvec_h.end(), lonvec_d.begin());
 
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, xvec_h.begin(), xvec_h.end(), xvec_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, yvec_h.begin(), yvec_h.end(), yvec_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, zvec_h.begin(), zvec_h.end(), zvec_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, rho_h.begin(), rho_h.end(), rho_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, theta_h.begin(), theta_h.end(), theta_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, uvel_h.begin(), uvel_h.end(), uvel_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, vvel_h.begin(), vvel_h.end(), vvel_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, wvel_h.begin(), wvel_h.end(), wvel_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qv_h.begin(), qv_h.end(), qv_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qc_h.begin(), qc_h.end(), qc_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qr_h.begin(), qr_h.end(), qr_d.begin());
 
    amrex::Gpu::streamSynchronize();
 
    Real* latvec_d_ptr = latvec_d.data();
    Real* lonvec_d_ptr = lonvec_d.data();
    Real* xvec_d_ptr = xvec_d.data();
    Real* yvec_d_ptr = yvec_d.data();
    Real* zvec_d_ptr = zvec_d.data();
    Real* rho_d_ptr   = rho_d.data();
    Real* uvel_d_ptr  = uvel_d.data();
    Real* vvel_d_ptr  = vvel_d.data();
    Real* wvel_d_ptr  = wvel_d.data();
    Real* theta_d_ptr = theta_d.data();
    Real* qv_d_ptr = qv_d.data();
    Real* qc_d_ptr = qc_d.data();
    Real* qr_d_ptr = qr_d.data();
 
    MultiFab& erf_mf_cons   = forecast_state[lev][Vars::cons];
    MultiFab& erf_mf_xvel   = forecast_state[lev][Vars::xvel];
    MultiFab& erf_mf_yvel   = forecast_state[lev][Vars::yvel];
    MultiFab& erf_mf_zvel   = forecast_state[lev][Vars::zvel];
    MultiFab& erf_mf_latlon = forecast_state[lev][4];
 
    erf_mf_cons.setVal(0.0);
    erf_mf_xvel.setVal(0.0);
    erf_mf_yvel.setVal(0.0);
    erf_mf_zvel.setVal(0.0);
    erf_mf_latlon.setVal(0.0);
 
    // Interpolate the data on to the ERF mesh
 
     for (MFIter mfi(erf_mf_cons); mfi.isValid(); ++mfi) {
        const auto z_arr    = (a_z_phys_nd) ? a_z_phys_nd->const_array(mfi) :
                                            Array4<const Real> {};
        const Array4<Real> &fine_cons_arr = erf_mf_cons.array(mfi);
        const Array4<Real> &fine_xvel_arr = erf_mf_xvel.array(mfi);
        const Array4<Real> &fine_yvel_arr = erf_mf_yvel.array(mfi);
        const Array4<Real> &fine_zvel_arr = erf_mf_zvel.array(mfi);
        const Array4<Real> &fine_latlon_arr = erf_mf_latlon.array(mfi);
 
 
        const Box& gbx = mfi.growntilebox(); // tilebox + ghost cells
 
        const Box &gtbx = mfi.tilebox(IntVect(1,0,0));
        const Box &gtby = mfi.tilebox(IntVect(0,1,0));
        const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
        const auto prob_lo  = geom[lev].ProbLoArray();
        const auto dx       = geom[lev].CellSizeArray();
       //const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
 
        ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
            // Geometry (note we must include these here to get the data on device)
            const Real x        = prob_lo[0] + (i + 0.5) * dx[0];
            const Real y        = prob_lo[1] + (j + 0.5) * dx[1];
            //const Real z        = prob_lo[2] + (k + 0.5) * dx[2];
            const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/2.0;
 
            // First interpolate where the weather data is available from
            Real tmp_rho, tmp_theta, tmp_qv, tmp_qc, tmp_qr, tmp_lat, tmp_lon;
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   rho_d_ptr, tmp_rho);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   theta_d_ptr, tmp_theta);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   qv_d_ptr, tmp_qv);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   qc_d_ptr, tmp_qc);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   qr_d_ptr, tmp_qr);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, 1,
                                   x, y, 0.0,
                                   latvec_d_ptr, tmp_lat);
 
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, 1,
                                   x, y, 0.0,
                                   lonvec_d_ptr, tmp_lon);
 
            fine_cons_arr(i,j,k,Rho_comp) = tmp_rho;
            fine_latlon_arr(i,j,k,0) = tmp_lat;
            fine_latlon_arr(i,j,k,1) = tmp_lon;
        });
 
        ParallelFor(gtbx, gtby, gtbz,
        [=] AMREX_GPU_DEVICE(int i, int j, int k) {
             // Physical location of the fine node
            Real x = prob_lo_erf[0] + i       * dx_erf[0];
            Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
            //Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
            const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/2.0;
 
            Real tmp_uvel;
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   uvel_d_ptr, tmp_uvel);
 
            fine_xvel_arr(i, j, k, 0) = tmp_uvel;
        },
        [=] AMREX_GPU_DEVICE(int i, int j, int k) {
             // Physical location of the fine node
            Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
            Real y = prob_lo_erf[1] + j       * dx_erf[1];
            //Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
            const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/2.0;
 
            Real tmp_vvel;
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   vvel_d_ptr, tmp_vvel);
 
            fine_yvel_arr(i, j, k, 0) = tmp_vvel;
        },
        [=] AMREX_GPU_DEVICE(int i, int j, int k) {
             // Physical location of the fine node
            Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
            Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
            Real z = prob_lo_erf[2] + k       * dx_erf[2];
            //const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/2.0;
 
            Real tmp_wvel;
            bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                   dxvec, dyvec,
                                   nx, ny, nz,
                                   x, y, z,
                                   wvel_d_ptr, tmp_wvel);
 
            fine_zvel_arr(i, j, k, 0) = tmp_wvel;
        });
    }
 
    /*Vector<std::string> varnames = {
    "rho", "uvel", "vvel", "wvel", "theta", "qv", "qc", "qr"
    }; // Customize variable names
 
     Vector<std::string> varnames_cons = {
    "rho", "rhotheta", "ke", "sc", "rhoqv", "rhoqc", "rhoqr"
    }; // Customize variable names
 
    Vector<std::string> varnames_plot_mf = {
    "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
    }; // Customize variable names
 
    const Real time = 0.0;
 
    std::string pltname = "plt_interp";
 
    MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
                     10, 0);
 
    plot_mf.setVal(0.0);
 
    for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
        const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
        const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
        const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
        const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
        const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
        const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
 
        const Box& bx = mfi.validbox();
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
            plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
            plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
            plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
            plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
            plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
 
            plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/2.0;
            plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/2.0;
            plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/2.0;
 
            plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
            plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
        });
    }
 
 
    WriteSingleLevelPlotfile(
            pltname,
            plot_mf,
            varnames_plot_mf,
            geom[0],
            time,
            0 // level
        );*/
}

Here is the call graph for this function:

FillIntermediatePatch()

void ERF::FillIntermediatePatch ( int  lev, amrex::Real  time, const amrex::Vector< amrex::MultiFab * > &  mfs_vel, const amrex::Vector< amrex::MultiFab * > &  mfs_mom, int  ng_cons, int  ng_vel, bool  cons_only, int  icomp_cons, int  ncomp_cons  )

private

{
    BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
    Interpolater* mapper;
 
    PhysBCFunctNoOp null_bc;
 
    //
    // ***************************************************************************
    // The first thing we do is interpolate the momenta on the "valid" faces of
    // the fine grids (where the interface is coarse/fine not fine/fine) -- this
    // will not be over-written by interpolation below because the FillPatch
    // operators see these as valid faces.  But we must have these interpolated
    // values in the fine data before we call FillPatchTwoLevels.
    //
    // Also -- note that we might be filling values by interpolation at physical boundaries
    //         here but that's ok because we will overwrite those values when we impose
    //         the physical bc's below
    // ***************************************************************************
    if (lev>0) {
        if (cf_set_width > 0) {
            // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
            FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
        }
        if ( !cons_only && (cf_set_width >= 0) ) {
            FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
            FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
            FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
        }
    }
 
    // amrex::Print() << "LEVEL " << lev << " CONS ONLY   " << cons_only <<
    //                   " ICOMP NCOMP " << icomp_cons << " " << ncomp_cons << " NGHOST " << ng_cons << std::endl;
 
    if (!cons_only) {
        AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
        AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
    }
 
    // Enforce no penetration for thin immersed body
    if (!cons_only) {
        // Enforce no penetration for thin immersed body
        if (xflux_imask[lev]) {
            ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
        }
        if (yflux_imask[lev]) {
            ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
        }
        if (zflux_imask[lev]) {
            ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
        }
    }
 
    //
    // We now start working on conserved quantities + VELOCITY
    //
    if (lev == 0)
    {
        // We don't do anything here because we will call the physbcs routines below,
        // which calls FillBoundary and fills other domain boundary conditions
        // Physical boundaries will be filled below
 
        if (!cons_only)
        {
            // ***************************************************************************
            // We always come in to this call with updated momenta but we need to create updated velocity
            //    in order to impose the rest of the bc's
            // ***************************************************************************
            const MultiFab* c_vfrac = nullptr;
            if (solverChoice.terrain_type == TerrainType::EB) {
                c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
            }
 
            // This only fills VALID region of velocity
            MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(), domain_bcs_type, c_vfrac);
        }
    }
    else
    {
        //
        // We must fill a temporary then copy it back so we don't double add/subtract
        //
        MultiFab mf(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
                    mfs_vel[Vars::cons]->nComp()   ,mfs_vel[Vars::cons]->nGrowVect());
        //
        // Set all components to 1.789e19, then copy just the density from *mfs_vel[Vars::cons]
        //
        mf.setVal(1.789e19);
        MultiFab::Copy(mf,*mfs_vel[Vars::cons],Rho_comp,Rho_comp,1,mf.nGrowVect());
 
        Vector<MultiFab*> fmf = {mfs_vel[Vars::cons],mfs_vel[Vars::cons]};
        Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
        Vector<Real> ctime    = {t_old[lev-1], t_new[lev-1]};
        Vector<Real> ftime    = {time,time};
 
        if (interpolation_type == StateInterpType::Perturbational)
        {
            if (icomp_cons+ncomp_cons > 1)
            {
                // Divide (rho theta) by rho to get theta
                MultiFab::Divide(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
 
                // Subtract theta_0 from theta
                MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
 
                if (!amrex::almostEqual(time,ctime[1])) {
                    MultiFab::Divide(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                                     Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                    MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                       BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                }
                if (!amrex::almostEqual(time,ctime[0])) {
                    MultiFab::Divide(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                                     Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                    MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                       BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                }
            }
 
            // Subtract rho_0 from rho before we interpolate -- note we only subtract
            //    on valid region of mf since the ghost cells will be filled below
            if (icomp_cons == 0)
            {
                MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
 
                if (!amrex::almostEqual(time,ctime[1])) {
                    MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                       BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                }
                if (!amrex::almostEqual(time,ctime[0])) {
                    MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                       BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                }
            }
        } // interpolation_type == StateInterpType::Perturbational
 
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        mapper = &cell_cons_interp;
        FillPatchTwoLevels(mf, IntVect{ng_cons}, IntVect(0,0,0),
                           time, cmf, ctime, fmf, ftime,
                           icomp_cons, icomp_cons, ncomp_cons, geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper, domain_bcs_type,
                           icomp_cons);
 
        if (interpolation_type == StateInterpType::Perturbational)
        {
            if (icomp_cons == 0)
            {
                // Restore the coarse values to what they were
                if (!amrex::almostEqual(time,ctime[1])) {
                    MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                }
                if (!amrex::almostEqual(time,ctime[0])) {
                    MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                }
 
                // Set values in the cells outside the domain boundary so that we can do the Add
                //     without worrying about uninitialized values outside the domain -- these
                //     will be filled in the physbcs call
                mf.setDomainBndry(1.234e20,Rho_comp,1,geom[lev]);
 
                // Add rho_0 back to rho after we interpolate -- on all the valid + ghost region
                MultiFab::Add(mf, base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{ng_cons});
            }
 
            if (icomp_cons+ncomp_cons > 1)
            {
                // Add theta_0 to theta
                if (!amrex::almostEqual(time,ctime[1])) {
                    MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                    MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                                       Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                }
                if (!amrex::almostEqual(time,ctime[0])) {
                    MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                    MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                                       Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                }
 
                // Multiply theta by rho to get (rho theta)
                MultiFab::Multiply(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
 
                // Add theta_0 to theta
                MultiFab::Add(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
 
                // Add theta_0 back to theta
                MultiFab::Add(mf,base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{ng_cons});
 
                // Multiply (theta) by rho to get (rho theta)
                MultiFab::Multiply(mf,mf,Rho_comp,RhoTheta_comp,1,IntVect{ng_cons});
            }
        } // interpolation_type == StateInterpType::Perturbational
 
        // Impose physical bc's on fine data (note time and 0 are not used)
        // Note that we do this after the FillPatch because imposing physical bc's on fine ghost
        //      cells that need to be filled from coarse requires that we have done the interpolation first
        bool do_fb = true; bool do_terrain_adjustment = false;
        (*physbcs_cons[lev])(mf,*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                             icomp_cons,ncomp_cons,IntVect{ng_cons},time,BCVars::cons_bc,
                             do_fb, do_terrain_adjustment);
 
        // Make sure to only copy back the components we worked on
        MultiFab::Copy(*mfs_vel[Vars::cons],mf,icomp_cons,icomp_cons,ncomp_cons,IntVect{ng_cons});
 
        // *****************************************************************************************
 
        if (!cons_only)
        {
            // ***************************************************************************
            // We always come in to this call with updated momenta but we need to create updated velocity
            //    in order to impose the rest of the bc's
            // ***************************************************************************
            const MultiFab* c_vfrac = nullptr;
            if (solverChoice.terrain_type == TerrainType::EB) {
                c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
            }
 
            // This only fills VALID region of velocity
            MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(), domain_bcs_type, c_vfrac);
 
            mapper = &face_cons_linear_interp;
 
            //
            // NOTE: All interpolation here happens on velocities not momenta;
            //       note we only do the interpolation and FillBoundary here,
            //       physical bc's are imposed later
            //
            // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
            //       unlike the FillSet calls above which filled momenta on the coarse/fine bdy
            //
 
            MultiFab& mfu = *mfs_vel[Vars::xvel];
 
            fmf = {&mfu,&mfu};
            cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mfu, IntVect{ng_vel}, IntVect(0,0,0),
                               time, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::xvel_bc);
 
            // *****************************************************************************************
 
            MultiFab& mfv = *mfs_vel[Vars::yvel];
 
            fmf = {&mfv,&mfv};
            cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mfv, IntVect{ng_vel}, IntVect(0,0,0),
                               time, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::yvel_bc);
 
            // *****************************************************************************************
 
            MultiFab& mfw = *mfs_vel[Vars::zvel];
 
            fmf = {&mfw,&mfw};
            cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mfw, IntVect{ng_vel}, IntVect(0,0,0),
                               time, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::zvel_bc);
        } // !cons_only
    } // lev > 0
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
    IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
 
    bool do_fb = true;
 
#ifdef ERF_USE_NETCDF
    // We call this here because it is an ERF routine
    if (solverChoice.use_real_bcs && (lev==0)) {
        if (solverChoice.upwind_real_bcs) {
            fill_from_realbdy_upwind(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
        } else {
            fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
        }
        do_fb = false;
    }
#endif
 
    if (m_r2d) fill_from_bndryregs(mfs_vel,time);
 
    // We call this even if use_real_bcs is true because these will fill the vertical bcs
    (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                         icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
    if (!cons_only) {
        (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::xvel_bc, do_fb);
        (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::yvel_bc, do_fb);
        (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::zvel_bc, do_fb);
    }
    // ***************************************************************************
 
    // We always come in to this call with momenta so we need to leave with momenta!
    // We need to make sure we convert back on all ghost cells/faces because this is
    // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
    if (!cons_only)
    {
        IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
        IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
        IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
 
        const MultiFab* c_vfrac = nullptr;
        if (solverChoice.terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
        }
 
        VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
                           *mfs_vel[Vars::yvel], ngv,
                           *mfs_vel[Vars::zvel], ngw,
                           *mfs_vel[Vars::cons],
                           *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                           Geom(lev).Domain(),
                           domain_bcs_type, c_vfrac);
    }
 
    // NOTE: There are not FillBoundary calls here for the following reasons:
    // Removal of the FillBoundary (FB) calls has bee completed for the following reasons:
    //
    // 1. physbc_cons is called before VelocityToMomentum and a FB is completed in that functor.
    //    Therefore, the conserved CC vars have their inter-rank ghost cells filled and then their
    //    domain ghost cells filled from the BC operations. We should not call FB on this MF again.
    //
    // 2. physbc_u/v/w is also called before VelocityToMomentum and a FB is completed those functors.
    //    Furthermore, VelocityToMomentum operates on a growntilebox so we exit that routine with momentum
    //    filled everywhere---i.e., physbc_u/v/w fills velocity ghost cells (inter-rank and domain)
    //    and then V2M does the conversion to momenta everywhere; so there is again no need to do a FB on momenta.
}

Here is the call graph for this function:

FillPatchCrseLevel()

void ERF::FillPatchCrseLevel ( int  lev, amrex::Real  time, const amrex::Vector< amrex::MultiFab * > &  mfs_vel, bool  cons_only = false  )

private

{
    BL_PROFILE_VAR("ERF::FillPatchCrseLevel()",ERF_FillPatchCrseLevel);
 
    AMREX_ALWAYS_ASSERT(lev == 0);
 
    IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
    IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
 
    Vector<Real> ftime    = {t_old[lev], t_new[lev]};
 
    //
    // Below we call FillPatchSingleLevel which does NOT fill ghost cells outside the domain
    //
 
    Vector<MultiFab*> fmf;
    Vector<MultiFab*> fmf_u;
    Vector<MultiFab*> fmf_v;
    Vector<MultiFab*> fmf_w;
 
    if (amrex::almostEqual(time,ftime[0])) {
        fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
    } else if (amrex::almostEqual(time,ftime[1])) {
        fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
    } else {
        fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
    }
 
    const int  ncomp = mfs_vel[Vars::cons]->nComp();
 
    FillPatchSingleLevel(*mfs_vel[Vars::cons], ngvect_cons, time, fmf, IntVect(0,0,0), ftime,
                         0, 0, ncomp, geom[lev]);
 
    if (!cons_only) {
        if (amrex::almostEqual(time,ftime[0])) {
            fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
            fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
            fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
        } else if (amrex::almostEqual(time,ftime[1])) {
            fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
            fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
            fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
        } else {
            fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
            fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
            fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
        }
        FillPatchSingleLevel(*mfs_vel[Vars::xvel], ngvect_vels, time, fmf_u,
                             IntVect(0,0,0), ftime,  0, 0, 1, geom[lev]);
 
        FillPatchSingleLevel(*mfs_vel[Vars::yvel], ngvect_vels, time, fmf_v,
                             IntVect(0,0,0), ftime,  0, 0, 1, geom[lev]);
 
        FillPatchSingleLevel(*mfs_vel[Vars::zvel], ngvect_vels, time, fmf_w,
                             IntVect(0,0,0), ftime,  0, 0, 1, geom[lev]);
    } // !cons_only
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    int icomp_cons = 0;
    int ncomp_cons = mfs_vel[Vars::cons]->nComp();
 
    bool do_fb = true;
 
#ifdef ERF_USE_NETCDF
    // We call this here because it is an ERF routine
    if(solverChoice.use_real_bcs && (lev==0)) {
        if (solverChoice.upwind_real_bcs) {
            fill_from_realbdy_upwind(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
        } else {
            fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
        }
        do_fb = false;
    }
#endif
 
    if (m_r2d) fill_from_bndryregs(mfs_vel,time);
 
    // We call this even if use_real_bcs is true because these will fill the vertical bcs
    // Note that we call FillBoundary inside the physbcs call
    (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
    if (!cons_only) {
        (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::xvel_bc, do_fb);
        (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::yvel_bc, do_fb);
        (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::zvel_bc, do_fb);
    }
}

FillPatchFineLevel()

void ERF::FillPatchFineLevel ( int  lev, amrex::Real  time, const amrex::Vector< amrex::MultiFab * > &  mfs_vel, const amrex::Vector< amrex::MultiFab * > &  mfs_mom, const amrex::MultiFab &  old_base_state, const amrex::MultiFab &  new_base_state, bool  fillset = true, bool  cons_only = false  )

private

{
    BL_PROFILE_VAR("ERF::FillPatchFineLevel()",ERF_FillPatchFineLevel);
 
    AMREX_ALWAYS_ASSERT(lev > 0);
 
    Interpolater* mapper = nullptr;
 
    PhysBCFunctNoOp null_bc;
 
    //
    // ***************************************************************************
    // The first thing we do is interpolate the momenta on the "valid" faces of
    // the fine grids (where the interface is coarse/fine not fine/fine) -- this
    // will not be over-written below because the FillPatch operators see these as
    // valid faces.
    //
    // Note that we interpolate momentum not velocity, but all the other boundary
    // conditions are imposed on velocity, so we convert to momentum here then
    // convert back.
    // ***************************************************************************
    if (fillset) {
        if (cf_set_width > 0) {
            FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
        }
        if (cf_set_width >= 0 && !cons_only) {
 
            const MultiFab* c_vfrac = nullptr;
            if (solverChoice.terrain_type == TerrainType::EB) {
                c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
            }
 
            VelocityToMomentum(*mfs_vel[Vars::xvel], IntVect{0},
                               *mfs_vel[Vars::yvel], IntVect{0},
                               *mfs_vel[Vars::zvel], IntVect{0},
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom],
                               *mfs_mom[IntVars::ymom],
                               *mfs_mom[IntVars::zmom],
                               Geom(lev).Domain(),
                               domain_bcs_type, c_vfrac);
 
            FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
            FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
            FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
 
            MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                               *mfs_vel[Vars::cons],
                               *mfs_mom[IntVars::xmom],
                               *mfs_mom[IntVars::ymom],
                               *mfs_mom[IntVars::zmom],
                               Geom(lev).Domain(),
                               domain_bcs_type, c_vfrac);
        }
    }
 
    IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
    IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
 
    Vector<Real> ftime    = {t_old[lev  ], t_new[lev  ]};
    Vector<Real> ctime    = {t_old[lev-1], t_new[lev-1]};
 
    amrex::Real small_dt = 1.e-8 * (ftime[1] - ftime[0]);
 
    Vector<MultiFab*> fmf;
    if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
        fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
    } else if (amrex::almostEqual(time,ftime[1])) {
        fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
    } else {
        fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
    }
    Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
 
    // We must fill a temporary then copy it back so we don't double add/subtract
    MultiFab mf_c(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
                  mfs_vel[Vars::cons]->nComp()   ,mfs_vel[Vars::cons]->nGrowVect());
 
    mapper = &cell_cons_interp;
 
    if (interpolation_type == StateInterpType::Perturbational)
    {
        // Divide (rho theta) by rho to get theta (before we subtract rho0 from rho!)
        if (!amrex::almostEqual(time,ctime[1])) {
            MultiFab::Divide(vars_old[lev-1][Vars::cons],vars_old[lev-1][Vars::cons],
                             Rho_comp,RhoTheta_comp,1,ngvect_cons);
            MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
                               BaseState::r0_comp,Rho_comp,1,ngvect_cons);
            MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
                               BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
        }
        if (!amrex::almostEqual(time,ctime[0])) {
            MultiFab::Divide(vars_new[lev-1][Vars::cons],vars_new[lev-1][Vars::cons],
                             Rho_comp,RhoTheta_comp,1,ngvect_cons);
            MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
                               BaseState::r0_comp,Rho_comp,1,ngvect_cons);
            MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
                               BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
        }
 
        if (!amrex::almostEqual(time,ftime[1])) {
            MultiFab::Divide(vars_old[lev  ][Vars::cons],vars_old[lev  ][Vars::cons],
                             Rho_comp,RhoTheta_comp,1,IntVect{0});
            MultiFab::Subtract(vars_old[lev  ][Vars::cons],old_base_state,
                               BaseState::r0_comp,Rho_comp,1,IntVect{0});
            MultiFab::Subtract(vars_old[lev  ][Vars::cons],old_base_state,
                               BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        }
        if (!amrex::almostEqual(time,ftime[0])) {
            MultiFab::Divide(vars_new[lev  ][Vars::cons],vars_new[lev  ][Vars::cons],
                             Rho_comp,RhoTheta_comp,1,IntVect{0});
            MultiFab::Subtract(vars_new[lev  ][Vars::cons],old_base_state,
                               BaseState::r0_comp,Rho_comp,1,IntVect{0});
            MultiFab::Subtract(vars_new[lev  ][Vars::cons],old_base_state,
                               BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
        }
    }
 
    // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
    FillPatchTwoLevels(mf_c, ngvect_cons, IntVect(0,0,0),
                       time, cmf, ctime, fmf, ftime,
                       0, 0, mf_c.nComp(), geom[lev-1], geom[lev],
                       refRatio(lev-1), mapper, domain_bcs_type,
                       BCVars::cons_bc);
 
    if (interpolation_type == StateInterpType::Perturbational)
    {
        // Restore the coarse values to what they were
        if (!amrex::almostEqual(time,ctime[1])) {
            MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                          BaseState::r0_comp,Rho_comp,1,ngvect_cons);
            MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                          BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
            MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                               Rho_comp,RhoTheta_comp,1,ngvect_cons);
        }
        if (!amrex::almostEqual(time,ctime[0])) {
            MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                          BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
            MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                          BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
            MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                               Rho_comp,RhoTheta_comp,1,ngvect_cons);
        }
 
        if (!amrex::almostEqual(time,ftime[1])) {
            MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev  ],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
            MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev  ],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
            MultiFab::Multiply(vars_old[lev][Vars::cons], vars_old[lev][Vars::cons],
                                   Rho_comp,RhoTheta_comp,1,ngvect_cons);
        }
        if (!amrex::almostEqual(time,ftime[0])) {
            MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
            MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
            MultiFab::Multiply(vars_new[lev][Vars::cons], vars_new[lev][Vars::cons],
                               Rho_comp,RhoTheta_comp,1,ngvect_cons);
        }
 
        // Set values in the cells outside the domain boundary so that we can do the Add
        //     without worrying about uninitialized values outside the domain -- these
        //     will be filled in the physbcs call
        mf_c.setDomainBndry(1.234e20,0,2,geom[lev]); // Do both rho and (rho theta) together
 
        // Add rho_0 back to rho and theta_0 back to theta
        MultiFab::Add(mf_c, new_base_state,BaseState::r0_comp,Rho_comp,1,ngvect_cons);
        MultiFab::Add(mf_c, new_base_state,BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
 
        // Multiply (theta) by rho to get (rho theta)
        MultiFab::Multiply(mf_c,mf_c,Rho_comp,RhoTheta_comp,1,ngvect_cons);
    }
 
    MultiFab::Copy(*mfs_vel[Vars::cons],mf_c,0,0,mf_c.nComp(),mf_c.nGrowVect());
 
    // ***************************************************************************************
 
    if (!cons_only)
    {
            mapper = &face_cons_linear_interp;
 
            MultiFab& mf_u = *mfs_vel[Vars::xvel];
            MultiFab& mf_v = *mfs_vel[Vars::yvel];
            MultiFab& mf_w = *mfs_vel[Vars::zvel];
 
            Vector<MultiFab*> fmf_u; Vector<MultiFab*> fmf_v; Vector<MultiFab*> fmf_w;
            Vector<MultiFab*> cmf_u; Vector<MultiFab*> cmf_v; Vector<MultiFab*> cmf_w;
 
            // **********************************************************************
 
            if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
                fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
                fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
                fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
            } else if ( amrex::almostEqual(time,ftime[1]) ) {
                fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
                fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
                fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
            } else {
                fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
                fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
                fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
            }
            cmf_u = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
            cmf_v = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
            cmf_w = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mf_u, ngvect_vels, IntVect(0,0,0),
                               time, cmf_u, ctime, fmf_u, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::xvel_bc);
 
            FillPatchTwoLevels(mf_v, ngvect_vels, IntVect(0,0,0),
                               time, cmf_v, ctime, fmf_v, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::yvel_bc);
 
            // We put these here because these may be used in constructing omega outside the
            // domain when fillpatching w
            bool do_fb = true;
            (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                              ngvect_vels,time,BCVars::xvel_bc, do_fb);
            (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                              ngvect_vels,time,BCVars::yvel_bc, do_fb);
 
            // **********************************************************************
 
            // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mf_w, ngvect_vels, IntVect(0,0,0),
                               time, cmf_w, ctime, fmf_w, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::zvel_bc);
    } // !cons_only
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
    int icomp_cons = 0;
    int ncomp_cons = mfs_vel[Vars::cons]->nComp();
 
    bool do_fb = true;
 
    if (m_r2d) fill_from_bndryregs(mfs_vel,time);
 
    // We call this even if use_real_bcs is true because these will fill the vertical bcs
    // Note that we call FillBoundary inside the physbcs call
    (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                         icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
    if (!cons_only) {
        // Note that we need to fill u and v in the case of terrain because we will use
        //      these in the call of WFromOmega in lateral ghost cells of the fine grid
        // (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
        //                   ngvect_vels,time,BCVars::xvel_bc, do_fb);
        // (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
        //                   ngvect_vels,time,BCVars::yvel_bc, do_fb);
        (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          ngvect_vels,time,BCVars::zvel_bc, do_fb);
    }
}

Here is the call graph for this function:

FillSurfaceStateMultiFabs()

void ERF::FillSurfaceStateMultiFabs	(	const int	lev,
		const std::string &	filename,
		amrex::Vector< amrex::MultiFab > &	surface_state
	)

{
       // Open the binary file in input mode
    std::ifstream infile(filename, std::ios::binary);
    if (!infile) {
        std::cerr << "Error: Could not open file " << filename << std::endl;
    }
    Vector<Real> xvec_h, yvec_h, zvec_h;
    Vector<Real> sst_h, q_star_h, t_star_h, u_star_h, ls_mask_h;
 
    int nx, ny, nz, ndata;
    float value;
 
    // Read the four integers
    infile.read(reinterpret_cast<char*>(&nx), sizeof(int));
    infile.read(reinterpret_cast<char*>(&ny), sizeof(int));
    infile.read(reinterpret_cast<char*>(&nz), sizeof(int));
    infile.read(reinterpret_cast<char*>(&ndata), sizeof(int));
 
    amrex::Gpu::DeviceVector<Real> xvec_d(nx*ny*nz), yvec_d(nx*ny*nz), zvec_d(nz);
    for(int i=0; i<nx; i++) {
        infile.read(reinterpret_cast<char*>(&value), sizeof(float));
        xvec_h.emplace_back(value);
    }
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, xvec_h.begin(), xvec_h.end(), xvec_d.begin());
 
    for(int j=0; j<ny; j++) {
        infile.read(reinterpret_cast<char*>(&value), sizeof(float));
        yvec_h.emplace_back(value);
    }
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, yvec_h.begin(), yvec_h.end(), yvec_d.begin());
 
    for(int k=0; k<nz; k++) {
        infile.read(reinterpret_cast<char*>(&value), sizeof(float));
        zvec_h.emplace_back(value);
    }
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, zvec_h.begin(), zvec_h.end(), zvec_d.begin());
 
         // Vector to store the data
 
    Vector<Real>* data_h = nullptr; // Declare pointer outside the loop
 
    Real* xvec_d_ptr = xvec_d.data();
    Real* yvec_d_ptr = yvec_d.data();
 
    Real dxvec = (xvec_h[nx-1]-xvec_h[0])/(nx-1);
    Real dyvec = (yvec_h[ny-1]-yvec_h[0])/(ny-1);
 
    // Read the file
    for(int idx=0; idx<ndata; idx++){
        if(idx == 0){
            data_h = &sst_h;
        } else if (idx==1) {
            data_h = &q_star_h;
        } else if (idx==2) {
            data_h = &t_star_h;
        } else if (idx==3) {
            data_h = &u_star_h;
        } else if(idx==4) {
            data_h = &ls_mask_h;
        }
        for(int k=0; k<nz; k++) {
            for(int j=0; j<ny; j++) {
                for(int i=0; i<nx; i++) {
                    infile.read(reinterpret_cast<char*>(&value), sizeof(float));
                    //if(idx == 3) {
                        //printf("theta is %0.15g, %0.15g, %0.15g %0.15g\n", xvec_h[i], yvec_h[j], zvec_h[k], value);
                    //}
                    data_h->emplace_back(value);
                }
            }
        }
    }
 
    infile.close();
 
    amrex::Gpu::DeviceVector<Real> ls_mask_d(nx*ny*nz), sst_d(nx*ny*nz);
 
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, ls_mask_h.begin(), ls_mask_h.end(), ls_mask_d.begin());
    amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, sst_h.begin(), sst_h.end(), sst_d.begin());
 
    Real* ls_mask_d_ptr = ls_mask_d.data();
    Real* sst_d_ptr   = sst_d.data();
 
    const auto prob_lo  = geom[lev].ProbLo();
    const auto dx       = geom[lev].CellSize();
 
    for (amrex::MFIter mfi(surface_state[lev]); mfi.isValid(); ++mfi) {
        const Box gbx = mfi.growntilebox();
        const Array4<Real>& surf_arr = surface_state[lev].array(mfi);
 
        ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
 
            if(k == 0) {
                const Real x        = prob_lo[0] + (i + 0.5) * dx[0];
                const Real y        = prob_lo[1] + (j + 0.5) * dx[1];
 
                // First interpolate where the weather data is available from
                Real tmp_ls_mask, tmp_sst;
 
                bilinear_interpolation_2d(xvec_d_ptr, yvec_d_ptr,
                                          dxvec, dyvec,
                                          nx, ny,
                                          x, y,
                                          ls_mask_d_ptr, tmp_ls_mask);
 
                bilinear_interpolation_2d(xvec_d_ptr, yvec_d_ptr,
                                          dxvec, dyvec,
                                          nx, ny,
                                          x, y,
                                          sst_d_ptr, tmp_sst);
 
                surf_arr(i, j, 0) = std::min(tmp_ls_mask, 1.0);
                surf_arr(i, j, 1) = tmp_sst;
            }
        });
    }
 
}

Here is the call graph for this function:

FindInitialEye()

bool ERF::FindInitialEye	(	int	lev,
		const amrex::MultiFab &	cc_vel,
		const amrex::Real	velmag_threshold,
		amrex::Real &	eye_x,
		amrex::Real &	eye_y
	)

{
    const auto dx = geom[levc].CellSizeArray();
    const auto prob_lo = geom[levc].ProbLoArray();
 
    Gpu::DeviceVector<Real> d_coords(2, 0.0);
    Gpu::DeviceVector<int>  d_found(1,0);
 
    Real* d_coords_ptr = d_coords.data();
    int*   d_found_ptr = d_found.data();
 
    for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi)
    {
        const Box& box = mfi.validbox();
        const Array4<const Real>& vel_arr = mf_cc_vel.const_array(mfi);
 
        ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k)
        {
            Real magnitude = std::sqrt(vel_arr(i,j,k,0) * vel_arr(i,j,k,0) +
                                       vel_arr(i,j,k,1) * vel_arr(i,j,k,1) +
                                       vel_arr(i,j,k,2) * vel_arr(i,j,k,2));
 
            magnitude *= 3.6;
 
            Real z = prob_lo[2] + (k + 0.5) * dx[2];
 
            // Check if magnitude exceeds threshold
            if (z < 2000. && magnitude > velmag_threshold) {
                // Use atomic operations to set found flag and store coordinates
                Gpu::Atomic::Add(&d_found_ptr[0], 1); // Mark as found
 
                Real x = prob_lo[0] + (i + 0.5) * dx[0];
                Real y = prob_lo[1] + (j + 0.5) * dx[1];
 
                // Store coordinates
                Gpu::Atomic::Add(&d_coords_ptr[0],x); // Store x index
                Gpu::Atomic::Add(&d_coords_ptr[1],y); // Store x index
            }
        });
    }
 
    // Synchronize to ensure all threads complete their execution
    amrex::Gpu::streamSynchronize(); // Wait for all GPU threads to finish
 
    Vector<int> h_found(1,0);
    Gpu::copy(Gpu::deviceToHost, d_found.begin(), d_found.end(), h_found.begin());
    ParallelAllReduce::Sum(h_found.data(), h_found.size(), ParallelContext::CommunicatorAll());
 
    // Broadcast coordinates if found
    if (h_found[0] > 0) {
        Vector<Real> h_coords(2,-1e10);
        Gpu::copy(Gpu::deviceToHost, d_coords.begin(), d_coords.end(), h_coords.begin());
 
        ParallelAllReduce::Sum(h_coords.data(), h_coords.size(), ParallelContext::CommunicatorAll());
 
        eye_x = h_coords[0]/h_found[0];
        eye_y = h_coords[1]/h_found[0];
 
    } else {
        // Random large negative numbers so we don't trigger refinement in this case
        eye_x = -1.e20;
        eye_y = -1.e20;
    }
 
    return (h_found[0] > 0);
}

get_eb()

eb_ const& ERF::get_eb ( int  lev ) const

inlineprivatenoexcept

                                                             {
        AMREX_ASSERT(lev >= 0 && lev < eb.size() && eb[lev] != nullptr);
        return *eb[lev];
    }

getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int  lev )

inlineprivate

    {
        return advflux_reg[lev];
    }

getCPUTime()

static amrex::Real ERF::getCPUTime ( )

inlinestaticprivate

    {
      int numCores = amrex::ParallelDescriptor::NProcs();
#ifdef _OPENMP
      numCores = numCores * omp_get_max_threads();
#endif
 
      amrex::Real T =
        numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
        previousCPUTimeUsed;
 
      return T;
    }

GotoNextLine()

void ERF::GotoNextLine ( std::istream &  is )

staticprivate

Utility to skip to next line in Header file input stream.

{
    constexpr std::streamsize bl_ignore_max { 100000 };
    is.ignore(bl_ignore_max, '\n');
}

HurricaneTracker()

void ERF::HurricaneTracker	(	int	lev,
		amrex::Real	time,
		const amrex::MultiFab &	cc_vel,
		const amrex::Real	velmag_threshold,
		amrex::TagBoxArray *	tags = nullptr
	)

{
    bool is_found;
 
    Real eye_x, eye_y;
 
    if (time==0.0) {
        is_found = FindInitialEye(levc, mf_cc_vel, velmag_threshold, eye_x, eye_y);
    } else {
        is_found = true;
        const auto& last = hurricane_eye_track_xy.back();
        eye_x = last[0];
        eye_y = last[1];
    }
 
    if (is_found) {
        Real rad_tag = 4.e5 * std::pow(2, max_level-1-levc);
        tag_on_distance_from_eye(geom[levc], tags, eye_x, eye_y, rad_tag);
    }
}

Here is the call graph for this function:

ImposeBCsOnPhi()

void ERF::ImposeBCsOnPhi	(	int	lev,
		amrex::MultiFab &	phi,
		const amrex::Box &	subdomain
	)

Impose bc's on the pressure that comes out of the solve

{
    BL_PROFILE("ERF::ImposeBCsOnPhi()");
 
    auto const sub_lo = lbound(subdomain);
    auto const sub_hi = ubound(subdomain);
 
    auto const dom_lo = lbound(geom[lev].Domain());
    auto const dom_hi = ubound(geom[lev].Domain());
 
    phi.setBndry(1.e25);
    phi.FillBoundary(geom[lev].periodicity());
 
    // ****************************************************************************
    // Impose bc's on pprime
    // ****************************************************************************
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        Array4<Real> const& pp_arr  = phi.array(mfi);
        Box const& bx    = mfi.tilebox();
        auto const bx_lo = lbound(bx);
        auto const bx_hi = ubound(bx);
 
        auto bc_type_xlo = domain_bc_type[Orientation(0,Orientation::low)];
        auto bc_type_xhi = domain_bc_type[Orientation(0,Orientation::high)];
        auto bc_type_ylo = domain_bc_type[Orientation(1,Orientation::low)];
        auto bc_type_yhi = domain_bc_type[Orientation(1,Orientation::high)];
        auto bc_type_zhi = domain_bc_type[Orientation(2,Orientation::high)];
 
        if ( (bx_lo.x == dom_lo.x) && (bc_type_xlo == "Outflow" || bc_type_xlo == "Open") && !solverChoice.use_real_bcs) {
            ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i-1,j,k) = -pp_arr(i,j,k);
            });
        } else if (bx_lo.x == sub_lo.x) {
            ParallelFor(makeSlab(bx,0,sub_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i-1,j,k) = pp_arr(i,j,k);
            });
        }
 
        if ( (bx_hi.x == dom_hi.x) && (bc_type_xhi == "Outflow" || bc_type_xhi == "Open") && !solverChoice.use_real_bcs) {
            ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i+1,j,k) = -pp_arr(i,j,k);
            });
        } else if (bx_hi.x == sub_hi.x) {
            ParallelFor(makeSlab(bx,0,sub_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i+1,j,k) = pp_arr(i,j,k);
            });
        }
 
        if ( (bx_lo.y == dom_lo.y) && (bc_type_ylo == "Outflow" || bc_type_ylo == "Open") && !solverChoice.use_real_bcs) {
            ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j-1,k) = -pp_arr(i,j,k);
            });
        } else if (bx_lo.y == sub_lo.y) {
            ParallelFor(makeSlab(bx,1,sub_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j-1,k) = pp_arr(i,j,k);
            });
        }
 
        if ( (bx_hi.y == dom_hi.y) && (bc_type_yhi == "Outflow" || bc_type_yhi == "Open") && !solverChoice.use_real_bcs) {
            ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j+1,k) = -pp_arr(i,j,k);
            });
        } else if (bx_hi.y == sub_hi.y) {
            ParallelFor(makeSlab(bx,1,sub_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j+1,k) = pp_arr(i,j,k);
            });
        }
 
        // At low z we are always Neumann whether the box touches the bottom boundary or not
        Box zbx(bx); zbx.grow(0,1); zbx.grow(1,1); // Grow in x-dir and y-dir because we have filled that above
        if (bx_lo.z == sub_lo.z) {
            ParallelFor(makeSlab(zbx,2,dom_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j,k-1) = pp_arr(i,j,k);
            });
        }
 
        if ( (bx_hi.z == dom_hi.z) && (bc_type_zhi == "Outflow" || bc_type_zhi == "Open") ) {
            ParallelFor(makeSlab(bx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j,k+1) = -pp_arr(i,j,k);
            });
        } else if (bx_hi.z == sub_hi.z) {
            ParallelFor(makeSlab(bx,2,sub_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                pp_arr(i,j,k+1) = pp_arr(i,j,k);
            });
        }
    } // mfi
 
    // Now overwrite with periodic fill outside domain and fine-fine fill inside
    phi.FillBoundary(geom[lev].periodicity());
}

init1DArrays()

void ERF::init1DArrays ( )

private

init_bcs()

void ERF::init_bcs ( )

private

{
    bool rho_read = false;
    bool read_prim_theta = true;
 
    init_phys_bcs(rho_read, read_prim_theta);
 
    Vector<Real> cons_dir_init(NBCVAR_max,0.0);
    cons_dir_init[BCVars::Rho_bc_comp] = 1.0;
    cons_dir_init[BCVars::RhoTheta_bc_comp] = -1.0;
 
    bool keqn_dir = (solverChoice.turbChoice[max_level].rans_type == RANSType::kEqn &&
                     solverChoice.turbChoice[max_level].dirichlet_k == true);
    if (keqn_dir) {
        // Need to change wall BC type, assume for now that all levels are RANS
        for (int lev = 0; lev < max_level; ++lev) {
            if (solverChoice.turbChoice[lev].rans_type != RANSType::kEqn) {
                Error("If using one-eqn RANS, all levels must be RANS for now");
            }
        }
        Print() << "Using dirichlet BC for k equation" << std::endl;
    }
 
    // *****************************************************************************
    //
    // Here we translate the physical boundary conditions -- one type per face --
    //     into logical boundary conditions for each velocity component
    //
    // *****************************************************************************
    {
        domain_bcs_type.resize(AMREX_SPACEDIM+NBCVAR_max);
        domain_bcs_type_d.resize(AMREX_SPACEDIM+NBCVAR_max);
 
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            int dir = ori.coordDir();
            Orientation::Side side = ori.faceDir();
            auto const bct = phys_bc_type[ori];
            if ( bct == ERF_BC::symmetry )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::reflect_even);
                    }
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::reflect_odd);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::reflect_even);
                    }
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::reflect_odd);
                }
            }
            else if (bct == ERF_BC::outflow or bct == ERF_BC::ho_outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                    if (!solverChoice.anelastic[0]) {
                        domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::neumann_int);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                    if (!solverChoice.anelastic[0]) {
                        domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::neumann_int);
                    }
                }
            }
            else if (bct == ERF_BC::open)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::open);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::open);
                }
            }
            else if (bct == ERF_BC::inflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                        if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                            domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                        }
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                        if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                            domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                        }
                    }
                }
            }
            else if (bct == ERF_BC::inflow_outflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_upwind);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_upwind);
                    }
                }
            }
            else if (bct == ERF_BC::no_slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                    }
                }
            }
            else if (bct == ERF_BC::slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::ext_dir);
 
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::ext_dir);
                }
            }
            else if (bct == ERF_BC::periodic)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::int_dir);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::int_dir);
                    }
                }
            }
            else if ( bct == ERF_BC::surface_layer )
            {
                AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
                domain_bcs_type[BCVars::xvel_bc+0].setLo(dir, ERFBCType::hoextrap);
                domain_bcs_type[BCVars::xvel_bc+1].setLo(dir, ERFBCType::hoextrap);
                domain_bcs_type[BCVars::xvel_bc+2].setLo(dir, ERFBCType::ext_dir);
            }
        }
    }
 
    // *****************************************************************************
    //
    // Here we translate the physical boundary conditions -- one type per face --
    //     into logical boundary conditions for each cell-centered variable
    //     (including the base state variables)
    // NOTE: all "scalars" share the same type of boundary condition
    //
    // *****************************************************************************
    {
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            int dir = ori.coordDir();
            Orientation::Side side = ori.faceDir();
            auto const bct = phys_bc_type[ori];
            if ( bct == ERF_BC::symmetry )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::reflect_even);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::reflect_even);
                    }
                }
            }
            else if ( bct == ERF_BC::outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                    }
                }
            }
            else if ( bct == ERF_BC::ho_outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::hoextrap);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::hoextrap);
                    }
                }
            }
            else if ( bct == ERF_BC::open )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::open);
                } else {
                    for (int i = 0; i < NBCVAR_max; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::open);
                }
            }
            else if ( bct == ERF_BC::no_slip_wall )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                            if (rho_read) {
                                domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                            } else {
                                domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
                            }
                        }
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                            if (rho_read) {
                                domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                            } else {
                                domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
                            }
                        }
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
                    }
                }
            }
            else if (bct == ERF_BC::slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                            if (rho_read) {
                                domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                            } else {
                                domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
                            }
                        }
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::Rho_bc_comp].setLo(dir, ERFBCType::neumann);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                            if (rho_read) {
                                domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                            } else {
                                domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
                            }
                        }
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
                    }
                    if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
                        domain_bcs_type[BCVars::Rho_bc_comp].setHi(dir, ERFBCType::neumann);
                    }
                }
            }
            else if (bct == ERF_BC::inflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                        if ((BCVars::cons_bc+i == RhoTheta_comp) &&
                            (th_bc_data[0].data() != nullptr))
                        {
                            if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
                        }
                        else if (input_bndry_planes && dir < 2 && (
                           ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )) )
                        {
                            domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                        }
                        else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        }
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                        if ((BCVars::cons_bc+i == RhoTheta_comp) &&
                            (th_bc_data[0].data() != nullptr))
                        {
                            if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
                        }
                        else if (input_bndry_planes && dir < 2 && (
                           ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                           ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )
                           ) )
                        {
                            domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                        }
                        else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        }
                    }
                }
            }
            else if (bct == ERF_BC::inflow_outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_upwind);
                        if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                        }
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_upwind);
                        if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                            domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                        }
                    }
                }
            }
            else if (bct == ERF_BC::periodic)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::int_dir);
                    }
                } else {
                    for (int i = 0; i < NBCVAR_max; i++) {
                       domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::int_dir);
                    }
                }
            }
            else if ( bct == ERF_BC::surface_layer )
            {
                AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
                for (int i = 0; i < NBCVAR_max; i++) {
                    domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::hoextrap);
                }
                if (keqn_dir) {
                    Print() << "Setting surface layer logical BC to dirichlet for RANS with k model" << std::endl;
                    domain_bcs_type[BCVars::RhoKE_bc_comp].setLo(dir, ERFBCType::ext_dir);
                }
            }
        }
    }
 
    // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
    Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
}

init_custom()

void ERF::init_custom ( int  lev )

private

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
 
    MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
 
    MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
                       lev_new[Vars::cons].nComp()   , lev_new[Vars::cons].nGrow());
    MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
    MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
    MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
 
    // Default all perturbations to zero
    cons_pert.setVal(0.);
    xvel_pert.setVal(0.);
    yvel_pert.setVal(0.);
    zvel_pert.setVal(0.);
 
    // If the initial condition needs to have spatially correlated perturbations superimposed onto
    // the base state, then populate the "pert" multifabs with random perturbations,
    // and then apply a Gaussian smoothing to make the perturbations spatially correlated
    if(solverChoice.is_init_with_correlated_pert) {
        create_random_perturbations(lev, cons_pert, xvel_pert, yvel_pert, zvel_pert);
        apply_gaussian_smoothing_to_perturbations(lev, cons_pert, xvel_pert, yvel_pert, zvel_pert);
    }
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
    {
        const Box &bx  = mfi.tilebox();
        const Box &xbx = mfi.tilebox(IntVect(1,0,0));
        const Box &ybx = mfi.tilebox(IntVect(0,1,0));
        const Box &zbx = mfi.tilebox(IntVect(0,0,1));
 
        const auto &cons_pert_arr = cons_pert.array(mfi);
        const auto &xvel_pert_arr = xvel_pert.array(mfi);
        const auto &yvel_pert_arr = yvel_pert.array(mfi);
        const auto &zvel_pert_arr = zvel_pert.array(mfi);
 
        Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
        Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
        Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
 
        // Here we arbitrarily choose the x-oriented map factor -- this should be generalized
        Array4<Real const> mf_m     = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        Array4<Real const> mf_u     = mapfac[lev][MapFacType::u_x]->const_array(mfi);
        Array4<Real const> mf_v     = mapfac[lev][MapFacType::v_y]->const_array(mfi);
 
        Array4<Real> r_hse_arr = r_hse.array(mfi);
        Array4<Real> p_hse_arr = p_hse.array(mfi);
 
        prob->init_custom_pert(bx, xbx, ybx, zbx, cons_arr, cons_pert_arr,
                               xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
                               r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
                               geom[lev].data(), mf_m, mf_u, mf_v,
                               solverChoice, lev);
    } //mfi
 
    // Add problem-specific perturbation to background flow if not doing anelastic with fixed-in-time density
    if (!solverChoice.fixed_density[lev]) {
        MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp,      Rho_comp,             1, cons_pert.nGrow());
    }
    MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp,        1, cons_pert.nGrow());
    MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,NSCALARS, cons_pert.nGrow());
 
    // RhoKE is relevant if using Deardorff with LES, k-equation for RANS, or MYNN with PBL
    if (solverChoice.turbChoice[lev].use_tke) {
        MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp,    RhoKE_comp,    1, cons_pert.nGrow());
    }
 
    if (solverChoice.moisture_type != MoistureType::None) {
        int qstate_size = micro->Get_Qstate_Size();
        for (int q_offset(0); q_offset<qstate_size; ++q_offset) {
            int q_idx = RhoQ1_comp+q_offset;
            MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
        }
    }
 
    MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0,             0,             1, xvel_pert.nGrowVect());
    MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0,             0,             1, yvel_pert.nGrowVect());
    MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0,             0,             1, zvel_pert.nGrowVect());
}

Here is the call graph for this function:

init_Dirichlet_bc_data()

void ERF::init_Dirichlet_bc_data ( const std::string  input_file )

private

{
    // Read the dirichlet_input file
    Print() << "dirichlet_input file location : " << input_file << std::endl;
    std::ifstream input_reader(input_file);
    if (!input_reader.is_open()) {
        amrex::Abort("Error opening the dirichlet_input file.\n");
    }
 
    Print() << "Successfully opened the dirichlet_input file. Now reading... " << std::endl;
    std::string line;
 
    // Size of Ninp (number of z points in input file)
    Vector<Real> z_inp_tmp, u_inp_tmp, v_inp_tmp, w_inp_tmp, th_inp_tmp;
 
    // Top and bot for domain
    const int  klo  = geom[0].Domain().smallEnd()[2];
    const int  khi  = geom[0].Domain().bigEnd()[2];
    const Real zbot = zlevels_stag[0][klo];
    const Real ztop = zlevels_stag[0][khi+1];
 
    // Flag if theta input
    Real th_init = -300.0;
    bool th_read{false};
 
    // Add surface
    z_inp_tmp.push_back(zbot); // height above sea level [m]
    u_inp_tmp.push_back(0.);
    v_inp_tmp.push_back(0.);
    w_inp_tmp.push_back(0.);
    th_inp_tmp.push_back(th_init);
 
    // Read the vertical profile at each given height
    Real z, u, v, w, th;
    while(std::getline(input_reader, line)) {
        std::istringstream iss_z(line);
 
        Vector<Real> rval_v;
        Real rval;
        while (iss_z >> rval) {
            rval_v.push_back(rval);
        }
        z = rval_v[0];
        u = rval_v[1];
        v = rval_v[2];
        w = rval_v[3];
 
        // Format without theta
        if (rval_v.size() == 4) {
            if (z == zbot) {
                u_inp_tmp[0] = u;
                v_inp_tmp[0] = v;
                w_inp_tmp[0] = w;
            } else {
                AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
                z_inp_tmp.push_back(z);
                u_inp_tmp.push_back(u);
                v_inp_tmp.push_back(v);
                w_inp_tmp.push_back(w);
                if (z >= ztop) break;
            }
        } else if (rval_v.size() == 5) {
            th_read = true;
            th = rval_v[4];
            if (z == zbot) {
                u_inp_tmp[0]  = u;
                v_inp_tmp[0]  = v;
                w_inp_tmp[0]  = w;
                th_inp_tmp[0] = th;
            } else {
                AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
                z_inp_tmp.push_back(z);
                u_inp_tmp.push_back(u);
                v_inp_tmp.push_back(v);
                w_inp_tmp.push_back(w);
                th_inp_tmp.push_back(th);
                if (z >= ztop) break;
            }
        } else {
            Abort("Unknown inflow file format!");
        }
    }
 
    // Ensure we set a reasonable theta surface
    if (th_read) {
        if (th_inp_tmp[0] == th_init) {
            Real slope = (th_inp_tmp[2] - th_inp_tmp[1]) / (z_inp_tmp[2] - z_inp_tmp[1]);
            Real dz    = z_inp_tmp[0] - z_inp_tmp[1];
            th_inp_tmp[0] = slope * dz + th_inp_tmp[1];
        }
    }
 
    amrex::Print() << "Successfully read and interpolated the dirichlet_input file..." << std::endl;
    input_reader.close();
 
    for (int lev = 0; lev <= max_level; lev++) {
 
        const int Nz  = geom[lev].Domain().size()[2];
 
        // Size of Nz (domain grid)
        Vector<Real> zcc_inp(Nz  );
        Vector<Real> znd_inp(Nz+1);
        Vector<Real> u_inp(Nz  ); xvel_bc_data[lev].resize(Nz  ,0.0);
        Vector<Real> v_inp(Nz  ); yvel_bc_data[lev].resize(Nz  ,0.0);
        Vector<Real> w_inp(Nz+1); zvel_bc_data[lev].resize(Nz+1,0.0);
        Vector<Real> th_inp;
        if (th_read) {
            th_inp.resize(Nz);
            th_bc_data[lev].resize(Nz, 0.0);
        }
 
        // At this point, we have an input from zbot up to
        // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights
        const int Ninp = z_inp_tmp.size();
        for (int k(0); k<Nz; ++k) {
            zcc_inp[k] = 0.5 * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
            znd_inp[k] = zlevels_stag[lev][k+1];
            u_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), u_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
            v_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), v_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
            w_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), znd_inp[k], Ninp);
            if (th_read) {
                th_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), th_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
            }
        }
        znd_inp[Nz] = ztop;
        w_inp[Nz]   = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), ztop, Ninp);
 
        // Copy host data to the device
        Gpu::copy(Gpu::hostToDevice, u_inp.begin(), u_inp.end(), xvel_bc_data[lev].begin());
        Gpu::copy(Gpu::hostToDevice, v_inp.begin(), v_inp.end(), yvel_bc_data[lev].begin());
        Gpu::copy(Gpu::hostToDevice, w_inp.begin(), w_inp.end(), zvel_bc_data[lev].begin());
        if (th_read) {
           Gpu::copy(Gpu::hostToDevice, th_inp.begin(), th_inp.end(), th_bc_data[lev].begin());
        }
 
        // NOTE: These device vectors are passed to the PhysBC constructors when that
        //       class is instantiated in ERF_MakeNewArrays.cpp.
    } // lev
}

Here is the call graph for this function:

init_from_hse()

void ERF::init_from_hse

(

int

lev

)

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

• call prob->erf_init_dens_hse(...)
  • call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
  • save r_hse
• call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse and th_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
 
    MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
    {
        const Box &gbx = mfi.growntilebox(1);
        const Array4<Real      >& cons_arr  = lev_new[Vars::cons].array(mfi);
        const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
        const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
 
        ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            cons_arr(i,j,k,Rho_comp)      = r_hse_arr(i,j,k);
            cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
        });
    } //mfi
}

Here is the call graph for this function:

init_from_input_sounding()

void ERF::init_from_input_sounding

(

int

lev

)

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters

lev	Integer specifying the current level

{
    // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
    if (lev == 0) {
        if (input_sounding_data.input_sounding_file.empty()) {
            Error("input_sounding file name must be provided via input");
        }
 
        input_sounding_data.resize_arrays();
 
        // this will interpolate the input profiles to the nominal height levels
        // (ranging from 0 to the domain top)
        for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
            input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n);
        }
 
        // this will calculate the hydrostatically balanced density and pressure
        // profiles following WRF ideal.exe
        if (solverChoice.sounding_type == SoundingType::Ideal) {
            input_sounding_data.calc_rho_p(0);
        } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
                   solverChoice.sounding_type == SoundingType::DryIsentropic) {
            input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
            input_sounding_data.calc_rho_p_isentropic(0);
        }
 
    } else {
        //
        // We need to do this interp from coarse level in order to set the values of
        // the base state inside the domain but outside of the fine region
        //
        base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
        //
        // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
        //       have been pre-filled - this includes ghost cells both inside and outside
        //       the domain
        //
        InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                              IntVect(0,0,0), // do not fill ghost cells outside the domain
                              base_state[lev-1], 0, 0, base_state[lev].nComp(),
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BaseBCVars::rho0_bc_comp);
 
         // We need to do this here because the interpolation above may leave corners unfilled
         //    when the corners need to be filled by, for example, reflection of the fine ghost
         //    cell outside the fine region but inide the domain.
         (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
    }
 
    auto& lev_new = vars_new[lev];
 
    // updated if sounding is ideal (following WRF) or isentropic
    const bool l_isentropic = (solverChoice.sounding_type == SoundingType::Isentropic ||
                               solverChoice.sounding_type == SoundingType::DryIsentropic);
    const bool sounding_ideal_or_isentropic = (solverChoice.sounding_type == SoundingType::Ideal ||
                                               l_isentropic);
    MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
    MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
    MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
    MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
 
    const Real l_gravity = solverChoice.gravity;
    const Real l_rdOcp   = solverChoice.rdOcp;
    const bool l_moist   = (solverChoice.moisture_type != MoistureType::None);
 
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box &bx = mfi.tilebox();
        const auto &cons_arr = lev_new[Vars::cons].array(mfi);
        const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
        const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
        const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
        Array4<Real>  r_hse_arr =  r_hse.array(mfi);
        Array4<Real>  p_hse_arr =  p_hse.array(mfi);
        Array4<Real> pi_hse_arr = pi_hse.array(mfi);
        Array4<Real> th_hse_arr = th_hse.array(mfi);
        Array4<Real> qv_hse_arr = qv_hse.array(mfi);
 
        Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
        Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
 
        if (sounding_ideal_or_isentropic)
        {
            // HSE will be initialized here, interpolated from values previously
            // calculated by calc_rho_p or calc_rho_p_isentropic
            init_bx_scalars_from_input_sounding_hse(
                bx, cons_arr,
                r_hse_arr, p_hse_arr, pi_hse_arr, th_hse_arr, qv_hse_arr,
                geom[lev].data(), z_cc_arr,
                l_gravity, l_rdOcp, l_moist, input_sounding_data,
                l_isentropic);
        }
        else
        {
            // This assumes rho_0 = 1.0
            // HSE will be calculated later with call to initHSE
            init_bx_scalars_from_input_sounding(
                bx, cons_arr,
                geom[lev].data(), z_cc_arr,
                l_moist, input_sounding_data);
        }
 
        init_bx_velocities_from_input_sounding(
            bx, xvel_arr, yvel_arr, zvel_arr,
            geom[lev].data(), z_nd_arr,
            input_sounding_data);
 
    } //mfi
}

Here is the call graph for this function:

init_geo_wind_profile()

void ERF::init_geo_wind_profile ( const std::string  input_file, amrex::Vector< amrex::Real > &  u_geos, amrex::Gpu::DeviceVector< amrex::Real > &  u_geos_d, amrex::Vector< amrex::Real > &  v_geos, amrex::Gpu::DeviceVector< amrex::Real > &  v_geos_d, const amrex::Geometry &  lgeom, const amrex::Vector< amrex::Real > &  zlev_stag  )

private

{
    const int klo = 0;
    const int khi = lgeom.Domain().bigEnd()[AMREX_SPACEDIM-1];
    const amrex::Real dz = lgeom.CellSize()[AMREX_SPACEDIM-1];
 
    const bool grid_stretch = (zlev_stag.size() > 0);
    const Real zbot = (grid_stretch) ? zlev_stag[klo]   : lgeom.ProbLo(AMREX_SPACEDIM-1);
    const Real ztop = (grid_stretch) ? zlev_stag[khi+1] : lgeom.ProbHi(AMREX_SPACEDIM-1);
 
    amrex::Print() << "Reading geostrophic wind profile from " << input_file << std::endl;
    std::ifstream profile_reader(input_file);
    if(!profile_reader.is_open()) {
        amrex::Error("Error opening the abl_geo_wind_table\n");
    }
 
    // First, read the input data into temp vectors
    std::string line;
    Vector<Real> z_inp, Ug_inp, Vg_inp;
    Real z, Ug, Vg;
    amrex::Print() << "z  Ug  Vg" << std::endl;
    while(std::getline(profile_reader, line)) {
        std::istringstream iss(line);
        iss >> z >> Ug >> Vg;
        amrex::Print() << z << " " << Ug << " " << Vg << std::endl;
        z_inp.push_back(z);
        Ug_inp.push_back(Ug);
        Vg_inp.push_back(Vg);
        if (z >= ztop) break;
    }
 
    const int Ninp = z_inp.size();
    AMREX_ALWAYS_ASSERT(z_inp[0] <= zbot);
    AMREX_ALWAYS_ASSERT(z_inp[Ninp-1] >= ztop);
 
    // Now, interpolate vectors to the cell centers
    for (int k = 0; k <= khi; k++) {
        z = (grid_stretch) ? 0.5 * (zlev_stag[k] + zlev_stag[k+1])
                           : zbot + (k + 0.5) * dz;
        u_geos[k] = interpolate_1d(z_inp.dataPtr(), Ug_inp.dataPtr(), z, Ninp);
        v_geos[k] = interpolate_1d(z_inp.dataPtr(), Vg_inp.dataPtr(), z, Ninp);
    }
 
    // Copy from host version to device version
    Gpu::copy(Gpu::hostToDevice, u_geos.begin(), u_geos.end(), u_geos_d.begin());
    Gpu::copy(Gpu::hostToDevice, v_geos.begin(), v_geos.end(), v_geos_d.begin());
 
    profile_reader.close();
}

Here is the call graph for this function:

init_immersed_forcing()

void ERF::init_immersed_forcing

(

int

lev

)

Set velocities in cells that are immersed to be 0 (or a very small number)

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
    MultiFab* terrain_blank = terrain_blanking[lev].get();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
    {
        const Box &xbx = mfi.tilebox(IntVect(1,0,0));
        const Box &ybx = mfi.tilebox(IntVect(0,1,0));
        const Box &zbx = mfi.tilebox(IntVect(0,0,1));
        const Real epsilon = 1e-2;
 
        const Array4<const Real>& t_blank_arr = terrain_blank->const_array(mfi);
 
        const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
        const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
        const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
 
        // Set the x,y,z-velocities
        ParallelFor(xbx, ybx, zbx,
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
            const Real t_blank = 0.5 * (t_blank_arr(i, j, k) + t_blank_arr(i-1, j, k));
            if (t_blank == 1.0) { xvel_arr(i, j, k) = epsilon; }
        },
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
            const Real t_blank = 0.5 * (t_blank_arr(i, j, k) + t_blank_arr(i, j-1, k));
            if (t_blank == 1.0) { yvel_arr(i, j, k) = epsilon; }
        },
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
            const Real t_blank = 0.5 * (t_blank_arr(i, j, k) + t_blank_arr(i, j, k-1));
            if (t_blank == 1.0) { zvel_arr(i, j, k) = epsilon; }
        });
    } //mfi
}

Here is the call graph for this function:

init_only()

void ERF::init_only	(	int	lev,
		amrex::Real	time
	)

{
    t_new[lev] = time;
    t_old[lev] = time - 1.e200;
 
    auto& lev_new = vars_new[lev];
    auto& lev_old = vars_old[lev];
 
    // Loop over grids at this level to initialize our grid data
    lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
    lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
    lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
    lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
 
    // Initialize background flow (optional)
    if (solverChoice.init_type == InitType::Input_Sounding) {
        // The physbc's need the terrain but are needed for initHSE
        // We have already made the terrain in the call to init_zphys
        //    in MakeNewLevelFromScratch
        make_physbcs(lev);
 
        // Now init the base state and the data itself
        init_from_input_sounding(lev);
 
        // The base state has been initialized by integrating vertically
        // through the sounding for ideal (like WRF) or isentropic approaches
        if (solverChoice.sounding_type == SoundingType::Ideal ||
            solverChoice.sounding_type == SoundingType::Isentropic ||
            solverChoice.sounding_type == SoundingType::DryIsentropic) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
                "Gravity should be on to be consistent with sounding initialization.");
        } else { // SoundingType::ConstantDensity
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!solverChoice.use_gravity || (solverChoice.anelastic[lev] == 1),
                "Constant density probably doesn't make sense for compressible flow with gravity");
            initHSE();
        }
 
#ifdef ERF_USE_NETCDF
    }
    else if (solverChoice.init_type == InitType::WRFInput && !nc_init_file[lev].empty())
    {
        // The base state is initialized from WRF wrfinput data, output by
        // ideal.exe or real.exe
 
        init_from_wrfinput(lev, *mf_C1H, *mf_C2H, *mf_MUB, *mf_PSFC[lev]);
 
        // The physbc's need the terrain but are needed for initHSE
        if (!solverChoice.use_real_bcs) {
            make_physbcs(lev);
        }
    }
    else if (solverChoice.init_type == InitType::WRFInput && nc_init_file[lev].empty())
    {
        amrex::Abort("This pathway is not quite implemented yet");
    }
    else if (solverChoice.init_type == InitType::NCFile)
    {
        // The state is initialized by reading from a Netcdf file
        init_from_ncfile(lev);
 
        // The physbc's need the terrain but are needed for initHSE
        make_physbcs(lev);
    }
    else if (solverChoice.init_type == InitType::Metgrid)
    {
        // The base state is initialized from data output by WPS metgrid;
        // we will rebalance after interpolation
        init_from_metgrid(lev);
#endif
    } else if (solverChoice.init_type == InitType::Uniform) {
        // Initialize a uniform background field and base state based on the
        // problem-specified reference density and temperature
 
        // The physbc's need the terrain but are needed for initHSE
        make_physbcs(lev);
 
        init_uniform(lev);
        initHSE(lev);
    } else {
        // No background flow initialization specified, initialize the
        // background field to be equal to the base state, calculated from the
        // problem-specific erf_init_dens_hse
 
        // The bc's need the terrain but are needed for initHSE
        make_physbcs(lev);
 
        // We will initialize the state from the background state so must set that first
        initHSE(lev);
        init_from_hse(lev);
    }
 
    // Add problem-specific flow features
    //
    // Notes:
    // - This calls init_custom_pert that is defined for each problem
    // - This may modify the base state
    // - The fields set by init_custom_pert are **perturbations** to the
    //   background flow set based on init_type
    if (solverChoice.init_type != InitType::NCFile) {
        init_custom(lev);
    }
 
    // Ensure that the face-based data are the same on both sides of a periodic domain.
    // The data associated with the lower grid ID is considered the correct value.
    lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
    lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
    lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
 
   if(solverChoice.spongeChoice.sponge_type == "input_sponge"){
        input_sponge(lev);
   }
 
    // Initialize turbulent perturbation
    if (solverChoice.pert_type == PerturbationType::Source ||
        solverChoice.pert_type == PerturbationType::Direct ||
        solverChoice.pert_type == PerturbationType::CPM) {
        turbPert_update(lev, 0.);
        turbPert_amplitude(lev);
    }
 
    // Set initial velocity field for immersed cells to be close to 0
    if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
        solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
        init_immersed_forcing(lev);
    }
}

init_phys_bcs()

void ERF::init_phys_bcs ( bool &  rho_read, bool &  read_prim_theta  )

private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, InflowOutflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

{
    auto f = [this,&rho_read,&read_prim_theta] (std::string const& bcid, Orientation ori)
    {
        // These are simply defaults for Dirichlet faces -- they should be over-written below
        m_bc_extdir_vals[BCVars::Rho_bc_comp][ori]       =  1.0;
        m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori]  = -1.0; // It is important to set this negative
                                                                 // because the sign is tested on below
        for (int n = BCVars::RhoKE_bc_comp; n < BCVars::xvel_bc; n++) {
            m_bc_extdir_vals[n][ori]                     = 0.0;
        }
 
        m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.0; // default
        m_bc_extdir_vals[BCVars::yvel_bc][ori] = 0.0;
        m_bc_extdir_vals[BCVars::zvel_bc][ori] = 0.0;
 
        // These are simply defaults for Neumann gradients -- they should be over-written below
        m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]       = 0.0;
        m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]  = 0.0;
 
        m_bc_neumann_vals[BCVars::RhoKE_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoScalar_bc_comp][ori] = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ1_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ2_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ3_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ4_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ5_bc_comp][ori]     = 0.0;
        m_bc_neumann_vals[BCVars::RhoQ6_bc_comp][ori]     = 0.0;
 
        m_bc_neumann_vals[BCVars::xvel_bc][ori] = 0.0;
        m_bc_neumann_vals[BCVars::yvel_bc][ori] = 0.0;
        m_bc_neumann_vals[BCVars::zvel_bc][ori] = 0.0;
 
        std::string pp_text = pp_prefix + "." + bcid;
        ParmParse pp(pp_text);
 
        std::string bc_type_in;
        if (pp.query("type", bc_type_in) <= 0)
        {
            pp_text = bcid;
            pp = ParmParse(pp_text);
            pp.query("type", bc_type_in);
        }
 
        std::string bc_type = amrex::toLower(bc_type_in);
 
        if (bc_type == "symmetry")
        {
            // Print() << bcid << " set to symmetry.\n";
              phys_bc_type[ori] = ERF_BC::symmetry;
            domain_bc_type[ori] = "Symmetry";
        }
        else if (bc_type == "outflow")
        {
            // Print() << bcid << " set to outflow.\n";
              phys_bc_type[ori] = ERF_BC::outflow;
            domain_bc_type[ori] = "Outflow";
        }
        else if (bc_type == "open")
        {
            // Print() << bcid << " set to open.\n";
            AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
              phys_bc_type[ori] = ERF_BC::open;
            domain_bc_type[ori] = "Open";
        }
        else if (bc_type == "ho_outflow")
        {
            phys_bc_type[ori] = ERF_BC::ho_outflow;
            domain_bc_type[ori] = "HO_Outflow";
        }
 
        else if (bc_type == "inflow" || bc_type == "inflow_outflow")
        {
            if (bc_type == "inflow") {
                // Print() << bcid << " set to inflow.\n";
                  phys_bc_type[ori] = ERF_BC::inflow;
                domain_bc_type[ori] = "Inflow";
            } else {
                // Print() << bcid << " set to inflow_outflow.\n";
                  phys_bc_type[ori] = ERF_BC::inflow_outflow;
                domain_bc_type[ori] = "InflowOutflow";
            }
 
            std::vector<Real> v;
            if (input_bndry_planes && m_r2d->ingested_velocity()) {
                m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.;
                m_bc_extdir_vals[BCVars::yvel_bc][ori] = 0.;
                m_bc_extdir_vals[BCVars::zvel_bc][ori] = 0.;
            } else {
                // Test for input data file if at xlo face
                std::string dirichlet_file;
                auto file_exists = pp.query("dirichlet_file", dirichlet_file);
                if (file_exists) {
                    pp.query("read_prim_theta", read_prim_theta);
                    init_Dirichlet_bc_data(dirichlet_file);
                } else {
                    pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
                    m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                    m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                    m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
                }
            }
 
            Real rho_in = 0.;
            if (input_bndry_planes && m_r2d->ingested_density()) {
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = 0.;
            } else {
                if (!pp.query("density", rho_in)) {
                    amrex::Print() << "Using interior values to set conserved vars" << std::endl;
                }
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
            }
 
            bool th_read  = (th_bc_data[0].data()!=nullptr);
            Real theta_in = 0.;
            if (input_bndry_planes && m_r2d->ingested_theta()) {
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = 0.;
            } else if (!th_read) {
                if (rho_in > 0) {
                    pp.get("theta", theta_in);
                }
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
            }
 
            Real scalar_in = 0.;
            if (input_bndry_planes && m_r2d->ingested_scalar()) {
                m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = 0.;
            } else {
                if (pp.query("scalar", scalar_in))
                m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
            }
 
            if (solverChoice.moisture_type != MoistureType::None) {
                Real qv_in = 0.;
                if (input_bndry_planes && m_r2d->ingested_q1()) {
                    m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = 0.;
                } else {
                    if (pp.query("qv", qv_in))
                    m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
                }
                Real qc_in = 0.;
                if (input_bndry_planes && m_r2d->ingested_q2()) {
                    m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = 0.;
                } else {
                    if (pp.query("qc", qc_in))
                    m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
                }
            }
 
            Real KE_in = 0.;
            if (input_bndry_planes && m_r2d->ingested_KE()) {
                m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = 0.;
            } else {
                if (pp.query("KE", KE_in))
                m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
            }
        }
        else if (bc_type == "noslipwall")
        {
            // Print() << bcid <<" set to no-slip wall.\n";
              phys_bc_type[ori] = ERF_BC::no_slip_wall;
            domain_bc_type[ori] = "NoSlipWall";
 
            std::vector<Real> v;
 
            // The values of m_bc_extdir_vals default to 0.
            // But if we find "velocity" in the inputs file, use those values instead.
            if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
            {
                v[ori.coordDir()] = 0.0;
                m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
            }
 
            Real rho_in;
            rho_read = pp.query("density", rho_in);
            if (rho_read)
            {
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
            }
 
            Real theta_in;
            if (pp.query("theta", theta_in))
            {
               m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
            }
 
            Real theta_grad_in;
            if (pp.query("theta_grad", theta_grad_in))
            {
                m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
            }
 
            Real qv_in;
            if (pp.query("qv", qv_in))
            {
                m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = qv_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
            }
        }
        else if (bc_type == "slipwall")
        {
            // Print() << bcid <<" set to slip wall.\n";
 
              phys_bc_type[ori] = ERF_BC::slip_wall;
            domain_bc_type[ori] = "SlipWall";
 
            Real rho_in;
            rho_read = pp.query("density", rho_in);
            if (rho_read)
            {
                m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
            }
 
            Real theta_in;
            if (pp.query("theta", theta_in))
            {
               m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
            }
 
            Real rho_grad_in;
            if (pp.query("density_grad", rho_grad_in))
            {
               m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = rho_grad_in;
            }
 
            Real theta_grad_in;
            if (pp.query("theta_grad", theta_grad_in))
            {
               m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
            }
        }
        else if (bc_type == "surface_layer")
        {
              phys_bc_type[ori] = ERF_BC::surface_layer;
            domain_bc_type[ori] = "surface_layer";
        }
        else
        {
            phys_bc_type[ori] = ERF_BC::undefined;
        }
 
        if (geom[0].isPeriodic(ori.coordDir())) {
            domain_bc_type[ori] = "Periodic";
            if (phys_bc_type[ori] == ERF_BC::undefined)
            {
                phys_bc_type[ori] = ERF_BC::periodic;
            } else {
                Abort("Wrong BC type for periodic boundary");
            }
        }
 
        if (phys_bc_type[ori] == ERF_BC::undefined)
        {
             Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
             Abort("This BC type is unknown");
        }
    };
 
    f("xlo", Orientation(Direction::x,Orientation::low));
    f("xhi", Orientation(Direction::x,Orientation::high));
    f("ylo", Orientation(Direction::y,Orientation::low));
    f("yhi", Orientation(Direction::y,Orientation::high));
    f("zlo", Orientation(Direction::z,Orientation::low));
    f("zhi", Orientation(Direction::z,Orientation::high));
}

Here is the call graph for this function:

init_stuff()

void ERF::init_stuff ( int  lev, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm, amrex::Vector< amrex::MultiFab > &  lev_new, amrex::Vector< amrex::MultiFab > &  lev_old, amrex::MultiFab &  tmp_base_state, std::unique_ptr< amrex::MultiFab > &  tmp_zphys_nd  )

private

{
    // ********************************************************************************************
    // Base state holds r_0, pres_0, pi_0, th_0 (in that order)
    //
    // Here is where we set the number of ghost cells for the base state!
    // ********************************************************************************************
    int ngb = (solverChoice.terrain_type == TerrainType::EB) ? ComputeGhostCells(solverChoice)+1 : 3;
    tmp_base_state.define(ba,dm,BaseState::num_comps,ngb);
    tmp_base_state.setVal(0.);
 
    if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
        base_state_new[lev].define(ba,dm,BaseState::num_comps,base_state[lev].nGrowVect());
        base_state_new[lev].setVal(0.);
    }
 
    // ********************************************************************************************
    // Allocate terrain arrays
    // ********************************************************************************************
 
    BoxArray ba_nd(ba);
    ba_nd.surroundingNodes();
 
    // NOTE: this is where we actually allocate z_phys_nd -- but here it's called "tmp_zphys_nd"
    // We need this to be one greater than the ghost cells to handle levels > 0
 
    int ngrow = ComputeGhostCells(solverChoice) + 2;
    tmp_zphys_nd = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
 
    z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
    init_default_zphys(lev, geom[lev], *tmp_zphys_nd, *z_phys_cc[lev]);
 
    if (solverChoice.terrain_type == TerrainType::MovingFittedMesh)
    {
        detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
        detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
 
        ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
        ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
        az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
 
        z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
 
        z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
        z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
        z_phys_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
    }
    else
    {
        z_phys_nd_new[lev] = nullptr;
          detJ_cc_new[lev] = nullptr;
 
        z_phys_nd_src[lev] = nullptr;
        z_phys_cc_src[lev] = nullptr;
          detJ_cc_src[lev] = nullptr;
 
               z_t_rk[lev] = nullptr;
    }
 
    if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
        solverChoice.buildings_type == BuildingsType::ImmersedForcing)
    {
        terrain_blanking[lev] = std::make_unique<MultiFab>(ba,dm,1,ngrow);
        terrain_blanking[lev]->setVal(1.0);
    }
 
    // We use these area arrays regardless of terrain, EB or none of the above
    detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
         ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
         ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
         az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
 
    detJ_cc[lev]->setVal(1.0);
         ax[lev]->setVal(1.0);
         ay[lev]->setVal(1.0);
         az[lev]->setVal(1.0);
 
    // ********************************************************************************************
    // Create wall distance array for RANS modeling
    // ********************************************************************************************
    if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
        walldist[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
        walldist[lev]->setVal(1e23);
    } else {
        walldist[lev] = nullptr;
    }
 
    // ********************************************************************************************
    // These are the persistent containers for the old and new data
    // ********************************************************************************************
    int ncomp;
    if (lev > 0) {
        ncomp = vars_new[lev-1][Vars::cons].nComp();
    } else {
        int n_qstate   = micro->Get_Qstate_Size();
        ncomp = NDRY + NSCALARS + n_qstate;
    }
 
    // ********************************************************************************************
    // The number of ghost cells for density must be 1 greater than that for velocity
    //     so that we can go back in forth between velocity and momentum on all faces
    // ********************************************************************************************
    int ngrow_state = ComputeGhostCells(solverChoice) + 1;
    int ngrow_vels  = ComputeGhostCells(solverChoice);
 
    // ********************************************************************************************
    // New solution data containers
    // ********************************************************************************************
    if (solverChoice.terrain_type != TerrainType::EB) {
        lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
        lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
    } else {
        // EB: Define the MultiFabs with the EBFactory
        lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
        lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
    }
    lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
    lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
 
    lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
    lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
 
    gradp[lev][GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp[lev][GpVars::gpx].setVal(0.);
    gradp[lev][GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp[lev][GpVars::gpy].setVal(0.);
    gradp[lev][GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp[lev][GpVars::gpz].setVal(0.);
 
    // Note that we need the ghost cells in the z-direction if we are doing any
    // kind of domain decomposition in the vertical (at level 0 or above)
    lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
    lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
 
    if ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) {
        pp_inc[lev].define(ba, dm, 1, 1);
        pp_inc[lev].setVal(0.0);
    }
 
    // We use this in the fast substepping only
    if (solverChoice.anelastic[lev] == 0) {
        lagged_delta_rt[lev].define(ba, dm, 1, 1);
        lagged_delta_rt[lev].setVal(0.0);
    }
 
    // We use these for advecting the slow variables, whether anelastic or compressible
    avg_xmom[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, 1);
    avg_ymom[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, 1);
    avg_zmom[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, 1);
    avg_xmom[lev].setVal(0.0); avg_ymom[lev].setVal(0.0); avg_zmom[lev].setVal(0.0);
 
    // ********************************************************************************************
    // These are just used for scratch in the time integrator but we might as well define them here
    // ********************************************************************************************
    if (solverChoice.terrain_type != TerrainType::EB) {
        rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
        rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
 
        rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
        rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
 
        rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
        rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
    } else {
        // EB: Define the MultiFabs with the EBFactory
        rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
        rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
 
        rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
        rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
 
        rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
        rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
    }
 
    if (lev > 0) {
        //xmom_crse_rhs[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, IntVect{0});
        //ymom_crse_rhs[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, IntVect{0});
        zmom_crse_rhs[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect{0});
    }
 
    // We do this here just so they won't be undefined in the initial FillPatch
    rU_old[lev].setVal(1.2e21);
    rV_old[lev].setVal(3.4e22);
    rW_old[lev].setVal(5.6e23);
    rU_new[lev].setVal(1.2e21);
    rV_new[lev].setVal(3.4e22);
    rW_new[lev].setVal(5.6e23);
 
    // ********************************************************************************************
    // These are just time averaged fields for diagnostics
    // ********************************************************************************************
 
    // NOTE: We are not completing a fillpach call on the time averaged data;
    //       which would copy on intersection and interpolate from coarse.
    //       Therefore, we are restarting the averaging when the ba changes,
    //       this may give poor statistics for dynamic mesh refinement.
    vel_t_avg[lev] = nullptr;
    if (solverChoice.time_avg_vel) {
        vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
        vel_t_avg[lev]->setVal(0.0);
        t_avg_cnt[lev] = 0.0;
    }
 
    // ********************************************************************************************
    // Initialize flux registers whenever we create/re-create a level
    // ********************************************************************************************
    if (solverChoice.coupling_type == CouplingType::TwoWay) {
        if (lev == 0) {
            advflux_reg[0] = nullptr;
        } else {
            int ncomp_reflux = vars_new[0][Vars::cons].nComp();
            advflux_reg[lev] = new YAFluxRegister(ba       , grids[lev-1],
                                                  dm       ,  dmap[lev-1],
                                                  geom[lev],  geom[lev-1],
                                                  ref_ratio[lev-1], lev, ncomp_reflux);
        }
    }
 
    // ********************************************************************************************
    // Define Theta_prim storage if using surface_layer BC
    // ********************************************************************************************
    if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
        Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
        if (solverChoice.moisture_type != MoistureType::None) {
            Qv_prim[lev]    = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
            Qr_prim[lev]    = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
        } else {
            Qv_prim[lev]    = nullptr;
            Qr_prim[lev]    = nullptr;
        }
    } else {
        Theta_prim[lev] = nullptr;
        Qv_prim[lev]    = nullptr;
        Qr_prim[lev]    = nullptr;
    }
 
    // ********************************************************************************************
    // Map factors
    // ********************************************************************************************
    BoxList bl2d_mf = ba.boxList();
    for (auto& b : bl2d_mf) {
        b.setRange(2,0);
    }
    BoxArray ba2d_mf(std::move(bl2d_mf));
 
    mapfac[lev].resize(MapFacType::num);
    mapfac[lev][MapFacType::m_x] = std::make_unique<MultiFab>(                        ba2d_mf,dm,1,IntVect(3,3,0));
    mapfac[lev][MapFacType::u_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,IntVect(3,3,0));
    mapfac[lev][MapFacType::v_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,IntVect(3,3,0));
 
#if 0
    // For now we comment this out to avoid CI failures but we will need to re-enable
    //     this if using non-conformal mappings
    if (MapFacType::m_y != MapFacType::m_x) {
        mapfac[lev][MapFacType::m_y] = std::make_unique<MultiFab>(ba2d_mf,dm,1,IntVect(3,3,0));
    }
    if (MapFacType::u_y != MapFacType::u_x) {
        mapfac[lev][MapFacType::u_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,IntVect(3,3,0));
    }
    if (MapFacType::v_y != MapFacType::v_x) {
        mapfac[lev][MapFacType::v_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,IntVect(3,3,0));
    }
#endif
 
    if (solverChoice.test_mapfactor) {
        for (int i = 0; i < 3; i++) {
            mapfac[lev][i]->setVal(0.5);
        }
        for (int i = 3; i < mapfac[lev].size(); i++) {
            mapfac[lev][i]->setVal(0.25);
        }
    } else {
        for (int i = 0; i < mapfac[lev].size(); i++) {
            mapfac[lev][i]->setVal(1.0);
        }
    }
 
    // ********************************************************************************************
    // Build 1D BA and 2D BA
    // ********************************************************************************************
    BoxList bl1d = ba.boxList();
    for (auto& b : bl1d) {
        b.setRange(0,0);
        b.setRange(1,0);
    }
    ba1d[lev]  = BoxArray(std::move(bl1d));
 
    // Build 2D BA
    BoxList bl2d = ba.boxList();
    for (auto& b : bl2d) {
        b.setRange(2,0);
    }
    ba2d[lev]  = BoxArray(std::move(bl2d));
 
    IntVect ng  = vars_new[lev][Vars::cons].nGrowVect();
 
    if (lev == 0) {
        mf_C1H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
        mf_C2H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
        mf_MUB = std::make_unique<MultiFab>(ba2d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
    }
 
    mf_PSFC[lev] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ng);
 
    //*********************************************************
    // Variables for Fitch model for windfarm parametrization
    //*********************************************************
#if defined(ERF_USE_WINDFARM)
    if (solverChoice.windfarm_type == WindFarmType::Fitch){
        vars_windfarm[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
    }
    if (solverChoice.windfarm_type == WindFarmType::EWP){
        vars_windfarm[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
    }
    if (solverChoice.windfarm_type == WindFarmType::SimpleAD) {
        vars_windfarm[lev].define(ba, dm, 2, ngrow_state);// dudt, dvdt
    }
    if (solverChoice.windfarm_type == WindFarmType::GeneralAD) {
        vars_windfarm[lev].define(ba, dm, 3, ngrow_state);// dudt, dvdt, dwdt
    }
        Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
        SMark[lev].define(ba, dm, 2, 1); // Free stream velocity/source term
                                                   // sampling marker in a cell - 2 components
#endif
 
    if(solverChoice.init_type == InitType::HindCast and
        solverChoice.hindcast_lateral_forcing) {
 
        int ncomp_extra = 2;
        int nvars = vars_new[lev].size();
 
        // Resize all containers
        forecast_state_1[lev].resize(nvars + 1);
        forecast_state_2[lev].resize(nvars + 1);
        forecast_state_interp[lev].resize(nvars + 1);
 
        // Define the "normal" components
        for (int comp = 0; comp < nvars; ++comp) {
            const MultiFab& src = vars_new[lev][comp];
            ncomp = src.nComp();
            ngrow = src.nGrow();
 
            forecast_state_1[lev][comp].define(ba, dm, ncomp, ng);
            forecast_state_2[lev][comp].define(ba, dm, ncomp, ng);
            forecast_state_interp[lev][comp].define(ba, dm, ncomp, ng);
        }
 
        // Define the "extra" component (last slot)
        {
            const MultiFab& src0 = vars_new[lev][0];
            ngrow = src0.nGrow();
            int idx = nvars;
 
            forecast_state_1[lev][idx].define(ba, dm, ncomp_extra, ngrow);
            forecast_state_2[lev][idx].define(ba, dm, ncomp_extra, ngrow);
            forecast_state_interp[lev][idx].define(ba, dm, ncomp_extra, ngrow);
        }
        bool regrid_forces_file_read = true;
        WeatherDataInterpolation(lev, t_new[0],z_phys_nd, regrid_forces_file_read);
    }
 
 
    if(solverChoice.init_type == InitType::HindCast and
        solverChoice.hindcast_surface_bcs)
 
    {
        const MultiFab& src = vars_new[lev][0];
        BoxArray ba_hc = src.boxArray();
        BoxList bl2d_hc = ba_hc.boxList();
        for (auto& b : bl2d_hc) {
            b.setRange(2, 0);
        }
        BoxArray ba2d_hc(std::move(bl2d_hc));
        const amrex::DistributionMapping& dm_hc = src.DistributionMap();
 
        surface_state_1[lev].define(ba2d_hc, dm_hc, 2, src.nGrow());
        surface_state_2[lev].define(ba2d_hc, dm_hc, 2, src.nGrow());
        surface_state_interp[lev].define(ba2d_hc, dm_hc, 2, src.nGrow());
 
        bool regrid_forces_file_read = true;
        SurfaceDataInterpolation(lev, t_new[0], z_phys_nd, regrid_forces_file_read);
    }
 
#ifdef ERF_USE_WW3_COUPLING
    // create a new BoxArray and DistributionMapping for a MultiFab with 1 box
    BoxArray ba_onegrid(geom[lev].Domain());
    BoxList bl2d_onegrid = ba_onegrid.boxList();
    for (auto& b : bl2d_onegrid) {
        b.setRange(2,0);
    }
    BoxArray ba2d_onegrid(std::move(bl2d_onegrid));
    Vector<int> pmap;
    pmap.resize(1);
    pmap[0]=0;
    DistributionMapping dm_onegrid(ba2d_onegrid);
    dm_onegrid.define(pmap);
 
    Hwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
    Lwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
 
    BoxList bl2d_wave = ba.boxList();
    for (auto& b : bl2d_wave) {
        b.setRange(2,0);
    }
    BoxArray ba2d_wave(std::move(bl2d_wave));
 
    Hwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
    Lwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
 
    std::cout<<ba_onegrid<<std::endl;
    std::cout<<ba2d_onegrid<<std::endl;
    std::cout<<dm_onegrid<<std::endl;
#endif
 
 
    //*********************************************************
    // Radiation heating source terms
    //*********************************************************
    if (solverChoice.rad_type != RadiationType::None)
    {
        qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, 0);
        rad_fluxes[lev]     = std::make_unique<MultiFab>(ba, dm, 4, 0);
        qheating_rates[lev]->setVal(0.);
        rad_fluxes[lev]->setVal(0.);
    }
 
    //*********************************************************
    // Radiation fluxes for coupling to LSM
    //*********************************************************
 
    // NOTE: Finer levels do not need to coincide with the bottom domain boundary
    //       at k=0. We make slabs here with the kmin for a given box. Therefore,
    //       care must be taken before applying these fluxes to an LSM model. For
 
    // Radiative fluxes for LSM
    if (solverChoice.lsm_type != LandSurfaceType::None &&
        solverChoice.rad_type != RadiationType::None)
    {
        BoxList m_bl = ba.boxList();
        for (auto& b : m_bl) {
            int kmin = b.smallEnd(2);
            b.setRange(2,kmin);
        }
        BoxArray m_ba(std::move(m_bl));
 
        sw_lw_fluxes[lev] = std::make_unique<MultiFab>(m_ba, dm, 6, 0); // DIR/DIF VIS/NIR (4), NET SW (1), LW (1)
        solar_zenith[lev] = std::make_unique<MultiFab>(m_ba, dm, 1, 0);
 
        sw_lw_fluxes[lev]->setVal(0.);
        solar_zenith[lev]->setVal(0.);
    }
 
    //*********************************************************
    // Turbulent perturbation region initialization
    //*********************************************************
    if (solverChoice.pert_type == PerturbationType::Source ||
        solverChoice.pert_type == PerturbationType::Direct ||
        solverChoice.pert_type == PerturbationType::CPM)
    {
        amrex::Box bnd_bx = ba.minimalBox();
        turbPert.init_tpi_type(solverChoice.pert_type);
        turbPert.init_tpi(lev, bnd_bx.smallEnd(), bnd_bx.bigEnd(), geom[lev].CellSizeArray(),
                          ba, dm, ngrow_state, pp_prefix, refRatio(), max_level);
    }
 
    //
    // Define the land mask here and set it to all land by default
    // NOTE: the logic below will BREAK if we have any grids not touching the bottom boundary
    //
    {
    lmask_lev[lev].resize(1);
    auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0;
    BoxList bl2d_mask = ba.boxList();
    for (auto& b : bl2d_mask) {
        b.setRange(2,0);
    }
    BoxArray ba2d_mask(std::move(bl2d_mask));
    lmask_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
    lmask_lev[lev][0]->setVal(1);
    lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
 
    land_type_lev[lev].resize(1);
    land_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
    land_type_lev[lev][0]->setVal(0);
    land_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
 
    soil_type_lev[lev].resize(1);
    soil_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
    soil_type_lev[lev][0]->setVal(0);
    soil_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
 
    urb_frac_lev[lev].resize(1);
    urb_frac_lev[lev][0] = std::make_unique<MultiFab>(ba2d_mask,dm,1,ngv);
    urb_frac_lev[lev][0]->setVal(1.0);
    urb_frac_lev[lev][0]->FillBoundary(geom[lev].periodicity());
    }
 
    // Read in tables needed for windfarm simulations
    // fill in Nturb multifab - number of turbines in each mesh cell
    // write out the vtk files for wind turbine location and/or
    // actuator disks
    #ifdef ERF_USE_WINDFARM
        //init_windfarm(lev);
    #endif
 
    if (lev > 0) {
        fine_mask[lev] = std::make_unique<MultiFab>(grids[lev-1], dmap[lev-1], 1, 0);
        build_fine_mask(lev, *fine_mask[lev].get());
    }
}

Here is the call graph for this function:

init_thin_body()

void ERF::init_thin_body	(	int	lev,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

{
    //********************************************************************************************
    // Thin immersed body
    // *******************************************************************************************
#if 0
    if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
        (solverChoice.advChoice.zero_yflux.size() > 0) ||
        (solverChoice.advChoice.zero_zflux.size() > 0))
    {
        overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
        overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
    }
#endif
 
    if (solverChoice.advChoice.zero_xflux.size() > 0) {
        amrex::Print() << "Setting up thin immersed body for "
            << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
        BoxArray ba_xf(ba);
        ba_xf.surroundingNodes(0);
        thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
        thin_xforce[lev]->setVal(0.0);
        xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
        xflux_imask[lev]->setVal(1);
        for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
        {
            Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
            //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
            Box xbx = mfi.nodaltilebox(0);
            for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
                const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
                if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
                    (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
                    (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
                {
                    imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
                    amrex::AllPrint() << "  mask xface at " << faceidx << std::endl;
                }
            }
        }
    } else {
        thin_xforce[lev] = nullptr;
        xflux_imask[lev] = nullptr;
    }
 
    if (solverChoice.advChoice.zero_yflux.size() > 0) {
        amrex::Print() << "Setting up thin immersed body for "
            << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
        BoxArray ba_yf(ba);
        ba_yf.surroundingNodes(1);
        thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
        thin_yforce[lev]->setVal(0.0);
        yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
        yflux_imask[lev]->setVal(1);
        for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
        {
            Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
            //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
            Box ybx = mfi.nodaltilebox(1);
            for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
                const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
                if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
                    (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
                    (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
                {
                    imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
                    amrex::AllPrint() << "  mask yface at " << faceidx << std::endl;
                }
            }
        }
    } else {
        thin_yforce[lev] = nullptr;
        yflux_imask[lev] = nullptr;
    }
 
    if (solverChoice.advChoice.zero_zflux.size() > 0) {
        amrex::Print() << "Setting up thin immersed body for "
            << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
        BoxArray ba_zf(ba);
        ba_zf.surroundingNodes(2);
        thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
        thin_zforce[lev]->setVal(0.0);
        zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
        zflux_imask[lev]->setVal(1);
        for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
        {
            Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
            //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
            Box zbx = mfi.nodaltilebox(2);
            for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
                const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
                if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
                    (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
                    (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
                {
                    imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                    //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
                    amrex::AllPrint() << "  mask zface at " << faceidx << std::endl;
                }
            }
        }
    } else {
        thin_zforce[lev] = nullptr;
        zflux_imask[lev] = nullptr;
    }
}

init_uniform()

void ERF::init_uniform ( int  lev )

private

Use problem-specific reference density and temperature to set the background state to a uniform value.

Parameters

lev	Integer specifying the current level

{
    auto& lev_new = vars_new[lev];
    for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box &gbx = mfi.growntilebox(1);
        const auto &cons_arr = lev_new[Vars::cons].array(mfi);
        prob->init_uniform(gbx, cons_arr);
    }
}

init_zphys()

void ERF::init_zphys	(	int	lev,
		amrex::Real	time
	)

{
    if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
    {
        if (lev > 0) {
            //
            // First interpolate from coarser level if there is one
            // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
            //       have been pre-filled - this includes ghost cells both inside and outside
            //       the domain
            //
            InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
                                  IntVect(0,0,0), // do NOT fill ghost cells outside the domain
                                  *z_phys_nd[lev-1], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &node_bilinear_interp,
                                  domain_bcs_type, BCVars::cons_bc);
        }
 
        int ngrow = ComputeGhostCells(solverChoice) + 2;
        Box bx(surroundingNodes(Geom(lev).Domain())); bx.grow(ngrow);
        FArrayBox terrain_fab(makeSlab(bx,2,0),1);
 
        //
        // If we are using fitted mesh then we use the surface as defined above
        // If we are not using fitted mesh but are using z_levels, we still need z_phys (for now)
        //    but we need to use a flat terrain for the mesh itself (the EB data has already been made
        //    from the correct terrain)
        //
        if (solverChoice.terrain_type != TerrainType::StaticFittedMesh &&
            solverChoice.terrain_type != TerrainType::MovingFittedMesh) {
                terrain_fab.template setVal<RunOn::Device>(0.0);
        } else {
            //
            // Fill the values of the terrain height at k=0 only
            //
            prob->init_terrain_surface(geom[lev],terrain_fab,time);
        }
 
        for (MFIter mfi(*z_phys_nd[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            Box isect = terrain_fab.box() & (*z_phys_nd[lev])[mfi].box();
            if (!isect.isEmpty()) {
                (*z_phys_nd[lev])[mfi].template copy<RunOn::Device>(terrain_fab,isect,0,isect,0,1);
            }
        }
 
        make_terrain_fitted_coords(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type);
 
        z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
 
        if (lev == 0) {
            Real zmax = z_phys_nd[0]->max(0,0,false);
            Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
            if (rel_diff < 1.e-8) {
                amrex::Print() << "max of zphys_nd " << zmax << std::endl;
                amrex::Print() << "max of zlevels  " << zlevels_stag[0][zlevels_stag[0].size()-1] << std::endl;
                AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < 1.e-8, "Terrain is taller than domain top!");
            }
        } // lev == 0
 
    } else {
        // NOTE: If a WRFInput file is NOT provided for a finer level,
        //       we simply interpolate from the coarse. This is necessary
        //       since we average_down the terrain (ERF_MakeNewLevel.cpp L351).
        //       If a WRFInput file IS present, it overwrites the terrain data.
        if (lev > 0) {
            //
            // First interpolate from coarser level if there is one
            // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
            //       have been pre-filled - this includes ghost cells both inside and outside
            //       the domain
            //
            InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
                                  z_phys_nd[lev]->nGrowVect(), // DO fill ghost cells outside the domain
                                  *z_phys_nd[lev-1], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &node_bilinear_interp,
                                  domain_bcs_type, BCVars::cons_bc);
        }
    } // init_type
 
    if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
        solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
        terrain_blanking[lev]->setVal(1.0);
        MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ComputeGhostCells(solverChoice) + 2);
        terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
        init_immersed_forcing(lev); // needed for real cases
    }
 
    // Compute the min dz and pass to the micro model
    Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
    micro->Set_dzmin(lev, dzmin);
}

Here is the call graph for this function:

InitData()

void ERF::InitData

(

)

{
    BL_PROFILE_VAR("ERF::InitData()", InitData);
    InitData_pre();
    InitData_post();
    BL_PROFILE_VAR_STOP(InitData);
}

Referenced by main().

Here is the caller graph for this function:

InitData_post()

void ERF::InitData_post

(

)

{
    if (solverChoice.advChoice.have_zero_flux_faces)
    {
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
            "Thin immersed body with refinement not currently supported.");
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            amrex::Print() << "NOTE: Thin immersed body with non-constant dz has not been tested." << std::endl;
        }
    }
 
    if (!restart_chkfile.empty()) {
        restart();
    }
 
    //
    // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one and if two way coupling
    //
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        if (solverChoice.coupling_type == CouplingType::TwoWay) {
            for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
                average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
                average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
            }
        }
        for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
              detJ_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
            z_phys_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
        }
    }
 
#ifdef ERF_IMPLICIT_W
    if (SolverChoice::mesh_type == MeshType::VariableDz &&
        (solverChoice.vert_implicit_fac[0] > 0 ||
         solverChoice.vert_implicit_fac[1] > 0 ||
         solverChoice.vert_implicit_fac[2] > 0  )       &&
        solverChoice.implicit_momentum_diffusion)
    {
        Warning("Doing implicit solve for u, v, and w with terrain -- this has not been tested");
    }
#endif
 
    //
    // Copy vars_new into vars_old, then use vars_old to fill covered cells in vars_new during AverageDown
    //
    if (SolverChoice::terrain_type == TerrainType::EB) {
        for (int lev = 0; lev <= finest_level; lev++) {
            int ncomp_cons = vars_new[lev][Vars::cons].nComp();
            MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,ncomp_cons,vars_new[lev][Vars::cons].nGrowVect());
        }
    }
 
    if (restart_chkfile.empty()) {
        if (solverChoice.coupling_type == CouplingType::TwoWay) {
            AverageDown();
        }
    }
 
#ifdef ERF_USE_PARTICLES
    if (restart_chkfile.empty()) {
        if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
            if (solverChoice.moisture_tight_coupling) {
                Warning("Tight coupling has not been tested with Lagrangian microphysics");
            }
 
            for (int lev = 0; lev <= finest_level; lev++) {
                dynamic_cast<LagrangianMicrophysics&>(*micro).initParticles(z_phys_nd[lev]);
            }
        }
    }
#endif
 
    if (!restart_chkfile.empty()) { // Restart from a checkpoint
 
        // Create the physbc objects for {cons, u, v, w, base state}
        // We fill the additional base state ghost cells just in case we have read the old format
        for (int lev(0); lev <= finest_level; ++lev) {
            make_physbcs(lev);
            (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
        }
 
        if (solverChoice.do_forest_drag) {
            for (int lev(0); lev <= finest_level; ++lev) {
                m_forest_drag[lev]->define_drag_field(grids[lev], dmap[lev], geom[lev],
                                                      z_phys_cc[lev].get(), z_phys_nd[lev].get());
            }
        }
 
#ifdef ERF_USE_NETCDF
        //
        // Create the needed bdy_data_xlo etc ... since we don't read it in from checkpoint any more
        // This follows init_from_wrfinput()
        //
        bool use_moist = (solverChoice.moisture_type != MoistureType::None);
        if (solverChoice.use_real_bcs) {
 
            if ( geom[0].isPeriodic(0) || geom[0].isPeriodic(1) ) {
                 amrex::Error("Cannot set periodic lateral boundary conditions when reading in real boundary values");
            }
 
            bdy_time_interval = read_times_from_wrfbdy(nc_bdy_file,
                                                       bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                                       start_bdy_time);
            Real dT = bdy_time_interval;
 
            int n_time_old = static_cast<int>(t_new[0] /  dT);
 
            int lev = 0;
 
            int ntimes = std::min(n_time_old+3, static_cast<int>(bdy_data_xlo.size()));
 
            for (int itime = n_time_old; itime < ntimes; itime++)
            {
                read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
                                 bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                 real_width);
                convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
                                        *mf_MUB, *mf_C1H, *mf_C2H,
                                        vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel], vars_new[lev][Vars::cons],
                                        geom[lev], use_moist);
            } // itime
        } // use_real_bcs
 
        if (!nc_low_file.empty())
        {
            low_time_interval = read_times_from_wrflow(nc_low_file,
                                                       low_data_zlo,
                                                       start_low_time);
            Real dT = low_time_interval;
 
            int lev = 0;
            sst_lev[lev].resize(low_data_zlo.size());
            tsk_lev[lev].resize(low_data_zlo.size());
 
            int n_time_old = static_cast<int>(t_new[0] /  dT);
 
            int ntimes = std::min(n_time_old+2, static_cast<int>(low_data_zlo.size()));
 
            for (int itime = n_time_old; itime < ntimes; itime++)
            {
                read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
 
                // Need to read PSFC
                FArrayBox NC_fab_var_file;
                for (int idx = 0; idx < num_boxes_at_level[lev]; idx++) {
                    int success, use_theta_m;
                    read_from_wrfinput(lev, boxes_at_level[lev][idx], nc_init_file[lev][0],
                                       NC_fab_var_file, "PSFC", geom[lev],
                                       use_theta_m, success);
                    auto& var_fab = NC_fab_var_file;
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                    for ( MFIter mfi(*mf_PSFC[lev], false); mfi.isValid(); ++mfi )
                    {
                        FArrayBox &cur_fab = (*mf_PSFC[lev])[mfi];
                        cur_fab.template copy<RunOn::Device>(var_fab, 0, 0, 1);
                    }
                    var_fab.clear();
                }
 
                update_sst_tsk(itime, geom[lev], ba2d[lev],
                               sst_lev[lev], tsk_lev[lev],
                               m_SurfaceLayer, low_data_zlo,
                               vars_new[lev][Vars::cons], *mf_PSFC[lev],
                               solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
            } // itime
        }
#endif
    } // end restart
 
#ifdef ERF_USE_PARTICLES
    /* If using a Lagrangian microphysics model, its particle container has now been
       constructed and initialized (calls to micro->Init). So, add its pointer to
       ERF::particleData and remove its name from list of unallocated particle containers. */
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
        const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
        const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
        particleData.pushBack(pc_name, pc_ptr);
        particleData.getNamesUnalloc().remove(pc_name);
    }
#endif
 
    if (input_bndry_planes) {
        // Read the "time.dat" file to know what data is available
        m_r2d->read_time_file();
 
        // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
        Real dt_dummy = 0.0;
        m_r2d->read_input_files(t_new[0],dt_dummy,m_bc_extdir_vals);
    }
 
    if (solverChoice.custom_rhotheta_forcing)
    {
        rhotheta_src.resize(max_level+1);
        for (int lev = 0; lev <= finest_level; lev++) {
            BoxList bl_src = vars_new[lev][Vars::cons].boxArray().boxList();
            for (auto& b : bl_src) {
                if (!solverChoice.spatial_rhotheta_forcing)
                {
                    // source is only defined in Z
                    b.setRange(0, 0, 1);
                    b.setRange(1, 0, 1);
                }
            }
            BoxArray ba_src(std::move(bl_src));
            rhotheta_src[lev] = std::make_unique<MultiFab>(ba_src, vars_new[lev][Vars::cons].DistributionMap(), 1, 0);
            rhotheta_src[lev]->setVal(0.);
            prob->update_rhotheta_sources(t_new[0],
                                          rhotheta_src[lev].get(),
                                          geom[lev], z_phys_cc[lev]);
        }
    }
 
    if (solverChoice.have_geo_wind_profile)
    {
        h_u_geos.resize(max_level+1, Vector<Real>(0));
        d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        h_v_geos.resize(max_level+1, Vector<Real>(0));
        d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        for (int lev = 0; lev <= finest_level; lev++) {
            const int domlen = geom[lev].Domain().length(2);
            h_u_geos[lev].resize(domlen, 0.0_rt);
            d_u_geos[lev].resize(domlen, 0.0_rt);
            h_v_geos[lev].resize(domlen, 0.0_rt);
            d_v_geos[lev].resize(domlen, 0.0_rt);
            if (solverChoice.custom_geostrophic_profile) {
                prob->update_geostrophic_profile(t_new[0],
                                                 h_u_geos[lev], d_u_geos[lev],
                                                 h_v_geos[lev], d_v_geos[lev],
                                                 geom[lev], z_phys_cc[lev]);
            } else {
                if (SolverChoice::mesh_type == MeshType::VariableDz) {
                    amrex::Print() << "Note: 1-D geostrophic wind profile input is not defined for real terrain" << std::endl;
                }
                init_geo_wind_profile(solverChoice.abl_geo_wind_table,
                                      h_u_geos[lev], d_u_geos[lev],
                                      h_v_geos[lev], d_v_geos[lev],
                                      geom[lev],
                                      zlevels_stag[0]);
            }
        }
    }
 
    if (solverChoice.custom_moisture_forcing)
    {
        rhoqt_src.resize(max_level+1);
        for (int lev = 0; lev <= finest_level; lev++) {
            BoxList bl_src = vars_new[lev][Vars::cons].boxArray().boxList();
            for (auto& b : bl_src) {
                if (!solverChoice.spatial_moisture_forcing)
                {
                    // source is only defined in Z
                    b.setRange(0, 0, 1);
                    b.setRange(1, 0, 1);
                }
            }
            BoxArray ba_src(std::move(bl_src));
            rhoqt_src[lev] = std::make_unique<MultiFab>(ba_src, vars_new[lev][Vars::cons].DistributionMap(), 1, 0);
            rhoqt_src[lev]->setVal(0.);
            prob->update_rhoqt_sources(t_new[0],
                                       rhoqt_src[lev].get(),
                                       geom[lev], z_phys_cc[lev]);
        }
    }
 
    if (solverChoice.custom_w_subsidence)
    {
        h_w_subsid.resize(max_level+1, Vector<Real>(0));
        d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
        for (int lev = 0; lev <= finest_level; lev++) {
            const int domlen = geom[lev].Domain().length(2) + 1; // lives on z-faces
            h_w_subsid[lev].resize(domlen, 0.0_rt);
            d_w_subsid[lev].resize(domlen, 0.0_rt);
            prob->update_w_subsidence(t_new[0],
                                      h_w_subsid[lev], d_w_subsid[lev], base_state[lev],
                                      geom[lev], z_phys_nd[lev]);
        }
    }
 
    if (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
        solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T)
    {
        initRayleigh();
        if (solverChoice.init_type == InitType::Input_Sounding)
        {
            // Overwrite ubar, vbar, and thetabar with input profiles;
            // wbar is assumed to be 0. Note: the tau coefficient set by
            // prob->erf_init_rayleigh() is still used
            bool restarting = (!restart_chkfile.empty());
            setRayleighRefFromSounding(restarting);
        }
    }
 
    // Read in sponge data from input file
    if(solverChoice.spongeChoice.sponge_type == "input_sponge")
    {
        initSponge();
        bool restarting = (!restart_chkfile.empty());
        setSpongeRefFromSounding(restarting);
    }
 
    if (solverChoice.pert_type == PerturbationType::Source ||
        solverChoice.pert_type == PerturbationType::Direct ||
        solverChoice.pert_type == PerturbationType::CPM) {
        if (is_it_time_for_action(istep[0], t_new[0], dt[0], pert_interval, -1.)) {
            turbPert.debug(t_new[0]);
        }
    }
 
    // We only write the file at level 0 for now
    if (output_bndry_planes)
    {
        // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
        m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
 
        Real time = 0.;
        if (time >= bndry_output_planes_start_time) {
            bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
            m_w2d->write_planes(0, time, vars_new, is_moist);
        }
    }
 
    // Fill boundary conditions in vars_new
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        auto& lev_new = vars_new[lev];
 
        // ***************************************************************************
        // Physical bc's at domain boundary
        // ***************************************************************************
        IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
        IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
 
        int ncomp_cons = lev_new[Vars::cons].nComp();
        bool do_fb     = true;
 
#ifdef ERF_USE_NETCDF
        // We call this here because it is an ERF routine
        if (solverChoice.use_real_bcs && (lev==0)) {
            int icomp_cons = 0;
            bool cons_only = false;
            Vector<MultiFab*> mfs_vec = {&lev_new[Vars::cons],&lev_new[Vars::xvel],
                                         &lev_new[Vars::yvel],&lev_new[Vars::zvel]};
            if (solverChoice.upwind_real_bcs) {
                fill_from_realbdy_upwind(mfs_vec,t_new[lev],cons_only,icomp_cons,
                                         ncomp_cons,ngvect_cons,ngvect_vels);
            } else {
                fill_from_realbdy(mfs_vec,t_new[lev],cons_only,icomp_cons,
                                  ncomp_cons,ngvect_cons,ngvect_vels);
            }
            do_fb = false;
    }
#endif
 
        (*physbcs_cons[lev])(lev_new[Vars::cons],lev_new[Vars::xvel],lev_new[Vars::yvel],0,ncomp_cons,
                             ngvect_cons,t_new[lev],BCVars::cons_bc,do_fb);
        (   *physbcs_u[lev])(lev_new[Vars::xvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                             ngvect_vels,t_new[lev],BCVars::xvel_bc,do_fb);
        (   *physbcs_v[lev])(lev_new[Vars::yvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                             ngvect_vels,t_new[lev],BCVars::yvel_bc,do_fb);
        (   *physbcs_w[lev])(lev_new[Vars::zvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                             ngvect_vels,t_new[lev],BCVars::zvel_bc,do_fb);
    }
 
    //
    // If we are starting from scratch, we have the option to project the initial velocity field
    //    regardless of how we initialized.  Note that project_initial_velocity operates on vars_new.
    // pp_inc is used as scratch space here; we zero it out after the projection
    //
    if (restart_chkfile == "")
    {
        for (int lev = 0; lev <= finest_level; ++lev)
        {
            if (solverChoice.project_initial_velocity[lev] == 1) {
                Real dummy_dt = 1.0;
                if (verbose > 0) {
                    amrex::Print() << "Projecting initial velocity field at level " << lev << std::endl;
                }
 
                project_initial_velocity(lev, t_new[lev], dummy_dt);
 
                pp_inc[lev].setVal(0.);
                gradp[lev][GpVars::gpx].setVal(0.);
                gradp[lev][GpVars::gpy].setVal(0.);
                gradp[lev][GpVars::gpz].setVal(0.);
            } // project
        } // lev
    }
 
    // Copy from new into old just in case (after filling boundary conditions and possibly projecting)
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        int nc = vars_new[lev][Vars::cons].nComp();
 
        MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,nc,vars_new[lev][Vars::cons].nGrowVect());
        MultiFab::Copy(vars_old[lev][Vars::xvel],vars_new[lev][Vars::xvel],0,0, 1,vars_new[lev][Vars::xvel].nGrowVect());
        MultiFab::Copy(vars_old[lev][Vars::yvel],vars_new[lev][Vars::yvel],0,0, 1,vars_new[lev][Vars::yvel].nGrowVect());
        MultiFab::Copy(vars_old[lev][Vars::zvel],vars_new[lev][Vars::zvel],0,0, 1,vars_new[lev][Vars::zvel].nGrowVect());
    }
 
    // Compute the minimum dz in the domain at each level (to be used for setting the timestep)
    dz_min.resize(max_level+1);
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        dz_min[lev] = geom[lev].CellSize(2);
        if ( SolverChoice::mesh_type != MeshType::ConstantDz ) {
            dz_min[lev] *= (*detJ_cc[lev]).min(0);
        }
    }
 
    // We don't need to recompute dt[lev] on restart because we read it in from the checkpoint file.
    if (restart_chkfile.empty()) {
        ComputeDt();
    }
 
    // Check the viscous limit
    DiffChoice dc = solverChoice.diffChoice;
    if (dc.molec_diff_type == MolecDiffType::Constant ||
        dc.molec_diff_type == MolecDiffType::ConstantAlpha) {
        Real delta = std::min({geom[finest_level].CellSize(0),
                               geom[finest_level].CellSize(1),
                               dz_min[finest_level]});
        if (dc.dynamic_viscosity == 0) {
            Print() << "Note: Molecular diffusion specified but dynamic_viscosity has not been specified" << std::endl;
        } else {
            Real nu = dc.dynamic_viscosity / dc.rho0_trans;
            Real viscous_limit = 2.0 * delta*delta / nu;
            Print() << "smallest grid spacing at level " << finest_level << " = " << delta << std::endl;
            Print() << "dt at level " << finest_level << " = " << dt[finest_level] << std::endl;
            Print() << "Viscous CFL is " << dt[finest_level] / viscous_limit << std::endl;
            if (fixed_dt[finest_level] >= viscous_limit) {
                Warning("Specified fixed_dt is above the viscous limit");
            } else if (dt[finest_level] >= viscous_limit) {
                Warning("Adaptive dt based on convective CFL only is above the viscous limit");
            }
        }
    }
 
    // Fill ghost cells/faces
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        if (lev > 0 && cf_width >= 0) {
            Construct_ERFFillPatchers(lev);
        }
 
        auto& lev_new = vars_new[lev];
 
        //
        // Fill boundary conditions -- not sure why we need this here
        //
        bool fillset = false;
        if (lev == 0) {
            FillPatchCrseLevel(lev, t_new[lev],
                      {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]});
        } else {
            FillPatchFineLevel(lev, t_new[lev],
                      {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
                      {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                      base_state[lev], base_state[lev],
                      fillset);
        }
 
        //
        // We do this here to make sure level (lev-1) boundary conditions are filled
        // before we interpolate to level (lev) ghost cells
        //
        if (lev < finest_level) {
            auto& lev_old = vars_old[lev];
            MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
            MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
            MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
            MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
        }
 
        //
        // We fill the ghost cell values of the base state in case it wasn't done in the initialization
        //
        base_state[lev].FillBoundary(geom[lev].periodicity());
 
        // For moving terrain only
        if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
            MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
            base_state_new[lev].FillBoundary(geom[lev].periodicity());
        }
 
    }
 
    // Allow idealized cases over water, used to set lmask
    ParmParse pp("erf");
    int is_land;
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        if (pp.query("is_land", is_land, lev)) {
            if (is_land == 1) {
                amrex::Print() << "Level " << lev << " is land" << std::endl;
            } else if (is_land == 0) {
                amrex::Print() << "Level " << lev << " is water" << std::endl;
            } else {
                Error("is_land should be 0 or 1");
            }
            lmask_lev[lev][0]->setVal(is_land);
            lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
        }
    }
 
    // If lev > 0, we need to fill bc's by interpolation from coarser grid
    for (int lev = 1; lev <= finest_level; ++lev)
    {
        Interp2DArrays(lev,grids[lev],dmap[lev]);
    } // lev
 
#ifdef ERF_USE_WW3_COUPLING
    int my_lev = 0;
    amrex::Print() <<  " About to call send_to_ww3 from ERF.cpp" << std::endl;
    send_to_ww3(my_lev);
    amrex::Print() <<  " About to call read_waves from ERF.cpp"  << std::endl;
    read_waves(my_lev);
   // send_to_ww3(my_lev);
#endif
 
    // Create wall distance field for RANS model
    for (int lev = 0; lev <= finest_level; lev++) {
        if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
            // Handle bottom boundary
            poisson_wall_dist(lev);
 
            // Correct the wall distance for immersed bodies
            if (solverChoice.advChoice.have_zero_flux_faces) {
                thinbody_wall_dist(walldist[lev],
                                   solverChoice.advChoice.zero_xflux,
                                   solverChoice.advChoice.zero_yflux,
                                   solverChoice.advChoice.zero_zflux,
                                   geom[lev],
                                   z_phys_cc[lev]);
            }
        }
    }
 
    // Configure SurfaceLayer params if used
    // NOTE: we must set up the MOST routine after calling FillPatch
    //       in order to have lateral ghost cells filled (MOST + terrain interp).
    if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
    {
        bool has_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
                          (solverChoice.turbChoice[0].les_type  != LESType::None)          ||
                          (solverChoice.turbChoice[0].rans_type != RANSType::None)         ||
                          (solverChoice.turbChoice[0].pbl_type  != PBLType::None) );
        AMREX_ALWAYS_ASSERT(has_diff);
 
        bool rotate = solverChoice.use_rotate_surface_flux;
        if (rotate) {
            Print() << "Using surface layer model with stress rotations" << std::endl;
        }
 
        //
        // This constructor will make the SurfaceLayer object but not allocate the arrays at each level.
        //
        m_SurfaceLayer = std::make_unique<SurfaceLayer>(geom, rotate, pp_prefix, Qv_prim,
                                                        z_phys_nd,
                                                        solverChoice.mesh_type,
                                                        solverChoice.terrain_type,
                                                        solverChoice.turbChoice[finest_level],
                                                        start_time, stop_time
#ifdef ERF_USE_NETCDF
                                                        , bdy_time_interval
#endif
                                                        );
        // This call will allocate the arrays at each level. If we regrid later, either changing
        // the number of levels or just the grids at each existing level, we will call an update routine
        // to redefine the internal arrays in m_SurfaceLayer.
        for (int lev = 0; lev <= finest_level; lev++)
        {
            Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
                                         &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
            m_SurfaceLayer->make_SurfaceLayer_at_level(lev,finest_level+1,
                                                       mfv_old, Theta_prim[lev], Qv_prim[lev],
                                                       Qr_prim[lev], z_phys_nd[lev],
                                                       Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
                                                       lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
                                                       sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
        }
 
        // If initializing from an input_sounding, make sure the surface layer
        // is using the same surface conditions
        if (solverChoice.init_type == InitType::Input_Sounding) {
            const Real theta0 = input_sounding_data.theta_ref_inp_sound;
            const Real qv0    = input_sounding_data.qv_ref_inp_sound;
            for (int lev = 0; lev <= finest_level; lev++) {
                m_SurfaceLayer->set_t_surf(lev, theta0);
                m_SurfaceLayer->set_q_surf(lev, qv0);
            }
        }
 
        if (restart_chkfile != "") {
            // Update surface fields if needed (and available)
            ReadCheckpointFileSurfaceLayer();
        }
 
        // We now configure ABLMost params here so that we can print the averages at t=0
        // Note we don't fill ghost cells here because this is just for diagnostics
        for (int lev = 0; lev <= finest_level; ++lev)
        {
            Real time  = t_new[lev];
            IntVect ng = Theta_prim[lev]->nGrowVect();
 
            MultiFab::Copy(  *Theta_prim[lev], vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, ng);
            MultiFab::Divide(*Theta_prim[lev], vars_new[lev][Vars::cons],      Rho_comp, 0, 1, ng);
 
            if (solverChoice.moisture_type != MoistureType::None) {
                ng = Qv_prim[lev]->nGrowVect();
 
                MultiFab::Copy(  *Qv_prim[lev], vars_new[lev][Vars::cons], RhoQ1_comp, 0, 1, ng);
                MultiFab::Divide(*Qv_prim[lev], vars_new[lev][Vars::cons],   Rho_comp, 0, 1, ng);
 
                int rhoqr_comp = solverChoice.moisture_indices.qr;
                if (rhoqr_comp > -1) {
                    MultiFab::Copy(  *Qr_prim[lev], vars_new[lev][Vars::cons], rhoqr_comp, 0, 1, ng);
                    MultiFab::Divide(*Qr_prim[lev], vars_new[lev][Vars::cons],   Rho_comp, 0, 1, ng);
                } else {
                    Qr_prim[lev]->setVal(0.0);
                }
            }
            m_SurfaceLayer->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim, Qr_prim);
 
            if (restart_chkfile == "") {
                // Only do this if starting from scratch; if restarting, then
                // we don't want to call update_fluxes multiple times because
                // it will change u* and theta* from their previous values
                m_SurfaceLayer->update_pblh(lev, vars_new, z_phys_cc[lev].get(),
                                            solverChoice.moisture_indices);
                m_SurfaceLayer->update_fluxes(lev, time,
                                              vars_new[lev][Vars::cons],
                                              z_phys_nd[lev],
                                              walldist[lev]);
 
                // Initialize tke(x,y,z) as a function of u*(x,y)
                if (solverChoice.turbChoice[lev].init_tke_from_ustar) {
                    Real qkefac = 1.0;
                    if (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25 ||
                        solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF)
                    {
                        // https://github.com/NCAR/MYNN-EDMF/blob/90f36c25259ec1960b24325f5b29ac7c5adeac73/module_bl_mynnedmf.F90#L1325-L1333
                        const Real B1 = solverChoice.turbChoice[lev].pbl_mynn.B1;
                        qkefac = 1.5 * std::pow(B1, 2.0/3.0);
                    }
                    m_SurfaceLayer->init_tke_from_ustar(lev, vars_new[lev][Vars::cons], z_phys_nd[lev], qkefac);
                }
            }
        }
    } // end if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
 
    // Update micro vars and finish moisture model initializations before first plot file
    if (solverChoice.moisture_type != MoistureType::None) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons]);
            micro->FinishInit(lev, vars_new[lev][Vars::cons], z_phys_nd);
        }
    }
 
    // Fill time averaged velocities before first plot file
    if (solverChoice.time_avg_vel) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
                              vars_new[lev][Vars::xvel],
                              vars_new[lev][Vars::yvel],
                              vars_new[lev][Vars::zvel]);
        }
    }
 
    // check for additional plotting variables that are available after particle containers
    // are setup.
    const std::string& pv3d_1 = "plot_vars_1"  ; appendPlotVariables(pv3d_1,plot3d_var_names_1);
    const std::string& pv3d_2 = "plot_vars_2"  ; appendPlotVariables(pv3d_2,plot3d_var_names_2);
    const std::string& pv2d_1 = "plot2d_vars_1"; appendPlotVariables(pv2d_1,plot2d_var_names_1);
    const std::string& pv2d_2 = "plot2d_vars_2"; appendPlotVariables(pv2d_2,plot2d_var_names_2);
 
    if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > 0.) )
    {
        WriteCheckpointFile();
        last_check_file_step = 0;
        if (m_check_per > 0.) {last_check_file_time += m_check_per;}
    }
 
    if ( (restart_chkfile.empty()) ||
         (!restart_chkfile.empty() && plot_file_on_restart) )
    {
        if (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.)
        {
            Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
            if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 += m_plot3d_per_1;}
            last_plot3d_file_step_1 = istep[0];
        }
        if (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.)
        {
            Write3DPlotFile(2,plotfile3d_type_2,plot3d_var_names_2);
            if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 += m_plot3d_per_2;}
            last_plot3d_file_step_2 = istep[0];
        }
        if (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.)
        {
            Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
            if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 += m_plot2d_per_1;}
            last_plot2d_file_step_1 = istep[0];
        }
        if (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.)
        {
            Write2DPlotFile(2,plotfile2d_type_2,plot2d_var_names_2);
            if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 += m_plot2d_per_2;}
            last_plot2d_file_step_2 = istep[0];
        }
        for (int i = 0; i < m_subvol_int.size(); i++) {
            if (m_subvol_int[i] > 0 || m_subvol_per[i] > 0.) {
                WriteSubvolume(i,subvol3d_var_names);
                last_subvol_step[i] = istep[0];
                if (m_subvol_per[i] > 0.) {last_subvol_time[i] += m_subvol_per[i];}
            }
        }
    }
 
    // Set these up here because we need to know which MPI rank "cell" is on...
    if (pp.contains("data_log"))
    {
        int num_datalogs = pp.countval("data_log");
        datalog.resize(num_datalogs);
        datalogname.resize(num_datalogs);
        pp.queryarr("data_log",datalogname,0,num_datalogs);
        for (int i = 0; i < num_datalogs; i++) {
            setRecordDataInfo(i,datalogname[i]);
        }
    }
 
    if (pp.contains("der_data_log"))
    {
        int num_der_datalogs = pp.countval("der_data_log");
        der_datalog.resize(num_der_datalogs);
        der_datalogname.resize(num_der_datalogs);
        pp.queryarr("der_data_log",der_datalogname,0,num_der_datalogs);
        for (int i = 0; i < num_der_datalogs; i++) {
            setRecordDerDataInfo(i,der_datalogname[i]);
        }
    }
 
    if (pp.contains("energy_data_log"))
    {
        int num_energy_datalogs = pp.countval("energy_data_log");
        tot_e_datalog.resize(num_energy_datalogs);
        tot_e_datalogname.resize(num_energy_datalogs);
        pp.queryarr("energy_data_log",tot_e_datalogname,0,num_energy_datalogs);
        for (int i = 0; i < num_energy_datalogs; i++) {
            setRecordEnergyDataInfo(i,tot_e_datalogname[i]);
        }
    }
 
    if (solverChoice.rad_type != RadiationType::None)
    {
        // Create data log for radiation model if requested
        rad[0]->setupDataLog();
    }
 
 
    if (restart_chkfile.empty() && profile_int > 0) {
        if (destag_profiles) {
            // all variables cell-centered
            write_1D_profiles(t_new[0]);
        } else {
            // some variables staggered
            write_1D_profiles_stag(t_new[0]);
        }
    }
 
    if (pp.contains("sample_point_log") && pp.contains("sample_point"))
    {
        int lev = 0;
 
        int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
        if (num_samplepts > 0) {
            Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
            samplepoint.resize(num_samplepts);
 
            pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
            for (int i = 0; i < num_samplepts; i++) {
                IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                samplepoint[i] = iv;
            }
        }
 
        int num_sampleptlogs = pp.countval("sample_point_log");
        AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
        if (num_sampleptlogs > 0) {
            sampleptlog.resize(num_sampleptlogs);
            sampleptlogname.resize(num_sampleptlogs);
            pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
 
            for (int i = 0; i < num_sampleptlogs; i++) {
                setRecordSamplePointInfo(i,lev,samplepoint[i],sampleptlogname[i]);
            }
        }
 
    }
 
    if (pp.contains("sample_line_log") && pp.contains("sample_line"))
    {
        int lev = 0;
 
        int num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
        if (num_samplelines > 0) {
            Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
            sampleline.resize(num_samplelines);
 
            pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
            for (int i = 0; i < num_samplelines; i++) {
                IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                sampleline[i] = iv;
            }
        }
 
        int num_samplelinelogs = pp.countval("sample_line_log");
        AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
        if (num_samplelinelogs > 0) {
            samplelinelog.resize(num_samplelinelogs);
            samplelinelogname.resize(num_samplelinelogs);
            pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
 
            for (int i = 0; i < num_samplelinelogs; i++) {
                setRecordSampleLineInfo(i,lev,sampleline[i],samplelinelogname[i]);
            }
        }
 
    }
 
    if (is_it_time_for_action(istep[0], t_new[0], dt[0], sum_interval, sum_per)) {
        sum_integrated_quantities(t_new[0]);
        sum_derived_quantities(t_new[0]);
        sum_energy_quantities(t_new[0]);
    }
 
    // Create object to do line and plane sampling if needed
    bool do_line = false; bool do_plane = false;
    pp.query("do_line_sampling",do_line); pp.query("do_plane_sampling",do_plane);
    if (do_line) {
        if (line_sampling_interval < 0 && line_sampling_per < 0) {
            Abort("Need to specify line_sampling_interval or line_sampling_per");
        }
        line_sampler = std::make_unique<LineSampler>();
        line_sampler->write_coords(z_phys_cc);
    }
    if (do_plane) {
        if (plane_sampling_interval < 0 && plane_sampling_per < 0) {
            Abort("Need to specify plane_sampling_interval or plane_sampling_per");
        }
        plane_sampler = std::make_unique<PlaneSampler>();
    }
 
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing  ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing )
    {
        bool write_eb_surface = false;
        pp.query("write_eb_surface", write_eb_surface);
        if (write_eb_surface) {
            if (verbose > 0) {
                amrex::Print() << "Writing the geometry to a vtp file.\n" << std::endl;
            }
            WriteEBSurface(grids[finest_level],dmap[finest_level],Geom(finest_level),&EBFactory(finest_level));
        }
    }
 
}

Here is the call graph for this function:

InitData_pre()

void ERF::InitData_pre

(

)

{
    // Initialize the start time for our CPU-time tracker
    startCPUTime = ParallelDescriptor::second();
 
    // Create the ReadBndryPlanes object so we can read boundary plane data
    // m_r2d is used by init_bcs so we must instantiate this class before
    if (input_bndry_planes) {
        Print() << "Defining r2d for the first time " << std::endl;
        m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
    }
 
    if (restart_chkfile.empty()) {
        // Start simulation from the beginning
        InitFromScratch(0.0);
    } else {
        // For initialization this is done in init_only; it is done here for restart
        init_bcs();
    }
 
    solverChoice.check_params(max_level,geom,phys_bc_type);
}

initHSE() [1/2]

void ERF::initHSE ( )

private

Initialize HSE.

{
    for (int lev = 0; lev <= finest_level; lev++)
    {
        initHSE(lev);
    }
}

initHSE() [2/2]

void ERF::initHSE ( int  lev )

private

Initialize density and pressure base state in hydrostatic equilibrium.

{
    // This integrates up through column to update p_hse, pi_hse, th_hse;
    // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
 
    MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
    MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
    MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
    MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
    MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
 
    bool all_boxes_touch_bottom = true;
    Box domain(geom[lev].Domain());
 
    int icomp = 0; int ncomp = BaseState::num_comps;
 
    if (lev == 0) {
        BoxArray ba(base_state[lev].boxArray());
        for (int i = 0; i < ba.size(); i++) {
            if (ba[i].smallEnd(2) != domain.smallEnd(2)) {
                all_boxes_touch_bottom = false;
            }
        }
    }
    else
    {
        //
        // We need to do this interp from coarse level in order to set the values of
        // the base state inside the domain but outside of the fine region
        //
        base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
        //
        // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
        //       have been pre-filled - this includes ghost cells both inside and outside
        //       the domain
        //
        InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                              IntVect(0,0,0), // do not fill ghost cells outside the domain
                              base_state[lev-1], icomp, icomp, ncomp,
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
 
         // We need to do this here because the interpolation above may leave corners unfilled
         //    when the corners need to be filled by, for example, reflection of the fine ghost
         //    cell outside the fine region but inide the domain.
         (*physbcs_base[lev])(base_state[lev],icomp,ncomp,base_state[lev].nGrowVect());
    }
 
    if (all_boxes_touch_bottom || lev > 0) {
 
        // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
        if (solverChoice.use_moist_background){
            prob->erf_init_dens_hse_moist(r_hse, z_phys_nd[lev], geom[lev]);
        } else {
            prob->erf_init_dens_hse(r_hse, z_phys_nd[lev], z_phys_cc[lev], geom[lev]);
        }
 
        erf_enforce_hse(lev, r_hse, p_hse, pi_hse, th_hse, qv_hse, z_phys_cc[lev]);
 
        //
        // Impose physical bc's on the base state
        //
        (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
 
    } else {
 
        BoxArray ba_new(domain);
 
        ChopGrids2D(ba_new, domain, ParallelDescriptor::NProcs());
 
        DistributionMapping dm_new(ba_new);
 
        MultiFab new_base_state(ba_new, dm_new, BaseState::num_comps, base_state[lev].nGrowVect());
        new_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
                                    base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
 
        MultiFab new_r_hse (new_base_state, make_alias, BaseState::r0_comp, 1);
        MultiFab new_p_hse (new_base_state, make_alias, BaseState::p0_comp, 1);
        MultiFab new_pi_hse(new_base_state, make_alias, BaseState::pi0_comp, 1);
        MultiFab new_th_hse(new_base_state, make_alias, BaseState::th0_comp, 1);
        MultiFab new_qv_hse(new_base_state, make_alias, BaseState::qv0_comp, 1);
 
        std::unique_ptr<MultiFab> new_z_phys_cc;
        std::unique_ptr<MultiFab> new_z_phys_nd;
        if (solverChoice.mesh_type != MeshType::ConstantDz) {
            new_z_phys_cc = std::make_unique<MultiFab>(ba_new,dm_new,1,1);
            new_z_phys_cc->ParallelCopy(*z_phys_cc[lev],0,0,1,1,1);
 
            BoxArray ba_new_nd(ba_new);
            ba_new_nd.surroundingNodes();
            new_z_phys_nd = std::make_unique<MultiFab>(ba_new_nd,dm_new,1,1);
            new_z_phys_nd->ParallelCopy(*z_phys_nd[lev],0,0,1,1,1);
        }
 
        // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
        if (solverChoice.use_moist_background){
            prob->erf_init_dens_hse_moist(new_r_hse, new_z_phys_nd, geom[lev]);
        } else {
            prob->erf_init_dens_hse(new_r_hse, new_z_phys_nd, new_z_phys_cc, geom[lev]);
        }
 
        erf_enforce_hse(lev, new_r_hse, new_p_hse, new_pi_hse, new_th_hse, new_qv_hse, new_z_phys_cc);
 
        //
        // Impose physical bc's on the base state (we must make new, temporary bcs object because the z_phys_nd is different)
        //
        ERFPhysBCFunct_base* temp_physbcs_base =
            new ERFPhysBCFunct_base(lev, geom[lev], domain_bcs_type, domain_bcs_type_d, new_z_phys_nd,
                                    (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
        (*temp_physbcs_base)(new_base_state,0,new_base_state.nComp(),new_base_state.nGrowVect());
        delete temp_physbcs_base;
 
        // Now copy back into the original arrays
        base_state[lev].ParallelCopy(new_base_state,0,0,base_state[lev].nComp(),
                                     base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
    }
 
    //
    // Impose physical bc's on the base state -- the values outside the fine region
    //   but inside the domain have already been filled in the call above to InterpFromCoarseLevel
    //
    (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
}

Here is the call graph for this function:

initialize_integrator()

void ERF::initialize_integrator ( int  lev, amrex::MultiFab &  cons_mf, amrex::MultiFab &  vel_mf  )

private

{
    const BoxArray& ba(cons_mf.boxArray());
    const DistributionMapping& dm(cons_mf.DistributionMap());
 
    int ncomp_cons = cons_mf.nComp();
 
    // Initialize the integrator memory
    Vector<MultiFab> int_state; // integration state data structure example
    int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons));         // cons
    int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
    int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
    int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
 
    mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
    mri_integrator_mem[lev]->setNoSubstepping((solverChoice.substepping_type[lev] == SubsteppingType::None));
    mri_integrator_mem[lev]->setAnelastic(solverChoice.anelastic[lev]);
    mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
    mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
}

InitializeFromFile()

void ERF::InitializeFromFile ( )

private

InitializeLevelFromData()

void ERF::InitializeLevelFromData ( int  lev, const amrex::MultiFab &  initial_data  )

private

initializeMicrophysics()

void ERF::initializeMicrophysics ( const int &  a_nlevsmax )

private

Parameters

a_nlevsmax

number of AMR levels

{
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
 
        micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
 
    } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
#ifdef ERF_USE_PARTICLES
        micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
        /* Lagrangian microphysics models will have a particle container; it needs to be added
           to ERF::particleData */
        const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
        /* The particle container has not yet been constructed and initialized, so just add
           its name here for now (so that functions to set plotting variables can see it). */
        particleData.addName( pc_name );
 
#else
        Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
#endif
    }
 
    qmoist.resize(a_nlevsmax);
    return;
}

Here is the call graph for this function:

initRayleigh()

void ERF::initRayleigh ( )

private

Initialize Rayleigh damping profiles.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

{
    const int khi = geom[0].Domain().bigEnd(2);
    solverChoice.dampingChoice.rayleigh_ztop = (solverChoice.terrain_type == TerrainType::None) ?  geom[0].ProbHi(2) : zlevels_stag[0][khi+1];
 
    h_rayleigh_ptrs.resize(max_level+1);
    d_rayleigh_ptrs.resize(max_level+1);
 
    h_sinesq_ptrs.resize(max_level+1);
    d_sinesq_ptrs.resize(max_level+1);
 
    h_sinesq_stag_ptrs.resize(max_level+1);
    d_sinesq_stag_ptrs.resize(max_level+1);
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        // These have 4 components: ubar, vbar, wbar, thetabar
        h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
        d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
 
        const int zlen_rayleigh = geom[lev].Domain().length(2);
 
        // Allocate space for these 1D vectors
        for (int n = 0; n < Rayleigh::nvars; n++) {
            h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
            d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
        }
 
        h_sinesq_ptrs[lev].resize(zlen_rayleigh);
        d_sinesq_ptrs[lev].resize(zlen_rayleigh);
 
        h_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
        d_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
 
        Real ztop     = solverChoice.dampingChoice.rayleigh_ztop;
        Real zdamp    = solverChoice.dampingChoice.rayleigh_zdamp;
 
        for (int k = 0; k < zlen_rayleigh; k++) {
            Real z = 0.5 * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
            if (z > (ztop - zdamp)) {
                Real zfrac = 1.0 - (ztop - z) / zdamp;
                Real s = std::sin(PIoTwo*zfrac);
                h_sinesq_ptrs[lev][k] = s*s;
            } else {
                h_sinesq_ptrs[lev][k] = 0.0;
            }
        }
 
        for (int k = 0; k < zlen_rayleigh+1; k++) {
            Real z = zlevels_stag[lev][k];
            if (z > (ztop - zdamp)) {
                Real zfrac = 1.0 - (ztop - z) / zdamp;
                Real s = std::sin(PIoTwo*zfrac);
                h_sinesq_stag_ptrs[lev][k] = s*s;
            } else {
                h_sinesq_stag_ptrs[lev][k] = 0.0;
            }
        }
 
        // Init the host vectors for the reference states
        prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_nd[lev], solverChoice.dampingChoice.rayleigh_zdamp);
 
        // Copy from host vectors to device vectors
        for (int n = 0; n < Rayleigh::nvars; n++) {
            Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
                      d_rayleigh_ptrs[lev][n].begin());
        }
        Gpu::copy(Gpu::hostToDevice, h_sinesq_ptrs[lev].begin(), h_sinesq_ptrs[lev].end(), d_sinesq_ptrs[lev].begin());
        Gpu::copy(Gpu::hostToDevice, h_sinesq_stag_ptrs[lev].begin(), h_sinesq_stag_ptrs[lev].end(), d_sinesq_stag_ptrs[lev].begin());
    }
}

initSponge()

void ERF::initSponge ( )

private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

{
    h_sponge_ptrs.resize(max_level+1);
    d_sponge_ptrs.resize(max_level+1);
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        // These have 2 components: ubar, vbar
        h_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
        d_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
 
        const int zlen_sponge = geom[lev].Domain().length(2);
 
        // Allocate space for these 1D vectors
        for (int n = 0; n < Sponge::nvars_sponge; n++) {
            h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
            d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
        }
 
    }
}

input_sponge()

void ERF::input_sponge

(

int

lev

)

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters

lev	Integer specifying the current level

{
    // We only want to read the file once
    if (lev == 0) {
        if (input_sponge_data.input_sponge_file.empty())
            Error("input_sounding file name must be provided via input");
 
        // this will interpolate the input profiles to the nominal height levels
        // (ranging from 0 to the domain top)
        input_sponge_data.read_from_file(geom[lev], zlevels_stag[lev]);
    }
}

Interp2DArrays()

void ERF::Interp2DArrays	(	int	lev,
		const amrex::BoxArray &	my_ba2d,
		const amrex::DistributionMapping &	my_dm
	)

{
    if (lon_m[lev-1] && !lon_m[lev]) {
        auto ngv = lon_m[lev-1]->nGrowVect(); ngv[2] = 0;
        lon_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
        InterpFromCoarseLevel(*lon_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                              *lon_m[lev-1], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
    }
    if (lat_m[lev-1] && !lat_m[lev]) {
        auto ngv = lat_m[lev-1]->nGrowVect(); ngv[2] = 0;
        lat_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
        InterpFromCoarseLevel(*lat_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                              *lat_m[lev-1], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
    }
    if (sinPhi_m[lev-1] && !sinPhi_m[lev]) {
        auto ngv = sinPhi_m[lev-1]->nGrowVect(); ngv[2] = 0;
        sinPhi_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
        InterpFromCoarseLevel(*sinPhi_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                              *sinPhi_m[lev-1], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
    }
    if (cosPhi_m[lev-1] && !cosPhi_m[lev]) {
        auto ngv = cosPhi_m[lev-1]->nGrowVect(); ngv[2] = 0;
        cosPhi_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
        InterpFromCoarseLevel(*cosPhi_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                              *cosPhi_m[lev-1], 0, 0, 1,
                              geom[lev-1], geom[lev],
                              refRatio(lev-1), &cell_cons_interp,
                              domain_bcs_type, BCVars::cons_bc);
    }
    if (sst_lev[lev-1][0] && !sst_lev[lev][0]) {
        int ntimes = sst_lev[lev-1].size();
        sst_lev[lev].resize(ntimes);
        auto ngv = sst_lev[lev-1][0]->nGrowVect(); ngv[2] = 0;
        for (int n = 0; n < ntimes; n++) {
            sst_lev[lev][n] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
            InterpFromCoarseLevel(*sst_lev[lev][n], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                                  *sst_lev[lev-1][n], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &cell_cons_interp,
                                  domain_bcs_type, BCVars::cons_bc);
        }
    }
    if (tsk_lev[lev-1][0] && !tsk_lev[lev][0]) {
        int ntimes = tsk_lev[lev-1].size();
        tsk_lev[lev].resize(ntimes);
        auto ngv = tsk_lev[lev-1][0]->nGrowVect(); ngv[2] = 0;
        for (int n = 0; n < ntimes; n++) {
            tsk_lev[lev][n] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
            InterpFromCoarseLevel(*tsk_lev[lev][n], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                                  *tsk_lev[lev-1][n], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &cell_cons_interp,
                                  domain_bcs_type, BCVars::cons_bc);
        }
    }
 
    Real time_for_fp = 0.; // This is not actually used
    Vector<Real> ftime    = {time_for_fp, time_for_fp};
    Vector<Real> ctime    = {time_for_fp, time_for_fp};
    if (lat_m[lev]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        Vector<MultiFab*> fmf = {lat_m[lev  ].get(), lat_m[lev  ].get()};
        Vector<MultiFab*> cmf = {lat_m[lev-1].get(), lat_m[lev-1].get()};
        IntVect ngv = lat_m[lev]->nGrowVect(); ngv[2] = 0;
        Interpolater* mapper = &cell_cons_interp;
        FillPatchTwoLevels(*lat_m[lev].get(), ngv, IntVect(0,0,0),
                           time_for_fp, cmf, ctime, fmf, ftime,
                           0, 0, 1, geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper, domain_bcs_type,
                           BCVars::cons_bc);
    }
    if (lon_m[lev]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        Vector<MultiFab*> fmf = {lon_m[lev  ].get(), lon_m[lev  ].get()};
        Vector<MultiFab*> cmf = {lon_m[lev-1].get(), lon_m[lev-1].get()};
        IntVect ngv = lon_m[lev]->nGrowVect(); ngv[2] = 0;
        Interpolater* mapper = &cell_cons_interp;
        FillPatchTwoLevels(*lon_m[lev].get(), ngv, IntVect(0,0,0),
                           time_for_fp, cmf, ctime, fmf, ftime,
                           0, 0, 1, geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper, domain_bcs_type,
                           BCVars::cons_bc);
    } // lon_m
    if (sinPhi_m[lev]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        Vector<MultiFab*> fmf = {sinPhi_m[lev  ].get(), sinPhi_m[lev  ].get()};
        Vector<MultiFab*> cmf = {sinPhi_m[lev-1].get(), sinPhi_m[lev-1].get()};
        IntVect ngv = sinPhi_m[lev]->nGrowVect(); ngv[2] = 0;
        Interpolater* mapper = &cell_cons_interp;
        FillPatchTwoLevels(*sinPhi_m[lev].get(), ngv, IntVect(0,0,0),
                           time_for_fp, cmf, ctime, fmf, ftime,
                           0, 0, 1, geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper, domain_bcs_type,
                           BCVars::cons_bc);
    } // sinPhi
    if (cosPhi_m[lev]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
        Vector<MultiFab*> fmf = {cosPhi_m[lev  ].get(), cosPhi_m[lev  ].get()};
        Vector<MultiFab*> cmf = {cosPhi_m[lev-1].get(), cosPhi_m[lev-1].get()};
        IntVect ngv = cosPhi_m[lev]->nGrowVect(); ngv[2] = 0;
        Interpolater* mapper = &cell_cons_interp;
        FillPatchTwoLevels(*cosPhi_m[lev].get(), ngv, IntVect(0,0,0),
                           time_for_fp, cmf, ctime, fmf, ftime,
                           0, 0, 1, geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper, domain_bcs_type,
                           BCVars::cons_bc);
    } // cosPhi
    if (sst_lev[lev][0]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
    int ntimes = sst_lev[lev].size();
    for (int n = 0; n < ntimes; n++) {
            Vector<MultiFab*> fmf = {sst_lev[lev  ][n].get(), sst_lev[lev  ][n].get()};
            Vector<MultiFab*> cmf = {sst_lev[lev-1][n].get(), sst_lev[lev-1][n].get()};
            IntVect ngv = sst_lev[lev][n]->nGrowVect(); ngv[2] = 0;
            Interpolater* mapper = &cell_cons_interp;
            FillPatchTwoLevels(*sst_lev[lev][n].get(), ngv, IntVect(0,0,0),
                               time_for_fp, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::cons_bc);
        } // ntimes
    } // sst_lev
    if (tsk_lev[lev][0]) {
        // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
    int ntimes = tsk_lev[lev].size();
    for (int n = 0; n < ntimes; n++) {
            Vector<MultiFab*> fmf = {tsk_lev[lev  ][n].get(), tsk_lev[lev  ][n].get()};
            Vector<MultiFab*> cmf = {tsk_lev[lev-1][n].get(), tsk_lev[lev-1][n].get()};
            IntVect ngv = tsk_lev[lev][n]->nGrowVect(); ngv[2] = 0;
            Interpolater* mapper = &cell_cons_interp;
            FillPatchTwoLevels(*tsk_lev[lev][n].get(), ngv, IntVect(0,0,0),
                               time_for_fp, cmf, ctime, fmf, ftime,
                               0, 0, 1, geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BCVars::cons_bc);
        } // ntimes
    } // tsk_lev
}

is_it_time_for_action()

bool ERF::is_it_time_for_action ( int  nstep, amrex::Real  time, amrex::Real  dt, int  action_interval, amrex::Real  action_per  )

static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters

nstep	Timestep number
time	Current time
dtlev	Timestep for the current level
action_interval	Interval in number of timesteps for taking action
action_per	Interval in simulation time for taking action

{
  bool int_test = (action_interval > 0 && nstep % action_interval == 0);
 
  bool per_test = false;
  if (action_per > 0.0) {
    const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
    const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
 
    if (num_per_old != num_per_new) {
      per_test = true;
    }
  }
 
  return int_test || per_test;
}

make_eb_box()

void ERF::make_eb_box

(

)

make_eb_regular()

void ERF::make_eb_regular

(

)

make_physbcs()

void ERF::make_physbcs ( int  lev )

private

{
    if (SolverChoice::mesh_type == MeshType::VariableDz) {
        AMREX_ALWAYS_ASSERT(z_phys_nd[lev] != nullptr);
    }
 
    physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                               m_bc_extdir_vals, m_bc_neumann_vals,
                                                               z_phys_nd[lev], solverChoice.use_real_bcs, th_bc_data[lev].data());
    physbcs_u[lev]    = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                            m_bc_extdir_vals, m_bc_neumann_vals,
                                                            z_phys_nd[lev], solverChoice.use_real_bcs, xvel_bc_data[lev].data());
    physbcs_v[lev]    = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                            m_bc_extdir_vals, m_bc_neumann_vals,
                                                            z_phys_nd[lev], solverChoice.use_real_bcs, yvel_bc_data[lev].data());
    physbcs_w[lev]    = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                            m_bc_extdir_vals, m_bc_neumann_vals,
                                                            solverChoice.terrain_type, mapfac[lev], z_phys_nd[lev],
                                                            solverChoice.use_real_bcs, zvel_bc_data[lev].data());
    physbcs_base[lev] = std::make_unique<ERFPhysBCFunct_base> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d, z_phys_nd[lev],
                                                               (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
}

make_subdomains()

void ERF::make_subdomains	(	const amrex::BoxList &	ba,
		amrex::Vector< amrex::BoxArray > &	bins
	)

{
    Vector<BoxList> bins_bl;
 
    // Clear out any old bins
    bins.clear();
 
    // Iterate over boxes
    for (auto bx : bl)
    {
        bool added = false;
 
        // Try to add box to existing bin
        for (int j = 0; j < bins_bl.size(); ++j) {
            BoxList& bin = bins_bl[j];
            bool touches = false;
 
            for (auto& b : bin)
            {
                Box gbx(bx); gbx.grow(1);
                if (gbx.intersects(b)) {
                    touches = true;
                    break;
                }
            }
 
            if (touches) {
                bin.push_back(bx);
                added = true;
                break;
            }
        }
 
        // If box couldn't be added to existing bin, create new bin
        if (!added) {
            BoxList new_bin;
            new_bin.push_back(bx);
            bins_bl.push_back(new_bin);
        }
    }
 
    // Convert the BoxLists to BoxArrays
    for (int i = 0; i < bins_bl.size(); ++i) {
        bins.push_back(BoxArray(bins_bl[i]));
    }
}

MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage	(	amrex::Vector< amrex::Real > &	h_havg,
		amrex::MultiFab &	S,
		int	n
	)

{
    // Get the number of cells in z at level 0
    int dir_z = AMREX_SPACEDIM-1;
    auto domain = geom[0].Domain();
    int size_z = domain.length(dir_z);
    int start_z = domain.smallEnd()[dir_z];
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
 
    // resize the level 0 horizontal average vectors
    h_havg.resize(size_z, 0.0_rt);
 
    // Get the cell centered data and construct sums
#ifdef _OPENMP
#pragma omp parallel if (Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(S); mfi.isValid(); ++mfi) {
        const Box& box = mfi.validbox();
        const IntVect& se = box.smallEnd();
        const IntVect& be = box.bigEnd();
 
        auto      fab_arr = S[mfi].array();
 
        FArrayBox fab_reduce(box, 1, The_Async_Arena());
        auto arr_reduce   = fab_reduce.array();
 
        ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
            arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
        });
 
        for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
            Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
            h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
        }
    }
 
    // combine sums from different MPI ranks
    ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), h_havg.size());
 
    // divide by the total number of cells we are averaging over
    for (int k = 0; k < size_z; ++k) {
        h_havg[k]     /= area_z;
    }
}

MakeEBGeometry()

void ERF::MakeEBGeometry

(

)

MakeFilename_EyeTracker_latlon()

std::string ERF::MakeFilename_EyeTracker_latlon

(

int

nstep

)

                                             {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/latlon";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_track_latlon" << std::setw(7) << std::setfill('0') << nstep << ".txt";
    return oss.str();
}

MakeFilename_EyeTracker_maxvel()

std::string ERF::MakeFilename_EyeTracker_maxvel

(

int

nstep

)

                                             {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/maxvel";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_maxvel_" << std::setw(7) << std::setfill('0') << nstep << ".txt";
    return oss.str();
}

MakeFilename_EyeTracker_minpressure()

std::string ERF::MakeFilename_EyeTracker_minpressure

(

int

nstep

)

                                                  {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/minpressure";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_minpressure_" << std::setw(7) << std::setfill('0') << nstep << ".txt";
    return oss.str();
}

MakeHorizontalAverages()

void ERF::MakeHorizontalAverages

(

)

{
    int lev = 0;
 
    // First, average down all levels (if doing two-way coupling)
    if (solverChoice.coupling_type == CouplingType::TwoWay) {
        AverageDown();
    }
 
    MultiFab mf(grids[lev], dmap[lev], 5, 0);
 
    int zdir = 2;
    auto domain = geom[0].Domain();
 
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
    bool is_anelastic = (solverChoice.anelastic[lev] == 1);
 
    for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
        const Box& bx = mfi.validbox();
        auto  fab_arr = mf.array(mfi);
        auto const  hse_arr = base_state[lev].const_array(mfi);
        auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
            Real dens = cons_arr(i, j, k, Rho_comp);
            fab_arr(i, j, k, 0) = dens;
            fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
            if (!use_moisture) {
                if (is_anelastic) {
                    fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
                } else {
                    fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp));
                }
            }
        });
    }
 
    if (use_moisture)
    {
        for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.validbox();
            auto  fab_arr = mf.array(mfi);
            auto const  hse_arr = base_state[lev].const_array(mfi);
            auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
            int ncomp = vars_new[lev][Vars::cons].nComp();
 
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                Real dens = cons_arr(i, j, k, Rho_comp);
                if (is_anelastic) {
                    fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
                } else {
                    Real qv = cons_arr(i, j, k, RhoQ1_comp) / dens;
                    fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
                }
                fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : 0.0);
                fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : 0.0);
            });
        }
 
        Gpu::HostVector<Real> h_avg_qv          = sumToLine(mf,3,1,domain,zdir);
        Gpu::HostVector<Real> h_avg_qc          = sumToLine(mf,4,1,domain,zdir);
    }
 
    // Sum in the horizontal plane
    Gpu::HostVector<Real> h_avg_density     = sumToLine(mf,0,1,domain,zdir);
    Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
    Gpu::HostVector<Real> h_avg_pressure    = sumToLine(mf,2,1,domain,zdir);
 
    // Divide by the total number of cells we are averaging over
    int size_z = domain.length(zdir);
    Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
    int klen = static_cast<int>(h_avg_density.size());
 
    for (int k = 0; k < klen; ++k) {
        h_havg_density[k]     /= area_z;
        h_havg_temperature[k] /= area_z;
        h_havg_pressure[k]    /= area_z;
        if (solverChoice.moisture_type != MoistureType::None)
        {
            h_havg_qc[k]          /= area_z;
            h_havg_qv[k]          /= area_z;
        }
    } // k
 
    // resize device vectors
    d_havg_density.resize(size_z, 0.0_rt);
    d_havg_temperature.resize(size_z, 0.0_rt);
    d_havg_pressure.resize(size_z, 0.0_rt);
 
    // copy host vectors to device vectors
    Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
    Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
    Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
 
    if (solverChoice.moisture_type != MoistureType::None)
    {
        d_havg_qv.resize(size_z, 0.0_rt);
        d_havg_qc.resize(size_z, 0.0_rt);
        Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
        Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
    }
}

Here is the call graph for this function:

MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

override

{
    AMREX_ALWAYS_ASSERT(lev > 0);
 
    if (verbose) {
        amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
    }
 
    //
    // Grow the subdomains vector and build the subdomains vector at this level
    //
    subdomains.resize(lev+1);
    //
    // Create subdomains at each level within the domain such that
    // 1) all boxes in a given subdomain are "connected"
    // 2) no boxes in a subdomain touch any boxes in any other subdomain
    //
    if ( (solverChoice.anelastic[lev] == 0) && (solverChoice.project_initial_velocity[lev] == 0) ) {
        BoxArray dom(geom[lev].Domain());
        subdomains[lev].push_back(dom);
    } else {
        make_subdomains(ba.simplified_list(), subdomains[lev]);
    }
 
    if (lev == 0) init_bcs();
 
    //********************************************************************************************
    // This allocates all kinds of things, including but not limited to: solution arrays,
    //      terrain arrays, ba2d, metric terms and base state.
    // *******************************************************************************************
    init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev], base_state[lev], z_phys_nd[lev]);
 
    t_new[lev] = time;
    t_old[lev] = time - 1.e200;
 
    // ********************************************************************************************
    // Build the data structures for metric quantities used with terrain-fitted coordinates
    // ********************************************************************************************
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing)
    {
        const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
        const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
        if (solverChoice.terrain_type == TerrainType::EB) {
            eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
        } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                   solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
            eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
        }
    }
    init_zphys(lev, time);
    update_terrain_arrays(lev);
 
    //
    // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
    //     *and* if there is two-way coupling
    //
    if ( (SolverChoice::mesh_type != MeshType::ConstantDz) && (solverChoice.coupling_type == CouplingType::TwoWay) ) {
        for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
            average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
            average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
        }
    }
 
    // ********************************************************************************************
    // Build the data structures for canopy model (depends upon z_phys)
    // ********************************************************************************************
    if (solverChoice.do_forest_drag) {
        m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
    }
 
    //********************************************************************************************
    // Radiation
    // *******************************************************************************************
    if (solverChoice.rad_type != RadiationType::None)
    {
        rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
    }
 
    // *****************************************************************************************************
    // Initialize the boundary conditions (after initializing the terrain but before calling
    //     initHSE or FillCoarsePatch)
    // *****************************************************************************************************
    make_physbcs(lev);
 
    // ********************************************************************************************
    // Update the base state at this level by interpolation from coarser level
    // ********************************************************************************************
    InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                          IntVect(0,0,0), // do not fill ghost cells outside the domain
                          base_state[lev-1], 0, 0, base_state[lev].nComp(),
                          geom[lev-1], geom[lev],
                          refRatio(lev-1), &cell_cons_interp,
                          domain_bcs_type, BCVars::cons_bc);
 
    // Impose bc's outside the domain
    (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
 
    //********************************************************************************************
    // Microphysics
    // *******************************************************************************************
    int q_size  = micro->Get_Qmoist_Size(lev);
    qmoist[lev].resize(q_size);
    micro->Define(lev, solverChoice);
    if (solverChoice.moisture_type != MoistureType::None)
    {
        micro->Init(lev, vars_new[lev][Vars::cons],
                    grids[lev], Geom(lev), 0.0,
                    z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
    }
    for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
        qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
    }
 
    // ********************************************************************************************
    // Build the data structures for calculating diffusive/turbulent terms
    // ********************************************************************************************
    update_diffusive_arrays(lev, ba, dm);
 
    // ********************************************************************************************
    // Build the data structures for holding sea surface temps and skin temps
    // ********************************************************************************************
    sst_lev[lev].resize(1);     sst_lev[lev][0] = nullptr;
    tsk_lev[lev].resize(1);     tsk_lev[lev][0] = nullptr;
 
    // ********************************************************************************************
    // Fill data at the new level by interpolation from the coarser level
    // Note that internal to FillCoarsePatch we will convert velocity to momentum,
    //      then interpolate momentum, then convert momentum back to velocity
    // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
    // ********************************************************************************************
 
#ifdef ERF_USE_NETCDF
    if ( ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) ) &&
         !nc_init_file[lev].empty() )
    {
        // Just making sure that ghost cells aren't uninitialized...
        vars_new[lev][Vars::cons].setVal(0.0); vars_old[lev][Vars::cons].setVal(0.0);
        vars_new[lev][Vars::xvel].setVal(0.0); vars_old[lev][Vars::xvel].setVal(0.0);
        vars_new[lev][Vars::yvel].setVal(0.0); vars_old[lev][Vars::yvel].setVal(0.0);
        vars_new[lev][Vars::zvel].setVal(0.0); vars_old[lev][Vars::zvel].setVal(0.0);
 
        AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
        if (solverChoice.init_type == InitType::Metgrid) {
            init_from_metgrid(lev);
        } else if (solverChoice.init_type == InitType::WRFInput) {
            init_from_wrfinput(lev, *mf_C1H, *mf_C2H, *mf_MUB, *mf_PSFC[lev]);
        }
        init_zphys(lev, time);
        update_terrain_arrays(lev);
        make_physbcs(lev);
 
        dz_min[lev] = (*detJ_cc[lev]).min(0) * geom[lev].CellSize(2);
 
    } else {
#endif
    //
    // Interpolate the solution data
    //
    FillCoarsePatch(lev, time);
 
    //
    // Interpolate the 2D arrays at the lower boundary
    // Note that ba2d is constructed already in init_stuff, but we have not yet defined dmap[lev]
    // so we must explicitly pass dm.
    Interp2DArrays(lev,ba2d[lev],dm);
#ifdef ERF_USE_NETCDF
    }
#endif
 
    // ********************************************************************************************
    // Initialize the integrator class
    // ********************************************************************************************
    dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
    initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
 
    // ********************************************************************************************
    // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
    // ********************************************************************************************
    if (lev > 0 && cf_width >= 0) {
        Construct_ERFFillPatchers(lev);
           Define_ERFFillPatchers(lev);
    }
 
    //********************************************************************************************
    // Land Surface Model
    // *******************************************************************************************
    int lsm_data_size  = lsm.Get_Data_Size();
    int lsm_flux_size  = lsm.Get_Flux_Size();
    lsm_data[lev].resize(lsm_data_size);
    lsm_data_name.resize(lsm_data_size);
    lsm_flux[lev].resize(lsm_flux_size);
    lsm_flux_name.resize(lsm_flux_size);
    lsm.Define(lev, solverChoice);
    if (solverChoice.lsm_type != LandSurfaceType::None)
    {
        lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
    }
    for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
        lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
        lsm_data_name[mvar] = lsm.Get_DataName(mvar);
    }
    for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
        lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
        lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
    }
 
    // ********************************************************************************************
    // Create the SurfaceLayer arrays at this (new) level
    // ********************************************************************************************
    if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
        Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
                                     &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
        m_SurfaceLayer->make_SurfaceLayer_at_level(lev,lev+1,
                                                   mfv_old, Theta_prim[lev], Qv_prim[lev],
                                                   Qr_prim[lev], z_phys_nd[lev],
                                                   Hwave[lev].get(), Lwave[lev].get(), eddyDiffs_lev[lev].get(),
                                                   lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
                                                   sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
    }
 
#ifdef ERF_USE_PARTICLES
    // particleData.Redistribute();
#endif
}

MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

override

{
    BoxArray ba;
    DistributionMapping dm;
    Box domain(Geom(0).Domain());
    if (lev == 0 && restart_chkfile.empty() &&
        (max_grid_size[0][0] >= domain.length(0)) &&
        (max_grid_size[0][1] >= domain.length(1)) &&
        ba_in.size() != ParallelDescriptor::NProcs())
    {
        // We only decompose in z if max_grid_size_z indicates we should
        bool decompose_in_z = (max_grid_size[0][2] < domain.length(2));
 
        ba = ERFPostProcessBaseGrids(Geom(0).Domain(),decompose_in_z);
        dm = DistributionMapping(ba);
    } else {
        ba = ba_in;
        dm = dm_in;
    }
 
    // ********************************************************************************************
    // Define grids[lev] to be ba
    // ********************************************************************************************
    SetBoxArray(lev, ba);
 
    // ********************************************************************************************
    // Define dmap[lev] to be dm
    // ********************************************************************************************
    SetDistributionMap(lev, dm);
 
    if (verbose) {
        amrex::Print() <<            "BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
        // amrex::Print() <<" SIMPLIFIED BA FROM SCRATCH AT LEVEL " << lev << " " << ba.simplified_list() << std::endl;
    }
 
    subdomains.resize(lev+1);
    if ( (lev == 0) || (
         (solverChoice.anelastic[lev] == 0) && (solverChoice.project_initial_velocity[lev] == 0) &&
         (solverChoice.init_type != InitType::WRFInput) && (solverChoice.init_type != InitType::Metgrid) ) ) {
        BoxArray dom(geom[lev].Domain());
        subdomains[lev].push_back(dom);
    } else {
        //
        // Create subdomains at each level within the domain such that
        // 1) all boxes in a given subdomain are "connected"
        // 2) no boxes in a subdomain touch any boxes in any other subdomain
        //
        make_subdomains(ba.simplified_list(), subdomains[lev]);
    }
 
    if (lev == 0) init_bcs();
 
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing)
    {
        const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
        const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
        if (solverChoice.terrain_type == TerrainType::EB) {
            eb[lev]->make_all_factories(lev, geom[lev], grids[lev], dmap[lev], eb_level);
        } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                   solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
            eb[lev]->make_cc_factory(lev, geom[lev], grids[lev], dmap[lev], eb_level);
        }
    }
 
    auto& lev_new = vars_new[lev];
    auto& lev_old = vars_old[lev];
 
    //********************************************************************************************
    // This allocates all kinds of things, including but not limited to: solution arrays,
    //      terrain arrays, metric terms and base state.
    // *******************************************************************************************
    init_stuff(lev, ba, dm, lev_new, lev_old, base_state[lev], z_phys_nd[lev]);
 
    //********************************************************************************************
    // Land Surface Model
    // *******************************************************************************************
    int lsm_data_size  = lsm.Get_Data_Size();
    int lsm_flux_size  = lsm.Get_Flux_Size();
    lsm_data[lev].resize(lsm_data_size);
    lsm_data_name.resize(lsm_data_size);
    lsm_flux[lev].resize(lsm_flux_size);
    lsm_flux_name.resize(lsm_flux_size);
    lsm.Define(lev, solverChoice);
    if (solverChoice.lsm_type != LandSurfaceType::None)
    {
        lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
    }
    for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
        lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
        lsm_data_name[mvar] = lsm.Get_DataName(mvar);
    }
    for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
        lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
        lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
    }
 
 
 
    // ********************************************************************************************
    // Build the data structures for calculating diffusive/turbulent terms
    // ********************************************************************************************
    update_diffusive_arrays(lev, ba, dm);
 
    // ********************************************************************************************
    // Build the data structures for holding sea surface temps and skin temps
    // ********************************************************************************************
    sst_lev[lev].resize(1);     sst_lev[lev][0] = nullptr;
    tsk_lev[lev].resize(1);     tsk_lev[lev][0] = nullptr;
 
    // ********************************************************************************************
    // Thin immersed body
    // *******************************************************************************************
    init_thin_body(lev, ba, dm);
 
    // ********************************************************************************************
    // Initialize the integrator class
    // ********************************************************************************************
    initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
 
    // ********************************************************************************************
    // Initialize the data itself
    // If (init_type == InitType::WRFInput) then we are initializing terrain and the initial data in
    //                                      the same call so we must call init_only before update_terrain_arrays
    // If (init_type != InitType::WRFInput) then we want to initialize the terrain before the initial data
    //                                      since we may need to use the grid information before constructing
    //                                      initial idealized data
    // ********************************************************************************************
    if (restart_chkfile.empty()) {
        if ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) )
        {
            AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
            init_only(lev, start_time+time);
            init_zphys(lev, start_time+time);
            update_terrain_arrays(lev);
            make_physbcs(lev);
        } else {
            init_zphys(lev, time);
            update_terrain_arrays(lev);
            // Note that for init_type != InitType::WRFInput and != InitType::Metgrid,
            // make_physbcs is called inside init_only
            init_only(lev, time);
        }
    } else {
        // if restarting and nudging from input sounding, load the input sounding files
        if (lev == 0 && solverChoice.init_type == InitType::Input_Sounding && solverChoice.nudging_from_input_sounding)
        {
            if (input_sounding_data.input_sounding_file.empty()) {
                Error("input_sounding file name must be provided via input");
            }
 
            input_sounding_data.resize_arrays();
 
            // this will interpolate the input profiles to the nominal height levels
            // (ranging from 0 to the domain top)
            for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
                input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n);
            }
 
            // this will calculate the hydrostatically balanced density and pressure
            // profiles following WRF ideal.exe
            if (solverChoice.sounding_type == SoundingType::Ideal) {
                input_sounding_data.calc_rho_p(0);
            } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
                       solverChoice.sounding_type == SoundingType::DryIsentropic) {
                input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
                input_sounding_data.calc_rho_p_isentropic(0);
            }
        }
 
        // We re-create terrain_blanking on restart rather than storing it in the checkpoint
        if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
            solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
            int ngrow = ComputeGhostCells(solverChoice) + 2;
            terrain_blanking[lev]->setVal(1.0);
            MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
            terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
        }
    }
 
     // Read in tables needed for windfarm simulations
    // fill in Nturb multifab - number of turbines in each mesh cell
    // write out the vtk files for wind turbine location and/or
    // actuator disks
    #ifdef ERF_USE_WINDFARM
        init_windfarm(lev);
    #endif
 
    // ********************************************************************************************
    // Build the data structures for canopy model (depends upon z_phys)
    // ********************************************************************************************
    if (restart_chkfile.empty()) {
        if (solverChoice.do_forest_drag) {
            m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
        }
    }
 
    //********************************************************************************************
    // Microphysics
    // *******************************************************************************************
    int q_size  = micro->Get_Qmoist_Size(lev);
    qmoist[lev].resize(q_size);
    micro->Define(lev, solverChoice);
    if (solverChoice.moisture_type != MoistureType::None)
    {
        micro->Init(lev, vars_new[lev][Vars::cons],
                    grids[lev], Geom(lev), 0.0,
                    z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
    }
    for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
        qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
    }
 
    //********************************************************************************************
    // Radiation
    // *******************************************************************************************
    if (solverChoice.rad_type != RadiationType::None)
    {
        rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
    }
 
    // ********************************************************************************************
    // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
    // ********************************************************************************************
    if (lev > 0 && cf_width >= 0) {
        Construct_ERFFillPatchers(lev);
           Define_ERFFillPatchers(lev);
    }
 
#ifdef ERF_USE_PARTICLES
    if (restart_chkfile.empty()) {
        if (lev == 0) {
            initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,time);
        } else {
            particleData.Redistribute();
        }
    }
#endif
}

Here is the call graph for this function:

MakeVTKFilename()

std::string ERF::MakeVTKFilename

(

int

nstep

)

                              {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker";
    if (!fs::exists(dir)) {
        fs::create_directory(dir);
    }
 
    std::ostringstream oss;
    oss << dir << "/storm_track_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
    return oss.str();
}

MakeVTKFilename_EyeTracker_xy()

std::string ERF::MakeVTKFilename_EyeTracker_xy

(

int

nstep

)

                                            {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/xy";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_track_xy_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
    return oss.str();
}

MakeVTKFilename_TrackerCircle()

std::string ERF::MakeVTKFilename_TrackerCircle

(

int

nstep

)

                                            {
    // Ensure output directory exists
    const std::string dir = "Output_StormTracker/tracker_circle";
    if (!fs::exists(dir)) {
        fs::create_directories(dir);
    }
 
    // Construct filename with zero-padded step
    std::ostringstream oss;
    oss << dir << "/storm_tracker_circle_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
    return oss.str();
}

nghost_eb_basic()

static int ERF::nghost_eb_basic ( )

inlinestaticprivate

    { return 5; }

nghost_eb_full()

static int ERF::nghost_eb_full ( )

inlinestaticprivate

    { return 4; }

nghost_eb_volume()

static int ERF::nghost_eb_volume ( )

inlinestaticprivate

    { return 5; }

NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )

inlineprivatenoexcept

    {
        return datalog.size();
    }

NumDerDataLogs()

AMREX_FORCE_INLINE int ERF::NumDerDataLogs ( )

inlineprivatenoexcept

    {
        return der_datalog.size();
    }

NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )

inlineprivatenoexcept

    {
        return samplelinelog.size();
    }

NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )

inlineprivatenoexcept

    {
        return sampleline.size();
    }

NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )

inlineprivatenoexcept

    {
        return sampleptlog.size();
    }

NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )

inlineprivatenoexcept

    {
        return samplepoint.size();
    }

operator=() [1/2]

ERF& ERF::operator= ( const ERF &  other )

delete

operator=() [2/2]

ERF& ERF::operator= ( ERF &&  other )

deletenoexcept

ParameterSanityChecks()

void ERF::ParameterSanityChecks ( )

private

{
    AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt[0] > 0.);
 
    // We don't allow use_real_bcs to be true if init_type is not either InitType::WRFInput or InitType::Metgrid
    AMREX_ALWAYS_ASSERT( !solverChoice.use_real_bcs ||
                        ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) );
 
    AMREX_ALWAYS_ASSERT(real_width >= 0);
    AMREX_ALWAYS_ASSERT(real_set_width >= 0);
    AMREX_ALWAYS_ASSERT(real_width >= real_set_width);
 
    if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
        Abort("You must set cf_width >= cf_set_width >= 0");
    }
    if (max_level > 0 && cf_set_width > 0) {
        for (int lev = 1; lev <= max_level; lev++) {
            if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
                cf_set_width%ref_ratio[lev-1][1] != 0 ||
                cf_set_width%ref_ratio[lev-1][2] != 0 ) {
                Abort("You must set cf_width to be a multiple of ref_ratio");
            }
        }
    }
 
    // If fixed_mri_dt_ratio is set, it must be even
    if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
    {
        Abort("If you specify fixed_mri_dt_ratio, it must be even");
    }
 
    for (int lev = 0; lev <= max_level; lev++)
    {
        // We ignore fixed_fast_dt if not substepping
        if (solverChoice.substepping_type[lev] == SubsteppingType::None) {
            fixed_fast_dt[lev] = -1.0;
        }
 
        // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
        if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio <= 0)
        {
            Real eps = 1.e-12;
            int ratio = static_cast<int>( ( (1.0+eps) * fixed_dt[lev] ) / fixed_fast_dt[lev] );
            if (fixed_dt[lev] / fixed_fast_dt[lev] != ratio)
            {
                Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
            }
        }
 
        // If all three are specified, they must be consistent
        if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. &&  fixed_mri_dt_ratio > 0)
        {
            if (fixed_dt[lev] / fixed_fast_dt[lev] != fixed_mri_dt_ratio)
            {
                Abort("Dt is over-specfied");
            }
        }
    } // lev
 
    if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
        Abort("For two-way coupling you must set cf_width = 0");
    }
}

PlotFileName()

std::string ERF::PlotFileName ( int  lev ) const

private

PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string >  plot_var_names )

staticprivate

{
    Vector<std::string> names;
 
    names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
 
    return names;
 
}

poisson_wall_dist()

void ERF::poisson_wall_dist

(

int

lev

)

Calculate wall distances using the Poisson equation

The zlo boundary is assumed to correspond to the land surface. If there are no boundary walls, then the other use case is to calculate wall distances for immersed boundaries (embedded or thin body).

See Tucker, P. G. (2003). Differential equation-based wall distance computation for DES and RANS. Journal of Computational Physics, 190(1), 229–248. https://doi.org/10.1016/S0021-9991(03)00272-9

{
    BL_PROFILE("ERF::poisson_wall_dist()");
 
    bool havewall{false};
    Orientation zlo(Direction::z, Orientation::low);
    if ( ( phys_bc_type[zlo] == ERF_BC::surface_layer                      ) ||
         ( phys_bc_type[zlo] == ERF_BC::no_slip_wall                       ) )/*||
         ((phys_bc_type[zlo] == ERF_BC::slip_wall) && (dom_hi.z > dom_lo.z)) )*/
    {
        havewall = true;
    }
 
    auto const& geomdata = geom[lev];
    auto const& dxinv    = geomdata.InvCellSizeArray();
 
    auto const& zphys_arr = z_phys_nd[lev]->const_arrays();
 
    if (havewall) {
#if 1
        // Bypass wall dist calc in the trivial cases
 
        if (solverChoice.mesh_type == MeshType::ConstantDz) {
            Print() << "Directly calculating direct wall distance for constant dz" << std::endl;
            const Real* prob_lo = geomdata.ProbLo();
            const Real* dx = geomdata.CellSize();
            for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.validbox();
                auto dist_arr = walldist[lev]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    dist_arr(i, j, k) = prob_lo[2] + (k + 0.5) * dx[2];
                });
            }
            return;
        }
 
        if (solverChoice.mesh_type == MeshType::StretchedDz) {
            Print() << "Directly calculating direct wall distance for stretched dz" << std::endl;
            for (MFIter mfi(*walldist[lev],TileNoZ()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.validbox();
                auto dist_arr = walldist[lev]->array(mfi);
                const auto zcc_arr = z_phys_cc[lev]->const_array(mfi);
                const auto znd_arr = z_phys_nd[lev]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    dist_arr(i, j, k) = zcc_arr(i, j, k) - znd_arr(i, j, 0);
                });
            }
            return;
        }
#endif
    }
    else
    {
        Error("No solid boundaries in the computational domain");
    }
 
    Print() << "Calculating Poisson wall distance for general terrain" << std::endl;
 
    // Make sure the solver only sees the levels over which we are solving
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(walldist[lev]->boxArray());
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(walldist[lev]->DistributionMap());
 
    Vector<MultiFab> rhs;
    Vector<MultiFab> phi;
 
    if (solverChoice.terrain_type == TerrainType::EB) {
        amrex::Error("Wall dist calc not implemented for EB");
    } else {
        rhs.resize(1);   rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
        phi.resize(1);   phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
    }
 
    rhs[0].setVal(-1.0);
 
    auto const dom_lo = lbound(geom[lev].Domain());
    auto const dom_hi = ubound(geom[lev].Domain());
 
    // ****************************************************************************
    // Initialize phi
    // (It is essential that we do this in order to fill the corners; this is
    // used if we include blanking.)
    // ****************************************************************************
    phi[0].setVal(0.0);
 
    // ****************************************************************************
    // Interior boundaries are marked with phi=0
    // ****************************************************************************
#if 0
    // Define an overset mask (0 or 1) to set dirichlet nodes on walls
    // 1 means the node is an unknown. 0 means it's known.
    iMultiFab mask(ba_tmp[0], dm_tmp[0], 1, 0);
    Vector<const iMultiFab*> overset_mask = {&mask};
 
    mask.setVal(1);
    if (solverChoice.advChoice.have_zero_flux_faces) {
        Warning("Poisson distance is inaccurate for bodies in open domains that are small compared to the domain size, skipping");
        return;
 
        Gpu::DeviceVector<IntVect> xfacelist, yfacelist, zfacelist;
 
        xfacelist.resize(solverChoice.advChoice.zero_xflux.size());
        yfacelist.resize(solverChoice.advChoice.zero_yflux.size());
        zfacelist.resize(solverChoice.advChoice.zero_zflux.size());
 
        if (xfacelist.size() > 0) {
            Gpu::copy(amrex::Gpu::hostToDevice,
                      solverChoice.advChoice.zero_xflux.begin(),
                      solverChoice.advChoice.zero_xflux.end(),
                      xfacelist.begin());
            Print() << "  masking interior xfaces" << std::endl;
        }
        if (yfacelist.size() > 0) {
            Gpu::copy(amrex::Gpu::hostToDevice,
                      solverChoice.advChoice.zero_yflux.begin(),
                      solverChoice.advChoice.zero_yflux.end(),
                      yfacelist.begin());
            Print() << "  masking interior yfaces" << std::endl;
        }
        if (zfacelist.size() > 0) {
            Gpu::copy(amrex::Gpu::hostToDevice,
                      solverChoice.advChoice.zero_zflux.begin(),
                      solverChoice.advChoice.zero_zflux.end(),
                      zfacelist.begin());
            Print() << "  masking interior zfaces" << std::endl;
        }
 
        for (MFIter mfi(phi[0]); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.validbox();
 
            auto phi_arr  = phi[0].array(mfi);
            auto mask_arr = mask.array(mfi);
 
            if (xfacelist.size() > 0) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    for (int iface=0; iface < xfacelist.size(); ++iface) {
                        if ((i == xfacelist[iface][0]) &&
                            (j == xfacelist[iface][1]) &&
                            (k == xfacelist[iface][2]))
                        {
                            mask_arr(i, j  , k  ) = 0;
                            mask_arr(i, j  , k+1) = 0;
                            mask_arr(i, j+1, k  ) = 0;
                            mask_arr(i, j+1, k+1) = 0;
                        }
                    }
                });
            }
 
            if (yfacelist.size() > 0) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    for (int iface=0; iface < yfacelist.size(); ++iface) {
                        if ((i == yfacelist[iface][0]) &&
                            (j == yfacelist[iface][1]) &&
                            (k == yfacelist[iface][2]))
                        {
                            mask_arr(i  , j, k  ) = 0;
                            mask_arr(i  , j, k+1) = 0;
                            mask_arr(i+1, j, k  ) = 0;
                            mask_arr(i+1, j, k+1) = 0;
                        }
                    }
                });
            }
 
            if (zfacelist.size() > 0) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                    for (int iface=0; iface < zfacelist.size(); ++iface) {
                        if ((i == xfacelist[iface][0]) &&
                            (j == xfacelist[iface][1]) &&
                            (k == xfacelist[iface][2]))
                        {
                            mask_arr(i  , j  , k) = 0;
                            mask_arr(i  , j+1, k) = 0;
                            mask_arr(i+1, j  , k) = 0;
                            mask_arr(i+1, j+1, k) = 0;
                        }
                    }
                });
            }
        }
    }
#endif
 
    // ****************************************************************************
    // Setup BCs, with solid domain boundaries being dirichlet
    // ****************************************************************************
    amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bc3d_lo, bc3d_hi;
    for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
        if (geom[0].isPeriodic(dir)) {
            bc3d_lo[dir] = LinOpBCType::Periodic;
            bc3d_hi[dir] = LinOpBCType::Periodic;
        } else {
            bc3d_lo[dir] = LinOpBCType::Neumann;
            bc3d_hi[dir] = LinOpBCType::Neumann;
        }
    }
    if (havewall) {
        Print() << "  Poisson zlo BC is dirichlet" << std::endl;
        bc3d_lo[2] = LinOpBCType::Dirichlet;
    }
    Print() << "  bc lo : " << bc3d_lo << std::endl;
    Print() << "  bc hi : " << bc3d_hi << std::endl;
 
    if (!solverChoice.advChoice.have_zero_flux_faces && !havewall) {
        Error("No solid boundaries in the computational domain");
    }
 
    LPInfo info; // defaults
 
/* Nodal solver cannot have hidden dimensions */
#if 0
    // Allow a hidden direction if the domain is one cell wide
    if (dom_lo.x == dom_hi.x) {
        info.setHiddenDirection(0);
        Print() << "  domain is 2D in yz" << std::endl;
    } else if (dom_lo.y == dom_hi.y) {
        info.setHiddenDirection(1);
        Print() << "  domain is 2D in xz" << std::endl;
    } else if (dom_lo.z == dom_hi.z) {
        info.setHiddenDirection(2);
        Print() << "  domain is 2D in xy" << std::endl;
    }
#endif
 
#if 0
    Vector<EBFArrayBoxFactory const*> factory_vec;
    factory_vec.push_back(static_cast<FabFactory<FArrayBox> const*>(&EBFactory(lev));
#endif
 
    // ****************************************************************************
    // Setup Poisson problem
    // (A \alpha - B \nabla \cdot \beta \nabla ) \phi = f
    //
    // In physical space:
    //   \nabla \cdot \nabla \phi = -1
    //
    // In computational space:
    //   grad(phi) = T^T \nabla \phi
    // and
    //   \nabla \cdot (h_zeta T (T^T \nabla \phi)) = -h_zeta
    // where T = inv(J), T^T is the transpose of inv(J)
    // ****************************************************************************
    constexpr Real constA = 0.0;
    constexpr Real constB = -1.0;
 
    MLABecLaplacian mlabec(geom_tmp, ba_tmp, dm_tmp, info);
 
    mlabec.setScalars(constA, constB);
    mlabec.setACoeffs(lev, 0.0);
#if 1
    // Set beta coefficients at faces
    Array<MultiFab, AMREX_SPACEDIM> beta;
 
    for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
        BoxArray ba_face = ba_tmp[0];
        ba_face.surroundingNodes(idim);  // Convert to face-centered in direction idim
        beta[idim].define(ba_face, dm_tmp[0], 1, 0);
    }
 
    auto beta0_arr = beta[0].arrays();
    auto beta1_arr = beta[1].arrays();
    auto beta2_arr = beta[2].arrays();
 
    // Note: This ignores the off-diagonal components of (h_zeta T T^T), which
    //       is equivalent to assuming that h_xi and h_eta are small.
 
    ParallelFor(beta[0], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
        beta0_arr[b](i, j, k) = Compute_h_zeta_AtIface(i, j, k, dxinv, zphys_arr[b]);;
    });
    ParallelFor(beta[1], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
        beta1_arr[b](i, j, k) = Compute_h_zeta_AtJface(i, j, k, dxinv, zphys_arr[b]);;
    });
    ParallelFor(beta[2], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
        Real inv_h_zeta = 1.0 / Compute_h_zeta_AtKface(i, j, k, dxinv, zphys_arr[b]);
        Real h_xi = Compute_h_xi_AtKface(i, j, k, dxinv, zphys_arr[b]);
        Real h_eta = Compute_h_eta_AtKface(i, j, k, dxinv, zphys_arr[b]);
        beta2_arr[b](i, j, k) = inv_h_zeta * (1 + h_xi*h_xi + h_eta*h_eta);
    });
 
    mlabec.setBCoeffs(lev, GetArrOfConstPtrs(beta));
 
    // Set RHS := -h_zeta
    auto rhs_arr = rhs[0].arrays();
    ParallelFor(rhs[0], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
        rhs_arr[b](i, j, k) = -Compute_h_zeta_AtCellCenter(i, j, k, dxinv, zphys_arr[b]);
    });
#else
    mlabec.setBCoeffs(lev, 1.0);
#endif
 
    mlabec.setDomainBC(bc3d_lo, bc3d_hi);
 
    if (lev > 0) {
        mlabec.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
    }
 
    // If we have inhomogeneous BCs -- do this after setCoarseFineBC
    mlabec.setLevelBC(0, nullptr);
 
    // ****************************************************************************
    // Solve Poisson problem with MLMG
    // ****************************************************************************
    const Real reltol = solverChoice.poisson_reltol;
    const Real abstol = solverChoice.poisson_abstol;
    const int n_corr = solverChoice.ncorr;
    constexpr int max_iter = 100;
 
    MLMG mlmg(mlabec);
    mlmg.setMaxIter(max_iter);
    mlmg.setVerbose(mg_verbose);
    mlmg.setBottomVerbose(0);
 
    for (int icorr=0; icorr <= n_corr; ++icorr) {
        Print()<< "Solving wall distance poisson, icorr=" << icorr << std::endl;
 
        mlmg.solve(GetVecOfPtrs(phi),
                   GetVecOfConstPtrs(rhs),
                   reltol, abstol);
 
        // ****************************************************************************
        // Apply BCs: dirichlet (odd) on zlo, neumann (even) / periodic elsewhere
        // ****************************************************************************
 
        // Overwrite with periodic fill outside domain and fine-fine fill inside
        phi[0].FillBoundary(geom[lev].periodicity());
 
        if (!geom[lev].isPeriodic(0)) {
            for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
            {
                Box bx = mfi.tilebox();
                const Array4<Real>& phi_arr = phi[0].array(mfi);
                if (bx.smallEnd(0) <= dom_lo.x) {
                    ParallelFor(makeSlab(bx,0,dom_lo.x),
                    [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        phi_arr(i-1,j,k) =  phi_arr(i,j,k); // even BC
                    });
                } // lo x
                if (bx.bigEnd(0) >= dom_hi.x) {
                    ParallelFor(makeSlab(bx,0,dom_hi.x),
                    [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        phi_arr(i+1,j,k) =  phi_arr(i,j,k); // even BC
                    });
                } // hi x
            } // mfi
        } // not periodic in x
 
        if (!geom[lev].isPeriodic(1)) {
            for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
            {
                Box bx = mfi.tilebox();
                Box bx2(bx); bx2.grow(0,1);
                const Array4<Real>& phi_arr = phi[0].array(mfi);
                if (bx.smallEnd(1) <= dom_lo.y) {
                    ParallelFor(makeSlab(bx2,1,dom_lo.y),
                    [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        phi_arr(i,j-1,k) =  phi_arr(i,j,k); // even BC
                    });
                } // lo y
                if (bx.bigEnd(1) >= dom_hi.y) {
                    ParallelFor(makeSlab(bx2,1,dom_hi.y),
                    [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        phi_arr(i,j+1,k) =  phi_arr(i,j,k); // even BC
                    });
                } // hi y
 
            } // mfi
        } // not periodic in y
 
        for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
        {
            Box bx = mfi.tilebox();
            Box bx3(bx); bx3.grow(0,1); bx3.grow(1,1);
            const Array4<Real>& phi_arr = phi[0].array(mfi);
            if (bx.smallEnd(2) <= dom_lo.z) {
                ParallelFor(makeSlab(bx3,2,dom_lo.z),
                [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    phi_arr(i,j,k-1) = -phi_arr(i,j,k); // ODD BC
                });
            } // lo z
            if (bx.bigEnd(2) >= dom_hi.z) {
                ParallelFor(makeSlab(bx3,2,dom_hi.z),
                [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    phi_arr(i,j,k+1) =  phi_arr(i,j,k); // even BC
                });
            } // hi z
        } // mfi
 
        // ****************************************************************************
        // Compute grad(phi) to get distances
        // ****************************************************************************
        auto const& phi_arr = phi[0].const_arrays();
        //auto rhs_arr = rhs[0].arrays();
        auto dist_arr = walldist[lev]->arrays();
 
        ParallelFor(*walldist[lev], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
            Real dpdx{0}, dpdy{0}, dpdz{0};
 
            dpdx = terrpoisson_flux_x(i, j, k, phi_arr[b], zphys_arr[b], dxinv[0]);
            dpdy = terrpoisson_flux_y(i, j, k, phi_arr[b], zphys_arr[b], dxinv[1]);
            if (k == dom_lo.z) {
                dpdz = terrpoisson_flux_zlo_dir(i, j, k, phi_arr[b], zphys_arr[b], dxinv[0], dxinv[1]);
            } else {
                // This returns 0 at the wall, hence the need for the separate calc above
                dpdz = terrpoisson_flux_z(i, j, k, phi_arr[b], zphys_arr[b], dxinv[0], dxinv[1]);
            }
 
            Real magsqr_dphi = dpdx*dpdx + dpdy*dpdy + dpdz*dpdz;
            Real mag_dphi = std::sqrt(magsqr_dphi);
#if 1
            // Tucker 2003 Eqn 2
            dist_arr[b](i, j, k) = -mag_dphi + std::sqrt(magsqr_dphi + 2*phi_arr[b](i, j, k));
#else
            // DEBUG: output phi instead
            if (i==0 && j==0) AllPrint() << "walldist"<<IntVect(i,j,k) << " = " << dist_arr[b](i,j,k) << std::endl;
            dist_arr[b](i, j, k) = phi_arr[b](i, j, k);
#endif
            // Update RHS source term to explicitly include cross-terms
            if (n_corr > 0) {
                // d/dxi ( h_xi * dphi/dzeta )
                Real phi_zeta_xlo = 0.25 * dxinv[2] * ( phi_arr[b](i  , j, k+1) - phi_arr[b](i  , j, k-1)
                                                      + phi_arr[b](i-1, j, k+1) - phi_arr[b](i-1, j, k-1) );
                Real phi_zeta_xhi = 0.25 * dxinv[2] * ( phi_arr[b](i  , j, k+1) - phi_arr[b](i  , j, k-1)
                                                      + phi_arr[b](i+1, j, k+1) - phi_arr[b](i+1, j, k-1) );
                Real h_xi_xlo = Compute_h_xi_AtIface(i  , j, k, dxinv, zphys_arr[b]);
                Real h_xi_xhi = Compute_h_xi_AtIface(i+1, j, k, dxinv, zphys_arr[b]);
 
                // d/deta ( h_eta * dphi/dzeta )
                Real phi_zeta_ylo = 0.25 * dxinv[2] * ( phi_arr[b](i, j  , k+1) - phi_arr[b](i, j  , k-1)
                                                      + phi_arr[b](i, j-1, k+1) - phi_arr[b](i, j-1, k-1) );
                Real phi_zeta_yhi = 0.25 * dxinv[2] * ( phi_arr[b](i, j  , k+1) - phi_arr[b](i, j  , k-1)
                                                      + phi_arr[b](i, j+1, k+1) - phi_arr[b](i, j+1, k-1) );
                Real h_eta_ylo = Compute_h_eta_AtJface(i, j  , k, dxinv, zphys_arr[b]);
                Real h_eta_yhi = Compute_h_eta_AtJface(i, j+1, k, dxinv, zphys_arr[b]);
 
                // d/dzeta ( h_xi * dphi/dxi )
                Real phi_xi_zlo = 0.25 * dxinv[0] * ( phi_arr[b](i+1, j, k  ) - phi_arr[b](i-1, j, k  )
                                                    + phi_arr[b](i+1, j, k-1) - phi_arr[b](i-1, j, k-1) );
                Real phi_xi_zhi = 0.25 * dxinv[0] * ( phi_arr[b](i+1, j, k  ) - phi_arr[b](i-1, j, k  )
                                                    + phi_arr[b](i+1, j, k+1) - phi_arr[b](i-1, j, k+1) );
                Real h_xi_zlo = Compute_h_xi_AtKface(i, j, k  , dxinv, zphys_arr[b]);
                Real h_xi_zhi = Compute_h_xi_AtKface(i, j, k+1, dxinv, zphys_arr[b]);
 
                // d/dzeta ( h_eta * dphi/deta )
                Real phi_eta_zlo = 0.25 * dxinv[1] * ( phi_arr[b](i, j+1, k  ) - phi_arr[b](i, j-1, k  )
                                                     + phi_arr[b](i, j+1, k-1) - phi_arr[b](i, j-1, k-1) );
                Real phi_eta_zhi = 0.25 * dxinv[1] * ( phi_arr[b](i, j+1, k  ) - phi_arr[b](i, j-1, k  )
                                                     + phi_arr[b](i, j+1, k+1) - phi_arr[b](i, j-1, k+1) );
                Real h_eta_zlo = Compute_h_eta_AtKface(i, j, k  , dxinv, zphys_arr[b]);
                Real h_eta_zhi = Compute_h_eta_AtKface(i, j, k+1, dxinv, zphys_arr[b]);
 
                Real detJ = Compute_h_zeta_AtCellCenter(i, j, k, dxinv, zphys_arr[b]);
 
                rhs_arr[b](i, j, k) = -detJ
                                    + dxinv[0] * ( h_xi_xhi * phi_zeta_xhi - h_xi_xlo * phi_zeta_xlo)
                                    + dxinv[1] * ( h_eta_yhi * phi_zeta_yhi - h_eta_ylo * phi_zeta_ylo)
                                    + dxinv[2] * ( h_xi_zhi * phi_xi_zhi - h_xi_zlo * phi_xi_zlo
                                                 + h_eta_zhi * phi_eta_zhi - h_eta_zlo * phi_eta_zlo);
            }
        });
    } // corrector loop
}

Here is the call graph for this function:

post_timestep()

void ERF::post_timestep	(	int	nstep,
		amrex::Real	time,
		amrex::Real	dt_lev
	)

{
    BL_PROFILE("ERF::post_timestep()");
 
#ifdef ERF_USE_PARTICLES
    particleData.Redistribute();
#endif
 
    if (solverChoice.coupling_type == CouplingType::TwoWay)
    {
        int ncomp = vars_new[0][Vars::cons].nComp();
        for (int lev = finest_level-1; lev >= 0; lev--)
        {
            // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
            // m is the map scale factor at cell centers
            // Here we pre-divide (rho S) by m^2 before refluxing
            for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<      Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
                const Array4<const Real>  mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<const Real>  mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                if (SolverChoice::mesh_type == MeshType::ConstantDz) {
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                    });
                } else {
                    const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                    });
                }
            } // mfi
 
            // This call refluxes all "slow" cell-centered variables
            // (i.e. not density or (rho theta) or velocities) from the lev/lev+1 interface onto lev
            getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 2, 2, ncomp-2);
 
            // Here we multiply (rho S) by m^2 after refluxing
            for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
                const Array4<const Real>  mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<const Real>  mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                if (SolverChoice::mesh_type == MeshType::ConstantDz) {
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                    });
                } else {
                    const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                    {
                        cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
                    });
                }
            } // mfi
 
            // We need to do this before anything else because refluxing changes the
            // values of coarse cells underneath fine grids with the assumption they'll
            // be over-written by averaging down
            int src_comp;
            if (solverChoice.anelastic[lev]) {
                src_comp = 1;
            } else {
                src_comp = 0;
            }
            int num_comp = ncomp - src_comp;
            AverageDownTo(lev,src_comp,num_comp);
        }
    }
 
    if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
        sum_integrated_quantities(time);
        sum_derived_quantities(time);
        sum_energy_quantities(time);
    }
 
    if (solverChoice.pert_type == PerturbationType::Source ||
        solverChoice.pert_type == PerturbationType::Direct ||
        solverChoice.pert_type == PerturbationType::CPM) {
        if (is_it_time_for_action(nstep, time, dt_lev0, pert_interval, -1.)) {
            turbPert.debug(time);
        }
    }
 
    if (profile_int > 0 && (nstep+1) % profile_int == 0) {
        if (destag_profiles) {
            // all variables cell-centered
            write_1D_profiles(time);
        } else {
            // some variables staggered
            write_1D_profiles_stag(time);
        }
    }
 
    if (solverChoice.rad_type != RadiationType::None)
    {
        if ( rad_datalog_int > 0 &&
             (((nstep+1) % rad_datalog_int == 0) || (nstep==0)) ) {
            if (rad[0]->hasDatalog()) {
                rad[0]->WriteDataLog(time+start_time);
            }
        }
    }
 
    if (output_1d_column) {
#ifdef ERF_USE_NETCDF
      if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
      {
         int lev_column = 0;
         for (int lev = finest_level; lev >= 0; lev--)
         {
            Real dx_lev = geom[lev].CellSize(0);
            Real dy_lev = geom[lev].CellSize(1);
            int i_lev = static_cast<int>(std::floor(column_loc_x / dx_lev));
            int j_lev = static_cast<int>(std::floor(column_loc_y / dy_lev));
            if (grids[lev].contains(IntVect(i_lev,j_lev,0))) lev_column = lev;
         }
         writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
      }
#else
      Abort("To output 1D column files ERF must be compiled with NetCDF");
#endif
    }
 
    if (output_bndry_planes)
    {
      if (is_it_time_for_action(istep[0], time, dt_lev0, bndry_output_planes_interval, bndry_output_planes_per) &&
          time >= bndry_output_planes_start_time)
      {
         bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
         m_w2d->write_planes(istep[0], time, vars_new, is_moist);
      }
    }
 
    // Write plane/line sampler data
    if (line_sampler && is_it_time_for_action(nstep+1, time, dt_lev0, line_sampling_interval, line_sampling_per)) {
        line_sampler->get_sample_data(geom, vars_new);
        line_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
    }
    if (plane_sampler && is_it_time_for_action(nstep+1, time, dt_lev0, plane_sampling_interval, plane_sampling_per)) {
        plane_sampler->get_sample_data(geom, vars_new);
        plane_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
    }
 
    // Moving terrain
    if ( solverChoice.terrain_type == TerrainType::MovingFittedMesh )
    {
      for (int lev = finest_level; lev >= 0; lev--)
      {
        // Copy z_phs_nd and detJ_cc at end of timestep
        MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
        MultiFab::Copy(  *detJ_cc[lev],   *detJ_cc_new[lev], 0, 0, 1,   detJ_cc[lev]->nGrowVect());
        MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
 
        make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
      }
    }
 
    if ( solverChoice.io_hurricane_eye_tracker and (nstep == 0 or (nstep+1)%m_plot3d_int_1 == 0) )
    {
        int levc=finest_level;
 
        HurricaneEyeTracker(geom[levc],
                            vars_new[levc],
                            solverChoice.moisture_type,
                            solverChoice.hindcast_lateral_forcing? &forecast_state_interp[levc] : nullptr,
                            solverChoice.hurricane_eye_latitude,
                            solverChoice.hurricane_eye_longitude,
                            hurricane_eye_track_xy,
                            hurricane_eye_track_latlon,
                            hurricane_tracker_circle);
 
        MultiFab& U_new = vars_new[levc][Vars::xvel];
        MultiFab& V_new = vars_new[levc][Vars::yvel];
        MultiFab& W_new = vars_new[levc][Vars::zvel];
 
        MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
        average_face_to_cellcenter(mf_cc_vel,0,{AMREX_D_DECL(&U_new,&V_new,&W_new)},0);
 
        HurricaneMaxVelTracker(geom[levc],
                               mf_cc_vel,
                               t_new[0],
                               hurricane_eye_track_xy,
                               hurricane_maxvel_vs_time);
 
        HurricaneMinPressureTracker(solverChoice.moisture_type,
                                    geom[levc],
                                    vars_new[levc][Vars::cons],
                                    t_new[0],
                                    hurricane_eye_track_xy,
                                    hurricane_minpressure_vs_time);
 
        std::string filename_tracker = MakeVTKFilename_TrackerCircle(nstep);
        std::string filename_xy      = MakeVTKFilename_EyeTracker_xy(nstep);
        std::string filename_latlon  = MakeFilename_EyeTracker_latlon(nstep);
        std::string filename_maxvel  = MakeFilename_EyeTracker_maxvel(nstep);
        std::string filename_minpressure  = MakeFilename_EyeTracker_minpressure(nstep);
 
        if (ParallelDescriptor::IOProcessor()) {
            WriteVTKPolyline(filename_tracker, hurricane_tracker_circle);
            WriteVTKPolyline(filename_xy, hurricane_eye_track_xy);
            WriteLinePlot(filename_latlon, hurricane_eye_track_latlon);
            WriteLinePlot(filename_maxvel, hurricane_maxvel_vs_time);
            WriteLinePlot(filename_minpressure, hurricane_minpressure_vs_time);
        }
    }
} // post_timestep

Here is the call graph for this function:

post_update()

void ERF::post_update ( amrex::MultiFab &  state_mf, amrex::Real  time, const amrex::Geometry &  geom  )

private

print_banner()

void ERF::print_banner ( MPI_Comm  comm, std::ostream &  out  )

static

                     : " << msg << std::endl;
}
 
void ERF::print_banner (MPI_Comm comm, std::ostream& out)
{
#ifdef AMREX_USE_MPI
    int irank = 0;
    int num_ranks = 1;
    MPI_Comm_size(comm, &num_ranks);
    MPI_Comm_rank(comm, &irank);
 
    // Only root process does the printing
    if (irank != 0) return;
#else
    amrex::ignore_unused(comm);
#endif
 
    auto etime = std::chrono::system_clock::now();
    auto etimet = std::chrono::system_clock::to_time_t(etime);
#ifndef _WIN32
    char time_buf[64];
    ctime_r(&etimet, time_buf);
    const std::string tstamp(time_buf);
#else
    char* time_buf = new char[64];
    ctime_s(time_buf, 64, &etimet);
    const std::string tstamp(time_buf);
#endif
 
    const char* githash1 = amrex::buildInfoGetGitHash(1);
    const char* githash2 = amrex::buildInfoGetGitHash(2);
 
    // clang-format off
    out << dbl_line
        << "                ERF (https://github.com/erf-model/ERF)"
        << std::endl << std::endl
        << "  ERF Git SHA      :: " << githash1 << std::endl
        << "  AMReX Git SHA    :: " << githash2 << std::endl
        << "  AMReX version    :: " << amrex::Version() << std::endl << std::endl
        << "  Exec. time       :: " << tstamp
        << "  Build time       :: " << amrex::buildInfoGetBuildDate() << std::endl
        << "  C++ compiler     :: " << amrex::buildInfoGetComp()
        << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
        << "  MPI              :: "
#ifdef AMREX_USE_MPI
        << "ON    (Num. ranks = " << num_ranks << ")" << std::endl
#else
        << "OFF " << std::endl
#endif
        << "  GPU              :: "
#ifdef AMREX_USE_GPU
        << "ON    "
#if defined(AMREX_USE_CUDA)
        << "(Backend: CUDA)"
#elif defined(AMREX_USE_HIP)
        << "(Backend: HIP)"
#elif defined(AMREX_USE_SYCL)
        << "(Backend: SYCL)"
#endif
        << std::endl
#else
        << "OFF" << std::endl
#endif
        << "  OpenMP           :: "
#ifdef AMREX_USE_OMP
        << "ON    (Num. threads = " << omp_get_max_threads() << ")" << std::endl
#else
        << "OFF" << std::endl
#endif
        << std::endl;
 

Referenced by main().

Here is the caller graph for this function:

print_error()

void ERF::print_error ( MPI_Comm  comm, const std::string &  msg  )

static

        :
    input_file   : Input file with simulation settings
 
Optional:
    param=value  : Overrides for parameters during runtime
)doc" << std::endl;
}
 
void ERF::print_error (MPI_Comm comm, const std::string& msg)
{
#ifdef AMREX_USE_MPI
    int irank = 0;
    int num_ranks = 1;
    MPI_Comm_size(comm, &num_ranks);
    MPI_Comm_rank(comm, &irank);
 

Referenced by main().

Here is the caller graph for this function:

print_summary()

static void ERF::print_summary ( std::ostream &  )

static

print_tpls()

void ERF::print_tpls ( std::ostream &  out )

static

                      ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
        << dash_line << std::endl;
    // clang-format on
}
 
void ERF::print_tpls (std::ostream& out)
{
    amrex::Vector<std::string> tpls;
 
#ifdef ERF_USE_NETCDF
    tpls.push_back(std::string("NetCDF    ") + NC_VERSION);
#endif
#ifdef AMREX_USE_SUNDIALS
    tpls.push_back(std::string("SUNDIALS     ") + SUNDIALS_VERSION);
#endif
 
    if (!tpls.empty()) {
        out << "  Enabled third-party libraries: ";
        for (const auto& val : tpls) {

print_usage()

void ERF::print_usage ( MPI_Comm  comm, std::ostream &  out  )

static

{
#ifdef AMREX_USE_MPI
    int irank = 0;
    int num_ranks = 1;
    MPI_Comm_size(comm, &num_ranks);
    MPI_Comm_rank(comm, &irank);
 
    // Only root process does the printing
    if (irank != 0) return;
#else
    amrex::ignore_unused(comm);
#endif
 
    out << R"doc(Usage:
    ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

project_initial_velocity()

void ERF::project_initial_velocity	(	int	lev,
		amrex::Real	time,
		amrex::Real	dt
	)

Project the single-level velocity field to enforce the anelastic constraint Note that the level may or may not be level 0.

{
    BL_PROFILE("ERF::project_initial_velocity()");
    // Impose FillBoundary on density since we use it in the conversion of velocity to momentum
    vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
 
    const MultiFab* c_vfrac = nullptr;
    if (solverChoice.terrain_type == TerrainType::EB) {
        c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
    }
 
    VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
                       vars_new[lev][Vars::yvel], IntVect{0},
                       vars_new[lev][Vars::zvel], IntVect{0},
                       vars_new[lev][Vars::cons],
                       rU_new[lev], rV_new[lev], rW_new[lev],
                       Geom(lev).Domain(), domain_bcs_type, c_vfrac);
 
    Vector<MultiFab> tmp_mom;
 
    tmp_mom.push_back(MultiFab(vars_new[lev][Vars::cons],make_alias,0,1));
    tmp_mom.push_back(MultiFab(rU_new[lev],make_alias,0,1));
    tmp_mom.push_back(MultiFab(rV_new[lev],make_alias,0,1));
    tmp_mom.push_back(MultiFab(rW_new[lev],make_alias,0,1));
 
    // If at lev > 0 we must first fill the velocities at the c/f interface -- this must
    //    be done *after* the projection at lev-1
    if (lev > 0) {
        int levc = lev-1;
 
        const MultiFab* c_vfrac_crse = nullptr;
        if (solverChoice.terrain_type == TerrainType::EB) {
            c_vfrac_crse = &((get_eb(levc).get_const_factory())->getVolFrac());
        }
 
        MultiFab& S_new_crse = vars_new[levc][Vars::cons];
        MultiFab& U_new_crse = vars_new[levc][Vars::xvel];
        MultiFab& V_new_crse = vars_new[levc][Vars::yvel];
        MultiFab& W_new_crse = vars_new[levc][Vars::zvel];
 
        VelocityToMomentum(U_new_crse, IntVect{0}, V_new_crse, IntVect{0}, W_new_crse, IntVect{0}, S_new_crse,
                           rU_new[levc], rV_new[levc], rW_new[levc],
                           Geom(levc).Domain(), domain_bcs_type, c_vfrac_crse);
 
        rU_new[levc].FillBoundary(geom[levc].periodicity());
        FPr_u[levc].RegisterCoarseData({&rU_new[levc], &rU_new[levc]}, {time, time+l_dt});
 
        rV_new[levc].FillBoundary(geom[levc].periodicity());
        FPr_v[levc].RegisterCoarseData({&rV_new[levc], &rV_new[levc]}, {time, time+l_dt});
 
        rW_new[levc].FillBoundary(geom[levc].periodicity());
        FPr_w[levc].RegisterCoarseData({&rW_new[levc], &rW_new[levc]}, {time, time+l_dt});
    }
 
    Real l_time = 0.0;
    project_momenta(lev, l_time, l_dt, tmp_mom);
 
    MomentumToVelocity(vars_new[lev][Vars::xvel],
                       vars_new[lev][Vars::yvel],
                       vars_new[lev][Vars::zvel],
                       vars_new[lev][Vars::cons],
                       rU_new[lev], rV_new[lev], rW_new[lev],
                       Geom(lev).Domain(), domain_bcs_type, c_vfrac);
 }

Here is the call graph for this function:

project_momenta()

void ERF::project_momenta	(	int	lev,
		amrex::Real	l_time,
		amrex::Real	l_dt,
		amrex::Vector< amrex::MultiFab > &	vars
	)

Project the single-level momenta to enforce the anelastic constraint Note that the level may or may not be level 0.

{
    BL_PROFILE("ERF::project_momenta()");
    //
    // If at lev > 0 we must first fill the momenta at the c/f interface with interpolated coarse values
    //
    if (lev > 0) {
        PhysBCFunctNoOp null_bc;
        FPr_u[lev-1].FillSet(mom_mf[IntVars::xmom], l_time, null_bc, domain_bcs_type);
        FPr_v[lev-1].FillSet(mom_mf[IntVars::ymom], l_time, null_bc, domain_bcs_type);
        FPr_w[lev-1].FillSet(mom_mf[IntVars::zmom], l_time, null_bc, domain_bcs_type);
    }
 
    // Make sure the solver only sees the levels over which we are solving
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(mom_mf[Vars::cons].boxArray());
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap());
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
 
    Box domain = geom[lev].Domain();
 
    MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
 
    Vector<MultiFab> rhs;
    Vector<MultiFab> phi;
 
    if (solverChoice.terrain_type == TerrainType::EB)
    {
        rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
        phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1, MFInfo(), EBFactory(lev));
    } else {
        rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
        phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
    }
 
    MultiFab rhs_lev(rhs[0], make_alias, 0, 1);
    MultiFab phi_lev(phi[0], make_alias, 0, 1);
 
    auto dx    = geom[lev].CellSizeArray();
    auto dxInv = geom[lev].InvCellSizeArray();
 
    // Inflow on an x-face -- note only the normal velocity is used in the projection
    if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow") {
        (*physbcs_u[lev])(vars_new[lev][Vars::xvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                        IntVect{1,0,0},t_new[lev],BCVars::xvel_bc,false);
    }
 
    // Inflow on a  y-face -- note only the normal velocity is used in the projection
    if (domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
        (*physbcs_v[lev])(vars_new[lev][Vars::yvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                          IntVect{0,1,0},t_new[lev],BCVars::yvel_bc,false);
    }
 
    if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow" ||
        domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
 
        const MultiFab* c_vfrac = nullptr;
        if (solverChoice.terrain_type == TerrainType::EB) {
            c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
        }
 
        VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
                            vars_new[lev][Vars::yvel], IntVect{0},
                            vars_new[lev][Vars::zvel], IntVect{0},
                            vars_new[lev][Vars::cons],
                            mom_mf[IntVars::xmom],
                            mom_mf[IntVars::ymom],
                            mom_mf[IntVars::zmom],
                            Geom(lev).Domain(),
                            domain_bcs_type, c_vfrac);
    }
 
    // If !fixed_density, we must convert (rho u) which came in
    // to (rho0 u) which is what we will project
    if (!solverChoice.fixed_density[lev]) {
        ConvertForProjection(mom_mf[Vars::cons], r_hse,
                             mom_mf[IntVars::xmom],
                             mom_mf[IntVars::ymom],
                             mom_mf[IntVars::zmom],
                             Geom(lev).Domain(),
                             domain_bcs_type);
    }
 
    //
    // ****************************************************************************
    // Now convert the rho0w MultiFab to hold Omega rather than rhow
    // ****************************************************************************
    //
    if (solverChoice.mesh_type == MeshType::VariableDz)
    {
        for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Array4<Real const>& rho0u_arr = mom_mf[IntVars::xmom].const_array(mfi);
            const Array4<Real const>& rho0v_arr = mom_mf[IntVars::ymom].const_array(mfi);
            const Array4<Real      >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
 
            const Array4<Real const>&     z_nd = z_phys_nd[lev]->const_array(mfi);
            const Array4<Real const>&     mf_u =  mapfac[lev][MapFacType::u_x]->const_array(mfi);
            const Array4<Real const>&     mf_v =  mapfac[lev][MapFacType::v_y]->const_array(mfi);
 
            //
            // Define Omega from (rho0 W) but store it in the same array
            //
            Box tbz = mfi.nodaltilebox(2);
            ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                if (k == 0) {
                    rho0w_arr(i,j,k) = Real(0.0);
                } else {
                    Real rho0w = rho0w_arr(i,j,k);
                    rho0w_arr(i,j,k) = OmegaFromW(i,j,k,rho0w,
                                                  rho0u_arr,rho0v_arr,
                                                  mf_u,mf_v,z_nd,dxInv);
                }
            });
        } // mfi
    }
 
    // ****************************************************************************
    // Allocate fluxes
    // ****************************************************************************
    Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
    fluxes.resize(1);
    for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
        if (solverChoice.terrain_type == TerrainType::EB) {
            fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
        } else {
            fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
        }
    }
 
    // ****************************************************************************
    // Initialize phi to 0
    // (It is essential that we do this in order to fill the corners; these are never
    //  used but the Saxpy requires the values to be initialized.)
    // ****************************************************************************
    phi_lev.setVal(0.0);
 
    // ****************************************************************************
    // Break into subdomains
    // ****************************************************************************
 
    std::map<int,int> index_map;
 
    BoxArray ba(grids[lev]);
 
    Vector<MultiFab> rhs_sub; rhs_sub.resize(1);
    Vector<MultiFab> phi_sub; phi_sub.resize(1);
    Vector<Array<MultiFab,AMREX_SPACEDIM>> fluxes_sub; fluxes_sub.resize(1);
 
    MultiFab ax_sub, ay_sub, az_sub, dJ_sub, znd_sub;
    MultiFab mfmx_sub, mfmy_sub;
 
    Array<MultiFab,AMREX_SPACEDIM> rho0_u_sub;
    Array<MultiFab const*, AMREX_SPACEDIM> rho0_u_const;
 
    // If we are going to solve with MLMG then we do not need to break this into subdomains
    bool will_solve_with_mlmg = false;
    if (solverChoice.mesh_type == MeshType::ConstantDz) {
        will_solve_with_mlmg = true;
#ifdef ERF_USE_FFT
        if (use_fft) {
            bool all_boxes_ok = true;
            for (int isub = 0; isub < subdomains[lev].size(); ++isub) {
                Box my_region(subdomains[lev][isub].minimalBox());
                bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
                if (!boxes_make_rectangle) {
                    all_boxes_ok = false;
                }
            } // isub
            if (all_boxes_ok) {
                will_solve_with_mlmg = false;
            }
        } // use_fft
#else
        if (use_fft) {
            amrex::Warning("You set use_fft=true but didn't build with USE_FFT = TRUE; defaulting to MLMG");
        }
#endif
    } // No terrain or grid stretching
 
    for (int isub = 0; isub < subdomains[lev].size(); ++isub)
    {
        BoxList bl_sub;
        Vector<int> dm_sub;
 
        for (int j = 0; j < ba.size(); j++)
        {
            if (subdomains[lev][isub].intersects(ba[j]))
            {
                //
                // Note that bl_sub.size() is effectively a counter which is
                // incremented above
                //
                // if (ParallelDescriptor::MyProc() == j) {
                // }
                index_map[bl_sub.size()] = j;
 
                bl_sub.push_back(grids[lev][j]);
                dm_sub.push_back(dmap[lev][j]);
            } // intersects
        } // loop over ba (j)
 
        BoxArray ba_sub(bl_sub);
 
        BoxList bl2d_sub = ba_sub.boxList();
        for (auto& b : bl2d_sub) {
            b.setRange(2,0);
        }
        BoxArray ba2d_sub(std::move(bl2d_sub));
 
        // Define MultiFabs that hold only the data in this particular subdomain
        if (solverChoice.terrain_type == TerrainType::EB) {
            if (ba_sub != ba) {
                amrex::Print() << "EB Solves with multiple regions is not yet supported" << std::endl;
            }
            rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs_lev.nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi_lev.nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
 
            mfmx_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_x]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            mfmy_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_y]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
              dJ_sub.define(ba_sub, DistributionMapping(dm_sub), 1, detJ_cc[lev]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
 
            for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
                fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
                                                   IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            }
            rho0_u_sub[0].define(convert(ba_sub, IntVect::TheDimensionVector(0)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::xmom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            rho0_u_sub[1].define(convert(ba_sub, IntVect::TheDimensionVector(1)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::ymom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
            rho0_u_sub[2].define(convert(ba_sub, IntVect::TheDimensionVector(2)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::zmom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
        } else {
            rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
            phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
 
            mfmx_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_x]->nGrowVect(), MFInfo{}.SetAlloc(false));
            mfmy_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_y]->nGrowVect(), MFInfo{}.SetAlloc(false));
              dJ_sub.define(ba_sub, DistributionMapping(dm_sub), 1, detJ_cc[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
 
            for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
                fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
                                                   IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false));
            }
            rho0_u_sub[0].define(convert(ba_sub, IntVect::TheDimensionVector(0)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::xmom].nGrowVect(), MFInfo{}.SetAlloc(false));
            rho0_u_sub[1].define(convert(ba_sub, IntVect::TheDimensionVector(1)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::ymom].nGrowVect(), MFInfo{}.SetAlloc(false));
            rho0_u_sub[2].define(convert(ba_sub, IntVect::TheDimensionVector(2)), DistributionMapping(dm_sub), 1,
                                         mom_mf[IntVars::zmom].nGrowVect(), MFInfo{}.SetAlloc(false));
        }
 
        // Link the new MultiFabs to the FABs in the original MultiFabs (no copy required)
        for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
        {
            int orig_index = index_map[mfi.index()];
            rhs_sub[0].setFab(mfi, FArrayBox(rhs_lev[orig_index], amrex::make_alias, 0, 1));
            phi_sub[0].setFab(mfi, FArrayBox(phi_lev[orig_index], amrex::make_alias, 0, 1));
 
            mfmx_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::m_x])[orig_index], amrex::make_alias, 0, 1));
            mfmy_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::m_y])[orig_index], amrex::make_alias, 0, 1));
 
            fluxes_sub[0][0].setFab(mfi,FArrayBox(fluxes[0][0][orig_index], amrex::make_alias, 0, 1));
            fluxes_sub[0][1].setFab(mfi,FArrayBox(fluxes[0][1][orig_index], amrex::make_alias, 0, 1));
            fluxes_sub[0][2].setFab(mfi,FArrayBox(fluxes[0][2][orig_index], amrex::make_alias, 0, 1));
 
            rho0_u_sub[0].setFab(mfi,FArrayBox(mom_mf[IntVars::xmom][orig_index], amrex::make_alias, 0, 1));
            rho0_u_sub[1].setFab(mfi,FArrayBox(mom_mf[IntVars::ymom][orig_index], amrex::make_alias, 0, 1));
            rho0_u_sub[2].setFab(mfi,FArrayBox(mom_mf[IntVars::zmom][orig_index], amrex::make_alias, 0, 1));
        }
 
        rho0_u_const[0] = &rho0_u_sub[0];
        rho0_u_const[1] = &rho0_u_sub[1];
        rho0_u_const[2] = &rho0_u_sub[2];
 
        if (solverChoice.mesh_type != MeshType::ConstantDz) {
            ax_sub.define(convert(ba_sub,IntVect(1,0,0)), DistributionMapping(dm_sub), 1,
                          ax[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
            ay_sub.define(convert(ba_sub,IntVect(0,1,0)), DistributionMapping(dm_sub), 1,
                          ay[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
            az_sub.define(convert(ba_sub,IntVect(0,0,1)), DistributionMapping(dm_sub), 1,
                          az[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
            znd_sub.define(convert(ba_sub,IntVect(1,1,1)), DistributionMapping(dm_sub), 1,
                           z_phys_nd[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
 
            for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
                int orig_index = index_map[mfi.index()];
                ax_sub.setFab(mfi, FArrayBox((*ax[lev])[orig_index], amrex::make_alias, 0, 1));
                ay_sub.setFab(mfi, FArrayBox((*ay[lev])[orig_index], amrex::make_alias, 0, 1));
                az_sub.setFab(mfi, FArrayBox((*az[lev])[orig_index], amrex::make_alias, 0, 1));
                znd_sub.setFab(mfi, FArrayBox((*z_phys_nd[lev])[orig_index], amrex::make_alias, 0, 1));
                dJ_sub.setFab(mfi, FArrayBox((*detJ_cc[lev])[orig_index], amrex::make_alias, 0, 1));
            }
        }
 
        if (solverChoice.terrain_type == TerrainType::EB) {
            for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
                int orig_index = index_map[mfi.index()];
                dJ_sub.setFab(mfi, FArrayBox((*detJ_cc[lev])[orig_index], amrex::make_alias, 0, 1));
            }
        }
 
        // ****************************************************************************
        // Compute divergence which will form RHS
        // Note that we replace "rho0w" with the contravariant momentum, Omega
        // ****************************************************************************
 
        compute_divergence(lev, rhs_sub[0], rho0_u_const, geom_tmp[0]);
 
        Real rhsnorm;
 
        // Max norm over the entire MultiFab
        rhsnorm = rhs_sub[0].norm0();
 
        if (mg_verbose > 0) {
            bool local = false;
            Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
            Print() << "Max/L2 norm of divergence before solve in subdomain " << isub << " at level " << lev << " : " << rhsnorm << " " <<
                        rhs_sub[0].norm2() << " and volume-weighted sum " << sum << std::endl;
        }
 
        if (lev == 0 && solverChoice.use_real_bcs)
        {
            // We always use VariableDz if use_real_bcs is true
            AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::VariableDz);
 
            // Note that we always impose the projections one level at a time so this will always be a vector of length 1
            Array<MultiFab*, AMREX_SPACEDIM> rho0_u_vec =
               {&mom_mf[IntVars::xmom], &mom_mf[IntVars::ymom], &mom_mf[IntVars::zmom]};
            Array<MultiFab*, AMREX_SPACEDIM> area_vec = {ax[lev].get(), ay[lev].get(), az[lev].get()};
            //
            // Modify ax,ay,ax to include the map factors as used in the divergence calculation
            // We do this here so that it is seen in the call to enforceInOutSolvability
            //
            for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
            {
                Box xbx = mfi.nodaltilebox(0);
                Box ybx = mfi.nodaltilebox(1);
                Box zbx = mfi.nodaltilebox(2);
                const Array4<Real      >& ax_ar = ax[lev]->array(mfi);
                const Array4<Real      >& ay_ar = ay[lev]->array(mfi);
                const Array4<Real      >& az_ar = az[lev]->array(mfi);
                const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(xbx,ybx,zbx,
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ax_ar(i,j,k) /= mf_uy(i,j,0);
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ay_ar(i,j,k) /= mf_vx(i,j,0);
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
                });
            } // mfi
 
            if (mg_verbose > 0) {
                Print() << "Calling enforceInOutSolvability" << std::endl;
            }
            enforceInOutSolvability(lev, rho0_u_vec, area_vec, geom[lev]);
 
            //
            // Return ax,ay,ax to their original definition
            //
            for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
            {
                Box xbx = mfi.nodaltilebox(0);
                Box ybx = mfi.nodaltilebox(1);
                Box zbx = mfi.nodaltilebox(2);
                const Array4<Real      >& ax_ar = ax[lev]->array(mfi);
                const Array4<Real      >& ay_ar = ay[lev]->array(mfi);
                const Array4<Real      >& az_ar = az[lev]->array(mfi);
                const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(xbx,ybx,zbx,
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ax_ar(i,j,k) *= mf_uy(i,j,0);
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ay_ar(i,j,k) *= mf_vx(i,j,0);
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
                });
            } // mfi
 
            compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
 
            // Re-define max norm over the entire MultiFab
            rhsnorm = rhs_lev.norm0();
 
            if (mg_verbose > 0)
            {
                bool local = false;
                Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
                Print() << "Max/L2 norm of divergence before solve at level " << lev << " : " << rhsnorm << " " <<
                            rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
            }
        } // lev 0 && use_real_bcs
 
        // *******************************************************************************************
        // Enforce solvability if the problem is singular (i.e all sides Neumann or periodic)
        // Note that solves at lev > 0 are always singular because we impose Neumann bc's on all sides
        // *******************************************************************************************
        bool is_singular = true;
        if (lev == 0) {
            if ( (domain_bc_type[0] == "Outflow" || domain_bc_type[0] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
            if ( (domain_bc_type[1] == "Outflow" || domain_bc_type[1] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
            if ( (domain_bc_type[3] == "Outflow" || domain_bc_type[3] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
            if ( (domain_bc_type[4] == "Outflow" || domain_bc_type[4] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
            if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open")                               ) is_singular = false;
        } else {
            Box my_region(subdomains[lev][isub].minimalBox());
            if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open") && (my_region.bigEnd(2) == domain.bigEnd(2)) ) is_singular = false;
        }
 
        if (is_singular)
        {
            bool local = false;
            Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
 
            Real vol;
            if (solverChoice.mesh_type == MeshType::ConstantDz) {
                vol = rhs_sub[0].boxArray().numPts();
            } else {
                vol = dJ_sub.sum();
            }
 
            sum /= (vol * dx[0] * dx[1] * dx[2]);
 
            for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
            {
                rhs_sub[0][mfi.index()].template minus<RunOn::Device>(sum);
            }
            if (mg_verbose > 0) {
                amrex::Print() << " Subtracting " << sum << " from rhs in subdomain " << isub << std::endl;
 
                sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
                Print() << "Sum after subtraction " << sum << " in subdomain " << isub << std::endl;
            }
 
        } // if is_singular
 
        rhsnorm = rhs_sub[0].norm0();
 
        // ****************************************************************************
        // No need to build the solver if RHS == 0
        // ****************************************************************************
        if (rhsnorm <= solverChoice.poisson_abstol) return;
 
        Real start_step = static_cast<Real>(ParallelDescriptor::second());
 
        if (mg_verbose > 0) {
            amrex::Print() << " Solving in subdomain " << isub << " of " << subdomains[lev].size() << " bins at level " << lev << std::endl;
        }
 
        if (solverChoice.mesh_type == MeshType::VariableDz) {
            //
            // Modify ax,ay,ax to include the map factors as used in the divergence calculation
            // We do this here to set the coefficients used in the stencil -- the extra factor
            // of the mapfac comes from the gradient
            //
            for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
            {
                Box xbx = mfi.nodaltilebox(0);
                Box ybx = mfi.nodaltilebox(1);
                Box zbx = mfi.nodaltilebox(2);
                const Array4<Real      >& ax_ar = ax_sub.array(mfi);
                const Array4<Real      >& ay_ar = ay_sub.array(mfi);
                const Array4<Real      >& az_ar = az_sub.array(mfi);
                const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
                const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
                const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(xbx,ybx,zbx,
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ax_ar(i,j,k) *= (mf_ux(i,j,0) / mf_uy(i,j,0));
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ay_ar(i,j,k) *= (mf_vy(i,j,0) / mf_vx(i,j,0));
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
                });
            } // mfi
        }
 
        if (solverChoice.terrain_type != TerrainType::EB) {
 
#ifdef ERF_USE_FFT
        Box my_region(subdomains[lev][isub].minimalBox());
#endif
 
        // ****************************************************************************
        // No terrain or grid stretching
        // ****************************************************************************
        if (solverChoice.mesh_type == MeshType::ConstantDz) {
            if (will_solve_with_mlmg) {
                solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub, geom[lev], ref_ratio, domain_bc_type,
                                mg_verbose, solverChoice.poisson_reltol, solverChoice.poisson_abstol);
            } else {
#ifdef ERF_USE_FFT
                solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
#endif
            }
        } // No terrain or grid stretching
        // ****************************************************************************
        // Grid stretching (flat terrain)
        // ****************************************************************************
        else if (solverChoice.mesh_type == MeshType::StretchedDz) {
#ifndef ERF_USE_FFT
            amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT solver");
#else
            bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
            if (!boxes_make_rectangle) {
                amrex::Abort("FFT won't work unless the union of boxes is rectangular");
            } else {
                if (!use_fft) {
                    amrex::Warning("Using FFT even though you didn't set use_fft to true; it's the best choice");
                }
                solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
            }
#endif
        } // grid stretching
 
        // ****************************************************************************
        // General terrain
        // ****************************************************************************
        else if (solverChoice.mesh_type == MeshType::VariableDz) {
#ifdef ERF_USE_FFT
            bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
            if (!boxes_make_rectangle) {
                amrex::Abort("FFT preconditioner for GMRES won't work unless the union of boxes is rectangular");
            } else {
                solve_with_gmres(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0], ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
            }
#else
            amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT preconditioner for GMRES");
#endif
 
            //
            // Restore ax,ay,ax to their original definitions
            //
            for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
            {
                Box xbx = mfi.nodaltilebox(0);
                Box ybx = mfi.nodaltilebox(1);
                Box zbx = mfi.nodaltilebox(2);
                const Array4<Real      >& ax_ar = ax_sub.array(mfi);
                const Array4<Real      >& ay_ar = ay_sub.array(mfi);
                const Array4<Real      >& az_ar = az_sub.array(mfi);
                const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
                const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
                const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(xbx,ybx,zbx,
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ax_ar(i,j,k) *= (mf_uy(i,j,0) / mf_ux(i,j,0));
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    ay_ar(i,j,k) *= (mf_vx(i,j,0) / mf_vy(i,j,0));
                },
                [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
                });
            } // mfi
 
        } // MeshType::VariableDz
 
        // ****************************************************************************
        // Print time in solve
        // ****************************************************************************
        Real end_step = static_cast<Real>(ParallelDescriptor::second());
        if (mg_verbose > 0) {
            amrex::Print() << "Time in solve " << end_step - start_step << std::endl;
        }
 
        } // not EB
    } // loop over subdomains (i)
 
    // ****************************************************************************
    // When using multigrid we can solve for all of the level at once, even if there
    //      are disjoint regions
    // ****************************************************************************
    if (solverChoice.terrain_type == TerrainType::EB) {
        Real start_step_eb = static_cast<Real>(ParallelDescriptor::second());
        solve_with_EB_mlmg(lev, rhs_sub, phi_sub, fluxes_sub,
                           *(get_eb(lev).get_const_factory()),
                           *(get_eb(lev).get_u_const_factory()),
                           *(get_eb(lev).get_v_const_factory()),
                           *(get_eb(lev).get_w_const_factory()),
                           geom[lev], ref_ratio, domain_bc_type,
                           mg_verbose, solverChoice.poisson_reltol, solverChoice.poisson_abstol);
        Real end_step_eb = static_cast<Real>(ParallelDescriptor::second());
        if (mg_verbose > 0) {
            amrex::Print() << "Time in solve " << end_step_eb - start_step_eb << std::endl;
        }
    }
 
    // ****************************************************************************
    // Subtract dt grad(phi) from the momenta (rho0u, rho0v, Omega)
    // ****************************************************************************
    MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0);
    MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0);
    MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0);
 
    // ****************************************************************************
    // Define gradp from fluxes -- note that fluxes is dt * change in Gp
    //   (weighted by map factor!)
    // ****************************************************************************
    MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
    MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
    MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
 
    gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
    gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
    gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
 
    //
    // This call is only to verify the divergence after the solve
    // It is important we do this before computing the rho0w_arr from Omega back to rho0w
    //
    // ****************************************************************************
    // THIS IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE
    // ****************************************************************************
    //
    if (mg_verbose > 0)
    {
        rho0_u_const[0] = &mom_mf[IntVars::xmom];
        rho0_u_const[1] = &mom_mf[IntVars::ymom];
        rho0_u_const[2] = &mom_mf[IntVars::zmom];
 
        compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
 
        bool local = false;
        Real sum = volWgtSumMF(lev,rhs_lev,0,*detJ_cc[lev],*mapfac[lev][MapFacType::m_x],*mapfac[lev][MapFacType::m_y],false,local);
 
        if (mg_verbose > 0) {
            Print() << "Max/L2 norm of divergence after  solve at level " << lev << " : " << rhs_lev.norm0() << " " <<
                        rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
        }
 
#if 0
         // FOR DEBUGGING ONLY
         for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
            const Array4<Real const>& rhs_arr = rhs_lev.const_array(mfi);
            Box bx = mfi.validbox();
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                if (std::abs(rhs_arr(i,j,k)) > 1.e-10) {
                    amrex::AllPrint() << "RHS after solve at " <<
                                          IntVect(i,j,k) << " " << rhs_arr(i,j,k) << std::endl;
                }
            });
         } // mfi
#endif
 
    } // mg_verbose
 
    //
    // ****************************************************************************
    // Now convert the rho0w MultiFab back to holding (rho0w) rather than Omega
    // ****************************************************************************
    //
    if (solverChoice.mesh_type == MeshType::VariableDz)
    {
        for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
             Box tbz = mfi.nodaltilebox(2);
             const Array4<Real      >& rho0u_arr = mom_mf[IntVars::xmom].array(mfi);
             const Array4<Real      >& rho0v_arr = mom_mf[IntVars::ymom].array(mfi);
             const Array4<Real      >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
             const Array4<Real const>&      z_nd = z_phys_nd[lev]->const_array(mfi);
             const Array4<Real const>&      mf_u =  mapfac[lev][MapFacType::u_x]->const_array(mfi);
             const Array4<Real const>&      mf_v =  mapfac[lev][MapFacType::v_y]->const_array(mfi);
             ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                 Real omega = rho0w_arr(i,j,k);
                 rho0w_arr(i,j,k) = WFromOmega(i,j,k,omega,
                                               rho0u_arr,rho0v_arr,
                                               mf_u,mf_v,z_nd,dxInv);
             });
        } // mfi
    }
 
    // If !fixed_density, we must convert (rho0 u) back
    // to (rho0 u) which is what we will pass back out
    if (!solverChoice.fixed_density[lev]) {
        ConvertForProjection(r_hse, mom_mf[Vars::cons],
                             mom_mf[IntVars::xmom],
                             mom_mf[IntVars::ymom],
                             mom_mf[IntVars::zmom],
                             Geom(lev).Domain(),
                             domain_bcs_type);
    }
 
    // ****************************************************************************
    // Update pressure variable with phi -- note that phi is dt * change in pressure
    // ****************************************************************************
    MultiFab::Saxpy(pp_inc[lev], 1.0/l_dt, phi_lev,0,0,1,1);
}

Here is the call graph for this function:

project_velocity_tb()

void ERF::project_velocity_tb	(	int	lev,
		amrex::Real	dt,
		amrex::Vector< amrex::MultiFab > &	vars
	)

Project the single-level velocity field to enforce incompressibility with a thin body

{
    BL_PROFILE("ERF::project_velocity_tb()");
    AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
 
    // Make sure the solver only sees the levels over which we are solving
    Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(vmf[Vars::cons].boxArray());
    Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(vmf[Vars::cons].DistributionMap());
    Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
 
    // Use the default settings
    LPInfo info;
    std::unique_ptr<MLPoisson> p_mlpoisson;
#if 0
    if (overset_imask[0]) {
        // Add overset mask around thin body
        p_mlpoisson = std::make_unique<MLPoisson>(geom, grids, dmap, GetVecOfConstPtrs(overset_imask), info);
    }
    else
#endif
    {
        // Use the default settings
        p_mlpoisson = std::make_unique<MLPoisson>(geom_tmp, ba_tmp, dm_tmp, info);
    }
 
    auto bclo = get_lo_projection_bc(geom[lev],domain_bc_type);
    auto bchi = get_hi_projection_bc(geom[lev],domain_bc_type);
 
    bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true;
    p_mlpoisson->setDomainBC(bclo, bchi);
 
    if (lev > 0) {
        p_mlpoisson->setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
    }
 
    p_mlpoisson->setLevelBC(0, nullptr);
 
    Vector<MultiFab> rhs;
    Vector<MultiFab> phi;
    Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
    Vector<Array<MultiFab,AMREX_SPACEDIM> > deltaf; // f^* - f^{n-1}
    Vector<Array<MultiFab,AMREX_SPACEDIM> > u_plus_dtdf; // u + dt*deltaf
 
    // Used to pass array of const MFs to ComputeDivergence
    Array<MultiFab const*, AMREX_SPACEDIM> u;
 
    rhs.resize(1);
    phi.resize(1);
    fluxes.resize(1);
    deltaf.resize(1);
    u_plus_dtdf.resize(1);
 
    rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
    phi[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
    rhs[0].setVal(0.0);
    phi[0].setVal(0.0);
 
    for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
             fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
        u_plus_dtdf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
 
             deltaf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
             deltaf[0][idim].setVal(0.0); // start with f^* == f^{n-1}
    }
 
#if 0
    // DEBUG
    u[0] = &(vmf[Vars::xvel]);
    u[1] = &(vmf[Vars::yvel]);
    u[2] = &(vmf[Vars::zvel]);
    computeDivergence(rhs[0], u, geom[0]);
    Print() << "Max norm of divergence before solve at level 0 : " << rhs[0].norm0() << std::endl;
#endif
 
    for (int itp = 0; itp < solverChoice.ncorr; ++itp)
    {
        // Calculate u + dt*deltaf
        for (int idim = 0; idim < 3; ++idim) {
            MultiFab::Copy(u_plus_dtdf[0][idim], deltaf[0][idim], 0, 0, 1, 0);
            u_plus_dtdf[0][0].mult(-l_dt,0,1,0);
        }
        MultiFab::Add(u_plus_dtdf[0][0], vmf[Vars::xvel], 0, 0, 1, 0);
        MultiFab::Add(u_plus_dtdf[0][1], vmf[Vars::yvel], 0, 0, 1, 0);
        MultiFab::Add(u_plus_dtdf[0][2], vmf[Vars::zvel], 0, 0, 1, 0);
 
        u[0] = &(u_plus_dtdf[0][0]);
        u[1] = &(u_plus_dtdf[0][1]);
        u[2] = &(u_plus_dtdf[0][2]);
        computeDivergence(rhs[0], u, geom_tmp[0]);
 
#if 0
        // DEBUG
        if (itp==0) {
            for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real const>& divU = rhs[0].const_array(mfi);
                const Array4<Real const>& uarr = vmf[Vars::xvel].const_array(mfi);
                const Array4<Real const>& varr = vmf[Vars::yvel].const_array(mfi);
                const Array4<Real const>& warr = vmf[Vars::zvel].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    if ((i>=120) && (i<=139) && (j==0) && ((k>=127)&&(k<=128))) {
                        amrex::AllPrint() << "before project div"<<IntVect(i,j,k)<<" = "<< divU(i,j,k)
                            << " u: " << uarr(i,j,k) << " " << uarr(i+1,j,k)
                            << " v: " << varr(i,j,k) << " " << varr(i,j+1,k)
                            << " w: " << warr(i,j,k) << " " << warr(i,j,k+1)
                            << std::endl;
                    }
                });
            }
        }
#endif
 
        // If all Neumann BCs, adjust RHS to make sure we can converge
        if (need_adjust_rhs) {
            bool local = false;
            Real offset = volWgtSumMF(lev,rhs[0],0,*detJ_cc[lev],*mapfac[lev][MapFacType::m_x],*mapfac[lev][MapFacType::m_y],false,local);
            // amrex::Print() << "Poisson solvability offset = " << offset << std::endl;
            rhs[0].plus(-offset, 0, 1);
        }
 
        // Initialize phi to 0
        phi[0].setVal(0.0);
 
        MLMG mlmg(*p_mlpoisson);
        int max_iter = 100;
        mlmg.setMaxIter(max_iter);
 
        mlmg.setVerbose(mg_verbose);
        //mlmg.setBottomVerbose(mg_verbose);
 
        // solve for dt*p
        mlmg.solve(GetVecOfPtrs(phi),
                   GetVecOfConstPtrs(rhs),
                   solverChoice.poisson_reltol,
                   solverChoice.poisson_abstol);
 
        mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
 
        // Calculate new intermediate body force with updated gradp
        if (thin_xforce[lev]) {
            MultiFab::Copy(   deltaf[0][0], fluxes[0][0], 0, 0, 1, 0);
            ApplyInvertedMask(deltaf[0][0], *xflux_imask[0]);
        }
        if (thin_yforce[lev]) {
            MultiFab::Copy(   deltaf[0][1], fluxes[0][1], 0, 0, 1, 0);
            ApplyInvertedMask(deltaf[0][1], *yflux_imask[0]);
        }
        if (thin_zforce[lev]) {
            MultiFab::Copy(   deltaf[0][2], fluxes[0][2], 0, 0, 1, 0);
            ApplyInvertedMask(deltaf[0][2], *zflux_imask[0]);
        }
 
        // DEBUG
        //        for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
        //        {
        //            const Box& bx = mfi.tilebox();
        //            const Array4<Real const>& dfz_arr = deltaf[0][2].const_array(mfi);
        //            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        //            {
        //                if ((i>=120) && (i<=139) && (j==0) && (k==128)) {
        //                    amrex::AllPrint()
        //                        << " piter" << itp
        //                        << " dfz"<<IntVect(i,j,k)<<" = "<< dfz_arr(i,j,k)
        //                        << std::endl;
        //                }
        //            });
        //        }
 
        // Update pressure variable with phi -- note that phi is change in pressure, not the full pressure
        MultiFab::Saxpy(pp_inc[lev], 1.0, phi[0],0,0,1,0);
 
        // Subtract grad(phi) from the velocity components
        Real beta = 1.0;
        MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
        MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
        MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
        if (thin_xforce[lev]) {
            ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
        }
        if (thin_yforce[lev]) {
            ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
        }
        if (thin_zforce[lev]) {
            ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
        }
    } // itp: pressure-force iterations
 
    // ****************************************************************************
    // Define gradp from fluxes -- note that fluxes is dt * change in Gp
    // ****************************************************************************
    MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
    MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
    MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
 
    gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
    gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
    gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
 
    // Subtract grad(phi) from the velocity components
//    Real beta = 1.0;
//    for (int ilev = lev_min; ilev <= lev_max; ++ilev) {
//        MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
//        MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
//        MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
//        if (thin_xforce[lev]) {
//            ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
//        }
//        if (thin_yforce[lev]) {
//            ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
//        }
//        if (thin_zforce[lev]) {
//            ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
//        }
//    }
 
#if 0
    // Confirm that the velocity is now divergence free
    u[0] = &(vmf[Vars::xvel]);
    u[1] = &(vmf[Vars::yvel]);
    u[2] = &(vmf[Vars::zvel]);
    computeDivergence(rhs[0], u, geom_tmp[0]);
    Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
 
#endif
}

Here is the call graph for this function:

ReadCheckpointFile()

void ERF::ReadCheckpointFile

(

)

ERF function for reading data from a checkpoint file during restart.

{
    Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
 
    // Header
    std::string File(restart_chkfile + "/Header");
 
    VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
 
    Vector<char> fileCharPtr;
    ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
    std::string fileCharPtrString(fileCharPtr.dataPtr());
    std::istringstream is(fileCharPtrString, std::istringstream::in);
 
    std::string line, word;
 
    int chk_ncomp_cons, chk_ncomp;
 
    // read in title line
    std::getline(is, line);
 
    // read in finest_level
    is >> finest_level;
    GotoNextLine(is);
 
    // read the number of components
    // for each variable we store
 
    // conservative, cell-centered vars
    is >> chk_ncomp_cons;
    GotoNextLine(is);
 
    // x-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // y-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // z-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // read in array of istep
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            istep[i++] = std::stoi(word);
        }
    }
 
    // read in array of dt
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            dt[i++] = std::stod(word);
        }
    }
 
    // read in array of t_new
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            t_new[i++] = std::stod(word);
        }
    }
 
    for (int lev = 0; lev <= finest_level; ++lev) {
        // read in level 'lev' BoxArray from Header
        BoxArray ba;
        ba.readFrom(is);
        GotoNextLine(is);
 
        // create a distribution mapping
        DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
 
        MakeNewLevelFromScratch (lev, t_new[lev], ba, dm);
    }
 
    // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
    // NOTE: Data is written over ncomp, so check that we match the header file
    int ncomp_cons = vars_new[0][Vars::cons].nComp();
 
    // NOTE: QKE was removed so this is for backward compatibility
    AMREX_ASSERT((chk_ncomp_cons==ncomp_cons) || ((chk_ncomp_cons-1)==ncomp_cons));
    //
    // See if we have a written separate file that tells how many components and how many ghost cells
    // we have of the base state
    //
    // If we can't find the file, then set the number of components to the original number = 3
    //
    int ncomp_base_to_read = 3;
    IntVect ng_base = IntVect{1};
    {
         std::string BaseStateFile(restart_chkfile + "/num_base_state_comps");
 
         if (amrex::FileExists(BaseStateFile))
         {
             Vector<char> BaseStatefileCharPtr;
             ParallelDescriptor::ReadAndBcastFile(BaseStateFile, BaseStatefileCharPtr);
             std::string BaseStatefileCharPtrString(BaseStatefileCharPtr.dataPtr());
 
             // We set this to the default value of 3 but allow it be larger if th0 and qv0 were written
             std::istringstream isb(BaseStatefileCharPtrString, std::istringstream::in);
             isb >> ncomp_base_to_read;
             isb >> ng_base;
         }
    }
 
    // read in the MultiFab data
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        // NOTE: For backward compatibility (chk file has QKE)
        if ((chk_ncomp_cons-1)==ncomp_cons) {
            MultiFab cons(grids[lev],dmap[lev],chk_ncomp_cons,0);
            VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
 
            // Copy up to RhoKE_comp
            MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,(RhoKE_comp+1),0);
 
            // Only if we have a PBL model do we need to copy QKE is src to KE in dst
            if ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
                 (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ) {
                MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+1),RhoKE_comp,1,0);
                vars_new[lev][Vars::cons].mult(0.5,RhoKE_comp,1,0);
            }
 
            // Copy other components
            int ncomp_remainder = ncomp_cons - (RhoKE_comp + 1);
            MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+2),(RhoKE_comp+1),ncomp_remainder,0);
 
            vars_new[lev][Vars::cons].setBndry(1.0e34);
        } else {
            MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
            VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
            MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
            vars_new[lev][Vars::cons].setBndry(1.0e34);
        }
 
        MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
        VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
        MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
        vars_new[lev][Vars::xvel].setBndry(1.0e34);
 
        MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
        VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
        MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
        vars_new[lev][Vars::yvel].setBndry(1.0e34);
 
        MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
        VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
        MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
        vars_new[lev][Vars::zvel].setBndry(1.0e34);
 
        if (solverChoice.anelastic[lev] == 1) {
            MultiFab ppinc(grids[lev],dmap[lev],1,0);
            VisMF::Read(ppinc, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PP_Inc"));
            MultiFab::Copy(pp_inc[lev],ppinc,0,0,1,0);
            pp_inc[lev].FillBoundary(geom[lev].periodicity());
 
            MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
            VisMF::Read(gpx, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpx"));
            MultiFab::Copy(gradp[lev][GpVars::gpx],gpx,0,0,1,0);
            gradp[lev][GpVars::gpx].FillBoundary(geom[lev].periodicity());
 
            MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
            VisMF::Read(gpy, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpy"));
            MultiFab::Copy(gradp[lev][GpVars::gpy],gpy,0,0,1,0);
            gradp[lev][GpVars::gpy].FillBoundary(geom[lev].periodicity());
 
            MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
            VisMF::Read(gpz, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpz"));
            MultiFab::Copy(gradp[lev][GpVars::gpz],gpz,0,0,1,0);
            gradp[lev][GpVars::gpz].FillBoundary(geom[lev].periodicity());
        }
 
        // Note that we read the ghost cells of the base state (unlike above)
 
        // The original base state only had 3 components and 1 ghost cell -- we read this
        // here to be consistent with the old style
        MultiFab base(grids[lev],dmap[lev],ncomp_base_to_read,ng_base);
        VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
 
        MultiFab::Copy(base_state[lev],base,0,0,ncomp_base_to_read,ng_base);
 
        // Create theta0 from p0, rh0
        if (ncomp_base_to_read < 4) {
            for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // We only compute theta_0 on valid cells since we will impose domain BC's after restart
                const Box& bx = mfi.tilebox();
                Array4<Real> const& fab = base_state[lev].array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    fab(i,j,k,BaseState::th0_comp) = getRhoThetagivenP(fab(i,j,k,BaseState::p0_comp))
                                                     / fab(i,j,k,BaseState::r0_comp);
                });
            }
        }
        // Default theta0 to 0
        if (ncomp_base_to_read < 5) {
            for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                // We only compute theta_0 on valid cells since we will impose domain BC's after restart
                const Box& bx = mfi.tilebox();
                Array4<Real> const& fab = base_state[lev].array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    fab(i,j,k,BaseState::qv0_comp) = 0.0;
                });
            }
        }
        base_state[lev].FillBoundary(geom[lev].periodicity());
 
        if (SolverChoice::mesh_type != MeshType::ConstantDz)  {
           // Note that we also read the ghost cells of z_phys_nd
           IntVect ng = z_phys_nd[lev]->nGrowVect();
           MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
           VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
           MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
           update_terrain_arrays(lev);
 
           // Compute the min dz and pass to the micro model
           Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
           micro->Set_dzmin(lev, dzmin);
 
           if (SolverChoice::mesh_type == MeshType::VariableDz) {
               MultiFab z_slab(convert(ba2d[lev],IntVect(1,1,1)),dmap[lev],1,0);
               int klo = geom[lev].Domain().smallEnd(2);
               for (MFIter mfi(z_slab); mfi.isValid(); ++mfi) {
                   Box nbx = mfi.tilebox();
                   Array4<Real const> const& z_arr      = z_phys_nd[lev]->const_array(mfi);
                   Array4<Real      > const& z_slab_arr = z_slab.array(mfi);
                   ParallelFor(nbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                   {
                       z_slab_arr(i,j,k) = z_arr(i,j,klo);
                   });
               }
               Real z_min = z_slab.min(0);
               Real z_max = z_slab.max(0);
 
               auto dz = geom[lev].CellSize()[2];
               if (z_max - z_min < 1.e-8 * dz) {
                   SolverChoice::set_mesh_type(MeshType::StretchedDz);
                   if (verbose > 0) {
                       amrex::Print() << "Resetting mesh type to StretchedDz since terrain is flat" << std::endl;
                   }
               }
           }
        }
 
        // Read in the moisture model restart variables
        std::vector<int> qmoist_indices;
        std::vector<std::string> qmoist_names;
        micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
        int qmoist_nvar = qmoist_indices.size();
        for (int var = 0; var < qmoist_nvar; var++) {
            const int ncomp  = 1;
            IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
            MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
            VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", qmoist_names[var]));
            MultiFab::Copy(*(qmoist[lev][qmoist_indices[var]]),moist_vars,0,0,ncomp,ng_moist);
        }
 
#if defined(ERF_USE_WINDFARM)
        if(solverChoice.windfarm_type == WindFarmType::Fitch or
           solverChoice.windfarm_type == WindFarmType::EWP or
           solverChoice.windfarm_type == WindFarmType::SimpleAD){
            IntVect ng = Nturb[lev].nGrowVect();
            MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
            VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
            MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
        }
#endif
 
        // Read the LSM data
        if (solverChoice.lsm_type != LandSurfaceType::None) {
            for (int ivar(0); ivar<lsm_data[lev].size(); ++ivar) {
                BoxArray ba = lsm_data[lev][ivar]->boxArray();
                DistributionMapping dm = lsm_data[lev][ivar]->DistributionMap();
                IntVect ng = lsm_data[lev][ivar]->nGrowVect();
                int nvar   = lsm_data[lev][ivar]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmData" + std::to_string(ivar)));
                MultiFab::Copy(*(lsm_data[lev][ivar]),lsm_vars,0,0,nvar,ng);
            }
            for (int iflux(0); iflux<lsm_flux[lev].size(); ++iflux) {
                BoxArray ba = lsm_flux[lev][iflux]->boxArray();
                DistributionMapping dm = lsm_flux[lev][iflux]->DistributionMap();
                IntVect ng = lsm_flux[lev][iflux]->nGrowVect();
                int nvar   = lsm_flux[lev][iflux]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmFlux" + std::to_string(iflux)));
                MultiFab::Copy(*(lsm_flux[lev][iflux]),lsm_vars,0,0,nvar,ng);
            }
        }
 
 
        IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
        MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
 
        std::string MapFacMFileName(restart_chkfile + "/Level_0/MapFactor_mx_H");
        if (amrex::FileExists(MapFacMFileName)) {
            VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_mx"));
        } else {
            VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
        }
        MultiFab::Copy(*mapfac[lev][MapFacType::m_x],mf_m,0,0,1,ng);
 
#if 0
        if (MapFacType::m_x != MapFacType::m_y) {
            VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_my"));
            MultiFab::Copy(*mapfac[lev][MapFacType::m_y],mf_m,0,0,1,ng);
        }
#endif
 
        ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
        MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
 
        std::string MapFacUFileName(restart_chkfile + "/Level_0/MapFactor_ux_H");
        if (amrex::FileExists(MapFacUFileName)) {
            VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_ux"));
        } else {
            VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
        }
        MultiFab::Copy(*mapfac[lev][MapFacType::u_x],mf_u,0,0,1,ng);
 
#if 0
        if (MapFacType::u_x != MapFacType::u_y) {
            VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_uy"));
            MultiFab::Copy(*mapfac[lev][MapFacType::u_y],mf_u,0,0,1,ng);
        }
#endif
 
        ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
        MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
 
        std::string MapFacVFileName(restart_chkfile + "/Level_0/MapFactor_vx_H");
        if (amrex::FileExists(MapFacVFileName)) {
            VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vx"));
        } else {
            VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
        }
        MultiFab::Copy(*mapfac[lev][MapFacType::v_x],mf_v,0,0,1,ng);
 
#if 0
        if (MapFacType::v_x != MapFacType::v_y) {
            VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vy"));
            MultiFab::Copy(*mapfac[lev][MapFacType::v_y],mf_v,0,0,1,ng);
        }
#endif
 
 
        // NOTE: We read MOST data in ReadCheckpointFileMOST (see below)!
 
        // See if we wrote out SST data
        std::string FirstSSTFileName(restart_chkfile + "/Level_0/SST_0_H");
        if (amrex::FileExists(FirstSSTFileName))
        {
            amrex::Print() << "Reading SST data" << std::endl;
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
            sst_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                VisMF::Read(sst_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
                                                             "SST_" + std::to_string(nt)));
                MultiFab::Copy(*sst_lev[lev][nt],sst_at_t,0,0,1,ng);
            }
        }
 
        // See if we wrote out TSK data
        std::string FirstTSKFileName(restart_chkfile + "/Level_0/TSK_0_H");
        if (amrex::FileExists(FirstTSKFileName))
        {
            amrex::Print() << "Reading TSK data" << std::endl;
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
            tsk_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                VisMF::Read(tsk_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
                                                             "TSK_" + std::to_string(nt)));
                MultiFab::Copy(*tsk_lev[lev][nt],tsk_at_t,0,0,1,ng);
            }
        }
 
        std::string LMaskFileName(restart_chkfile + "/Level_0/LMASK_0_H");
        if (amrex::FileExists(LMaskFileName))
        {
            amrex::Print() << "Reading LMASK data" << std::endl;
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                VisMF::Read(lmask_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
                                                              "LMASK_" + std::to_string(nt)));
                for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
                    const Box& bx = mfi.growntilebox();
                    Array4<int>  const& dst_arr = lmask_lev[lev][nt]->array(mfi);
                    Array4<Real> const& src_arr = lmask_at_t.array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        dst_arr(i,j,k) = int(src_arr(i,j,k));
                    });
                }
            }
        } else {
            // Allow idealized cases over water, used to set lmask
            ParmParse pp("erf");
            int is_land;
            if (pp.query("is_land", is_land, lev)) {
                if (is_land == 1) {
                    amrex::Print() << "Level " << lev << " is land" << std::endl;
                } else if (is_land == 0) {
                    amrex::Print() << "Level " << lev << " is water" << std::endl;
                } else {
                    Error("is_land should be 0 or 1");
                }
                lmask_lev[lev][0]->setVal(is_land);
            } else {
                // Default to land everywhere if not specified
                lmask_lev[lev][0]->setVal(1);
            }
            lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
        }
 
        IntVect ngv = ng; ngv[2] = 0;
 
        // Read lat/lon if it exists
        if (solverChoice.has_lat_lon) {
            amrex::Print() << "Reading Lat/Lon variables" << std::endl;
            MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
            MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
            VisMF::Read(lat, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LAT"));
            VisMF::Read(lon, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LON"));
            lat_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
            lon_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
            MultiFab::Copy(*lat_m[lev],lat,0,0,1,ngv);
            MultiFab::Copy(*lon_m[lev],lon,0,0,1,ngv);
        }
 
#ifdef ERF_USE_NETCDF
        // Read sinPhi and cosPhi if it exists
        if (solverChoice.variable_coriolis) {
            amrex::Print() << "Reading Coriolis factors" << std::endl;
            MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
            MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
            VisMF::Read(sphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SinPhi"));
            VisMF::Read(cphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "CosPhi"));
            sinPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
            cosPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
            MultiFab::Copy(*sinPhi_m[lev],sphi,0,0,1,ngv);
            MultiFab::Copy(*cosPhi_m[lev],cphi,0,0,1,ngv);
        }
 
        if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
            if (lev == 0) {
                MultiFab tmp1d(ba1d[0],dmap[0],1,0);
 
                VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C1H"));
                MultiFab::Copy(*mf_C1H,tmp1d,0,0,1,0);
 
                VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C2H"));
                MultiFab::Copy(*mf_C2H,tmp1d,0,0,1,0);
 
                MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
 
                VisMF::Read(tmp2d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MUB"));
                MultiFab::Copy(*mf_MUB,tmp2d,0,0,1,mf_MUB->nGrowVect());
            }
        }
#endif
    } // for lev
 
#ifdef ERF_USE_PARTICLES
    restartTracers((ParGDBBase*)GetParGDB(),restart_chkfile);
    if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
        dynamic_cast<LagrangianMicrophysics&>(*micro).restartParticles((ParGDBBase*)GetParGDB(),restart_chkfile);
    }
#endif
 
#if 0
#ifdef ERF_USE_NETCDF
    // Read bdy_data files
    if ( ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
         solverChoice.use_real_bcs )
    {
        int ioproc = ParallelDescriptor::IOProcessorNumber();  // I/O rank
        int num_time;
        int num_var;
        Vector<Box> bx_v;
        if (ParallelDescriptor::IOProcessor()) {
            // Open header file and read from it
            std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
            bdy_h_file >> num_time;
            bdy_h_file >> num_var;
            bdy_h_file >> start_bdy_time;
            bdy_h_file >> bdy_time_interval;
            bdy_h_file >> real_width;
            bx_v.resize(4*num_var);
            for (int ivar(0); ivar<num_var; ++ivar) {
                bdy_h_file >> bx_v[4*ivar  ];
                bdy_h_file >> bx_v[4*ivar+1];
                bdy_h_file >> bx_v[4*ivar+2];
                bdy_h_file >> bx_v[4*ivar+3];
            }
 
            // IO size the FABs
            bdy_data_xlo.resize(num_time);
            bdy_data_xhi.resize(num_time);
            bdy_data_ylo.resize(num_time);
            bdy_data_yhi.resize(num_time);
            for (int itime(0); itime<num_time; ++itime) {
                bdy_data_xlo[itime].resize(num_var);
                bdy_data_xhi[itime].resize(num_var);
                bdy_data_ylo[itime].resize(num_var);
                bdy_data_yhi[itime].resize(num_var);
                for (int ivar(0); ivar<num_var; ++ivar) {
                    bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
                    bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
                    bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
                    bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
                }
            }
 
            // Open data file and read from it
            std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
            for (int itime(0); itime<num_time; ++itime) {
                for (int ivar(0); ivar<num_var; ++ivar) {
                    bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
                }
            }
        } // IO
 
        // Broadcast the data
        ParallelDescriptor::Barrier();
        ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
        ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
        ParallelDescriptor::Bcast(&real_width,1,ioproc);
        ParallelDescriptor::Bcast(&num_time,1,ioproc);
        ParallelDescriptor::Bcast(&num_var,1,ioproc);
 
        // Everyone size their boxes
        bx_v.resize(4*num_var);
 
        ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
 
        // Everyone but IO size their FABs
        if (!ParallelDescriptor::IOProcessor()) {
          bdy_data_xlo.resize(num_time);
          bdy_data_xhi.resize(num_time);
          bdy_data_ylo.resize(num_time);
          bdy_data_yhi.resize(num_time);
          for (int itime(0); itime<num_time; ++itime) {
            bdy_data_xlo[itime].resize(num_var);
            bdy_data_xhi[itime].resize(num_var);
            bdy_data_ylo[itime].resize(num_var);
            bdy_data_yhi[itime].resize(num_var);
            for (int ivar(0); ivar<num_var; ++ivar) {
              bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
              bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
              bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
              bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
            }
          }
        }
 
        for (int itime(0); itime<num_time; ++itime) {
            for (int ivar(0); ivar<num_var; ++ivar) {
                ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
            }
        }
    } // init_type == WRFInput or Metgrid
#endif
#endif
}

Here is the call graph for this function:

ReadCheckpointFileSurfaceLayer()

void ERF::ReadCheckpointFileSurfaceLayer

(

)

ERF function for reading additional data for MOST from a checkpoint file during restart.

This is called after the ABLMost object is instantiated.

{
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        amrex::Print() << "Reading MOST variables" << std::endl;
 
        IntVect ng(1,1,0);
        MultiFab  m_var(ba2d[lev],dmap[lev],1,ng);
        MultiFab* dst = nullptr;
 
        // U*
        std::string UstarFileName(restart_chkfile + "/Level_0/Ustar_H");
        if (amrex::FileExists(UstarFileName)) {
            dst = m_SurfaceLayer->get_u_star(lev);
            VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Ustar"));
            MultiFab::Copy(*dst,m_var,0,0,1,ng);
        }
 
        // W*
        std::string WstarFileName(restart_chkfile + "/Level_0/Wstar_H");
        if (amrex::FileExists(WstarFileName)) {
            dst = m_SurfaceLayer->get_w_star(lev);
            VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Wstar"));
            MultiFab::Copy(*dst,m_var,0,0,1,ng);
        }
 
        // T*
        std::string TstarFileName(restart_chkfile + "/Level_0/Tstar_H");
        if (amrex::FileExists(TstarFileName)) {
            dst = m_SurfaceLayer->get_t_star(lev);
            VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Tstar"));
            MultiFab::Copy(*dst,m_var,0,0,1,ng);
        }
 
        // Q*
        std::string QstarFileName(restart_chkfile + "/Level_0/Qstar_H");
        if (amrex::FileExists(QstarFileName)) {
            dst = m_SurfaceLayer->get_q_star(lev);
            VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qstar"));
            MultiFab::Copy(*dst,m_var,0,0,1,ng);
        }
 
        // Olen
        std::string OlenFileName(restart_chkfile + "/Level_0/Olen_H");
        if (amrex::FileExists(OlenFileName)) {
            dst = m_SurfaceLayer->get_olen(lev);
            VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Olen"));
            MultiFab::Copy(*dst,m_var,0,0,1,ng);
        }
 
        // Qsurf
        std::string QsurfFileName(restart_chkfile + "/Level_0/Qsurf_H");
        if (amrex::FileExists(QsurfFileName)) {
            dst = m_SurfaceLayer->get_q_surf(lev);
            VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qsurf"));
            MultiFab::Copy(*dst,m_var,0,0,1,ng);
        }
 
        // PBLH
        std::string PBLHFileName(restart_chkfile + "/Level_0/PBLH_H");
        if (amrex::FileExists(PBLHFileName)) {
            dst = m_SurfaceLayer->get_pblh(lev);
            VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PBLH"));
            MultiFab::Copy(*dst,m_var,0,0,1,ng);
        }
 
        // Z0
        std::string Z0FileName(restart_chkfile + "/Level_0/Z0_H");
        if (amrex::FileExists(Z0FileName)) {
            dst = m_SurfaceLayer->get_z0(lev);
            VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z0"));
            MultiFab::Copy(*dst,m_var,0,0,1,ng);
        }
    }
}

ReadParameters()

void ERF::ReadParameters ( )

private

{
    std::string prob_name = "Unknown";
    ParmParse pp_pn("erf"); pp_pn.queryAdd("prob_name", prob_name);
    Print() << "Problem name (from inputs file) is " << prob_name << std::endl;
 
    ParmParse pp(pp_prefix);
    ParmParse pp_amr("amr");
    {
        pp.query("regrid_level_0_on_restart", regrid_level_0_on_restart);
        pp.query("regrid_int", regrid_int);
        pp.query("check_file", check_file);
 
        // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
        //    for those or "erf.restart" / "erf.m_check_int" with the former taking
        //    precedence if both are specified
        pp.query("check_int", m_check_int);
        pp.query("check_per", m_check_per);
        pp_amr.query("check_int", m_check_int);
        pp_amr.query("check_per", m_check_per);
 
        pp.query("restart", restart_chkfile);
        pp_amr.query("restart", restart_chkfile);
 
        // Verbosity
        pp.query("v", verbose);
        pp.query("mg_v", mg_verbose);
        pp.query("use_fft", use_fft);
#ifndef ERF_USE_FFT
        if (use_fft) {
            Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
        }
#endif
 
        // Check for NaNs?
        pp.query("check_for_nans", check_for_nans);
 
        // Frequency of diagnostic output
        pp.query("sum_interval", sum_interval);
        pp.query("sum_period"  , sum_per);
 
        pp.query("pert_interval", pert_interval);
 
        // Time step controls
        pp.query("cfl", cfl);
        pp.query("substepping_cfl", sub_cfl);
        pp.query("init_shrink", init_shrink);
        pp.query("change_max", change_max);
        pp.query("dt_max_initial", dt_max_initial);
        pp.query("dt_max", dt_max);
 
        fixed_dt.resize(max_level+1,-1.);
        fixed_fast_dt.resize(max_level+1,-1.);
 
        pp.query("fixed_dt", fixed_dt[0]);
        pp.query("fixed_fast_dt", fixed_fast_dt[0]);
 
        int nlevs_max = max_level + 1;
        istep.resize(nlevs_max, 0);
        nsubsteps.resize(nlevs_max, 1);
        // This is the default
        for (int lev = 1; lev <= max_level; ++lev) {
            nsubsteps[lev] = MaxRefRatio(lev-1);
        }
 
        if (max_level > 0) {
            ParmParse pp_erf("erf");
            int count = pp_erf.countval("dt_ref_ratio");
            if (count > 0) {
                Vector<int> nsub;
                nsub.resize(nlevs_max, 0);
                if (count == 1) {
                    pp_erf.queryarr("dt_ref_ratio", nsub, 0, 1);
                    for (int lev = 1; lev <= max_level; ++lev) {
                        nsubsteps[lev] = nsub[0];
                    }
                } else {
                    pp_erf.queryarr("dt_ref_ratio", nsub, 0, max_level);
                    for (int lev = 1; lev <= max_level; ++lev) {
                        nsubsteps[lev] = nsub[lev-1];
                    }
                }
            }
        }
 
        // Make sure we do this after we have defined nsubsteps above
        for (int lev = 1; lev <= max_level; lev++)
        {
            fixed_dt[lev]      = fixed_dt[lev-1]      / static_cast<Real>(nsubsteps[lev]);
            fixed_fast_dt[lev] = fixed_fast_dt[lev-1] / static_cast<Real>(nsubsteps[lev]);
        }
 
        pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
 
        // We use this to keep track of how many boxes we read in from WRF initialization
        num_files_at_level.resize(max_level+1,0);
 
        // We use this to keep track of how many boxes are specified thru the refinement indicators
        num_boxes_at_level.resize(max_level+1,0);
            boxes_at_level.resize(max_level+1);
 
        // We always have exactly one file at level 0
        num_boxes_at_level[0] = 1;
        boxes_at_level[0].resize(1);
        boxes_at_level[0][0] = geom[0].Domain();
 
#ifdef ERF_USE_NETCDF
        nc_init_file.resize(max_level+1);
        have_read_nc_init_file.resize(max_level+1);
 
        // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
        //        but we always have exactly one file at level 0
        for (int lev = 0; lev <= max_level; lev++) {
            const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
            if (pp.contains(nc_file_names.c_str())) {
                int num_files = pp.countval(nc_file_names.c_str());
                num_files_at_level[lev] = num_files;
                nc_init_file[lev].resize(num_files);
                have_read_nc_init_file[lev].resize(num_files);
                pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
                for (int j = 0; j < num_files; j++) {
                    Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
                    have_read_nc_init_file[lev][j] = 0;
                } // j
            } // if pp.contains
        } // lev
 
        // NetCDF wrfbdy lateral boundary file
        if (pp.query("nc_bdy_file", nc_bdy_file)) {
            Print() << "Reading NC bdy file name " << nc_bdy_file << std::endl;
        }
 
        // NetCDF wrflow lateral boundary file
        if (pp.query("nc_low_file", nc_low_file)) {
            Print() << "Reading NC low file name " << nc_low_file << std::endl;
        }
 
#endif
 
        // Options for vertical interpolation of met_em*.nc data.
        pp.query("metgrid_debug_quiescent",  metgrid_debug_quiescent);
        pp.query("metgrid_debug_isothermal", metgrid_debug_isothermal);
        pp.query("metgrid_debug_dry",        metgrid_debug_dry);
        pp.query("metgrid_debug_psfc",       metgrid_debug_psfc);
        pp.query("metgrid_debug_msf",        metgrid_debug_msf);
        pp.query("metgrid_interp_theta",     metgrid_interp_theta);
        pp.query("metgrid_basic_linear",     metgrid_basic_linear);
        pp.query("metgrid_use_below_sfc",    metgrid_use_below_sfc);
        pp.query("metgrid_use_sfc",          metgrid_use_sfc);
        pp.query("metgrid_retain_sfc",       metgrid_retain_sfc);
        pp.query("metgrid_proximity",        metgrid_proximity);
        pp.query("metgrid_order",            metgrid_order);
        pp.query("metgrid_force_sfc_k",      metgrid_force_sfc_k);
 
        // Set default to FullState for now ... later we will try Perturbation
        interpolation_type = StateInterpType::FullState;
        pp.query_enum_case_insensitive("interpolation_type"  ,interpolation_type);
 
        PlotFileType plotfile3d_type_temp = PlotFileType::None;
        pp.query_enum_case_insensitive("plotfile_type"  ,plotfile3d_type_temp);
        pp.query_enum_case_insensitive("plotfile_type_1",plotfile3d_type_1);
        pp.query_enum_case_insensitive("plotfile_type_2",plotfile3d_type_2);
 
        PlotFileType plotfile2d_type_temp = PlotFileType::None;
        pp.query_enum_case_insensitive("plotfile2d_type"  ,plotfile2d_type_temp);
        pp.query_enum_case_insensitive("plotfile2d_type_1",plotfile2d_type_1);
        pp.query_enum_case_insensitive("plotfile2d_type_2",plotfile2d_type_2);
        //
        // This option is for backward consistency -- if only plotfile_type is set,
        //     then it will be used for both 1 and 2 if and only if they are not set
        //
        // Default is native amrex if no type is specified
        //
        if (plotfile3d_type_temp == PlotFileType::None) {
            if (plotfile3d_type_1 == PlotFileType::None) {
                plotfile3d_type_1  = PlotFileType::Amrex;
            }
            if (plotfile3d_type_2 == PlotFileType::None) {
                plotfile3d_type_2  = PlotFileType::Amrex;
            }
        } else {
            if (plotfile3d_type_1 == PlotFileType::None) {
                plotfile3d_type_1  = plotfile3d_type_temp;
            } else {
                Abort("You must set either plotfile_type or plotfile_type_1, not both");
            }
            if (plotfile3d_type_2 == PlotFileType::None) {
                plotfile3d_type_2  = plotfile3d_type_temp;
            } else {
                Abort("You must set either plotfile_type or plotfile_type_2, not both");
            }
        }
        if (plotfile2d_type_temp == PlotFileType::None) {
            if (plotfile2d_type_1 == PlotFileType::None) {
                plotfile2d_type_1  = PlotFileType::Amrex;
            }
            if (plotfile2d_type_2 == PlotFileType::None) {
                plotfile2d_type_2  = PlotFileType::Amrex;
            }
        } else {
            if (plotfile2d_type_1 == PlotFileType::None) {
                plotfile2d_type_1  = plotfile2d_type_temp;
            } else {
                Abort("You must set either plotfile2d_type or plotfile2d_type_1, not both");
            }
            if (plotfile2d_type_2 == PlotFileType::None) {
                plotfile2d_type_2  = plotfile2d_type_temp;
            } else {
                Abort("You must set either plotfile2d_type or plotfile2d_type_2, not both");
            }
        }
#ifndef ERF_USE_NETCDF
        if (plotfile3d_type_1 == PlotFileType::Netcdf ||
            plotfile3d_type_2 == PlotFileType::Netcdf ||
            plotfile2d_type_1 == PlotFileType::Netcdf ||
            plotfile2d_type_2 == PlotFileType::Netcdf) {
            Abort("Plotfile type = Netcdf is not allowed without USE_NETCDF = TRUE");
        }
#endif
 
        pp.query("plot_file_1"  ,   plot3d_file_1);
        pp.query("plot_file_2"  ,   plot3d_file_2);
        pp.query("plot2d_file_1",   plot2d_file_1);
        pp.query("plot2d_file_2",   plot2d_file_2);
 
        pp.query("plot_int_1"   , m_plot3d_int_1);
        pp.query("plot_int_2"   , m_plot3d_int_2);
        pp.query("plot_per_1"   , m_plot3d_per_1);
        pp.query("plot_per_2"   , m_plot3d_per_2);
 
        pp.query("plot2d_int_1" , m_plot2d_int_1);
        pp.query("plot2d_int_2" , m_plot2d_int_2);
        pp.query("plot2d_per_1",  m_plot2d_per_1);
        pp.query("plot2d_per_2",  m_plot2d_per_2);
 
        pp.query("subvol_file",   subvol_file);
 
        // Should we use format like plt1970-01-01_00:00:00.000000 (if true) or plt00001 (if false)
        pp.query("use_real_time_in_pltname", use_real_time_in_pltname);
 
        // If use_real_time_in_pltname is false, how many digits should we use for the timestep?
        pp.query("file_name_digits", file_name_digits);
 
        // Default if subvol_int not specified
        m_subvol_int.resize(1); m_subvol_int[0] = -1;
        m_subvol_per.resize(1); m_subvol_per[0] = -1.0;
        last_subvol_step.resize(1);
        last_subvol_time.resize(1);
 
        int nsi = pp.countval("subvol_int");
        int nsr = pp.countval("subvol_per");
 
        // We must specify only subvol_int OR subvol_per
        AMREX_ALWAYS_ASSERT (!(nsi > 0 && nsr > 0));
 
        int nsub = -1;
        if (nsi > 0 || nsr > 0) {
            ParmParse pp_sv("erf.subvol");
            int n1 = pp_sv.countval("origin"); int n2 = pp_sv.countval("nxnynz"); int n3 = pp_sv.countval("dxdydz");
            if (n1 != n2 || n1 != n3 || n2 != n3) {
                Abort("WriteSubvolume: must have same number of entries in origin, nxnynz, and dxdydz.");
            }
            if ( n1%AMREX_SPACEDIM != 0) {
                Abort("WriteSubvolume: origin, nxnynz, and dxdydz must have multiples of AMReX_SPACEDIM");
            }
            nsub = n1/AMREX_SPACEDIM;
            m_subvol_int.resize(nsub);
            last_subvol_step.resize(nsub);
            last_subvol_time.resize(nsub);
            m_subvol_int.resize(nsub);
            m_subvol_per.resize(nsub);
        }
 
        if (nsi > 0) {
            for (int i = 1; i < nsub; i++) m_subvol_per[i] = -1.0;
            if ( nsi == 1) {
                m_subvol_int[0] = -1;
                pp.get("subvol_int" , m_subvol_int[0]);
            } else if ( nsi == nsub) {
                pp.getarr("subvol_int" , m_subvol_int);
            } else {
                Abort("There must either be a single value of subvol_int or one for every subdomain");
            }
        }
 
        if (nsr > 0) {
            for (int i = 1; i < nsub; i++) m_subvol_int[i] = -1.0;
            if ( nsr == 1) {
                m_subvol_per[0] = -1.0;
                pp.get("subvol_per" , m_subvol_per[0]);
            } else if ( nsr == nsub) {
                pp.getarr("subvol_per" , m_subvol_per);
            } else {
                Abort("There must either be a single value of subvol_per or one for every subdomain");
            }
        }
 
        setSubVolVariables("subvol_sampling_vars",subvol3d_var_names);
 
        pp.query("expand_plotvars_to_unif_rr",m_expand_plotvars_to_unif_rr);
 
        pp.query("plot_face_vels",m_plot_face_vels);
 
        if ( (m_plot3d_int_1 > 0 && m_plot3d_per_1 > 0) ||
             (m_plot3d_int_2 > 0 && m_plot3d_per_2 > 0.) ) {
            Abort("Must choose only one of plot_int or plot_per");
        }
        if ( (m_plot2d_int_1 > 0 && m_plot2d_per_1 > 0) ||
             (m_plot2d_int_2 > 0 && m_plot2d_per_2 > 0.) ) {
            Abort("Must choose only one of plot_int or plot_per");
        }
 
        pp.query("profile_int", profile_int);
        pp.query("destag_profiles", destag_profiles);
 
        pp.query("plot_lsm", plot_lsm);
#ifdef ERF_USE_RRTMGP
        pp.query("plot_rad", plot_rad);
#endif
        pp.query("profile_rad_int", rad_datalog_int);
 
        pp.query("output_1d_column", output_1d_column);
        pp.query("column_per", column_per);
        pp.query("column_interval", column_interval);
        pp.query("column_loc_x", column_loc_x);
        pp.query("column_loc_y", column_loc_y);
        pp.query("column_file_name", column_file_name);
 
        // Sampler output frequency
        pp.query("line_sampling_per", line_sampling_per);
        pp.query("line_sampling_interval", line_sampling_interval);
        pp.query("plane_sampling_per", plane_sampling_per);
        pp.query("plane_sampling_interval", plane_sampling_interval);
 
        // Specify information about outputting planes of data
        pp.query("output_bndry_planes", output_bndry_planes);
        pp.query("bndry_output_planes_interval", bndry_output_planes_interval);
        pp.query("bndry_output_planes_per", bndry_output_planes_per);
        pp.query("bndry_output_start_time", bndry_output_planes_start_time);
 
        // Specify whether ingest boundary planes of data
        pp.query("input_bndry_planes", input_bndry_planes);
 
        // Query the set and total widths for wrfbdy interior ghost cells
        pp.query("real_width", real_width);
        pp.query("real_set_width", real_set_width);
 
        // If using real boundaries, do we extrapolate w (or set to 0)
        pp.query("real_extrap_w", real_extrap_w);
 
        // Query the set and total widths for crse-fine interior ghost cells
        pp.query("cf_width", cf_width);
        pp.query("cf_set_width", cf_set_width);
 
        // AmrMesh iterate on grids?
        bool iterate(true);
        pp_amr.query("iterate_grids",iterate);
        if (!iterate) SetIterateToFalse();
    }
 
#ifdef ERF_USE_PARTICLES
    readTracersParams();
#endif
 
    solverChoice.init_params(max_level,pp_prefix);
 
    {
        ParmParse pp_no_prefix;  // Traditionally, max_step and stop_time do not have prefix.
        pp_no_prefix.query("max_step", max_step);
        if (max_step < 0) {
            max_step = std::numeric_limits<int>::max();
        }
 
        std::string start_datetime, stop_datetime;
        if (pp_no_prefix.query("start_datetime", start_datetime)) {
            if (start_datetime.length() == 16) { // YYYY-MM-DD HH:MM
                start_datetime += ":00"; // add seconds
            }
            if (start_datetime.length() != 19) {
                Print() << "Got start_datetime = \"" << start_datetime
                    << "\", format should be " << datetime_format << std::endl;
                exit(0);
            }
            start_time = getEpochTime(start_datetime, datetime_format);
 
#ifdef ERF_USE_NETCDF
            if (solverChoice.init_type == InitType::WRFInput) {
                // This is the start time as written in the wrfinput file
                Real start_time_from_wrfinput = read_start_time_from_wrfinput(0, nc_init_file[0][0]);
                if (start_time != start_time_from_wrfinput) {
                    amrex::Print() << "start_datetime from inputs file = "       << start_time <<
                                      " does not match SIMULATION START DATE from wrfinput = " <<
                                       start_time_from_wrfinput << std::endl;
                    amrex::Abort();
                }
            }
#endif
            Print() << "Start datetime   : " << start_datetime << std::endl;
 
            use_datetime = true;
 
        } else {
 
#ifdef ERF_USE_NETCDF
            if (solverChoice.init_type == InitType::WRFInput) {
                // This is the start time as written in the wrfinput file
                Real start_time_from_wrfinput = read_start_time_from_wrfinput(0, nc_init_file[0][0]);
                start_time = start_time_from_wrfinput;
 
                use_datetime = true;
 
                if (pp_no_prefix.query("start_time", start_time)) {
                    amrex::Print() << "start_time should not be set from inputs file; we are reading SIMULATION START DATE from wrfinput" << std::endl;
                    amrex::Abort();
                }
            }
#endif
        }
 
        if (pp_no_prefix.query("stop_datetime", stop_datetime)) {
            if (stop_datetime.length() == 16) { // YYYY-MM-DD HH:MM
                stop_datetime += ":00"; // add seconds
            }
            if (stop_datetime.length() != 19) {
                Print() << "Got stop_datetime = \"" << stop_datetime
                    << "\", format should be " << datetime_format << std::endl;
                exit(0);
            }
 
            stop_time = getEpochTime(stop_datetime, datetime_format);
            Print() << "Stop  datetime : " << start_datetime << std::endl;
 
        } else {
 
            if (pp_no_prefix.query("stop_time", stop_time)) {
                Print() << "Maximum simulation length based on stop_time: " << stop_time << " s (elapsed) " << std::endl;
                amrex::Print() <<" Adding stop time " << stop_time << " to start_time " << start_time << std::endl;
                stop_time += start_time;
            }
        }
    }
 
#ifndef ERF_USE_NETCDF
    AMREX_ALWAYS_ASSERT_WITH_MESSAGE(( (solverChoice.init_type != InitType::WRFInput) &&
                                       (solverChoice.init_type != InitType::Metgrid ) &&
                                       (solverChoice.init_type != InitType::NCFile  )  ),
                                     "init_type cannot be 'WRFInput', 'MetGrid' or 'NCFile' if we don't build with netcdf!");
#endif
 
    // Query the canopy model file name
    std::string forestfile;
    solverChoice.do_forest_drag = pp.query("forest_file", forestfile);
    if (solverChoice.do_forest_drag) {
        for (int lev = 0; lev <= max_level; ++lev) {
            m_forest_drag[lev] = std::make_unique<ForestDrag>(forestfile);
        }
    }
 
    // If init from WRFInput or Metgrid make sure a valid file name is present at level 0.
    // We allow for the possibility that finer levels may use native refinement rather than reading from a file
    if ((solverChoice.init_type == InitType::WRFInput) ||
        (solverChoice.init_type == InitType::Metgrid)  ||
        (solverChoice.init_type == InitType::NCFile) ) {
        int num_files = nc_init_file[0].size();
        AMREX_ALWAYS_ASSERT_WITH_MESSAGE(num_files>0, "A file name must be present at level 0 for init type WRFInput, Metgrid or NCFile.");
        for (int j = 0; j < num_files; j++) {
            AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!nc_init_file[0][j].empty(), "Valid file name must be present at level 0 for init type WRFInput, Metgrid or NCFile.");
        } //j
    } // InitType
 
    // What type of land surface model to use
    // NOTE: Must be checked after init_params
    if (solverChoice.lsm_type == LandSurfaceType::SLM) {
        lsm.SetModel<SLM>();
        Print() << "SLM land surface model!\n";
    } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
        lsm.SetModel<MM5>();
        Print() << "MM5 land surface model!\n";
#ifdef ERF_USE_NOAHMP
    } else if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
        lsm.SetModel<NOAHMP>();
        Print() << "Noah-MP land surface model!\n";
#endif
    } else if (solverChoice.lsm_type == LandSurfaceType::None) {
        lsm.SetModel<NullSurf>();
        Print() << "Null land surface model!\n";
    } else {
        Abort("Dont know this LandSurfaceType!") ;
    }
 
    if (verbose > 0) {
        solverChoice.display(max_level,pp_prefix);
    }
 
    ParameterSanityChecks();
}

Here is the call graph for this function:

refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )

private

Function to define the refinement criteria based on user input

{
    if (max_level > 0)
    {
        ParmParse pp(pp_prefix);
        Vector<std::string> refinement_indicators;
        pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
 
        for (int i=0; i<refinement_indicators.size(); ++i)
        {
            std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
 
            ParmParse ppr(ref_prefix);
            RealBox realbox;
            int lev_for_box;
 
            int num_real_lo = ppr.countval("in_box_lo");
            int num_indx_lo = ppr.countval("in_box_lo_indices");
            int num_real_hi = ppr.countval("in_box_hi");
            int num_indx_hi = ppr.countval("in_box_hi_indices");
 
            AMREX_ALWAYS_ASSERT(num_real_lo == num_real_hi);
            AMREX_ALWAYS_ASSERT(num_indx_lo == num_indx_hi);
 
            if ( !((num_real_lo >= AMREX_SPACEDIM-1 && num_indx_lo == 0) ||
                   (num_indx_lo >= AMREX_SPACEDIM-1 && num_real_lo == 0) ||
                   (num_indx_lo ==              0   && num_real_lo == 0)) )
            {
                amrex::Abort("Must only specify box for refinement using real OR index space");
            }
 
            if (num_real_lo > 0) {
                std::vector<Real> rbox_lo(3), rbox_hi(3);
                ppr.get("max_level",lev_for_box);
                if (lev_for_box <= max_level)
                {
                    if (n_error_buf[0] != IntVect::TheZeroVector()) {
                        amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
                    }
 
                    const Real* plo = geom[lev_for_box].ProbLo();
                    const Real* phi = geom[lev_for_box].ProbHi();
 
                    ppr.getarr("in_box_lo",rbox_lo,0,num_real_lo);
                    ppr.getarr("in_box_hi",rbox_hi,0,num_real_hi);
 
                    if (rbox_lo[0] < plo[0]) rbox_lo[0] = plo[0];
                    if (rbox_lo[1] < plo[1]) rbox_lo[1] = plo[1];
                    if (rbox_hi[0] > phi[0]) rbox_hi[0] = phi[0];
                    if (rbox_hi[1] > phi[1]) rbox_hi[1] = phi[1];
                    if (num_real_lo < AMREX_SPACEDIM) {
                        rbox_lo[2] = plo[2];
                        rbox_hi[2] = phi[2];
                    }
 
                    realbox = RealBox(&(rbox_lo[0]),&(rbox_hi[0]));
 
                    Print() << "Realbox read in and intersected laterally with domain is " << realbox << std::endl;
 
                    num_boxes_at_level[lev_for_box] += 1;
 
                    int ilo, jlo, klo;
                    int ihi, jhi, khi;
                    const auto* dx  = geom[lev_for_box].CellSize();
                    ilo = static_cast<int>((rbox_lo[0] - plo[0])/dx[0]);
                    jlo = static_cast<int>((rbox_lo[1] - plo[1])/dx[1]);
                    ihi = static_cast<int>((rbox_hi[0] - plo[0])/dx[0]-1);
                    jhi = static_cast<int>((rbox_hi[1] - plo[1])/dx[1]-1);
                    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                        // Search for k indices corresponding to nominal grid
                        // AGL heights
                        const Box& domain = geom[lev_for_box].Domain();
                        klo = domain.smallEnd(2) - 1;
                        khi = domain.smallEnd(2) - 1;
 
                        if (rbox_lo[2] <= zlevels_stag[lev_for_box][domain.smallEnd(2)])
                        {
                            klo = domain.smallEnd(2);
                        }
                        else
                        {
                            for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
                                if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
                                    klo = k-1;
                                    break;
                                }
                            }
                        }
                        AMREX_ASSERT(klo >= domain.smallEnd(2));
 
                        if (rbox_hi[2] >= zlevels_stag[lev_for_box][domain.bigEnd(2)+1])
                        {
                            khi = domain.bigEnd(2);
                        }
                        else
                        {
                            for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
                                if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
                                    khi = k-1;
                                    break;
                                }
                            }
                        }
                        AMREX_ASSERT((khi <= domain.bigEnd(2)) && (khi > klo));
 
                        // Need to update realbox because tagging is based on
                        // the initial _un_deformed grid
                        realbox = RealBox(plo[0]+ ilo   *dx[0], plo[1]+ jlo   *dx[1], plo[2]+ klo   *dx[2],
                                          plo[0]+(ihi+1)*dx[0], plo[1]+(jhi+1)*dx[1], plo[2]+(khi+1)*dx[2]);
                    } else {
                        klo = static_cast<int>((rbox_lo[2] - plo[2])/dx[2]);
                        khi = static_cast<int>((rbox_hi[2] - plo[2])/dx[2]-1);
                    }
 
                    Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
                    if ( (ilo%ref_ratio[lev_for_box-1][0] != 0) || ((ihi+1)%ref_ratio[lev_for_box-1][0] != 0) ||
                         (jlo%ref_ratio[lev_for_box-1][1] != 0) || ((jhi+1)%ref_ratio[lev_for_box-1][1] != 0) ||
                         (klo%ref_ratio[lev_for_box-1][2] != 0) || ((khi+1)%ref_ratio[lev_for_box-1][2] != 0) )
                    {
                        amrex::Print() << "Box : " << bx << std::endl;
                        amrex::Print() << "RealBox : " << realbox << std::endl;
                        amrex::Print() << "ilo, ihi+1, jlo, jhi+1, klo, khi+1 by ref_ratio : "
                                       << ilo%ref_ratio[lev_for_box-1][0] << " " << (ihi+1)%ref_ratio[lev_for_box-1][0] << " "
                                       << jlo%ref_ratio[lev_for_box-1][1] << " " << (jhi+1)%ref_ratio[lev_for_box-1][1] << " "
                                       << klo%ref_ratio[lev_for_box-1][2] << " " << (khi+1)%ref_ratio[lev_for_box-1][2] << std::endl;
                        amrex::Error("Fine box is not legit with this ref_ratio");
                    }
                    boxes_at_level[lev_for_box].push_back(bx);
                    Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                } // lev
 
                if (solverChoice.init_type == InitType::WRFInput) {
                    if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                        amrex::Error("Number of boxes doesn't match number of input files");
 
                    }
                }
 
            } else if (num_indx_lo > 0) {
 
                std::vector<int> box_lo(3), box_hi(3);
                ppr.get("max_level",lev_for_box);
                if (lev_for_box <= max_level)
                {
                    if (n_error_buf[0] != IntVect::TheZeroVector()) {
                        amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
                    }
 
                    ppr.getarr("in_box_lo_indices",box_lo,0,AMREX_SPACEDIM);
                    ppr.getarr("in_box_hi_indices",box_hi,0,AMREX_SPACEDIM);
 
                    Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
                    amrex::Print() << "BOX " << bx << std::endl;
 
                    const auto* dx  = geom[lev_for_box].CellSize();
                    const Real* plo = geom[lev_for_box].ProbLo();
                    realbox = RealBox(plo[0]+ box_lo[0]   *dx[0], plo[1]+ box_lo[1]   *dx[1], plo[2]+ box_lo[2]   *dx[2],
                                      plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
 
                    Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
                    num_boxes_at_level[lev_for_box] += 1;
 
                    if ( (box_lo[0]%ref_ratio[lev_for_box-1][0] != 0) || ((box_hi[0]+1)%ref_ratio[lev_for_box-1][0] != 0) ||
                         (box_lo[1]%ref_ratio[lev_for_box-1][1] != 0) || ((box_hi[1]+1)%ref_ratio[lev_for_box-1][1] != 0) ||
                         (box_lo[2]%ref_ratio[lev_for_box-1][2] != 0) || ((box_hi[2]+1)%ref_ratio[lev_for_box-1][2] != 0) )
                         amrex::Error("Fine box is not legit with this ref_ratio");
                    boxes_at_level[lev_for_box].push_back(bx);
                    Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                } // lev
 
                if (solverChoice.init_type == InitType::WRFInput) {
                    if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                        amrex::Error("Number of boxes doesn't match number of input files");
 
                    }
                }
            }
 
            AMRErrorTagInfo info;
 
            if (realbox.ok()) {
                info.SetRealBox(realbox);
            }
            if (ppr.countval("start_time") > 0) {
                Real ref_min_time; ppr.get("start_time",ref_min_time);
                info.SetMinTime(ref_min_time);
            }
            if (ppr.countval("end_time") > 0) {
                Real ref_max_time; ppr.get("end_time",ref_max_time);
                info.SetMaxTime(ref_max_time);
            }
            if (ppr.countval("max_level") > 0) {
                int ref_max_level; ppr.get("max_level",ref_max_level);
                info.SetMaxLevel(ref_max_level);
            }
 
            if (ppr.countval("value_greater")) {
                int num_val = ppr.countval("value_greater");
                Vector<Real> value(num_val);
                ppr.getarr("value_greater",value,0,num_val);
                std::string field; ppr.get("field_name",field);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
            }
            else if (ppr.countval("value_less")) {
                int num_val = ppr.countval("value_less");
                Vector<Real> value(num_val);
                ppr.getarr("value_less",value,0,num_val);
                std::string field; ppr.get("field_name",field);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
            }
            else if (ppr.countval("adjacent_difference_greater")) {
                int num_val = ppr.countval("adjacent_difference_greater");
                Vector<Real> value(num_val);
                ppr.getarr("adjacent_difference_greater",value,0,num_val);
                std::string field; ppr.get("field_name",field);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
            }
            else if (realbox.ok())
            {
                ref_tags.push_back(AMRErrorTag(info));
            } else if (refinement_indicators[i] != "storm_tracker") {
                Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
            }
        } // loop over criteria
    } // if max_level > 0
}

Here is the call graph for this function:

remake_zphys()

void ERF::remake_zphys	(	int	lev,
		amrex::Real	time,
		std::unique_ptr< amrex::MultiFab > &	temp_zphys_nd
	)

{
    if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
    {
        if (lev > 0)
        {
            //
            // First interpolate from coarser level
            // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
            //       have been pre-filled - this includes ghost cells both inside and outside
            //       the domain
            //
            InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
                                  IntVect(0,0,0), // do NOT fill ghost cells outside the domain
                                  *z_phys_nd[lev-1], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &node_bilinear_interp,
                                  domain_bcs_type, BCVars::cons_bc);
 
            // This recomputes the fine values using the bottom terrain at the fine resolution,
            //    and also fills values of z_phys_nd outside the domain
            make_terrain_fitted_coords(lev,geom[lev],*temp_zphys_nd,zlevels_stag[lev],phys_bc_type);
 
            std::swap(temp_zphys_nd, z_phys_nd[lev]);
        } // lev > 0
    } else {
        if (lev > 0)
        {
            //
            // First interpolate from coarser level
            // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
            //       have been pre-filled - this includes ghost cells both inside and outside
            //       the domain
            //
            InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
                                  z_phys_nd[lev]->nGrowVect(), // DO fill ghost cells outside the domain
                                  *z_phys_nd[lev-1], 0, 0, 1,
                                  geom[lev-1], geom[lev],
                                  refRatio(lev-1), &node_bilinear_interp,
                                  domain_bcs_type, BCVars::cons_bc);
 
            std::swap(temp_zphys_nd, z_phys_nd[lev]);
        } // lev > 0
    }
 
    if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
        solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
        //
        // This assumes we have already remade the EBGeometry
        //
        terrain_blanking[lev]->setVal(1.0);
        MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, z_phys_nd[lev]->nGrowVect());
    }
 
    // Compute the min dz and pass to the micro model
    Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
    micro->Set_dzmin(lev, dzmin);
}

Here is the call graph for this function:

RemakeLevel()

void ERF::RemakeLevel ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

override

{
    if (verbose) {
        amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
    }
 
    AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::MovingFittedMesh);
 
    BoxArray            ba_old(vars_new[lev][Vars::cons].boxArray());
    DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
 
    if (verbose) {
        amrex::Print() <<"               OLD BA AT LEVEL " << lev << " " << ba_old << std::endl;
    }
 
    //
    // Re-define subdomain at this level within the domain such that
    // 1) all boxes in a given subdomain are "connected"
    // 2) no boxes in a subdomain touch any boxes in any other subdomain
    //
    if (solverChoice.anelastic[lev] == 1) {
        make_subdomains(ba.simplified_list(), subdomains[lev]);
    }
 
    int     ncomp_cons  = vars_new[lev][Vars::cons].nComp();
    IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
 
    int ngrow_vels = ComputeGhostCells(solverChoice);
 
    Vector<MultiFab> temp_lev_new(Vars::NumTypes);
    Vector<MultiFab> temp_lev_old(Vars::NumTypes);
    MultiFab temp_base_state;
 
    std::unique_ptr<MultiFab> temp_zphys_nd;
 
    //********************************************************************************************
    // This allocates all kinds of things, including but not limited to: solution arrays,
    //      terrain arrays and metrics, and base state.
    // *******************************************************************************************
    init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old, temp_base_state, temp_zphys_nd);
 
    // ********************************************************************************************
    // Build the data structures for terrain-related quantities
    // ********************************************************************************************
    if ( solverChoice.terrain_type == TerrainType::EB ||
         solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing)
    {
        const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
        const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
        if (solverChoice.terrain_type == TerrainType::EB) {
            eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
        } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                   solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
            eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
        }
    }
    remake_zphys(lev, time, temp_zphys_nd);
    update_terrain_arrays(lev);
 
    // ********************************************************************************************
    // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
    // Note that this shouldn't be necessary because the fine grid is created by interpolation
    // from the coarse ... but just in case ...
    // ********************************************************************************************
    if ( (SolverChoice::mesh_type != MeshType::ConstantDz) && (solverChoice.coupling_type == CouplingType::TwoWay) ) {
        for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
            average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
            average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
        }
    }
 
    // ********************************************************************************************
    // Build the data structures for canopy model (depends upon z_phys)
    // ********************************************************************************************
    if (solverChoice.do_forest_drag) {
        m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
    }
 
    // *****************************************************************************************************
    // Create the physbcs objects (after initializing the terrain but before calling FillCoarsePatch
    // *****************************************************************************************************
    make_physbcs(lev);
 
    // ********************************************************************************************
    // Update the base state at this level by interpolation from coarser level AND copy
    //    from previous (pre-regrid) base_state array
    // ********************************************************************************************
    if (lev > 0) {
        Interpolater* mapper = &cell_cons_interp;
 
        Vector<MultiFab*> fmf = {&base_state[lev  ], &base_state[lev  ]};
        Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
        Vector<Real> ftime    = {time, time};
        Vector<Real> ctime    = {time, time};
 
        // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
        FillPatchTwoLevels(temp_base_state, temp_base_state.nGrowVect(), IntVect(0,0,0),
                           time, cmf, ctime, fmf, ftime,
                           0, 0, temp_base_state.nComp(), geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper, domain_bcs_type,
                           BaseBCVars::rho0_bc_comp);
 
        // Impose bc's outside the domain
        (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
 
        // *************************************************************************************************
        // This will fill the temporary MultiFabs with data from vars_new
        // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
        // NOTE: we must create the new base state before calling FillPatch because we will
        //       interpolate perturbational quantities
        // *************************************************************************************************
        FillPatchFineLevel(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
                           &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
                          {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                           base_state[lev], temp_base_state, false);
    } else {
        temp_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
                                     base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
        temp_lev_new[Vars::cons].ParallelCopy(vars_new[lev][Vars::cons],0,0,ncomp_cons,ngrow_state,ngrow_state);
        temp_lev_new[Vars::xvel].ParallelCopy(vars_new[lev][Vars::xvel],0,0,         1,ngrow_vels,ngrow_vels);
        temp_lev_new[Vars::yvel].ParallelCopy(vars_new[lev][Vars::yvel],0,0,         1,ngrow_vels,ngrow_vels);
 
        temp_lev_new[Vars::zvel].setVal(0.);
        temp_lev_new[Vars::zvel].ParallelCopy(vars_new[lev][Vars::zvel],0,0,         1,
                                              IntVect(ngrow_vels,ngrow_vels,0),IntVect(ngrow_vels,ngrow_vels,0));
    }
 
    // Now swap the pointers since we needed both old and new in the FillPatch
    std::swap(temp_base_state, base_state[lev]);
 
    // ********************************************************************************************
    // Copy from new into old just in case
    // ********************************************************************************************
    MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
    MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0,         1,ngrow_vels);
    MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0,         1,ngrow_vels);
    MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0,         1,IntVect(ngrow_vels,ngrow_vels,0));
 
    // ********************************************************************************************
    // Now swap the pointers
    // ********************************************************************************************
    for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
        std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
        std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
    }
 
    t_new[lev] = time;
    t_old[lev] = time - 1.e200;
 
    // ********************************************************************************************
    // Build the data structures for calculating diffusive/turbulent terms
    // ********************************************************************************************
    update_diffusive_arrays(lev, ba, dm);
 
    //********************************************************************************************
    // Microphysics
    // *******************************************************************************************
    int q_size = micro->Get_Qmoist_Size(lev);
    qmoist[lev].resize(q_size);
    micro->Define(lev, solverChoice);
    if (solverChoice.moisture_type != MoistureType::None)
    {
        micro->Init(lev, vars_new[lev][Vars::cons],
                    grids[lev], Geom(lev), 0.0,
                    z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
    }
    for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
        qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
    }
 
    //********************************************************************************************
    // Radiation
    // *******************************************************************************************
    if (solverChoice.rad_type != RadiationType::None)
    {
        rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
    }
 
    // ********************************************************************************************
    // Initialize the integrator class
    // ********************************************************************************************
    initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
 
    // We need to re-define the FillPatcher if the grids have changed
    if (lev > 0 && cf_width >= 0) {
        bool ba_changed = (ba != ba_old);
        bool dm_changed = (dm != dm_old);
        if (ba_changed || dm_changed) {
          Define_ERFFillPatchers(lev);
        }
    }
 
    // These calls are done in AmrCore::regrid if this is a regrid at lev > 0
    // For a level 0 regrid we must explicitly do them here
    if (lev == 0) {
        // Define grids[lev] to be ba
        SetBoxArray(lev, ba);
 
        // Define dmap[lev] to be dm
        SetDistributionMap(lev, dm);
    }
 
    // ********************************************************************************************
    // Initialize the 2D data structures
    // ********************************************************************************************
    // NOTE: 2D MFs must be filled before SurfaceLayer is defined since SL class uses sst/tsk
    // Clear the 2D arrays
    if (sst_lev[lev][0]) {
        for (int n = 0; n < sst_lev[lev].size(); n++) {
            sst_lev[lev][n].reset();
        }
    }
    if (tsk_lev[lev][0]) {
        for (int n = 0; n < tsk_lev[lev].size(); n++) {
            tsk_lev[lev][n].reset();
        }
    }
    if (lat_m[lev]) {
        lat_m[lev].reset();
    }
    if (lon_m[lev]) {
        lon_m[lev].reset();
    }
    if (sinPhi_m[lev]) {
        sinPhi_m[lev].reset();
    }
    if (cosPhi_m[lev]) {
        cosPhi_m[lev].reset();
    }
 
    //
    // Interpolate the 2D arrays at the lower boundary. We assume that since we created
    //     them by interpolation it is ok just to recreate them by interpolation.
    // Note that ba2d is constructed already in init_stuff, but we have not yet defined dmap[lev]
    //     so we must explicitly pass dm.
    Interp2DArrays(lev,ba2d[lev],dm);
 
    // ********************************************************************************************
    // Update the SurfaceLayer arrays at this level
    // ********************************************************************************************
    if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
        int nlevs = finest_level+1;
        Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
                                     &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
        m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
                                                   mfv_old, Theta_prim[lev], Qv_prim[lev],
                                                   Qr_prim[lev], z_phys_nd[lev],
                                                   Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
                                                   lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
                                                   sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
    }
 
#ifdef ERF_USE_PARTICLES
    particleData.Redistribute();
#endif
}

restart()

void ERF::restart

(

)

{
    auto dRestartTime0 = amrex::second();
 
    ReadCheckpointFile();
 
    if (regrid_level_0_on_restart) {
        //
        // Coarsening before we split the grids ensures that each resulting
        // grid will have an even number of cells in each direction.
        //
        BoxArray new_ba(amrex::coarsen(Geom(0).Domain(),2));
        //
        // Now split up into list of grids within max_grid_size[0] limit.
        //
        new_ba.maxSize(max_grid_size[0]/2);
        //
        // Now refine these boxes back to level 0.
        //
        new_ba.refine(2);
 
        if (refine_grid_layout) {
            ChopGrids(0, new_ba, ParallelDescriptor::NProcs());
        }
 
        if (new_ba != grids[0]) {
            DistributionMapping new_dm(new_ba);
            RemakeLevel(0,t_new[0],new_ba,new_dm);
        }
    }
 
#ifdef ERF_USE_PARTICLES
    // We call this here without knowing whether the particles have already been initialized or not
    initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,t_new[0]);
#endif
 
    Real cur_time = t_new[0];
    if (m_check_per    > 0.) {last_check_file_time    = cur_time;}
    if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 = std::floor(cur_time/m_plot2d_per_1) * m_plot2d_per_1;}
    if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 = std::floor(cur_time/m_plot2d_per_2) * m_plot2d_per_2;}
    if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 = std::floor(cur_time/m_plot3d_per_1) * m_plot3d_per_1;}
    if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 = std::floor(cur_time/m_plot3d_per_2) * m_plot3d_per_2;}
 
    if (m_check_int    > 0.) {last_check_file_step    = istep[0];}
    if (m_plot2d_int_1 > 0.) {last_plot2d_file_step_1 = istep[0];}
    if (m_plot2d_int_2 > 0.) {last_plot2d_file_step_2 = istep[0];}
    if (m_plot3d_int_1 > 0.) {last_plot3d_file_step_1 = istep[0];}
    if (m_plot3d_int_2 > 0.) {last_plot3d_file_step_2 = istep[0];}
 
    if (verbose > 0)
    {
        auto dRestartTime = amrex::second() - dRestartTime0;
        ParallelDescriptor::ReduceRealMax(dRestartTime,ParallelDescriptor::IOProcessorNumber());
        amrex::Print() << "Restart time = " << dRestartTime << " seconds." << '\n';
    }
}

sample_lines()

void ERF::sample_lines	(	int	lev,
		amrex::Real	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters

lev	Current level
time	Current time
cell	IntVect containing the x,y-dimension indices to sample along z
mf	MultiFab from which we sample the data

{
    int ifile = 0;
 
    const int ncomp = mf.nComp(); // cell-centered state vars
 
    MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
    average_face_to_cellcenter(mf_vels, 0,
                               Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
 
    //
    // Sample the data at a line (in direction "dir") in space
    // In this case we sample in the vertical direction so dir = 2
    // The "k" value of "cell" is ignored
    //
    int dir = 2;
    MultiFab my_line       = get_line_data(mf,              dir, cell);
    MultiFab my_line_vels  = get_line_data(mf_vels,         dir, cell);
    MultiFab my_line_tau11 = get_line_data(*Tau[lev][TauType::tau11], dir, cell);
    MultiFab my_line_tau12 = get_line_data(*Tau[lev][TauType::tau12], dir, cell);
    MultiFab my_line_tau13 = get_line_data(*Tau[lev][TauType::tau13], dir, cell);
    MultiFab my_line_tau22 = get_line_data(*Tau[lev][TauType::tau22], dir, cell);
    MultiFab my_line_tau23 = get_line_data(*Tau[lev][TauType::tau23], dir, cell);
    MultiFab my_line_tau33 = get_line_data(*Tau[lev][TauType::tau33], dir, cell);
 
    for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
    {
        // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
 
        std::ostream& sample_log = SampleLineLog(ifile);
        if (sample_log.good()) {
          sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
          const auto& my_line_arr = my_line[0].const_array();
          const auto& my_line_vels_arr = my_line_vels[0].const_array();
          const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
          const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
          const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
          const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
          const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
          const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
          const Box&  my_box = my_line[0].box();
          const int klo = my_box.smallEnd(2);
          const int khi = my_box.bigEnd(2);
          int i = cell[0];
          int j = cell[1];
          for (int n = 0; n < ncomp; n++) {
              for (int k = klo; k <= khi; k++) {
                  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
              }
          }
          for (int n = 0; n < AMREX_SPACEDIM; n++) {
              for (int k = klo; k <= khi; k++) {
                  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
              }
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
          }
          for (int k = klo; k <= khi; k++) {
              sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
          }
          sample_log << std::endl;
        } // if good
    } // mfi
}

sample_points()

void ERF::sample_points	(	int	lev,
		amrex::Real	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling MultiFab data at a specified cell index.

Parameters

lev	Level for the associated MultiFab data
time	Current time
cell	IntVect containing the indexes for the cell where we want to sample
mf	MultiFab from which we wish to sample data

{
    int ifile = 0;
 
    //
    // Sample the data at a single point in space
    //
    int ncomp = mf.nComp();
    Vector<Real> my_point = get_cell_data(mf, cell);
 
    if (!my_point.empty()) {
 
        // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
 
        std::ostream& sample_log = SamplePointLog(ifile);
        if (sample_log.good()) {
          sample_log << std::setw(datwidth) << time;
          for (int i = 0; i < ncomp; ++i)
          {
              sample_log << std::setw(datwidth) << my_point[i];
          }
          sample_log << std::endl;
        } // if good
    } // only write from processor that holds the cell
}

SampleLine()

amrex::IntVect& ERF::SampleLine ( int  i )

inlineprivate

    {
        return sampleline[i];
    }

SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int  i )

inlineprivate

    {
        return *samplelinelog[i];
    }

SampleLineLogName()

std::string ERF::SampleLineLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith samplelinelog file.

{ return samplelinelogname[i]; }

SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int  i )

inlineprivate

    {
        return samplepoint[i];
    }

SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int  i )

inlineprivate

    {
        return *sampleptlog[i];
    }

SamplePointLogName()

std::string ERF::SamplePointLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith sampleptlog file.

{ return sampleptlogname[i]; }

setPlotVariables()

void ERF::setPlotVariables ( const std::string &  pp_plot_var_names, amrex::Vector< std::string > &  plot_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    if (pp.contains(pp_plot_var_names.c_str()))
    {
        std::string nm;
 
        int nPltVars = pp.countval(pp_plot_var_names.c_str());
 
        for (int i = 0; i < nPltVars; i++)
        {
            pp.get(pp_plot_var_names.c_str(), nm, i);
 
            // Add the named variable to our list of plot variables
            // if it is not already in the list
            if (!containerHasElement(plot_var_names, nm)) {
                plot_var_names.push_back(nm);
            }
        }
    } else {
        //
        // The default is to add none of the variables to the list
        //
        plot_var_names.clear();
    }
 
    // Get state variables in the same order as we define them,
    // since they may be in any order in the input list
    Vector<std::string> tmp_plot_names;
 
    for (int i = 0; i < cons_names.size(); ++i) {
        if ( containerHasElement(plot_var_names, cons_names[i]) ) {
            if (solverChoice.moisture_type == MoistureType::None) {
                if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
                    cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                {
                    tmp_plot_names.push_back(cons_names[i]);
                }
            } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
                if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                {
                    tmp_plot_names.push_back(cons_names[i]);
                }
            } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
                        (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
                        (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
                if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
                    cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                {
                    tmp_plot_names.push_back(cons_names[i]);
                }
            } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                        (solverChoice.moisture_type == MoistureType::SAM_NoIce     ) ) { // allow rhoQ1, rhoQ2, rhoQ4
                if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                {
                    tmp_plot_names.push_back(cons_names[i]);
                }
            } else
            {
                // For moisture_type SAM and Morrison we have all six variables
                tmp_plot_names.push_back(cons_names[i]);
            }
        }
    }
 
    // check for velocity since it's not in cons_names
    // if we are asked for any velocity component, we will need them all
    if (containerHasElement(plot_var_names, "x_velocity") ||
        containerHasElement(plot_var_names, "y_velocity") ||
        containerHasElement(plot_var_names, "z_velocity")) {
        tmp_plot_names.push_back("x_velocity");
        tmp_plot_names.push_back("y_velocity");
        tmp_plot_names.push_back("z_velocity");
    }
 
    //
    // If the model we are running doesn't have the variable listed in the inputs file,
    //     just ignore it rather than aborting
    //
    for (int i = 0; i < derived_names.size(); ++i) {
        if ( containerHasElement(plot_var_names, derived_names[i]) ) {
            bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing || solverChoice.buildings_type == BuildingsType::ImmersedForcing ) ||
                               (derived_names[i] != "terrain_IB_mask") );
            ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                               (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                               (derived_names[i] != "detJ") );
            ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                               (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                               (derived_names[i] != "z_phys") );
#ifndef ERF_USE_WINDFARM
            ok_to_add     &= (derived_names[i] != "SMark0" && derived_names[i] != "SMark1");
#endif
            if (ok_to_add)
            {
                if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
                    if (derived_names[i] != "qv" && derived_names[i] != "qc"    && derived_names[i] != "qrain"  &&
                        derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                        derived_names[i] != "qt" && derived_names[i] != "qn"    && derived_names[i] != "qp" &&
                        derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                    {
                        tmp_plot_names.push_back(derived_names[i]);
                    }
                } else if ( (solverChoice.moisture_type == MoistureType::Kessler       ) ||
                            (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                            (solverChoice.moisture_type == MoistureType::SAM_NoIce     ) ) { // allow qv, qc, qrain
                    if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                        derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                    {
                        tmp_plot_names.push_back(derived_names[i]);
                    }
                } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
                            (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
                            (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
                    if (derived_names[i] != "qrain"  &&
                        derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                        derived_names[i] != "qp" &&
                        derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                    {
                        tmp_plot_names.push_back(derived_names[i]);
                    }
                } else
                {
                    // For moisture_type SAM and Morrison we have all moist quantities
                    tmp_plot_names.push_back(derived_names[i]);
                }
            } // use_terrain?
        } // hasElement
    }
 
#ifdef ERF_USE_WINDFARM
    for (int i = 0; i < derived_names.size(); ++i) {
        if ( containerHasElement(plot_var_names, derived_names[i]) ) {
            if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP) {
                if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0") {
                    tmp_plot_names.push_back(derived_names[i]);
                }
            }
            if( solverChoice.windfarm_type == WindFarmType::SimpleAD or
                solverChoice.windfarm_type == WindFarmType::GeneralAD ) {
                if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0" or derived_names[i] == "SMark1") {
                    tmp_plot_names.push_back(derived_names[i]);
                }
            }
        }
    }
#endif
 
#ifdef ERF_USE_PARTICLES
    const auto& particles_namelist( particleData.getNamesUnalloc() );
    for (auto it = particles_namelist.cbegin(); it != particles_namelist.cend(); ++it) {
        std::string tmp( (*it)+"_count" );
        if (containerHasElement(plot_var_names, tmp) ) {
            tmp_plot_names.push_back(tmp);
        }
    }
#endif
 
    plot_var_names = tmp_plot_names;
}

Here is the call graph for this function:

setPlotVariables2D()

void ERF::setPlotVariables2D ( const std::string &  pp_plot_var_names, amrex::Vector< std::string > &  plot_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    if (pp.contains(pp_plot_var_names.c_str()))
    {
        std::string nm;
 
        int nPltVars = pp.countval(pp_plot_var_names.c_str());
 
        for (int i = 0; i < nPltVars; i++)
        {
            pp.get(pp_plot_var_names.c_str(), nm, i);
 
            // Add the named variable to our list of plot variables
            // if it is not already in the list
            if (!containerHasElement(plot_var_names, nm)) {
                plot_var_names.push_back(nm);
            }
        }
    } else {
        //
        // The default is to add none of the variables to the list
        //
        plot_var_names.clear();
    }
 
    // Get state variables in the same order as we define them,
    // since they may be in any order in the input list
    Vector<std::string> tmp_plot_names;
 
    // 2D plot variables
    for (int i = 0; i < derived_names_2d.size(); ++i) {
        if (containerHasElement(plot_var_names, derived_names_2d[i]) ) {
            tmp_plot_names.push_back(derived_names_2d[i]);
        }
    }
 
    plot_var_names = tmp_plot_names;
}

Here is the call graph for this function:

setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool  restarting )

private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters

[in]

restarting

Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

{
    // If we are restarting then we haven't read the input_sounding file yet
    //    so we need to read it here
    // TODO: should we store this information in the checkpoint file instead?
    if (restarting) {
        input_sounding_data.resize_arrays();
        for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
            input_sounding_data.read_from_file(geom[0], zlevels_stag[0], n);
        }
    }
 
    const Real* z_inp_sound     = input_sounding_data.z_inp_sound[0].dataPtr();
    const Real* U_inp_sound     = input_sounding_data.U_inp_sound[0].dataPtr();
    const Real* V_inp_sound     = input_sounding_data.V_inp_sound[0].dataPtr();
    const Real* theta_inp_sound = input_sounding_data.theta_inp_sound[0].dataPtr();
    const int   inp_sound_size  = input_sounding_data.size(0);
 
    int refine_fac{1};
    for (int lev = 0; lev <= finest_level; lev++)
    {
        const int klo = geom[lev].Domain().smallEnd(2);
        const int khi = geom[lev].Domain().bigEnd(2);
        const int Nz = khi - klo + 1;
 
        Vector<Real> zcc(Nz);
        Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
                                 zlevels_stag[0].begin()+khi/refine_fac+2);
        expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
#if 0
        amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
        for (int k = 0; k < zlevels_sub.size(); k++) { amrex::AllPrint() << zlevels_sub[k] << " "; }
        amrex::AllPrint() << " --> ";
        for (int k = 0; k < Nz; k++) { amrex::AllPrint() << zcc[k] << " "; }
        amrex::AllPrint() << std::endl;
#endif
 
        for (int k = 0; k < Nz; k++)
        {
            h_rayleigh_ptrs[lev][Rayleigh::ubar][k]     = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
            h_rayleigh_ptrs[lev][Rayleigh::vbar][k]     = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
            h_rayleigh_ptrs[lev][Rayleigh::wbar][k]     = Real(0.0);
            h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
        }
 
        // Copy from host version to device version
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
        Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
                         d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
 
        refine_fac *= ref_ratio[lev][2];
    }
}

Here is the call graph for this function:

setRecordDataInfo()

void ERF::setRecordDataInfo ( int  i, const std::string &  filename  )

inlineprivate

    {
        if (amrex::ParallelDescriptor::IOProcessor())
        {
            datalog[i] = std::make_unique<std::fstream>();
            datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
            if (!datalog[i]->good()) {
                amrex::FileOpenFailed(filename);
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
    }

setRecordDerDataInfo()

void ERF::setRecordDerDataInfo ( int  i, const std::string &  filename  )

inlineprivate

    {
        if (amrex::ParallelDescriptor::IOProcessor())
        {
            der_datalog[i] = std::make_unique<std::fstream>();
            der_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
            if (!der_datalog[i]->good()) {
                amrex::FileOpenFailed(filename);
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordDerDataInfo");
    }

setRecordEnergyDataInfo()

void ERF::setRecordEnergyDataInfo ( int  i, const std::string &  filename  )

inlineprivate

    {
        if (amrex::ParallelDescriptor::IOProcessor())
        {
            tot_e_datalog[i] = std::make_unique<std::fstream>();
            tot_e_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
            if (!tot_e_datalog[i]->good()) {
                amrex::FileOpenFailed(filename);
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordEnergyDataInfo");
    }

setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo ( int  i, int  lev, amrex::IntVect &  cell, const std::string &  filename  )

inlineprivate

    {
        amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
        for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
        {
            const amrex::Box& bx = mfi.validbox();
            if (bx.contains(cell)) {
                samplelinelog[i] = std::make_unique<std::fstream>();
                samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                if (!samplelinelog[i]->good()) {
                    amrex::FileOpenFailed(filename);
                }
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
    }

setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo ( int  i, int  lev, amrex::IntVect &  cell, const std::string &  filename  )

inlineprivate

    {
        amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
        for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
        {
            const amrex::Box& bx = mfi.validbox();
            if (bx.contains(cell)) {
                sampleptlog[i] = std::make_unique<std::fstream>();
                sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                if (!sampleptlog[i]->good()) {
                    amrex::FileOpenFailed(filename);
                }
            }
        }
        amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
    }

setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool  restarting )

private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters

[in]

restarting

Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

{
    // If we are restarting then we haven't read the input_sponge file yet
    //    so we need to read it here
    // TODO: should we store this information in the checkpoint file instead?
    if (restarting) {
        input_sponge_data.read_from_file(geom[0], zlevels_stag[0]);
    }
 
    const Real* z_inp_sponge     = input_sponge_data.z_inp_sponge.dataPtr();
    const Real* U_inp_sponge     = input_sponge_data.U_inp_sponge.dataPtr();
    const Real* V_inp_sponge     = input_sponge_data.V_inp_sponge.dataPtr();
    const int   inp_sponge_size  = input_sponge_data.size();
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        const int khi = geom[lev].Domain().bigEnd()[2];
        Vector<Real> zcc(khi+1);
 
        if (z_phys_cc[lev]) {
            // use_terrain=1
            // calculate the damping strength based on the max height at each k
            reduce_to_max_per_height(zcc, z_phys_cc[lev]);
        } else {
            const auto *const prob_lo = geom[lev].ProbLo();
            const auto *const dx = geom[lev].CellSize();
            for (int k = 0; k <= khi; k++)
            {
                zcc[k] = prob_lo[2] + (k+0.5) * dx[2];
            }
        }
 
        for (int k = 0; k <= khi; k++)
        {
            h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
            h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
        }
 
        // Copy from host version to device version
        Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
                         d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
        Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
                         d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
    }
}

Here is the call graph for this function:

setSubVolVariables()

void ERF::setSubVolVariables ( const std::string &  pp_subvol_var_names, amrex::Vector< std::string > &  subvol_var_names  )

private

{
    ParmParse pp(pp_prefix);
 
    std::string nm;
 
    int nSubVolVars = pp.countval(pp_subvol_var_names.c_str());
 
    // We pre-populate the list with velocities, but allow these to be over-written
    //    by user input
    if (nSubVolVars == 0)
    {
        subvol_var_names.push_back("x_velocity");
        subvol_var_names.push_back("y_velocity");
        subvol_var_names.push_back("z_velocity");
 
    } else {
        for (int i = 0; i < nSubVolVars; i++)
        {
            pp.get(pp_subvol_var_names.c_str(), nm, i);
 
            // Add the named variable to our list of subvol variables
            // if it is not already in the list
            if (!containerHasElement(subvol_var_names, nm)) {
                subvol_var_names.push_back(nm);
            }
        }
    }
 
    // Get state variables in the same order as we define them,
    // since they may be in any order in the input list
    Vector<std::string> tmp_plot_names;
 
    for (int i = 0; i < cons_names.size(); ++i) {
        if ( containerHasElement(subvol_var_names, cons_names[i]) ) {
            if (solverChoice.moisture_type == MoistureType::None) {
                if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
                    cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                {
                    tmp_plot_names.push_back(cons_names[i]);
                }
            } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
                if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                {
                    tmp_plot_names.push_back(cons_names[i]);
                }
            } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
                        (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
                        (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
                if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
                    cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                {
                    tmp_plot_names.push_back(cons_names[i]);
                }
            } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                        (solverChoice.moisture_type == MoistureType::SAM_NoIce     ) ) { // allow rhoQ1, rhoQ2, rhoQ4
                if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                {
                    tmp_plot_names.push_back(cons_names[i]);
                }
            } else
            {
                // For moisture_type SAM and Morrison we have all six variables
                tmp_plot_names.push_back(cons_names[i]);
            }
        }
    }
 
    // Check for velocity since it's not in cons_names
    if (containerHasElement(subvol_var_names, "x_velocity")) {
        tmp_plot_names.push_back("x_velocity");
    }
    if (containerHasElement(subvol_var_names, "y_velocity")) {
        tmp_plot_names.push_back("y_velocity");
    }
    if (containerHasElement(subvol_var_names, "z_velocity")) {
        tmp_plot_names.push_back("z_velocity");
    }
 
    //
    // If the model we are running doesn't have the variable listed in the inputs file,
    //     just ignore it rather than aborting
    //
    for (int i = 0; i < derived_subvol_names.size(); ++i) {
        if ( containerHasElement(subvol_var_names, derived_names[i]) ) {
            bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing) ||
                               (derived_names[i] != "terrain_IB_mask") );
            ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                               (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                               (derived_names[i] != "detJ") );
            ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                               (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                               (derived_names[i] != "z_phys") );
            if (ok_to_add)
            {
                if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
                    if (derived_names[i] != "qv" && derived_names[i] != "qc"    && derived_names[i] != "qrain"  &&
                        derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                        derived_names[i] != "qt" && derived_names[i] != "qn"    && derived_names[i] != "qp" &&
                        derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                    {
                        tmp_plot_names.push_back(derived_names[i]);
                    }
                } else if ( (solverChoice.moisture_type == MoistureType::Kessler       ) ||
                            (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                            (solverChoice.moisture_type == MoistureType::SAM_NoIce     ) ) { // allow qv, qc, qrain
                    if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                        derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                    {
                        tmp_plot_names.push_back(derived_names[i]);
                    }
                } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
                            (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
                            (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
                    if (derived_names[i] != "qrain"  &&
                        derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                        derived_names[i] != "qp" &&
                        derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                    {
                        tmp_plot_names.push_back(derived_names[i]);
                    }
                } else
                {
                    // For moisture_type SAM and Morrison we have all moist quantities
                    tmp_plot_names.push_back(derived_names[i]);
                }
            } // use_terrain?
        } // hasElement
    }
 
    subvol_var_names = tmp_plot_names;
}

Here is the call graph for this function:

solve_with_gmres()

void ERF::solve_with_gmres	(	int	lev,
		const amrex::Box &	subdomain,
		amrex::MultiFab &	rhs,
		amrex::MultiFab &	p,
		amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &	fluxes,
		amrex::MultiFab &	ax_sub,
		amrex::MultiFab &	ay_sub,
		amrex::MultiFab &	az_sub,
		amrex::MultiFab &	,
		amrex::MultiFab &	znd_sub
	)

Solve the Poisson equation using FFT-preconditioned GMRES

{
#ifdef ERF_USE_FFT
    BL_PROFILE("ERF::solve_with_gmres()");
 
    Real reltol = solverChoice.poisson_reltol;
    Real abstol = solverChoice.poisson_abstol;
 
    auto const dom_lo = lbound(Geom(lev).Domain());
    auto const dom_hi = ubound(Geom(lev).Domain());
 
    auto const sub_lo = lbound(subdomain);
    auto const sub_hi = ubound(subdomain);
 
    auto dx    = Geom(lev).CellSizeArray();
 
    Geometry my_geom;
 
    Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
    if (Geom(lev).isPeriodic(0) && sub_lo.x == dom_lo.x && sub_hi.x == dom_hi.x) { is_per[0] = 1;}
    if (Geom(lev).isPeriodic(1) && sub_lo.y == dom_lo.y && sub_hi.y == dom_hi.y) { is_per[1] = 1;}
 
    int coord_sys = 0;
 
    // If subdomain == domain then we pass Geom(lev) to the FFT solver
    if (subdomain == Geom(lev).Domain()) {
        my_geom.define(Geom(lev).Domain(), Geom(lev).ProbDomain(), coord_sys, is_per);
    } else {
        // else we create a new geometry based only on the subdomain
        // The information in my_geom used by the FFT routines is:
        //   1) my_geom.Domain()
        //   2) my_geom.CellSize()
        //   3) my_geom.isAllPeriodic() / my_geom.periodicity()
        RealBox rb( sub_lo.x   *dx[0],  sub_lo.y   *dx[1],  sub_lo.z   *dx[2],
                   (sub_hi.x+1)*dx[0], (sub_hi.y+1)*dx[1], (sub_hi.z+1)*dx[2]);
        my_geom.define(subdomain, rb, coord_sys, is_per);
    }
 
    amrex::GMRES<MultiFab, TerrainPoisson> gmsolver;
 
    TerrainPoisson tp(my_geom, rhs.boxArray(), rhs.DistributionMap(), domain_bc_type,
                      stretched_dz_d[lev], ax_sub, ay_sub, az_sub, dJ_sub, &znd_sub,
                      solverChoice.use_real_bcs);
 
    gmsolver.define(tp);
 
    gmsolver.setVerbose(mg_verbose);
 
    gmsolver.setRestartLength(50);
 
    tp.usePrecond(true);
 
    gmsolver.solve(phi, rhs, reltol, abstol);
 
    tp.getFluxes(phi, fluxes);
 
    for (MFIter mfi(phi); mfi.isValid(); ++mfi)
    {
        Box xbx = mfi.nodaltilebox(0);
        Box ybx = mfi.nodaltilebox(1);
        const Array4<Real      >& fx_ar = fluxes[0].array(mfi);
        const Array4<Real      >& fy_ar = fluxes[1].array(mfi);
        const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
        const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
        ParallelFor(xbx,ybx,
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            fx_ar(i,j,k) *= mf_ux(i,j,0);
        },
        [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            fy_ar(i,j,k) *= mf_vy(i,j,0);
        });
    } // mfi
#else
    amrex::ignore_unused(lev, rhs, phi, fluxes, ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
#endif
 
    // ****************************************************************************
    // Impose bc's on pprime
    // ****************************************************************************
    ImposeBCsOnPhi(lev, phi, subdomain);
}

sum_derived_quantities()

void ERF::sum_derived_quantities

(

amrex::Real

time

)

{
    if (verbose <= 0 || NumDerDataLogs() <= 0) return;
 
    int lev = 0;
 
    AMREX_ALWAYS_ASSERT(lev == 0);
 
    auto& mfx0 = *mapfac[0][MapFacType::m_x];
    auto& mfy0 = *mapfac[0][MapFacType::m_x];
    auto&  dJ0 = *detJ_cc[0];
 
    // ************************************************************************
    // WARNING: we are not filling ghost cells other than periodic outside the domain
    // ************************************************************************
 
    MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
    mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
 
    // Average all three components of velocity (on faces) to the cell center
    average_face_to_cellcenter(mf_cc_vel,0,
                               Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                        &vars_new[lev][Vars::yvel],
                                                        &vars_new[lev][Vars::zvel]});
    mf_cc_vel.FillBoundary(geom[lev].periodicity());
 
    if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
        amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
    }
 
    MultiFab r_wted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
    MultiFab unwted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
    MultiFab     enstrophysq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
    MultiFab        theta_mf(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(unwted_magvelsq, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
        auto& src_fab = mf_cc_vel[mfi];
 
        auto& dest1_fab = unwted_magvelsq[mfi];
        derived::erf_dermagvelsq(bx, dest1_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
 
        auto& dest2_fab = enstrophysq[mfi];
        derived::erf_derenstrophysq(bx, dest2_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
    }
 
    // Copy the MF holding 1/2(u^2 + v^2 + w^2) into the MF that will hold 1/2 rho (u^2 + v^2 + w^2)d
    MultiFab::Copy(r_wted_magvelsq, unwted_magvelsq, 0, 0, 1, 0);
 
    // Multiply the MF holding 1/2(u^2 + v^2 + w^2) by rho to get  1/2 rho (u^2 + v^2 + w^2)
    MultiFab::Multiply(r_wted_magvelsq, vars_new[lev][Vars::cons], 0, 0, 1, 0);
 
    // Copy the MF holding (rho theta) into "theta_mf"
    MultiFab::Copy(theta_mf, vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, 0);
 
    // Divide (rho theta) by rho to get theta in the MF "theta_mf"
    MultiFab::Divide(theta_mf, vars_new[lev][Vars::cons], Rho_comp, 0, 1, 0);
 
    Real  unwted_avg = volWgtSumMF(lev, unwted_magvelsq, 0, dJ0, mfx0, mfy0, false);
    Real  r_wted_avg = volWgtSumMF(lev, r_wted_magvelsq, 0, dJ0, mfx0, mfy0, false);
    Real enstrsq_avg = volWgtSumMF(lev, enstrophysq,     0, dJ0, mfx0, mfy0, false);
    Real   theta_avg = volWgtSumMF(lev, theta_mf,        0, dJ0, mfx0, mfy0, false);
 
    // Get volume including terrain (consistent with volWgtSumMF routine)
    MultiFab volume(grids[lev], dmap[lev], 1, 0);
    auto const& dx = geom[lev].CellSizeArray();
    Real cell_vol  = dx[0]*dx[1]*dx[2];
    volume.setVal(cell_vol);
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
    }
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& tbx  = mfi.tilebox();
        auto dst        = volume.array(mfi);
        const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
        ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
        });
    }
    Real vol = volume.sum();
 
     unwted_avg /= vol;
     r_wted_avg /= vol;
    enstrsq_avg /= vol;
      theta_avg /= vol;
 
    const int nfoo = 4;
    Real foo[nfoo] = {unwted_avg,r_wted_avg,enstrsq_avg,theta_avg};
#ifdef AMREX_LAZY
    Lazy::QueueReduction([=]() mutable {
#endif
    ParallelDescriptor::ReduceRealSum(
        foo, nfoo, ParallelDescriptor::IOProcessorNumber());
 
      if (ParallelDescriptor::IOProcessor()) {
        int i = 0;
        unwted_avg  = foo[i++];
        r_wted_avg  = foo[i++];
        enstrsq_avg = foo[i++];
          theta_avg = foo[i++];
 
        std::ostream& data_log_der = DerDataLog(0);
 
        if (time == 0.0) {
            data_log_der << std::setw(datwidth) << "          time";
            data_log_der << std::setw(datwidth) << "        ke_den";
            data_log_der << std::setw(datwidth) << "         velsq";
            data_log_der << std::setw(datwidth) << "     enstrophy";
            data_log_der << std::setw(datwidth) << "    int_energy";
            data_log_der << std::endl;
        }
        data_log_der << std::setw(datwidth) << std::setprecision(timeprecision) << time;
        data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  <<  unwted_avg;
        data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  <<  r_wted_avg;
        data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  << enstrsq_avg;
        data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  <<   theta_avg;
        data_log_der << std::endl;
 
      } // if IOProcessor
#ifdef AMREX_LAZY
    }
#endif
}

Here is the call graph for this function:

sum_energy_quantities()

void ERF::sum_energy_quantities

(

amrex::Real

time

)

{
    if ( (verbose <= 0) || (tot_e_datalog.size() < 1) ) { return; }
 
    int lev = 0;
 
    auto& mfx0 = *mapfac[0][MapFacType::m_x];
    auto& mfy0 = *mapfac[0][MapFacType::m_x];
    auto&  dJ0 = *detJ_cc[0];
 
    AMREX_ALWAYS_ASSERT(lev == 0);
 
    bool local = true;
 
    // ************************************************************************
    // WARNING: we are not filling ghost cells other than periodic outside the domain
    // ************************************************************************
 
    MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
    mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
 
    // Average all three components of velocity (on faces) to the cell center
    average_face_to_cellcenter(mf_cc_vel,0,
                               Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                        &vars_new[lev][Vars::yvel],
                                                        &vars_new[lev][Vars::zvel]});
    mf_cc_vel.FillBoundary(geom[lev].periodicity());
 
    if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
        amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
    }
 
    MultiFab tot_mass  (grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
    MultiFab tot_energy(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
 
    auto const& dx = geom[lev].CellSizeArray();
    bool is_moist = (solverChoice.moisture_type != MoistureType::None);
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(tot_mass, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& bx = mfi.tilebox();
 
        const Array4<Real>& cc_vel_arr     = mf_cc_vel.array(mfi);
        const Array4<Real>& tot_mass_arr   = tot_mass.array(mfi);
        const Array4<Real>& tot_energy_arr = tot_energy.array(mfi);
        const Array4<const Real>& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
        const Array4<const Real>& z_arr    = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) :
                                                                Array4<const Real>{};
        ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
        {
            Real Qv   = (is_moist) ? cons_arr(i,j,k,RhoQ1_comp) : 0.0;
            Real Qc   = (is_moist) ? cons_arr(i,j,k,RhoQ2_comp) : 0.0;
            Real Qt   = Qv + Qc;
            Real Rhod = cons_arr(i,j,k,Rho_comp);
            Real Rhot = Rhod * (1.0 + Qt);
            Real Temp = getTgivenRandRTh(Rhod, cons_arr(i,j,k,RhoTheta_comp), Qv);
            Real TKE  = 0.5 * ( cc_vel_arr(i,j,k,0)*cc_vel_arr(i,j,k,0)
                              + cc_vel_arr(i,j,k,1)*cc_vel_arr(i,j,k,1)
                              + cc_vel_arr(i,j,k,2)*cc_vel_arr(i,j,k,2) );
            Real zval = (z_arr) ? z_arr(i,j,k) : Real(k)*dx[2];
 
            Real Cv   = Cp_d - R_d;
            Real Cvv  = Cp_v - R_v;
            Real Cpv  = Cp_v;
 
            tot_mass_arr(i,j,k)   = Rhot;
            tot_energy_arr(i,j,k) = Rhod * ( (Cv + Cvv*Qv + Cpv*Qc)*Temp - L_v*Qc
                                           + (1.0 + Qt)*TKE + (1.0 + Qt)*CONST_GRAV*zval );
 
        });
 
    }
 
    Real  tot_mass_avg   = volWgtSumMF(lev, tot_mass  , 0, dJ0, mfx0, mfy0, false, local);
    Real  tot_energy_avg = volWgtSumMF(lev, tot_energy, 0, dJ0, mfx0, mfy0, false, local);
 
    // Get volume including terrain (consistent with volWgtSumMF routine)
    MultiFab volume(grids[lev], dmap[lev], 1, 0);
    Real cell_vol  = dx[0]*dx[1]*dx[2];
    volume.setVal(cell_vol);
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
    }
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box& tbx  = mfi.tilebox();
        auto dst        = volume.array(mfi);
        const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
        const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
        ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
        {
            dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
        });
    }
    Real vol = volume.sum();
 
    // Divide by the volume
    tot_mass_avg   /= vol;
    tot_energy_avg /= vol;
 
    const int nfoo = 2;
    Real foo[nfoo] = {tot_mass_avg,tot_energy_avg};
#ifdef AMREX_LAZY
    Lazy::QueueReduction([=]() mutable {
#endif
    ParallelDescriptor::ReduceRealSum(
        foo, nfoo, ParallelDescriptor::IOProcessorNumber());
 
      if (ParallelDescriptor::IOProcessor()) {
        int i = 0;
        tot_mass_avg   = foo[i++];
        tot_energy_avg = foo[i++];
 
        std::ostream& data_log_energy = *tot_e_datalog[0];
 
        if (time == 0.0) {
            data_log_energy << std::setw(datwidth) << "          time";
            data_log_energy << std::setw(datwidth) << "      tot_mass";
            data_log_energy << std::setw(datwidth) << "    tot_energy";
            data_log_energy << std::endl;
        }
        data_log_energy << std::setw(datwidth) << std::setprecision(timeprecision) << time;
        data_log_energy << std::setw(datwidth) << std::setprecision(datprecision)  << tot_mass_avg;
        data_log_energy << std::setw(datwidth) << std::setprecision(datprecision)  << tot_energy_avg;
        data_log_energy << std::endl;
 
      } // if IOProcessor
#ifdef AMREX_LAZY
    }
#endif
}

Here is the call graph for this function:

sum_integrated_quantities()

void ERF::sum_integrated_quantities

(

amrex::Real

time

)

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters

time	Current time

{
    BL_PROFILE("ERF::sum_integrated_quantities()");
 
    if (verbose <= 0)
      return;
 
    // Single level sum
    Real mass_sl;
 
    // Multilevel sums
    Real mass_ml = 0.0;
    Real rhth_ml = 0.0;
    Real scal_ml = 0.0;
    Real mois_ml = 0.0;
 
    bool local = true;
 
    auto& mfx0 = *mapfac[0][MapFacType::m_x];
    auto& mfy0 = *mapfac[0][MapFacType::m_x];
    auto&  dJ0 = *detJ_cc[0];
 
    mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,dJ0,mfx0,mfy0,false,local);
 
    for (int lev = 0; lev <= finest_level; lev++) {
        auto& mfx = *mapfac[lev][MapFacType::m_x];
        auto& mfy = *mapfac[lev][MapFacType::m_x];
        auto&  dJ = *detJ_cc[lev];
        mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,dJ,mfx,mfy,true);
    }
 
    Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,dJ0,mfx0,mfy0,false);
    Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,dJ0,mfx0,mfy0,false);
    Real mois_sl = 0.0;
    if (solverChoice.moisture_type != MoistureType::None) {
        int n_qstate_moist = micro->Get_Qstate_Moist_Size();
        for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
            mois_sl += volWgtSumMF(0,vars_new[0][Vars::cons],RhoQ1_comp+qoff,dJ0,mfx0,mfy0,false);
        }
    }
 
    for (int lev = 0; lev <= finest_level; lev++) {
        auto& mfx = *mapfac[lev][MapFacType::m_x];
        auto& mfy = *mapfac[lev][MapFacType::m_x];
        auto&  dJ = *detJ_cc[lev];
        rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,dJ,mfx,mfy,true);
        scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,dJ,mfx,mfy,true);
        if (solverChoice.moisture_type != MoistureType::None) {
            int n_qstate_moist = micro->Get_Qstate_Moist_Size();
            for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
                mois_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoQ1_comp+qoff,dJ,mfx,mfy,false);
            }
        }
    }
 
    Gpu::HostVector<Real> h_avg_ustar; h_avg_ustar.resize(1);
    Gpu::HostVector<Real> h_avg_tstar; h_avg_tstar.resize(1);
    Gpu::HostVector<Real> h_avg_olen; h_avg_olen.resize(1);
    if ((m_SurfaceLayer != nullptr) && (NumDataLogs() > 0)) {
        Box domain = geom[0].Domain();
        int zdir = 2;
        h_avg_ustar = sumToLine(*m_SurfaceLayer->get_u_star(0),0,1,domain,zdir);
        h_avg_tstar = sumToLine(*m_SurfaceLayer->get_t_star(0),0,1,domain,zdir);
        h_avg_olen  = sumToLine(*m_SurfaceLayer->get_olen(0)  ,0,1,domain,zdir);
 
        // Divide by the total number of cells we are averaging over
        Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
        h_avg_ustar[0] /= area_z;
        h_avg_tstar[0] /= area_z;
        h_avg_olen[0]  /= area_z;
 
    } else {
        h_avg_ustar[0] = 0.;
        h_avg_tstar[0] = 0.;
        h_avg_olen[0]  = 0.;
    }
 
    const int nfoo = 8;
    Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mois_sl,mass_ml,rhth_ml,scal_ml,mois_ml};
#ifdef AMREX_LAZY
    Lazy::QueueReduction([=]() mutable {
#endif
    ParallelDescriptor::ReduceRealSum(
        foo, nfoo, ParallelDescriptor::IOProcessorNumber());
 
      if (ParallelDescriptor::IOProcessor()) {
        int i = 0;
        mass_sl = foo[i++];
        rhth_sl = foo[i++];
        scal_sl = foo[i++];
        mois_sl = foo[i++];
        mass_ml = foo[i++];
        rhth_ml = foo[i++];
        scal_ml = foo[i++];
        mois_ml = foo[i++];
 
        Print() << '\n';
        Print() << "TIME= " << std::setw(datwidth) << std::setprecision(timeprecision) << std::left << time << '\n';
        if (finest_level ==  0) {
#if 1
           Print() << " MASS       = " << mass_sl << '\n';
#else
           Print() << " PERT MASS  = " << mass_sl << '\n';
#endif
           Print() << " RHO THETA  = " << rhth_sl << '\n';
           Print() << " RHO SCALAR = " << scal_sl << '\n';
           Print() << " RHO QTOTAL = " << mois_sl << '\n';
        } else {
#if 1
           Print() << " MASS       SL/ML = " << mass_sl << " " << mass_ml << '\n';
#else
           Print() << " PERT MASS  SL/ML = " << mass_sl << " " << mass_ml << '\n';
#endif
           Print() << " RHO THETA  SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
           Print() << " RHO SCALAR SL/ML = " << scal_sl << " " << scal_ml << '\n';
           Print() << " RHO QTOTAL SL/ML = " << mois_sl << " " << mois_ml << '\n';
        }
 
        // The first data log only holds scalars
        if (NumDataLogs() > 0)
        {
            int n_d = 0;
            std::ostream& data_log1 = DataLog(n_d);
            if (data_log1.good()) {
                if (time == 0.0) {
                    data_log1 << std::setw(datwidth) << "          time";
                    data_log1 << std::setw(datwidth) << "          u_star";
                    data_log1 << std::setw(datwidth) << "          t_star";
                    data_log1 << std::setw(datwidth) << "          olen";
                    data_log1 << std::endl;
                } // time = 0
 
              // Write the quantities at this time
              data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time;
              data_log1 << std::setw(datwidth) << std::setprecision(datprecision)  << h_avg_ustar[0];
              data_log1 << std::setw(datwidth) << std::setprecision(datprecision)  << h_avg_tstar[0];
              data_log1 << std::setw(datwidth) << std::setprecision(datprecision)  << h_avg_olen[0];
              data_log1 << std::endl;
            } // if good
        } // loop over i
      } // if IOProcessor
#ifdef AMREX_LAZY
    });
#endif
 
    // This is just an alias for convenience
    int lev = 0;
    if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
        for (int i = 0; i < NumSamplePoints(); ++i)
        {
            sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
        }
    }
    if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
        for (int i = 0; i < NumSampleLines(); ++i)
        {
            sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
        }
    }
}

SurfaceDataInterpolation()

void ERF::SurfaceDataInterpolation	(	const int	nlevs,
		const amrex::Real	time,
		amrex::Vector< std::unique_ptr< amrex::MultiFab >> &	z_phys_nd,
		bool	regrid_forces_file_read
	)

{
 
    static amrex::Vector<Real> next_read_forecast_time;
    static amrex::Vector<Real> last_read_forecast_time;
 
    const int nlevs = a_z_phys_nd.size();
 
    Real hindcast_data_interval = solverChoice.hindcast_data_interval_in_hrs*3600.0;
 
    // Initialize static vectors once
    if (next_read_forecast_time.empty()) {
        next_read_forecast_time.resize(nlevs, -1.0);
        last_read_forecast_time.resize(nlevs, -1.0);
        Print() << "Initializing the time vector values here by " << lev << std::endl;
    }
 
    if (next_read_forecast_time[lev] < 0.0) {
        int next_multiple = static_cast<int>(time / hindcast_data_interval);
        next_read_forecast_time[lev] = next_multiple * hindcast_data_interval;
        last_read_forecast_time[lev] = next_read_forecast_time[lev];
    }
 
    if (time >= next_read_forecast_time[lev] or regrid_forces_file_read) {
 
        Print() << "Data reading happening at level " << lev << std::endl;
 
        std::string folder = solverChoice.hindcast_surface_data_dir;
 
        // Check if folder exists and is a directory
        if (!fs::exists(folder) || !fs::is_directory(folder)) {
            throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
        }
 
        std::vector<std::string> bin_files;
 
        for (const auto& entry : fs::directory_iterator(folder)) {
            if (!entry.is_regular_file()) continue;
 
            std::string fname = entry.path().filename().string();
            if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
                bin_files.push_back(entry.path().string());
            }
        }
        std::sort(bin_files.begin(), bin_files.end());
 
    // Check if no .bin files were found
        if (bin_files.empty()) {
            throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
        }
 
        std::string filename1, filename2;
 
        int idx1 = static_cast<int>(time / hindcast_data_interval);
        int idx2 = static_cast<int>(time / hindcast_data_interval)+1;
        Print() << "Reading surface data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
 
        if (idx2 >= static_cast<int>(bin_files.size())) {
            throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
        }
 
        filename1 = bin_files[idx1];
        filename2 = bin_files[idx2];
 
        FillSurfaceStateMultiFabs(lev, filename1, surface_state_1);
        FillSurfaceStateMultiFabs(lev, filename2, surface_state_2);
 
         // Create the time-interpolated forecast state
        //CreateForecastStateMultiFabs(forecast_state_interp);
        if(!regrid_forces_file_read){
            last_read_forecast_time[lev] = next_read_forecast_time[lev];
            next_read_forecast_time[lev] += hindcast_data_interval;
            Print() << "Next forecast time getting updated here " << std::endl;
        }
    }
 
    Real prev_read_time = last_read_forecast_time[lev];
    Real alpha1 = 1.0 - (time - prev_read_time)/hindcast_data_interval;
    Real alpha2 = 1.0 - alpha1;
 
    amrex::Print()<< "The values of alpha1 and alpha2 are " << alpha1 << " "<< alpha2 <<std::endl;
 
    if (alpha1 < 0.0 || alpha1 > 1.0 ||
    alpha2 < 0.0 || alpha2 > 1.0)
    {
        std::stringstream ss;
        ss << "Interpolation weights for hindcast files are incorrect: "
        << "alpha1 = " << alpha1 << ", alpha2 = " << alpha2;
       Abort(ss.str());
    }
 
    /*MultiFab& mf_surf_interp   = surface_state_interp[lev];
 
    // Fill the time-interpolated forecast states
    MultiFab::LinComb(surface_state_interp[lev],
                      alpha1, surface_state_1[lev], 0,
                      alpha2, surface_state_2[lev], 0,
                      0, mf_surf_interp.nComp(), mf_surf_interp.nGrow());
 
    std::string pltname = "plt_interp_surface";
    Vector<std::string> varnames_plot_mf = {"ls_mask", "SST"};
 
    const MultiFab& src = vars_new[0][0];
 
    MultiFab plot_mf(src.boxArray(),
                     src.DistributionMap(),
                     2, 0);
 
    plot_mf.setVal(0.0);
 
    for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
        const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
        const Array4<Real> &surf_mf_arr = surface_state_1[0].array(mfi);
 
        const Box& bx = mfi.validbox();
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
            plot_mf_arr(i,j,k,0) = surf_mf_arr(i,j,0);
            plot_mf_arr(i,j,k,1) = surf_mf_arr(i,j,1);
        });
    }
 
    WriteSingleLevelPlotfile(
            pltname,
            plot_mf,
            varnames_plot_mf,
            geom[0],
            time,
            0 // level
    );*/
}

timeStep()

void ERF::timeStep ( int  lev, amrex::Real  time, int  iteration  )

private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	iteration	time step counter

{
    //
    // We need to FillPatch the coarse level before assessing whether to regrid
    // We have not done the swap yet so we fill the "new" which will become the "old"
    //
    MultiFab& S_new = vars_new[lev][Vars::cons];
    MultiFab& U_new = vars_new[lev][Vars::xvel];
    MultiFab& V_new = vars_new[lev][Vars::yvel];
    MultiFab& W_new = vars_new[lev][Vars::zvel];
 
#ifdef ERF_USE_NETCDF
    //
    // Since we now only read in a subset of the time slices in wrfbdy and
    //     wrflowinp, we need to check whether it's time to read in more.
    //
    bool use_moist = (solverChoice.moisture_type != MoistureType::None);
    if (solverChoice.use_real_bcs && (lev==0)) {
        Real dT = bdy_time_interval;
 
        int n_time_old = static_cast<int>( (time        ) /  dT);
        int n_time_new = static_cast<int>( (time+dt[lev]) /  dT);
 
        int ntimes = bdy_data_xlo.size();
        for (int itime = 0; itime < ntimes; itime++)
        {
            //if (bdy_data_xlo[itime].size() > 0) {
            //    amrex::Print() << "HAVE  DATA AT TIME " << itime << std::endl;
            //} else {
            //    amrex::Print() << " NO   DATA AT TIME " << itime << std::endl;
            //}
 
            bool clear_itime = (itime < n_time_old);
 
            if (clear_itime && bdy_data_xlo[itime].size() > 0) {
                bdy_data_xlo[itime].clear();
                bdy_data_xhi[itime].clear();
                bdy_data_ylo[itime].clear();
                bdy_data_yhi[itime].clear();
                //amrex::Print() << "CLEAR BDY DATA AT TIME " << itime << std::endl;
            }
 
            bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
            //if (need_itime) amrex::Print()  << "NEED  BDY DATA AT TIME " << itime << std::endl;
 
            if (bdy_data_xlo[itime].size() == 0 && need_itime) {
                read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
                                 bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                 real_width);
 
                convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
                                        *mf_MUB, *mf_C1H, *mf_C2H,
                                    vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel], vars_new[lev][Vars::cons],
                                    geom[lev], use_moist);
           }
        } // itime
    } // use_real_bcs && lev == 0
 
    if (!nc_low_file.empty() && (lev==0)) {
        Real dT = low_time_interval;
 
        int n_time_old = static_cast<int>( (time        ) /  dT);
        int n_time_new = static_cast<int>( (time+dt[lev]) /  dT);
 
        int ntimes = bdy_data_xlo.size();
        for (int itime = 0; itime < ntimes; itime++)
        {
            bool clear_itime = (itime < n_time_old);
 
            if (clear_itime && low_data_zlo[itime].size() > 0) {
                low_data_zlo[itime].clear();
                //amrex::Print() << "CLEAR LOW DATA AT TIME " << itime << std::endl;
            }
 
            bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
            //if (need_itime) amrex::Print()  << "NEED  LOW DATA AT TIME " << itime << std::endl;
 
            if (low_data_zlo[itime].size() == 0 && need_itime) {
                read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
 
                update_sst_tsk(itime, geom[lev], ba2d[lev],
                               sst_lev[lev], tsk_lev[lev],
                               m_SurfaceLayer, low_data_zlo,
                               S_new, *mf_PSFC[lev],
                               solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
            }
        } // itime
    } // have nc_low_file && lev == 0
#endif
 
    //
    // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
    //
    if (lev == 0) {
        FillPatchCrseLevel(lev, time, {&S_new, &U_new, &V_new, &W_new});
    } else if (lev < finest_level) {
        FillPatchFineLevel(lev, time, {&S_new, &U_new, &V_new, &W_new},
                           {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                           base_state[lev], base_state[lev]);
    }
 
    if (regrid_int > 0)  // We may need to regrid
    {
        // help keep track of whether a level was already regridded
        // from a coarser level call to regrid
        static Vector<int> last_regrid_step(max_level+1, 0);
 
        // regrid changes level "lev+1" so we don't regrid on max_level
        // also make sure we don't regrid fine levels again if
        // it was taken care of during a coarser regrid
        if (lev < max_level)
        {
            if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
            {
                // regrid could add newly refine levels (if finest_level < max_level)
                // so we save the previous finest level index
                int old_finest = finest_level;
 
                regrid(lev, time);
 
#ifdef ERF_USE_PARTICLES
                if (finest_level != old_finest) {
                    particleData.Redistribute();
                }
#endif
 
                // mark that we have regridded this level already
                for (int k = lev; k <= finest_level; ++k) {
                    last_regrid_step[k] = istep[k];
                }
 
                // if there are newly created levels, set the time step
                for (int k = old_finest+1; k <= finest_level; ++k) {
                    dt[k] = dt[k-1] / static_cast<Real>(nsubsteps[k]);
                }
            } // if
        } // lev
    }
 
    // Update what we call "old" and "new" time
    t_old[lev] = t_new[lev];
    t_new[lev] += dt[lev];
 
    if (Verbose()) {
        amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
        amrex::Print() << std::setprecision(timeprecision)
                       << "ADVANCE from elapsed time = " << t_old[lev] << " to " << t_new[lev]
                       << " with dt = " << dt[lev] << std::endl;
    }
 
#ifdef ERF_USE_WW3_COUPLING
    amrex::Print() <<  " About to call send_to_ww3 from ERF_Timestep" << std::endl;
    send_to_ww3(lev);
    amrex::Print() <<  " About to call read_waves from ERF_Timestep"  << std::endl;
    read_waves(lev);
    //send_to_ww3(lev);
    //read_waves(lev);
    //send_to_ww3(lev);
#endif
 
    // Advance a single level for a single time step
    Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
 
    ++istep[lev];
 
    if (Verbose()) {
        amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
        amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
    }
 
    if (lev < finest_level)
    {
        // recursive call for next-finer level
        for (int i = 1; i <= nsubsteps[lev+1]; ++i)
        {
            Real strt_time_for_fine = time + (i-1)*dt[lev+1];
            timeStep(lev+1, strt_time_for_fine, i);
        }
    }
 
    if (verbose && lev == 0 && solverChoice.moisture_type != MoistureType::None) {
        amrex::Print() << "Cloud fraction " << time << "  " << cloud_fraction(time) << std::endl;
    }
}

turbPert_amplitude()

void ERF::turbPert_amplitude ( const int  lev )

private

{
    // Accessing data
    auto& lev_new = vars_new[lev];
 
    // Creating local data
    int ncons = lev_new[Vars::cons].nComp();
    MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
 
    // Defining BoxArray type
    auto m_ixtype = cons_data.boxArray().ixType();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
        const Box &bx  = mfi.validbox();
        const auto &cons_pert_arr = cons_data.array(mfi); // Address of perturbation array
        const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
 
        turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cons_pert_arr, pert_cell);
    } // mfi
}

Here is the call graph for this function:

turbPert_update()

void ERF::turbPert_update ( const int  lev, const amrex::Real  dt  )

private

{
    // Accessing data
    auto& lev_new = vars_new[lev];
 
    // Create aliases to state data to pass to calc_tpi_update
    int ncons = lev_new[Vars::cons].nComp();
    MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
    MultiFab xvel_data(lev_new[Vars::xvel], make_alias, 0, 1);
    MultiFab yvel_data(lev_new[Vars::yvel], make_alias, 0, 1);
 
    // Computing perturbation update time
    turbPert.calc_tpi_update(lev, local_dt, xvel_data, yvel_data, cons_data);
 
    Print() << "Successfully initialized turbulent perturbation update time and amplitude with type: "<< turbPert.pt_type <<"\n";
}

update_diffusive_arrays()

void ERF::update_diffusive_arrays ( int  lev, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

private

{
    // ********************************************************************************************
    // Diffusive terms
    // ********************************************************************************************
    bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None);
    bool l_use_kturb   = solverChoice.turbChoice[lev].use_kturb;
    bool l_use_diff    = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
                           l_use_kturb );
    bool l_need_SmnSmn = solverChoice.turbChoice[lev].use_keqn;
    bool l_use_moist   = (  solverChoice.moisture_type != MoistureType::None  );
    bool l_rotate      = (  solverChoice.use_rotate_surface_flux  );
 
    bool l_implicit_diff = (solverChoice.vert_implicit_fac[0] > 0 ||
                            solverChoice.vert_implicit_fac[1] > 0 ||
                            solverChoice.vert_implicit_fac[2] > 0);
 
    BoxArray ba12 = convert(ba, IntVect(1,1,0));
    BoxArray ba13 = convert(ba, IntVect(1,0,1));
    BoxArray ba23 = convert(ba, IntVect(0,1,1));
 
    Tau[lev].resize(9);
    Tau_corr[lev].resize(3);
 
    if (l_use_diff) {
        //
        // NOTE: We require ghost cells in the vertical when allowing grids that don't
        //       cover the entire vertical extent of the domain at this level
        //
        for (int i = 0; i < 3; i++) {
            Tau[lev][i] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
        }
        Tau[lev][TauType::tau12] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
        Tau[lev][TauType::tau13] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
        Tau[lev][TauType::tau23] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
        Tau[lev][TauType::tau12]->setVal(0.);
        Tau[lev][TauType::tau13]->setVal(0.);
        Tau[lev][TauType::tau23]->setVal(0.);
        if (l_use_terrain) {
            Tau[lev][TauType::tau21] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
            Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
            Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
            Tau[lev][TauType::tau21]->setVal(0.);
            Tau[lev][TauType::tau31]->setVal(0.);
            Tau[lev][TauType::tau32]->setVal(0.);
        } else if (l_implicit_diff) {
            Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
            Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
            Tau[lev][TauType::tau31]->setVal(0.);
            Tau[lev][TauType::tau32]->setVal(0.);
        } else {
            Tau[lev][TauType::tau21] = nullptr;
            Tau[lev][TauType::tau31] = nullptr;
            Tau[lev][TauType::tau32] = nullptr;
        }
 
        if (l_implicit_diff && solverChoice.implicit_momentum_diffusion)
        {
            Tau_corr[lev][0] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); // Tau31
            Tau_corr[lev][1] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); // Tau32
            Tau_corr[lev][0]->setVal(0.);
            Tau_corr[lev][1]->setVal(0.);
#ifdef ERF_IMPLICIT_W
            Tau_corr[lev][2] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) ); // Tau33
            Tau_corr[lev][2]->setVal(0.);
#else
            Tau_corr[lev][2] = nullptr;
#endif
        } else {
            Tau_corr[lev][0] = nullptr;
            Tau_corr[lev][1] = nullptr;
            Tau_corr[lev][2] = nullptr;
        }
 
        SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
        SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
        SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
        SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
        SFS_hfx1_lev[lev]->setVal(0.);
        SFS_hfx2_lev[lev]->setVal(0.);
        SFS_hfx3_lev[lev]->setVal(0.);
        SFS_diss_lev[lev]->setVal(0.);
        if (l_use_moist) {
            SFS_q1fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
            SFS_q2fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
            SFS_q1fx3_lev[lev]->setVal(0.0);
            SFS_q2fx3_lev[lev]->setVal(0.0);
            if (l_rotate) {
                SFS_q1fx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
                SFS_q1fx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
                SFS_q1fx1_lev[lev]->setVal(0.0);
                SFS_q1fx2_lev[lev]->setVal(0.0);
            } else {
                SFS_q1fx1_lev[lev] = nullptr;
                SFS_q1fx2_lev[lev] = nullptr;
            }
        } else {
            SFS_q1fx1_lev[lev] = nullptr;
            SFS_q1fx2_lev[lev] = nullptr;
            SFS_q1fx3_lev[lev] = nullptr;
            SFS_q2fx3_lev[lev] = nullptr;
        }
    } else {
        for (int i = 0; i < 9; i++) {
            Tau[lev][i] = nullptr;
        }
        SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
        SFS_diss_lev[lev] = nullptr;
    }
 
    if (l_use_kturb) {
        eddyDiffs_lev[lev] = std::make_unique<MultiFab>(ba, dm, EddyDiff::NumDiffs, 2);
        eddyDiffs_lev[lev]->setVal(0.0);
        if(l_need_SmnSmn) {
            SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
        } else {
            SmnSmn_lev[lev] = nullptr;
        }
    } else {
        eddyDiffs_lev[lev] = nullptr;
        SmnSmn_lev[lev]    = nullptr;
    }
}

update_terrain_arrays()

void ERF::update_terrain_arrays

(

int

lev

)

{
    if (SolverChoice::mesh_type == MeshType::StretchedDz ||
        SolverChoice::mesh_type == MeshType::VariableDz) {
        make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
        make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
        make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
    } else { // MeshType::ConstantDz
        if (SolverChoice::terrain_type == TerrainType::EB) {
            const auto& ebfact = *eb[lev]->get_const_factory();
            const MultiFab& volfrac = ebfact.getVolFrac();
            detJ_cc[lev] = std::make_unique<MultiFab>(volfrac, amrex::make_alias, 0, volfrac.nComp());
        }
    }
}

Here is the call graph for this function:

volWgtColumnSum()

void ERF::volWgtColumnSum	(	int	lev,
		const amrex::MultiFab &	mf,
		int	comp,
		amrex::MultiFab &	mf_2d,
		const amrex::MultiFab &	dJ
	)

{
    BL_PROFILE("ERF::volWgtSumColumnMF()");
 
    mf_2d.setVal(0.);
 
    // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
    // m is the map scale factor at cell centers
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(mf_to_be_summed, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx   = mfi.tilebox();
        const auto  dst_arr = mf_2d.array(mfi);
        const auto  src_arr = mf_to_be_summed.array(mfi);
        if (SolverChoice::mesh_type == MeshType::ConstantDz) {
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                amrex::HostDevice::Atomic::Add(&dst_arr(i,j,0),src_arr(i,j,k,comp));
            });
        } else {
            const auto& dJ_arr = dJ.const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                amrex::HostDevice::Atomic::Add(&dst_arr(i,j,0),src_arr(i,j,k,comp)*dJ_arr(i,j,k));
            });
        }
    } // mfi
 
    auto const& dx = geom[lev].CellSizeArray();
 
    mf_2d.mult(dx[2]);
}

volWgtSumMF()

Real ERF::volWgtSumMF	(	int	lev,
		const amrex::MultiFab &	mf,
		int	comp,
		const amrex::MultiFab &	dJ,
		const amrex::MultiFab &	mfx,
		const amrex::MultiFab &	mfy,
		bool	finemask,
		bool	local = true
	)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters

lev	Current level
mf_to_be_summed	: MultiFab on which we do the volume weighted sum
dJ	: volume weighting due to metric terms
mfmx	: map factor in x-direction at cell centers
mfmy	: map factor in y-direction at cell centers
comp	: Index of the component we want to sum
finemask	: If a finer level is available, determines whether we mask fine data
local	: Boolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)

{
    BL_PROFILE("ERF::volWgtSumMF()");
 
    Real sum = 0.0;
    MultiFab tmp(mf_to_be_summed.boxArray(), mf_to_be_summed.DistributionMap(), 1, 0);
 
    // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
    // m is the map scale factor at cell centers
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
        const Box& bx   = mfi.tilebox();
        const auto  dst_arr = tmp.array(mfi);
        const auto  src_arr = mf_to_be_summed.array(mfi);
        const auto& mfx_arr = mfmx.const_array(mfi);
        const auto& mfy_arr = mfmy.const_array(mfi);
 
        if (SolverChoice::terrain_type != TerrainType::EB) {
            if (SolverChoice::mesh_type == MeshType::ConstantDz) {
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    dst_arr(i,j,k,0) = src_arr(i,j,k,comp) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                });
            } else {
                const auto&  dJ_arr = dJ.const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                {
                    dst_arr(i,j,k,0) = src_arr(i,j,k,comp) * dJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                });
            }
        } else {
            const auto&  dJ_arr = dJ.const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                dst_arr(i,j,k,0) = src_arr(i,j,k,comp) * dJ_arr(i,j,k);
            });
        }
 
    } // mfi
 
    if (lev < finest_level && finemask) {
        MultiFab::Multiply(tmp, *fine_mask[lev+1].get(), 0, 0, 1, 0);
    }
 
    // If local = true  then "sum" will be the sum only over the FABs on each rank
    // If local = false then "sum" will be the sum over the whole MultiFab, and will be broadcast to all ranks
    sum = tmp.sum(0,local);
 
    auto const& dx = geom[lev].CellSizeArray();
 
    sum *= dx[0]*dx[1]*dx[2];
 
    return sum;
}

WeatherDataInterpolation()

void ERF::WeatherDataInterpolation	(	const int	nlevs,
		const amrex::Real	time,
		amrex::Vector< std::unique_ptr< amrex::MultiFab >> &	z_phys_nd,
		bool	regrid_forces_file_read
	)

{
 
    static amrex::Vector<Real> next_read_forecast_time;
    static amrex::Vector<Real> last_read_forecast_time;
 
    const int nlevs = a_z_phys_nd.size();
 
    Real hindcast_data_interval = solverChoice.hindcast_data_interval_in_hrs*3600.0;
 
    // Initialize static vectors once
    if (next_read_forecast_time.empty()) {
        next_read_forecast_time.resize(nlevs, -1.0);
        last_read_forecast_time.resize(nlevs, -1.0);
        Print() << "Initializing the time vector values here by " << lev << std::endl;
    }
 
    if (next_read_forecast_time[lev] < 0.0) {
        int next_multiple = static_cast<int>(time / hindcast_data_interval);
        next_read_forecast_time[lev] = next_multiple * hindcast_data_interval;
        last_read_forecast_time[lev] = next_read_forecast_time[lev];
    }
 
    if (time >= next_read_forecast_time[lev] or regrid_forces_file_read) {
 
        Print() << "Data reading happening at level " << lev << std::endl;
 
        std::string folder = solverChoice.hindcast_boundary_data_dir;
 
        // Check if folder exists and is a directory
        if (!fs::exists(folder) || !fs::is_directory(folder)) {
            throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
        }
 
        std::vector<std::string> bin_files;
 
        for (const auto& entry : fs::directory_iterator(folder)) {
            if (!entry.is_regular_file()) continue;
 
            std::string fname = entry.path().filename().string();
            if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
                bin_files.push_back(entry.path().string());
            }
        }
        std::sort(bin_files.begin(), bin_files.end());
 
    // Check if no .bin files were found
        if (bin_files.empty()) {
            throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
        }
 
        std::string filename1, filename2;
 
        int idx1 = static_cast<int>(time / hindcast_data_interval);
        int idx2 = static_cast<int>(time / hindcast_data_interval)+1;
        Print() << "Reading weather data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
 
        if (idx2 >= static_cast<int>(bin_files.size())) {
            throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
        }
 
        filename1 = bin_files[idx1];
        filename2 = bin_files[idx2];
 
        FillForecastStateMultiFabs(lev, filename1, a_z_phys_nd[lev], forecast_state_1);
        FillForecastStateMultiFabs(lev, filename2, a_z_phys_nd[lev], forecast_state_2);
 
         // Create the time-interpolated forecast state
        //CreateForecastStateMultiFabs(forecast_state_interp);
        if(!regrid_forces_file_read){
            last_read_forecast_time[lev] = next_read_forecast_time[lev];
            next_read_forecast_time[lev] += hindcast_data_interval;
            Print() << "Next forecast time getting updated here " << std::endl;
        }
    }
 
    Real prev_read_time = last_read_forecast_time[lev];
    Real alpha1 = 1.0 - (time - prev_read_time)/hindcast_data_interval;
    Real alpha2 = 1.0 - alpha1;
 
    amrex::Print()<< "The values of alpha1 and alpha2 are " << alpha1 << " "<< alpha2 <<std::endl;
 
    if (alpha1 < 0.0 || alpha1 > 1.0 ||
    alpha2 < 0.0 || alpha2 > 1.0)
    {
        std::stringstream ss;
        ss << "Interpolation weights for hindcast files are incorrect: "
        << "alpha1 = " << alpha1 << ", alpha2 = " << alpha2;
       Abort(ss.str());
    }
 
    MultiFab& erf_mf_cons   = forecast_state_interp[lev][Vars::cons];
    MultiFab& erf_mf_xvel   = forecast_state_interp[lev][Vars::xvel];
    MultiFab& erf_mf_yvel   = forecast_state_interp[lev][Vars::yvel];
    //MultiFab& erf_mf_zvel   = forecast_state_interp[0][Vars::zvel];
    MultiFab& erf_mf_latlon = forecast_state_interp[lev][4];
 
    // Fill the time-interpolated forecast states
    MultiFab::LinComb(forecast_state_interp[lev][Vars::cons],
                      alpha1, forecast_state_1[lev][Vars::cons], 0,
                      alpha2, forecast_state_2[lev][Vars::cons], 0,
                      0, erf_mf_cons.nComp(), forecast_state_interp[lev][Vars::cons].nGrow());
    MultiFab::LinComb(forecast_state_interp[lev][Vars::xvel],
                      alpha1, forecast_state_1[lev][Vars::xvel], 0,
                      alpha2, forecast_state_2[lev][Vars::xvel], 0,
                      0, erf_mf_xvel.nComp(), forecast_state_interp[lev][Vars::xvel].nGrow());
    MultiFab::LinComb(forecast_state_interp[lev][Vars::yvel],
                      alpha1, forecast_state_1[lev][Vars::yvel], 0,
                      alpha2, forecast_state_2[lev][Vars::yvel], 0,
                      0, erf_mf_yvel.nComp(), forecast_state_interp[lev][Vars::yvel].nGrow());
    MultiFab::LinComb(forecast_state_interp[lev][4],
                      alpha1, forecast_state_1[lev][4], 0,
                      alpha2, forecast_state_2[lev][4], 0,
                      0, erf_mf_latlon.nComp(), forecast_state_interp[lev][4].nGrow());
 
    /*Vector<std::string> varnames_plot_mf = {
    "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
    }; // Customize variable names
 
    std::string pltname = "plt_interp";
 
    MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
                     10, 0);
 
    plot_mf.setVal(0.0);
 
    for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
        const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
        const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
        const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
        const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
        const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
        const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
 
        const Box& bx = mfi.validbox();
 
        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
            plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
            plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
            plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
            plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
            plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
 
            plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/2.0;
            plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/2.0;
            plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/2.0;
 
            plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
            plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
        });
    }
 
 
    WriteSingleLevelPlotfile(
            pltname,
            plot_mf,
            varnames_plot_mf,
            geom[0],
            time,
            0 // level
        );*/
}

Write2DPlotFile()

void ERF::Write2DPlotFile	(	int	which,
		PlotFileType	plotfile_type,
		amrex::Vector< std::string >	plot_var_names
	)

{
    const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
    const int ncomp_mf = varnames.size();
 
    if (ncomp_mf == 0) return;
 
    // Vector of MultiFabs for cell-centered data
    Vector<MultiFab> mf(finest_level+1);
    for (int lev = 0; lev <= finest_level; ++lev) {
        mf[lev].define(ba2d[lev], dmap[lev], ncomp_mf, 0);
    }
 
 
    // **********************************************************************************************
    // (Effectively) 2D arrays
    // **********************************************************************************************
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        // Make sure getPgivenRTh and getTgivenRandRTh don't fail
        if (check_for_nans) {
            check_for_negative_theta(vars_new[lev][Vars::cons]);
        }
 
        int mf_comp = 0;
 
        // Set all components to zero in case they aren't defined below
        mf[lev].setVal(0.0);
 
        // Expose domain khi and klo at each level
        int klo = geom[lev].Domain().smallEnd(2);
        int khi = geom[lev].Domain().bigEnd(2);
 
        if (containerHasElement(plot_var_names, "z_surf")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf[lev].array(mfi);
                const Array4<const Real>& z_phys_arr = z_phys_nd[lev]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                   derdat(i, j, k, mf_comp) = Compute_Z_AtWFace(i, j, 0, z_phys_arr);
                });
            }
            mf_comp++;
        }
 
        if (containerHasElement(plot_var_names, "landmask")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf[lev].array(mfi);
                const Array4<const int>& lmask_arr = lmask_lev[lev][0]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                   derdat(i, j, k, mf_comp) = lmask_arr(i, j, 0);
                });
            }
            mf_comp++;
        }
 
        if (containerHasElement(plot_var_names, "mapfac")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf[lev].array(mfi);
                const Array4<Real>& mf_m   = mapfac[lev][MapFacType::m_x]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                   derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
                });
            }
            mf_comp++;
        }
 
        if (containerHasElement(plot_var_names, "lat_m")) {
            if (lat_m[lev]) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = lat_m[lev]->array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = data(i,j,0);
                    });
                }
            }
            mf_comp++;
        } // lat_m
 
        if (containerHasElement(plot_var_names, "lon_m")) {
            if (lon_m[lev]) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = lon_m[lev]->array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = data(i,j,0);
                    });
                }
            } else {
                mf[lev].setVal(0.0,mf_comp,1,0);
            }
 
            mf_comp++;
 
        } // lon_m
 
        ///////////////////////////////////////////////////////////////////////
        // These quantities are diagnosed by the surface layer
        if (containerHasElement(plot_var_names, "u_star")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (m_SurfaceLayer) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& ustar  = m_SurfaceLayer->get_u_star(lev)->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // ustar
 
        if (containerHasElement(plot_var_names, "w_star")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (m_SurfaceLayer) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& ustar  = m_SurfaceLayer->get_w_star(lev)->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // wstar
 
        if (containerHasElement(plot_var_names, "t_star")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (m_SurfaceLayer) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& ustar  = m_SurfaceLayer->get_t_star(lev)->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // tstar
 
        if (containerHasElement(plot_var_names, "q_star")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (m_SurfaceLayer) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& ustar  = m_SurfaceLayer->get_q_star(lev)->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // qstar
 
        if (containerHasElement(plot_var_names, "Olen")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (m_SurfaceLayer) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& ustar  = m_SurfaceLayer->get_olen(lev)->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // Olen
 
        if (containerHasElement(plot_var_names, "pblh")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (m_SurfaceLayer) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& ustar  = m_SurfaceLayer->get_pblh(lev)->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // pblh
 
        if (containerHasElement(plot_var_names, "t_surf")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (m_SurfaceLayer) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& tsurf  = m_SurfaceLayer->get_t_surf(lev)->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = tsurf(i, j, 0);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // tsurf
 
        if (containerHasElement(plot_var_names, "q_surf")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (m_SurfaceLayer) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& ustar  = m_SurfaceLayer->get_q_surf(lev)->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // qsurf
 
        if (containerHasElement(plot_var_names, "z0")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (m_SurfaceLayer) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& ustar  = m_SurfaceLayer->get_z0(lev)->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // z0
 
        if (containerHasElement(plot_var_names, "OLR")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (solverChoice.rad_type != RadiationType::None) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& olr    = rad_fluxes[lev]->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = olr(i, j, khi, 2);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // OLR
 
        if (containerHasElement(plot_var_names, "sens_flux")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (SFS_hfx3_lev[lev]) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& hfx_arr = SFS_hfx3_lev[lev]->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = hfx_arr(i, j, klo);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // sens_flux
 
        if (containerHasElement(plot_var_names, "laten_flux")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            if (SFS_hfx3_lev[lev]) {
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const auto& derdat = mf[lev].array(mfi);
                    const auto& qfx_arr = SFS_q1fx3_lev[lev]->const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = qfx_arr(i, j, klo);
                    });
                }
            } else {
                mf[lev].setVal(-999,mf_comp,1,0);
            }
            mf_comp++;
        } // laten_flux
 
        if (containerHasElement(plot_var_names, "surf_pres")) {
            bool moist = (solverChoice.moisture_type != MoistureType::None);
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const auto& derdat   = mf[lev].array(mfi);
                const auto& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    auto rt = cons_arr(i,j,klo,RhoTheta_comp);
                    auto qv = (moist) ? cons_arr(i,j,klo,RhoQ1_comp)/cons_arr(i,j,klo,Rho_comp)
                                      : 0.0;
                    derdat(i, j, k, mf_comp) = getPgivenRTh(rt, qv);
                });
            }
            mf_comp++;
        } // surf_pres
 
        if (containerHasElement(plot_var_names, "integrated_qv")) {
            MultiFab mf_qv_int(mf[lev],make_alias,mf_comp,1);
            if (solverChoice.moisture_type != MoistureType::None) {
                volWgtColumnSum(lev, vars_new[lev][Vars::cons], RhoQ1_comp, mf_qv_int, *detJ_cc[lev]);
            } else {
                mf_qv_int.setVal(0.);
            }
            mf_comp++;
        }
    } // lev
 
    std::string plotfilename;
    if (which == 1) {
       plotfilename = Concatenate(plot2d_file_1, istep[0], file_name_digits);
    } else if (which == 2) {
       plotfilename = Concatenate(plot2d_file_2, istep[0], file_name_digits);
    }
 
    Vector<Geometry> my_geom(finest_level+1);
 
    Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
    if (geom[0].isPeriodic(0)) { is_per[0] = 1;}
    if (geom[0].isPeriodic(1)) { is_per[1] = 1;}
 
    int coord_sys = 0;
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        Box slab = makeSlab(geom[lev].Domain(),2,0);
        auto const slab_lo = lbound(slab);
        auto const slab_hi = ubound(slab);
 
        // Create a new geometry based only on the 2D slab
        Real dz = geom[lev].CellSize(2);
        RealBox rb = geom[lev].ProbDomain();
        rb.setLo(2,  slab_lo.z   *dz);
        rb.setHi(2, (slab_hi.z+1)*dz);
        my_geom[lev].define(slab, rb, coord_sys, is_per);
    }
 
    if (plotfile_type == PlotFileType::Amrex)
    {
        Print() << "Writing 2D native plotfile " << plotfilename << "\n";
        WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                GetVecOfConstPtrs(mf),
                                varnames, my_geom, t_new[0], istep, refRatio());
        writeJobInfo(plotfilename);
 
#ifdef ERF_USE_NETCDF
    } else if (plotfile_type == PlotFileType::Netcdf) {
         int lev   = 0;
         int l_which = 0;
         const Real* p_lo = my_geom[lev].ProbLo();
         const Real* p_hi = my_geom[lev].ProbHi();
         const auto dx    = my_geom[lev].CellSize();
         writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
                         {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],dx[2]}, {dx[0],dx[1],dx[2]},
                         my_geom[lev].Domain(), t_new[0], start_bdy_time);
#endif
    } else {
        // Here we assume the plotfile_type is PlotFileType::None
        Print() << "Writing no 2D plotfile since plotfile_type is none" << std::endl;
    }
}

Here is the call graph for this function:

Write3DPlotFile()

void ERF::Write3DPlotFile	(	int	which,
		PlotFileType	plotfile_type,
		amrex::Vector< std::string >	plot_var_names
	)

{
    auto dPlotTime0 = amrex::second();
 
    const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
    const int ncomp_mf = varnames.size();
 
    int ncomp_cons = vars_new[0][Vars::cons].nComp();
 
    if (ncomp_mf == 0) return;
 
    // We Fillpatch here because some of the derived quantities require derivatives
    //     which require ghost cells to be filled.  We do not need to call FillPatcher
    //     because we don't need to set interior fine points.
    // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
 
    // Level 0 FillPatch
    FillPatchCrseLevel(0, t_new[0], {&vars_new[0][Vars::cons], &vars_new[0][Vars::xvel],
                       &vars_new[0][Vars::yvel], &vars_new[0][Vars::zvel]});
 
    for (int lev = 1; lev <= finest_level; ++lev) {
        bool fillset = false;
        FillPatchFineLevel(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
                           &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
                          {&vars_new[lev][Vars::cons], &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                           base_state[lev], base_state[lev], fillset);
    }
 
    // Get qmoist pointers if using moisture
    bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
    for (int lev = 0; lev <= finest_level; ++lev) {
        for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
            qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
        }
    }
 
    // Vector of MultiFabs for cell-centered data
    Vector<MultiFab> mf(finest_level+1);
    for (int lev = 0; lev <= finest_level; ++lev) {
        mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
    }
 
    // Vector of MultiFabs for nodal data
    Vector<MultiFab> mf_nd(finest_level+1);
    if ( SolverChoice::mesh_type != MeshType::ConstantDz) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
            mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
            mf_nd[lev].setVal(0.);
        }
    }
 
    // Vector of MultiFabs for face-centered velocity
    Vector<MultiFab> mf_u(finest_level+1);
    Vector<MultiFab> mf_v(finest_level+1);
    Vector<MultiFab> mf_w(finest_level+1);
    if (m_plot_face_vels) {
        for (int lev = 0; lev <= finest_level; ++lev) {
            BoxArray grid_stag_u(grids[lev]); grid_stag_u.surroundingNodes(0);
            BoxArray grid_stag_v(grids[lev]); grid_stag_v.surroundingNodes(1);
            BoxArray grid_stag_w(grids[lev]); grid_stag_w.surroundingNodes(2);
            mf_u[lev].define(grid_stag_u, dmap[lev], 1, 0);
            mf_v[lev].define(grid_stag_v, dmap[lev], 1, 0);
            mf_w[lev].define(grid_stag_w, dmap[lev], 1, 0);
            MultiFab::Copy(mf_u[lev],vars_new[lev][Vars::xvel],0,0,1,0);
            MultiFab::Copy(mf_v[lev],vars_new[lev][Vars::yvel],0,0,1,0);
            MultiFab::Copy(mf_w[lev],vars_new[lev][Vars::zvel],0,0,1,0);
        }
    }
 
    // Array of MultiFabs for cell-centered velocity
    Vector<MultiFab> mf_cc_vel(finest_level+1);
 
    if (containerHasElement(plot_var_names, "x_velocity" ) ||
        containerHasElement(plot_var_names, "y_velocity" ) ||
        containerHasElement(plot_var_names, "z_velocity" ) ||
        containerHasElement(plot_var_names, "magvel"     ) ||
        containerHasElement(plot_var_names, "vorticity_x") ||
        containerHasElement(plot_var_names, "vorticity_y") ||
        containerHasElement(plot_var_names, "vorticity_z") ) {
 
        for (int lev = 0; lev <= finest_level; ++lev) {
            mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
            mf_cc_vel[lev].setVal(-1.e20);
            average_face_to_cellcenter(mf_cc_vel[lev],0,
                                       Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                                &vars_new[lev][Vars::yvel],
                                                                &vars_new[lev][Vars::zvel]});
        } // lev
    } // if (vel or vort)
 
    // We need ghost cells if computing vorticity
    if ( containerHasElement(plot_var_names, "vorticity_x")||
         containerHasElement(plot_var_names, "vorticity_y") ||
         containerHasElement(plot_var_names, "vorticity_z") )
    {
        amrex::Interpolater* mapper = &cell_cons_interp;
        for (int lev = 1; lev <= finest_level; ++lev)
        {
            Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
            Vector<Real> ftime    = {t_new[lev], t_new[lev]};
            Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
            Vector<Real> ctime    = {t_new[lev], t_new[lev]};
 
            FillBdyCCVels(mf_cc_vel,lev-1);
 
            // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
            FillPatchTwoLevels(mf_cc_vel[lev], mf_cc_vel[lev].nGrowVect(), IntVect(0,0,0),
                               t_new[lev], cmf, ctime, fmf, ftime,
                               0, 0, mf_cc_vel[lev].nComp(), geom[lev-1], geom[lev],
                               refRatio(lev-1), mapper, domain_bcs_type,
                               BaseBCVars::rho0_bc_comp);
        } // lev
        FillBdyCCVels(mf_cc_vel);
    } // if (vort)
 
 
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        // Make sure getPgivenRTh and getTgivenRandRTh don't fail
        if (check_for_nans) {
            check_for_negative_theta(vars_new[lev][Vars::cons]);
        }
 
        int mf_comp = 0;
 
        BoxArray ba(vars_new[lev][Vars::cons].boxArray());
        DistributionMapping dm = vars_new[lev][Vars::cons].DistributionMap();
 
        // First, copy any of the conserved state variables into the output plotfile
        for (int i = 0; i < cons_names.size(); ++i) {
            if (containerHasElement(plot_var_names, cons_names[i])) {
                MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,0);
                mf_comp++;
            }
        }
 
        // Next, check for velocities
        if (containerHasElement(plot_var_names, "x_velocity")) {
            MultiFab::Copy(mf[lev], mf_cc_vel[lev], 0, mf_comp, 1, 0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "y_velocity")) {
            MultiFab::Copy(mf[lev], mf_cc_vel[lev], 1, mf_comp, 1, 0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "z_velocity")) {
            MultiFab::Copy(mf[lev], mf_cc_vel[lev], 2, mf_comp, 1, 0);
            mf_comp += 1;
        }
 
        // Create multifabs for HSE and pressure fields used to derive other quantities
        MultiFab  r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
        MultiFab  p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
        MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
 
        MultiFab pressure;
 
        if (solverChoice.anelastic[lev] == 0) {
            if (containerHasElement(plot_var_names, "pressure")     ||
                containerHasElement(plot_var_names, "pert_pres")    ||
                containerHasElement(plot_var_names, "dpdx")         ||
                containerHasElement(plot_var_names, "dpdy")         ||
                containerHasElement(plot_var_names, "dpdz")         ||
                containerHasElement(plot_var_names, "eq_pot_temp")  ||
                containerHasElement(plot_var_names, "qsat"))
            {
                int ng = (containerHasElement(plot_var_names, "dpdx") || containerHasElement(plot_var_names, "dpdy") ||
                          containerHasElement(plot_var_names, "dpdz")) ? 1 : 0;
 
                // Allocate space for pressure
                pressure.define(ba,dm,1,ng);
 
                if (ng > 0) {
                    // Default to p_hse as a way of filling ghost cells at domain boundaries
                    MultiFab::Copy(pressure,p_hse,0,0,1,1);
                }
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& gbx = mfi.growntilebox(IntVect(ng,ng,0));
 
                    const Array4<Real      >& p_arr = pressure.array(mfi);
                    const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                    const int ncomp = vars_new[lev][Vars::cons].nComp();
 
                    ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
                        const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                        p_arr(i, j, k) = getPgivenRTh(rhotheta,qv_for_p);
                    });
               } // mfi
               pressure.FillBoundary(geom[lev].periodicity());
            } // compute compressible pressure
        } // not anelastic
        else {
            if (containerHasElement(plot_var_names, "dpdx")         ||
                containerHasElement(plot_var_names, "dpdy")         ||
                containerHasElement(plot_var_names, "dpdz")         ||
                containerHasElement(plot_var_names, "eq_pot_temp")  ||
                containerHasElement(plot_var_names, "qsat"))
            {
                // Copy p_hse into pressure if using anelastic
                pressure.define(ba,dm,1,0);
                MultiFab::Copy(pressure,p_hse,0,0,1,0);
            }
        }
 
        // Finally, check for any derived quantities and compute them, inserting
        // them into our output multifab
        auto calculate_derived = [&](const std::string& der_name,
                                     MultiFab& src_mf,
                                     decltype(derived::erf_dernull)& der_function)
        {
            if (containerHasElement(plot_var_names, der_name)) {
                MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    auto& dfab = dmf[mfi];
                    auto& sfab = src_mf[mfi];
                    der_function(bx, dfab, 0, 1, sfab, Geom(lev), t_new[0], nullptr, lev);
                }
 
                mf_comp++;
            }
        };
 
        // *****************************************************************************************
        // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
        //       defined in ERF.H
        // *****************************************************************************************
 
        calculate_derived("soundspeed",  vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
        if (use_moisture) {
            calculate_derived("temp",        vars_new[lev][Vars::cons], derived::erf_dermoisttemp);
        } else {
            calculate_derived("temp",        vars_new[lev][Vars::cons], derived::erf_dertemp);
        }
        calculate_derived("theta",       vars_new[lev][Vars::cons], derived::erf_dertheta);
        calculate_derived("KE",          vars_new[lev][Vars::cons], derived::erf_derKE);
        calculate_derived("scalar",      vars_new[lev][Vars::cons], derived::erf_derscalar);
        calculate_derived("vorticity_x", mf_cc_vel[lev]           , derived::erf_dervortx);
        calculate_derived("vorticity_y", mf_cc_vel[lev]           , derived::erf_dervorty);
        calculate_derived("vorticity_z", mf_cc_vel[lev]           , derived::erf_dervortz);
        calculate_derived("magvel"     , mf_cc_vel[lev]           , derived::erf_dermagvel);
 
        if (containerHasElement(plot_var_names, "divU"))
        {
            // TODO TODO TODO  -- we need to convert w to omega here!!
            MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
            Array<MultiFab const*, AMREX_SPACEDIM> u;
            u[0] = &(vars_new[lev][Vars::xvel]);
            u[1] = &(vars_new[lev][Vars::yvel]);
            u[2] = &(vars_new[lev][Vars::zvel]);
            compute_divergence (lev, dmf, u, geom[lev]);
            mf_comp += 1;
        }
 
        if (containerHasElement(plot_var_names, "pres_hse"))
        {
            MultiFab::Copy(mf[lev],p_hse,0,mf_comp,1,0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "dens_hse"))
        {
            MultiFab::Copy(mf[lev],r_hse,0,mf_comp,1,0);
            mf_comp += 1;
        }
        if (containerHasElement(plot_var_names, "theta_hse"))
        {
            MultiFab::Copy(mf[lev],th_hse,0,mf_comp,1,0);
            mf_comp += 1;
        }
 
        if (containerHasElement(plot_var_names, "pressure"))
        {
            if (solverChoice.anelastic[lev] == 1) {
                MultiFab::Copy(mf[lev], p_hse, 0, mf_comp, 1, 0);
            } else {
                MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
            }
 
            mf_comp += 1;
        }
 
        if (containerHasElement(plot_var_names, "pert_pres"))
        {
            if (solverChoice.anelastic[lev] == 1) {
                MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0);
            } else {
                MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
                MultiFab::Subtract(mf[lev],p_hse,0,mf_comp,1,IntVect{0});
            }
            mf_comp += 1;
        }
 
        if (containerHasElement(plot_var_names, "pert_dens"))
        {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf[lev].array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
                });
            }
            mf_comp ++;
        }
 
        if (containerHasElement(plot_var_names, "eq_pot_temp"))
        {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat  = mf[lev].array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                const Array4<Real const>& p_arr = pressure.const_array(mfi);
                const int ncomp = vars_new[lev][Vars::cons].nComp();
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
                    Real qc = (use_moisture && (ncomp > RhoQ2_comp)) ? S_arr(i,j,k,RhoQ2_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
                    Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                    Real fac = Cp_d + Cp_l*(qv + qc);
                    Real pv = erf_esatw(T)*100.0;
 
                    derdat(i, j, k, mf_comp) = T*std::pow((p_arr(i,j,k) - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
                });
            }
            mf_comp ++;
        }
 
#ifdef ERF_USE_WINDFARM
        if ( containerHasElement(plot_var_names, "num_turb") and
             (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
              solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
        {
            MultiFab::Copy(mf[lev],Nturb[lev],0,mf_comp,1,0);
            mf_comp ++;
        }
 
        if ( containerHasElement(plot_var_names, "SMark0") and
             (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
              solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
        {
            MultiFab::Copy(mf[lev],SMark[lev],0,mf_comp,1,0);
            mf_comp ++;
        }
 
        if (containerHasElement(plot_var_names, "SMark1") and
           (solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD))
        {
            MultiFab::Copy(mf[lev],SMark[lev],1,mf_comp,1,0);
            mf_comp ++;
        }
#endif
 
        // **********************************************************************************************
        // Allocate space if we are computing any pressure gradients
        // **********************************************************************************************
 
        Vector<MultiFab> gradp_temp;  gradp_temp.resize(AMREX_SPACEDIM);
        if (containerHasElement(plot_var_names, "dpdx")       ||
            containerHasElement(plot_var_names, "dpdy")       ||
            containerHasElement(plot_var_names, "dpdz")       ||
            containerHasElement(plot_var_names, "pres_hse_x") ||
            containerHasElement(plot_var_names, "pres_hse_y"))
        {
            gradp_temp[GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp_temp[GpVars::gpx].setVal(0.);
            gradp_temp[GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp_temp[GpVars::gpy].setVal(0.);
            gradp_temp[GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp_temp[GpVars::gpz].setVal(0.);
        }
 
        // **********************************************************************************************
        // These are based on computing gradient of full pressure
        // **********************************************************************************************
 
        if (solverChoice.anelastic[lev] == 0) {
            if ( (containerHasElement(plot_var_names, "dpdx")) ||
                 (containerHasElement(plot_var_names, "dpdy")) ||
                 (containerHasElement(plot_var_names, "dpdz")) ) {
                compute_gradp(pressure, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
                              get_eb(lev), gradp_temp, solverChoice);
            }
        }
 
        if (containerHasElement(plot_var_names, "dpdx"))
        {
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real      >&   derdat  = mf[lev].array(mfi);
                const Array4<Real const>&   gpx_arr = (solverChoice.anelastic[lev] == 1) ?
                      gradp[lev][GpVars::gpx].array(mfi) : gradp_temp[GpVars::gpx].array(mfi);
                const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
                });
            }
            mf_comp ++;
        } // dpdx
        if (containerHasElement(plot_var_names, "dpdy"))
        {
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real      >&   derdat  = mf[lev].array(mfi);
                const Array4<Real const>&   gpy_arr = (solverChoice.anelastic[lev] == 1) ?
                      gradp[lev][GpVars::gpy].array(mfi) : gradp_temp[GpVars::gpy].array(mfi);
                const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
                });
            }
            mf_comp ++;
        } // dpdy
        if (containerHasElement(plot_var_names, "dpdz"))
        {
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real      >&  derdat  = mf[lev].array(mfi);
                const Array4<Real const>&  gpz_arr = (solverChoice.anelastic[lev] == 1) ?
                      gradp[lev][GpVars::gpz].array(mfi) : gradp_temp[GpVars::gpz].array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i ,j ,k, mf_comp) = 0.5 * (gpz_arr(i,j,k+1) + gpz_arr(i,j,k));
                });
            }
            mf_comp ++;
        } // dpdz
 
        // **********************************************************************************************
        // These are based on computing gradient of basestate pressure
        // **********************************************************************************************
 
        if ( (containerHasElement(plot_var_names, "pres_hse_x")) ||
             (containerHasElement(plot_var_names, "pres_hse_y")) ) {
            compute_gradp(p_hse, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
                          get_eb(lev), gradp_temp, solverChoice);
        }
 
        if (containerHasElement(plot_var_names, "pres_hse_x"))
        {
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real      >&  derdat  = mf[lev].array(mfi);
                const Array4<Real const>&  gpx_arr = gradp_temp[0].array(mfi);
                const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
                });
            }
            mf_comp += 1;
        } // pres_hse_x
 
        if (containerHasElement(plot_var_names, "pres_hse_y"))
        {
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real      >&  derdat  = mf[lev].array(mfi);
                const Array4<Real const>&  gpy_arr = gradp_temp[1].array(mfi);
                const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
                });
            }
            mf_comp += 1;
        } // pres_hse_y
 
        // **********************************************************************************************
        // Metric terms
        // **********************************************************************************************
 
        if (SolverChoice::mesh_type != MeshType::ConstantDz) {
            if (containerHasElement(plot_var_names, "z_phys"))
            {
                MultiFab::Copy(mf[lev],*z_phys_cc[lev],0,mf_comp,1,0);
                mf_comp ++;
            }
 
            if (containerHasElement(plot_var_names, "detJ"))
            {
                MultiFab::Copy(mf[lev],*detJ_cc[lev],0,mf_comp,1,0);
                mf_comp ++;
            }
        } // use_terrain
 
        if (containerHasElement(plot_var_names, "mapfac")) {
            amrex::Print() << "You are plotting a 3D version of mapfac; we suggest using the 2D plotfile instead" << std::endl;
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf[lev].array(mfi);
                const Array4<Real>& mf_m   = mapfac[lev][MapFacType::m_x]->array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                   derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
                });
            }
            mf_comp ++;
        }
 
        if (containerHasElement(plot_var_names, "lat_m")) {
            amrex::Print() << "You are plotting a 3D version of lat_m; we suggest using the 2D plotfile instead" << std::endl;
            if (lat_m[lev]) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = lat_m[lev]->array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = data(i,j,0);
                    });
                }
            } else {
                mf[lev].setVal(0.0,mf_comp,1,0);
            }
            mf_comp++;
        } // lat_m
 
        if (containerHasElement(plot_var_names, "lon_m")) {
            amrex::Print() << "You are plotting a 3D version of lon_m; we suggest using the 2D plotfile instead" << std::endl;
            if (lon_m[lev]) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = lon_m[lev]->array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                       derdat(i, j, k, mf_comp) = data(i,j,0);
                    });
                }
            } else {
                mf[lev].setVal(0.0,mf_comp,1,0);
            }
            mf_comp++;
        } // lon_m
 
        if (solverChoice.time_avg_vel) {
            if (containerHasElement(plot_var_names, "u_t_avg")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                    const Real norm = t_avg_cnt[lev];
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        derdat(i ,j ,k, mf_comp) = data(i,j,k,0) / norm;
                    });
                }
                mf_comp ++;
            }
 
            if (containerHasElement(plot_var_names, "v_t_avg")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                    const Real norm = t_avg_cnt[lev];
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        derdat(i ,j ,k, mf_comp) = data(i,j,k,1) / norm;
                    });
                }
                mf_comp ++;
            }
 
            if (containerHasElement(plot_var_names, "w_t_avg")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                    const Real norm = t_avg_cnt[lev];
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        derdat(i ,j ,k, mf_comp) = data(i,j,k,2) / norm;
                    });
                }
                mf_comp ++;
            }
 
            if (containerHasElement(plot_var_names, "umag_t_avg")) {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat = mf[lev].array(mfi);
                    const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                    const Real norm = t_avg_cnt[lev];
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        derdat(i ,j ,k, mf_comp) = data(i,j,k,3) / norm;
                    });
                }
                mf_comp ++;
            }
        }
 
        if (containerHasElement(plot_var_names, "nut")) {
            MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
            MultiFab cmf(vars_new[lev][Vars::cons], make_alias, 0, 1); // to provide rho only
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                auto       prim = dmf[mfi].array();
                auto const cons = cmf[mfi].const_array();
                auto const diff = (*eddyDiffs_lev[lev])[mfi].const_array();
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    const Real rho = cons(i, j, k, Rho_comp);
                    const Real Kmv = diff(i, j, k, EddyDiff::Mom_v);
                    prim(i,j,k) = Kmv / rho;
                });
            }
 
            mf_comp++;
        }
 
        if (containerHasElement(plot_var_names, "Kmv")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_v,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "Kmh")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "Khv")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_v,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "Khh")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "Lturb")) {
            MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Turb_lengthscale,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "walldist")) {
            MultiFab::Copy(mf[lev],*walldist[lev],0,mf_comp,1,0);
            mf_comp ++;
        }
        if (containerHasElement(plot_var_names, "diss")) {
            MultiFab::Copy(mf[lev],*SFS_diss_lev[lev],0,mf_comp,1,0);
            mf_comp ++;
        }
 
        // TODO: The size of the q variables can vary with different
        //       moisture models. Therefore, certain components may
        //       reside at different indices. For example, Kessler is
        //       warm but precipitating. This puts qp at index 3.
        //       However, SAM is cold and precipitating so qp is index 4.
        //       Need to built an external enum struct or a better pathway.
 
        // NOTE: Protect against accessing non-existent data
        if (use_moisture) {
            int n_qstate_moist   = micro->Get_Qstate_Moist_Size();
 
            // Moist density
            if(containerHasElement(plot_var_names, "moist_density"))
            {
                int n_start = RhoQ1_comp; // qv
                int n_end   = RhoQ2_comp; // qc
                if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
                for (int n_comp(n_start); n_comp <= n_end; ++n_comp) {
                    MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                }
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qv") && (n_qstate_moist >= 1))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ1_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qc") && (n_qstate_moist >= 2))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ2_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qi") && (n_qstate_moist >= 4))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ3_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qrain") && (n_qstate_moist >= 3))
            {
                int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], n_start , mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qsnow") && (n_qstate_moist >= 5))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ5_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons],   Rho_comp, mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if(containerHasElement(plot_var_names, "qgraup") && (n_qstate_moist >= 6))
            {
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ6_comp, mf_comp, 1, 0);
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons],   Rho_comp, mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            // Precipitating + non-precipitating components
            //--------------------------------------------------------------------------
            if(containerHasElement(plot_var_names, "qt"))
            {
                int n_start = RhoQ1_comp; // qv
                int n_end   = n_start + n_qstate_moist;
                MultiFab::Copy(mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
                for (int n_comp(n_start+1); n_comp < n_end; ++n_comp) {
                    MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                }
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            // Non-precipitating components
            //--------------------------------------------------------------------------
            if (containerHasElement(plot_var_names, "qn"))
            {
                int n_start = RhoQ1_comp; // qv
                int n_end   = RhoQ2_comp; // qc
                if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
                for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
                    MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                }
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            // Precipitating components
            //--------------------------------------------------------------------------
            if(containerHasElement(plot_var_names, "qp") && (n_qstate_moist >= 3))
            {
                int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
                int n_end   = ncomp_cons - 1;
                MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
                for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
                    MultiFab::Add(  mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                }
                MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                mf_comp += 1;
            }
 
            if (containerHasElement(plot_var_names, "qsat"))
            {
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
                for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                {
                    const Box& bx = mfi.tilebox();
                    const Array4<Real>& derdat  = mf[lev].array(mfi);
                    const Array4<Real const>& p_arr = pressure.array(mfi);
                    const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                    {
                        Real qv = S_arr(i,j,k,RhoQ1_comp) / S_arr(i,j,k,Rho_comp);
                        Real T  = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                        Real p  = p_arr(i,j,k) * Real(0.01);
                        erf_qsatw(T, p, derdat(i,j,k,mf_comp));
                    });
                }
                mf_comp ++;
            }
 
            if ( (solverChoice.moisture_type == MoistureType::Kessler) ||
                 (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                 (solverChoice.moisture_type == MoistureType::SAM_NoIce) )
            {
                int offset = (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ? 5 : 0;
                if (containerHasElement(plot_var_names, "rain_accum"))
                {
                    MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
                    mf_comp += 1;
                }
                if (containerHasElement(plot_var_names, "rel_humidity")) {
                    Print() << "Warning: plot variable \"rel_humidity\" is not available with Kessler moisture model.\n";
                    mf[lev].setVal(0.0, mf_comp, 1, 0);
                    mf_comp += 1;
                }
            }
            else if ( (solverChoice.moisture_type == MoistureType::SAM) ||
                      (solverChoice.moisture_type == MoistureType::Morrison) )
            {
                int offset = (solverChoice.moisture_type == MoistureType::Morrison) ? 5 : 0;
                if (containerHasElement(plot_var_names, "rain_accum"))
                {
                    MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
                    mf_comp += 1;
                }
                if (containerHasElement(plot_var_names, "snow_accum"))
                {
                    MultiFab::Copy(mf[lev],*(qmoist[lev][offset+1]),0,mf_comp,1,0);
                    mf_comp += 1;
                }
                if (containerHasElement(plot_var_names, "graup_accum"))
                {
                    MultiFab::Copy(mf[lev],*(qmoist[lev][offset+2]),0,mf_comp,1,0);
                    mf_comp += 1;
                }
                if (containerHasElement(plot_var_names, "rel_humidity")) {
                    Print() << "Warning: plot variable \"rel_humidity\" is not available with SAM moisture model.\n";
                    mf[lev].setVal(0.0, mf_comp, 1, 0);
                    mf_comp += 1;
                }
            }
            else if(solverChoice.moisture_type == MoistureType::SuperDroplets) {
                if (containerHasElement(plot_var_names, "rain_accum")) {
                    MultiFab::Copy(mf[lev],*(qmoist[lev][6]),0,mf_comp,1,0);
                    mf_comp += 1;
                }
                if (containerHasElement(plot_var_names, "rel_humidity")) {
                    MultiFab::Copy(mf[lev],*(qmoist[lev][5]),0,mf_comp,1,0);
                    mf_comp += 1;
                }
                if (containerHasElement(plot_var_names, "condensation_rate")) {
                    MultiFab::Copy(mf[lev],*(qmoist[lev][3]),0,mf_comp,1,0);
                    mf_comp += 1;
                }
            }
 
            if (containerHasElement(plot_var_names, "reflectivity")) {
                if (solverChoice.moisture_type == MoistureType::Morrison) {
 
                    for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                        const Box& bx = mfi.tilebox();
                        const Array4<Real>& derdat  = mf[lev].array(mfi);
                        const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                        ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
 
                            Real rho = S_arr(i,j,k,Rho_comp);
                            Real qv  = std::max(0.0,S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp));
                            Real qpr = std::max(0.0,S_arr(i,j,k,RhoQ4_comp)/S_arr(i,j,k,Rho_comp));
                            Real qps = std::max(0.0,S_arr(i,j,k,RhoQ5_comp)/S_arr(i,j,k,Rho_comp));
                            Real qpg = std::max(0.0,S_arr(i,j,k,RhoQ6_comp)/S_arr(i,j,k,Rho_comp));
 
                            Real temp  = getTgivenRandRTh(S_arr(i,j,k,Rho_comp),
                                                     S_arr(i,j,k,RhoTheta_comp),
                                                     qv);
                            derdat(i, j, k, mf_comp) = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
                                                                                1, 1, 1, 1) ;
                        });
                    }
                    mf_comp ++;
                }
            }
 
           if (solverChoice.moisture_type == MoistureType::Morrison) {
                if (containerHasElement(plot_var_names, "max_reflectivity")) {
                    for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                        const Box& bx = mfi.tilebox();
 
                        const Array4<Real>& derdat  = mf[lev].array(mfi);
                        const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
 
                        // collapse to i,j box (ignore vertical for now)
                        Box b2d = bx;
                        b2d.setSmall(2,0);
                        b2d.setBig(2,0);
 
                        ParallelFor(b2d, [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept {
 
                            Real max_dbz = -1.0e30;
 
                            // find max reflectivity over k
                            for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
                                Real rho = S_arr(i,j,k,Rho_comp);
                                Real qv  = std::max(0.0, S_arr(i,j,k,RhoQ1_comp)/rho);
                                Real qpr = std::max(0.0, S_arr(i,j,k,RhoQ4_comp)/rho);
                                Real qps = std::max(0.0, S_arr(i,j,k,RhoQ5_comp)/rho);
                                Real qpg = std::max(0.0, S_arr(i,j,k,RhoQ6_comp)/rho);
 
                                Real temp = getTgivenRandRTh(rho, S_arr(i,j,k,RhoTheta_comp), qv);
 
                                Real dbz = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
                                                                        1, 1, 1, 1);
                                max_dbz = amrex::max(max_dbz, dbz);
                            }
 
                            // store max_dbz into *all* levels for this (i,j)
                            for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
                                derdat(i, j, k, mf_comp) = max_dbz;
                            }
                        });
                    }
                    mf_comp++;
                }
            }
        } // use_moisture
 
        if (containerHasElement(plot_var_names, "terrain_IB_mask"))
        {
            MultiFab* terrain_blank = terrain_blanking[lev].get();
            MultiFab::Copy(mf[lev],*terrain_blank,0,mf_comp,1,0);
            mf_comp ++;
        }
 
        if (containerHasElement(plot_var_names, "volfrac")) {
            if ( solverChoice.terrain_type == TerrainType::EB ||
                 solverChoice.terrain_type == TerrainType::ImmersedForcing)
            {
                MultiFab::Copy(mf[lev], EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
            } else {
                mf[lev].setVal(1.0, mf_comp, 1, 0);
            }
            mf_comp += 1;
        }
 
#ifdef ERF_COMPUTE_ERROR
        // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
        if (containerHasElement(plot_var_names, "xvel_err") ||
            containerHasElement(plot_var_names, "yvel_err") ||
            containerHasElement(plot_var_names, "zvel_err"))
        {
            //
            // Moving terrain ANALYTICAL
            //
            Real H           = geom[lev].ProbHi()[2];
            Real Ampl        = 0.16;
            Real wavelength  = 100.;
            Real kp          = 2. * PI / wavelength;
            Real g           = CONST_GRAV;
            Real omega       = std::sqrt(g * kp);
            Real omega_t     = omega * t_new[lev];
 
            const auto dx = geom[lev].CellSizeArray();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.validbox();
                Box xbx(bx); xbx.surroundingNodes(0);
                const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
                const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
 
                const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
 
                ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    Real x = i * dx[0];
                    Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
 
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
                    z -= z_base;
 
                    Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
 
                    xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
                });
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    Real x   = (i + 0.5) * dx[0];
                    Real z   = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
 
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
                    z -= z_base;
 
                    Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
 
                    zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
                });
            }
 
            MultiFab temp_mf(mf[lev].boxArray(), mf[lev].DistributionMap(), AMREX_SPACEDIM, 0);
            average_face_to_cellcenter(temp_mf,0,
                Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
 
            if (containerHasElement(plot_var_names, "xvel_err")) {
                MultiFab::Copy(mf[lev],temp_mf,0,mf_comp,1,0);
                mf_comp += 1;
            }
            if (containerHasElement(plot_var_names, "yvel_err")) {
                MultiFab::Copy(mf[lev],temp_mf,1,mf_comp,1,0);
                mf_comp += 1;
            }
            if (containerHasElement(plot_var_names, "zvel_err")) {
                MultiFab::Copy(mf[lev],temp_mf,2,mf_comp,1,0);
                mf_comp += 1;
            }
 
            // Now restore the velocities to what they were
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.validbox();
                Box xbx(bx); xbx.surroundingNodes(0);
 
                const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
                const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
 
                const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
 
                ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    Real x = i * dx[0];
                    Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
 
                    z -= z_base;
 
                    Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                    xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
                });
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    Real x   = (i + 0.5) * dx[0];
                    Real z   = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
 
                    z -= z_base;
                    Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
 
                    zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
                });
            }
        } // end xvel_err, yvel_err, zvel_err
 
        if (containerHasElement(plot_var_names, "pp_err"))
        {
            // Moving terrain ANALYTICAL
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& bx = mfi.tilebox();
                const Array4<Real>& derdat = mf[lev].array(mfi);
                const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
                const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
 
                const auto dx = geom[lev].CellSizeArray();
                const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
                const Array4<Real const>&  p_arr = pressure.const_array(mfi);
                const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
 
                Real H           = geom[lev].ProbHi()[2];
                Real Ampl        = 0.16;
                Real wavelength  = 100.;
                Real kp          = 2. * PI / wavelength;
                Real g           = CONST_GRAV;
                Real omega       = std::sqrt(g * kp);
                Real omega_t     = omega * t_new[lev];
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                {
                    derdat(i, j, k, mf_comp) = p_arr(i,j,k) - p0_arr(i,j,k);
 
                    Real rho_hse     = r0_arr(i,j,k);
 
                    Real x   = (i + 0.5) * dx[0];
                    Real z   = 0.125 * ( z_nd(i,j,k  ) + z_nd(i+1,j,k  ) + z_nd(i,j+1,k  ) + z_nd(i+1,j+1,k  )
                                        +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
                    Real z_base = Ampl * std::sin(kp * x - omega_t);
 
                    z -= z_base;
                    Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                    Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
                                              std::sin(kp * x - omega_t) * r0_arr(i,j,k);
 
                    derdat(i,j,k,mf_comp) -= pprime_exact;
                });
            }
            mf_comp += 1;
        }
#endif
 
        if (solverChoice.rad_type != RadiationType::None) {
            if (containerHasElement(plot_var_names, "qsrc_sw")) {
                MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 0, mf_comp, 1, 0);
                mf_comp += 1;
            }
            if (containerHasElement(plot_var_names, "qsrc_lw")) {
                MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 1, mf_comp, 1, 0);
                mf_comp += 1;
            }
        }
 
        // *****************************************************************************************
        // End of derived variables corresponding to "derived_names" in ERF.H
        //
        // Particles and microphysics can provide additional outputs, which are handled below.
        // *****************************************************************************************
 
#ifdef ERF_USE_PARTICLES
        const auto& particles_namelist( particleData.getNames() );
 
        if (containerHasElement(plot_var_names, "tracer_particles_count")) {
            if (particles_namelist.size() == 0) {
                MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
                temp_dat.setVal(0);
                MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                mf_comp += 1;
            } else {
                for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++) {
                    if (containerHasElement(plot_var_names, std::string(particles_namelist[i]+"_count"))) {
                        MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
                        temp_dat.setVal(0);
                        if (particleData.HasSpecies(particles_namelist[i])) {
                            particleData[particles_namelist[i]]->Increment(temp_dat, lev);
                        }
                        MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                        mf_comp += 1;
                    }
                }
            }
        }
 
        Vector<std::string> particle_mesh_plot_names(0);
        particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
 
        for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
            std::string plot_var_name(particle_mesh_plot_names[i]);
            if (containerHasElement(plot_var_names, plot_var_name) ) {
                MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
                temp_dat.setVal(0);
                particleData.GetMeshPlotVar(plot_var_name, temp_dat, lev);
                MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                mf_comp += 1;
            }
        }
#endif
 
        {
            Vector<std::string> microphysics_plot_names;
            micro->GetPlotVarNames(microphysics_plot_names);
            for (auto& plot_name : microphysics_plot_names) {
                if (containerHasElement(plot_var_names, plot_name)) {
                    MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
                    temp_dat.setVal(0);
                    micro->GetPlotVar(plot_name, temp_dat, lev);
                    MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                    mf_comp += 1;
                }
            }
        }
 
 
    }
 
    if (solverChoice.terrain_type == TerrainType::EB)
    {
        for (int lev = 0; lev <= finest_level; ++lev) {
            EB_set_covered(mf[lev], 0.0);
        }
    }
 
    // Fill terrain distortion MF (nu_nd)
    if (SolverChoice::mesh_type != MeshType::ConstantDz) {
        for (int lev(0); lev <= finest_level; ++lev) {
            MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
            Real dz = Geom()[lev].CellSizeArray()[2];
            for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                Array4<Real> mf_arr = mf_nd[lev].array(mfi);
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    mf_arr(i,j,k,2) -= k * dz;
                });
            }
        }
    }
 
    std::string plotfilename;
    std::string plotfilenameU;
    std::string plotfilenameV;
    std::string plotfilenameW;
 
    if (which == 1) {
       if (use_real_time_in_pltname) {
           const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
           plotfilename = plot3d_file_1+"_"+getTimestamp(start_time+t_new[0], dt_format,false);
       } else {
           plotfilename  = Concatenate(plot3d_file_1, istep[0], file_name_digits);
       }
       plotfilenameU = Concatenate(plot3d_file_1+"U", istep[0], file_name_digits);
       plotfilenameV = Concatenate(plot3d_file_1+"V", istep[0], file_name_digits);
       plotfilenameW = Concatenate(plot3d_file_1+"W", istep[0], file_name_digits);
    } else if (which == 2) {
       if (use_real_time_in_pltname) {
           const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
           plotfilename = plot3d_file_2+"_"+getTimestamp(start_time+t_new[0], dt_format,false);
       } else {
           plotfilename  = Concatenate(plot3d_file_2, istep[0], file_name_digits);
       }
       plotfilenameU = Concatenate(plot3d_file_2+"U", istep[0], file_name_digits);
       plotfilenameV = Concatenate(plot3d_file_2+"V", istep[0], file_name_digits);
       plotfilenameW = Concatenate(plot3d_file_2+"W", istep[0], file_name_digits);
    }
 
    // LSM writes it's own data
    if (which==1 && plot_lsm) {
        lsm.Plot_Lsm_Data(t_new[0], istep, refRatio());
    }
 
#ifdef ERF_USE_RRTMGP
    /*
    // write additional RRTMGP data
    // TODO: currently single level only
    if (which==1 && plot_rad) {
        rad[0]->writePlotfile(plot_file_1, t_new[0], istep[0]);
    }
    */
#endif
 
    // Single level
    if (finest_level == 0)
    {
        if (plotfile_type == PlotFileType::Amrex)
        {
            Print() << "Writing native 3D plotfile " << plotfilename << "\n";
            if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                   GetVecOfConstPtrs(mf),
                                                   GetVecOfConstPtrs(mf_nd),
                                                   varnames,
                                                   Geom(), t_new[0], istep, refRatio());
            } else {
                WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                        GetVecOfConstPtrs(mf),
                                        varnames,
                                        Geom(), t_new[0], istep, refRatio());
            }
            writeJobInfo(plotfilename);
 
            if (m_plot_face_vels) {
                Print() << "Writing face velocities" << std::endl;
                WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
                                        GetVecOfConstPtrs(mf_u),
                                        {"x_velocity_stag"},
                                        Geom(), t_new[0], istep, refRatio());
                WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
                                        GetVecOfConstPtrs(mf_v),
                                        {"y_velocity_stag"},
                                        Geom(), t_new[0], istep, refRatio());
                WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
                                        GetVecOfConstPtrs(mf_w),
                                        {"z_velocity_stag"},
                                        Geom(), t_new[0], istep, refRatio());
            }
 
#ifdef ERF_USE_PARTICLES
            particleData.writePlotFile(plotfilename);
#endif
#ifdef ERF_USE_NETCDF
        } else if (plotfile_type == PlotFileType::Netcdf) {
             AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
             int lev   = 0;
             int l_which = 0;
             const Real* p_lo = geom[lev].ProbLo();
             const Real* p_hi = geom[lev].ProbHi();
             const auto dx    = geom[lev].CellSize();
             writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
                             {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
                             geom[lev].Domain(), t_new[0], start_bdy_time);
#endif
        } else {
            // Here we assume the plotfile_type is PlotFileType::None
            Print() << "Writing no 3D plotfile since plotfile_type is none" << std::endl;
        }
 
    } else { // Multilevel
 
        if (plotfile_type == PlotFileType::Amrex) {
 
            int lev0 = 0;
            int desired_ratio = std::max(std::max(ref_ratio[lev0][0],ref_ratio[lev0][1]),ref_ratio[lev0][2]);
            bool any_ratio_one = ( ( (ref_ratio[lev0][0] == 1) || (ref_ratio[lev0][1] == 1) ) ||
                                     (ref_ratio[lev0][2] == 1) );
            for (int lev = 1; lev < finest_level; lev++) {
                any_ratio_one = any_ratio_one ||
                                     ( ( (ref_ratio[lev][0] == 1) || (ref_ratio[lev][1] == 1) ) ||
                                         (ref_ratio[lev][2] == 1) );
            }
 
            if (any_ratio_one && m_expand_plotvars_to_unif_rr)
            {
                Vector<IntVect>   r2(finest_level);
                Vector<Geometry>  g2(finest_level+1);
                Vector<MultiFab> mf2(finest_level+1);
 
                mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
 
                // Copy level 0 as is
                MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
 
                // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
                Array<int,AMREX_SPACEDIM> periodicity =
                             {Geom()[lev0].isPeriodic(0),Geom()[lev0].isPeriodic(1),Geom()[lev0].isPeriodic(2)};
                g2[lev0].define(Geom()[lev0].Domain(),&(Geom()[lev0].ProbDomain()),0,periodicity.data());
 
                r2[0] = IntVect(desired_ratio/ref_ratio[lev0][0],
                                desired_ratio/ref_ratio[lev0][1],
                                desired_ratio/ref_ratio[lev0][2]);
 
                for (int lev = 1; lev <= finest_level; ++lev) {
                    if (lev > 1) {
                        r2[lev-1][0] = r2[lev-2][0] * desired_ratio / ref_ratio[lev-1][0];
                        r2[lev-1][1] = r2[lev-2][1] * desired_ratio / ref_ratio[lev-1][1];
                        r2[lev-1][2] = r2[lev-2][2] * desired_ratio / ref_ratio[lev-1][2];
                    }
 
                    mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
 
                    // Set the new problem domain
                    Box d2(Geom()[lev].Domain());
                    d2.refine(r2[lev-1]);
 
                    g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
                }
 
                //
                // We need to make a temporary that is the size of ncomp_mf
                // in order to not get an out of bounds error
                // even though the values will not be used
                //
                Vector<BCRec> temp_domain_bcs_type;
                temp_domain_bcs_type.resize(ncomp_mf);
 
                //
                // Do piecewise constant interpolation of mf into mf2
                //
                for (int lev = 1; lev <= finest_level; ++lev) {
                    Interpolater* mapper_c = &pc_interp;
                    InterpFromCoarseLevel(mf2[lev], t_new[lev], mf[lev],
                                          0, 0, ncomp_mf,
                                          geom[lev], g2[lev],
                                          null_bc_for_fill, 0, null_bc_for_fill, 0,
                                          r2[lev-1], mapper_c, temp_domain_bcs_type, 0);
                }
 
                // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
                Vector<IntVect> rr(finest_level);
                for (int lev = 0; lev < finest_level; ++lev) {
                    rr[lev] = IntVect(desired_ratio);
                }
 
               Print() << "Writing 3D plotfile " << plotfilename << "\n";
               if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                   WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                      GetVecOfConstPtrs(mf2),
                                                      GetVecOfConstPtrs(mf_nd),
                                                      varnames,
                                                      g2, t_new[0], istep, rr);
               } else {
                   WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                           GetVecOfConstPtrs(mf2), varnames,
                                           g2, t_new[0], istep, rr);
               }
 
            } else {
               if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                    WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                       GetVecOfConstPtrs(mf),
                                                       GetVecOfConstPtrs(mf_nd),
                                                       varnames,
                                                       geom, t_new[0], istep, ref_ratio);
                } else {
                    WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                            GetVecOfConstPtrs(mf), varnames,
                                            geom, t_new[0], istep, ref_ratio);
                }
                if (m_plot_face_vels) {
                    Print() << "Writing face velocities" << std::endl;
                    WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
                                            GetVecOfConstPtrs(mf_u),
                                            {"x_velocity_stag"},
                                            geom, t_new[0], istep, ref_ratio);
                    WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
                                            GetVecOfConstPtrs(mf_v),
                                            {"y_velocity_stag"},
                                            geom, t_new[0], istep, ref_ratio);
                    WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
                                            GetVecOfConstPtrs(mf_w),
                                            {"z_velocity_stag"},
                                            geom, t_new[0], istep, ref_ratio);
                }
            } // ref_ratio test
 
            writeJobInfo(plotfilename);
 
#ifdef ERF_USE_PARTICLES
            particleData.writePlotFile(plotfilename);
#endif
 
#ifdef ERF_USE_NETCDF
        } else if (plotfile_type == PlotFileType::Netcdf) {
             AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
             for (int lev = 0; lev <= finest_level; ++lev) {
                 for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
                     Box bounding_region = (lev == 0)  ? geom[lev].Domain() : boxes_at_level[lev][which_box];
                     const Real* p_lo = geom[lev].ProbLo();
                     const Real* p_hi = geom[lev].ProbHi();
                     const auto dx    = geom[lev].CellSizeArray();
                     writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
                                     {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
                                     bounding_region, t_new[0], start_bdy_time);
                 }
             }
#endif
        }
    } // end multi-level
 
    if (verbose > 0)
    {
        auto dPlotTime = amrex::second() - dPlotTime0;
        ParallelDescriptor::ReduceRealMax(dPlotTime,ParallelDescriptor::IOProcessorNumber());
        amrex::Print() << "3DPlotfile write time = " << dPlotTime << " seconds." << '\n';
    }
}

Here is the call graph for this function:

write_1D_profiles()

void ERF::write_1D_profiles

(

amrex::Real

time

)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters

time	Current time

{
    BL_PROFILE("ERF::write_1D_profiles()");
 
    if (NumDataLogs() > 1)
    {
        // Define the 1d arrays we will need
        Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
        Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
        Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
        Gpu::HostVector<Real> h_avg_wthv;
        Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
        Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
        Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
        Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
        Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
        Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
 
        if (NumDataLogs() > 1) {
            derive_diag_profiles(time,
                                 h_avg_u, h_avg_v, h_avg_w,
                                 h_avg_rho, h_avg_th, h_avg_ksgs,
                                 h_avg_Kmv, h_avg_Khv,
                                 h_avg_qv, h_avg_qc, h_avg_qr,
                                 h_avg_wqv, h_avg_wqc, h_avg_wqr,
                                 h_avg_qi, h_avg_qs, h_avg_qg,
                                 h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                 h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                 h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                 h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
                                 h_avg_wthv);
        }
 
        if (NumDataLogs() > 3 && time > 0.) {
            derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                   h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                   h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
                                   h_avg_sgsdiss);
        }
 
        int hu_size =  h_avg_u.size();
 
        auto const& dx = geom[0].CellSizeArray();
        if (ParallelDescriptor::IOProcessor()) {
            if (NumDataLogs() > 1) {
                std::ostream& data_log1 = DataLog(1);
                if (data_log1.good()) {
                  // Write the quantities at this time
                  for (int k = 0; k < hu_size; k++) {
                      Real z;
                      if (zlevels_stag[0].size() > 1) {
                          z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                      } else {
                          z = (k + 0.5)* dx[2];
                      }
                      data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_u[k]   << " " << h_avg_v[k]   << " " << h_avg_w[k]     << " "
                                << h_avg_rho[k] << " " << h_avg_th[k]  << " " << h_avg_ksgs[k] << " "
                                << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
                                << h_avg_qv[k]  << " " << h_avg_qc[k]  << " " << h_avg_qr[k]    << " "
                                << h_avg_qi[k]  << " " << h_avg_qs[k]  << " " << h_avg_qg[k]
                                << std::endl;
                  } // loop over z
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 2) {
                std::ostream& data_log2 = DataLog(2);
                if (data_log2.good()) {
                  // Write the perturbational quantities at this time
                  for (int k = 0; k < hu_size; k++) {
                      Real z;
                      if (zlevels_stag[0].size() > 1) {
                          z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                      } else {
                          z = (k + 0.5)* dx[2];
                      }
                      Real thv = h_avg_th[k] * (1 + 0.61*h_avg_qv[k] - h_avg_qc[k] - h_avg_qr[k]);
                      data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]  << " "
                                << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]  << " "
                                << h_avg_uw[k]   - h_avg_u[k]*h_avg_w[k]  << " "
                                << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]  << " "
                                << h_avg_vw[k]   - h_avg_v[k]*h_avg_w[k]  << " "
                                << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]  << " "
                                << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k] << " "
                                << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k] << " "
                                << h_avg_wth[k]  - h_avg_w[k]*h_avg_th[k] << " "
                                << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
                                // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                //                        -   <u_i u_i> * <u_j>
                                //                        - 2*<u_i> * <u_i u_j>
                                //                        + 2*<u_i>*<u_i> * <u_j>
                                << h_avg_uiuiu[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
                                 - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
                                   << " " // (u'_i u'_i)u'
                                << h_avg_uiuiv[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
                                 - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
                                   << " " // (u'_i u'_i)v'
                                << h_avg_uiuiw[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
                                 - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                   << " " // (u'_i u'_i)w'
                                << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k] << " "
                                << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k] << " "
                                << h_avg_pw[k]   - h_avg_p[k]*h_avg_w[k] << " "
                                << h_avg_wqv[k]  - h_avg_qv[k]*h_avg_w[k] << " "
                                << h_avg_wqc[k]  - h_avg_qc[k]*h_avg_w[k] << " "
                                << h_avg_wqr[k]  - h_avg_qr[k]*h_avg_w[k] << " "
                                << h_avg_wthv[k] - h_avg_w[k]*thv
                                << std::endl;
                  } // loop over z
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 3 && time > 0.) {
                std::ostream& data_log3 = DataLog(3);
                if (data_log3.good()) {
                  // Write the average stresses
                  for (int k = 0; k < hu_size; k++) {
                      Real z;
                      if (zlevels_stag[0].size() > 1) {
                          z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                      } else {
                          z = (k + 0.5)* dx[2];
                      }
                      data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_tau11[k]  << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                << h_avg_tau22[k]  << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                << h_avg_sgshfx[k] << " "
                                << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
                                << h_avg_sgsdiss[k]
                                << std::endl;
                  } // loop over z
                } // if good
            } // if (NumDataLogs() > 3)
        } // if IOProcessor
    } // if (NumDataLogs() > 1)
}

write_1D_profiles_stag()

void ERF::write_1D_profiles_stag

(

amrex::Real

time

)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Quantities are output at their native grid locations. Therefore, w and associated flux quantities <(•)'w'>, tau13, and tau23 (where '•' includes u, v, p, theta, ...) will be output at staggered heights (i.e., coincident with z faces) rather than cell-center heights to avoid performing additional averaging. Unstaggered (i.e., cell-centered) quantities are output alongside staggered quantities at the lower cell faces in the log file; these quantities will have a zero value at the big end, corresponding to k=Nz+1.

The structure of file should follow ERF_Write1DProfiles.cpp

Parameters

time	Current time

{
    BL_PROFILE("ERF::write_1D_profiles()");
 
    if (NumDataLogs() > 1)
    {
        // Define the 1d arrays we will need
        Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
        Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
        Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
        Gpu::HostVector<Real> h_avg_wthv;
        Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
        Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
        Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
        Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
        Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
        Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
 
        if (NumDataLogs() > 1) {
            derive_diag_profiles_stag(time,
                                      h_avg_u, h_avg_v, h_avg_w,
                                      h_avg_rho, h_avg_th, h_avg_ksgs,
                                      h_avg_Kmv, h_avg_Khv,
                                      h_avg_qv, h_avg_qc, h_avg_qr,
                                      h_avg_wqv, h_avg_wqc, h_avg_wqr,
                                      h_avg_qi, h_avg_qs, h_avg_qg,
                                      h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                      h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                      h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                      h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
                                      h_avg_wthv);
        }
 
        if (NumDataLogs() > 3 && time > 0.) {
            derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                        h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                        h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
                                        h_avg_sgsdiss);
        }
 
        int unstag_size =  h_avg_w.size() - 1; // _un_staggered heights
 
        auto const& dx = geom[0].CellSizeArray();
        if (ParallelDescriptor::IOProcessor()) {
            if (NumDataLogs() > 1) {
                std::ostream& data_log1 = DataLog(1);
                if (data_log1.good()) {
                  // Write the quantities at this time
                  for (int k = 0; k < unstag_size; k++) {
                      Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                      data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_u[k]   << " " << h_avg_v[k]   << " " << h_avg_w[k]     << " "
                                << h_avg_rho[k] << " " << h_avg_th[k]  << " " << h_avg_ksgs[k] << " "
                                << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
                                << h_avg_qv[k]  << " " << h_avg_qc[k]  << " " << h_avg_qr[k]    << " "
                                << h_avg_qi[k]  << " " << h_avg_qs[k]  << " " << h_avg_qg[k]
                                << std::endl;
                  } // loop over z
                  // Write top face values
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                            << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                            << 0 << " " << 0 << " " << h_avg_w[unstag_size] << " "
                            << 0 << " " << 0 << " " << 0 << " " // rho, theta, ksgs
                            << 0 << " " << 0 << " "             // Kmv, Khv
                            << 0 << " " << 0 << " " << 0 << " " // qv, qc, qr
                            << 0 << " " << 0 << " " << 0        // qi, qs, qg
                            << std::endl;
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 2) {
                std::ostream& data_log2 = DataLog(2);
                if (data_log2.good()) {
                  // Write the perturbational quantities at this time
                  // For surface values (k=0), assume w = uw = vw = ww = 0
                  Real w_cc  = h_avg_w[1] / 2;  // w at first cell center
                  Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
                  Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
                  Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
                  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                            << std::setw(datwidth) << std::setprecision(datprecision) << 0 << " "
                            << h_avg_uu[0]   - h_avg_u[0]*h_avg_u[0]   << " " // u'u'
                            << h_avg_uv[0]   - h_avg_u[0]*h_avg_v[0]   << " " // u'v'
                            << 0                                       << " " // u'w'
                            << h_avg_vv[0]   - h_avg_v[0]*h_avg_v[0]   << " " // v'v'
                            << 0                                       << " " // v'w'
                            << 0                                       << " " // w'w'
                            << h_avg_uth[0]  - h_avg_u[0]*h_avg_th[0]  << " " // u'th'
                            << h_avg_vth[0]  - h_avg_v[0]*h_avg_th[0]  << " " // v'th'
                            << 0                                       << " " // w'th'
                            << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
                            << h_avg_uiuiu[0]
                             - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
                             - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
                             + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
                               << " " // (u'_i u'_i)u'
                            << h_avg_uiuiv[0]
                             - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
                             - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
                             + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
                               << " " // (u'_i u'_i)v'
                            << 0 << " " // (u'_i u'_i)w'
                            << h_avg_pu[0]   - h_avg_p[0]*h_avg_u[0]   << " " // p'u'
                            << h_avg_pv[0]   - h_avg_p[0]*h_avg_v[0]   << " " // p'v'
                            << 0                                       << " " // p'w'
                            << 0                                       << " " // qv'w'
                            << 0                                       << " " // qc'w'
                            << 0                                       << " " // qr'w'
                            << 0                                              // thv'w'
                            << std::endl;
 
                  // For internal values, interpolate scalar quantities to faces
                  for (int k = 1; k < unstag_size; k++) {
                      Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                      Real uface  = 0.5*(h_avg_u[k]  + h_avg_u[k-1]);
                      Real vface  = 0.5*(h_avg_v[k]  + h_avg_v[k-1]);
                      Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]);
                      Real pface  = 0.5*(h_avg_p[k]  + h_avg_p[k-1]);
                      Real qvface = 0.5*(h_avg_qv[k] + h_avg_qv[k-1]);
                      Real qcface = 0.5*(h_avg_qc[k] + h_avg_qc[k-1]);
                      Real qrface = 0.5*(h_avg_qr[k] + h_avg_qr[k-1]);
                      Real uuface = 0.5*(h_avg_uu[k] + h_avg_uu[k-1]);
                      Real vvface = 0.5*(h_avg_vv[k] + h_avg_vv[k-1]);
                      Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
                      w_cc   = 0.5*(h_avg_w[k-1]  + h_avg_w[k]);
                      uw_cc  = 0.5*(h_avg_uw[k-1] + h_avg_uw[k]);
                      vw_cc  = 0.5*(h_avg_vw[k-1] + h_avg_vw[k]);
                      ww_cc  = 0.5*(h_avg_ww[k-1] + h_avg_ww[k]);
                      data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]   << " " // u'u'
                                << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]   << " " // u'v'
                                << h_avg_uw[k]   -      uface*h_avg_w[k]   << " " // u'w'
                                << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]   << " " // v'v'
                                << h_avg_vw[k]   -      vface*h_avg_w[k]   << " " // v'w'
                                << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]   << " " // w'w'
                                << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k]  << " " // u'th'
                                << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k]  << " " // v'th'
                                << h_avg_wth[k]  - h_avg_w[k]*thface       << " " // w'th'
                                << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
                                // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                //                        -   <u_i u_i> * <u_j>
                                //                        - 2*<u_i> * <u_i u_j>
                                //                        + 2*<u_i>*<u_i> * <u_j>
                                << h_avg_uiuiu[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
                                 - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
                                   << " " // cell-centered (u'_i u'_i)u'
                                << h_avg_uiuiv[k]
                                 - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
                                 - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
                                 + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
                                   << " " // cell-centered (u'_i u'_i)v'
                                << h_avg_uiuiw[k]
                                 - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                                 - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                 + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                   << " " // face-centered (u'_i u'_i)w'
                                << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k]   << " " // cell-centered p'u'
                                << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k]   << " " // cell-centered p'v'
                                << h_avg_pw[k]   -      pface*h_avg_w[k]   << " " // face-centered p'w'
                                << h_avg_wqv[k]  -     qvface*h_avg_w[k]   << " "
                                << h_avg_wqc[k]  -     qcface*h_avg_w[k]   << " "
                                << h_avg_wqr[k]  -     qrface*h_avg_w[k]   << " "
                                << h_avg_wthv[k] -    thvface*h_avg_w[k]
                                << std::endl;
                  } // loop over z
 
                  // Write top face values, extrapolating scalar quantities
                  const int k = unstag_size;
                  Real uface  = 1.5*h_avg_u[k-1]  - 0.5*h_avg_u[k-2];
                  Real vface  = 1.5*h_avg_v[k-1]  - 0.5*h_avg_v[k-2];
                  Real thface = 1.5*h_avg_th[k-1] - 0.5*h_avg_th[k-2];
                  Real pface  = 1.5*h_avg_p[k-1]  - 0.5*h_avg_p[k-2];
                  Real qvface = 1.5*h_avg_qv[k-1] - 0.5*h_avg_qv[k-2];
                  Real qcface = 1.5*h_avg_qc[k-1] - 0.5*h_avg_qc[k-2];
                  Real qrface = 1.5*h_avg_qr[k-1] - 0.5*h_avg_qr[k-2];
                  Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2];
                  Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2];
                  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                            << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                            << 0                                     << " " // u'u'
                            << 0                                     << " " // u'v'
                            << h_avg_uw[k]   -      uface*h_avg_w[k] << " " // u'w'
                            << 0                                     << " " // v'v'
                            << h_avg_vw[k]   -      vface*h_avg_w[k] << " " // v'w'
                            << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k] << " " // w'w'
                            << 0                                     << " " // u'th'
                            << 0                                     << " " // v'th'
                            << h_avg_wth[k]  -     thface*h_avg_w[k] << " " // w'th'
                            << 0                                     << " " // th'th'
                            << 0                                     << " " // (u'_i u'_i)u'
                            << 0                                     << " " // (u'_i u'_i)v'
                            << h_avg_uiuiw[k]
                             - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                             - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                             + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                               << " " // (u'_i u'_i)w'
                            << 0                                     << " " // pu'
                            << 0                                     << " " // pv'
                            << h_avg_pw[k]   -      pface*h_avg_w[k] << " " // pw'
                            << h_avg_wqv[k]  -     qvface*h_avg_w[k] << " "
                            << h_avg_wqc[k]  -     qcface*h_avg_w[k] << " "
                            << h_avg_wqr[k]  -     qrface*h_avg_w[k] << " "
                            << h_avg_wthv[k] -    thvface*h_avg_w[k]
                            << std::endl;
                } // if good
            } // NumDataLogs
 
            if (NumDataLogs() > 3 && time > 0.) {
                std::ostream& data_log3 = DataLog(3);
                if (data_log3.good()) {
                  // Write the average stresses
                  for (int k = 0; k < unstag_size; k++) {
                      Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                      data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                << h_avg_tau11[k]  << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                << h_avg_tau22[k]  << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                << h_avg_sgshfx[k] << " "
                                << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
                                << h_avg_sgsdiss[k]
                                << std::endl;
                  } // loop over z
                  // Write top face values
                  Real NANval = 0.0;
                  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                            << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                            << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
                            << NANval << " " << h_avg_tau23[unstag_size] << " " << NANval << " "
                            << h_avg_sgshfx[unstag_size] << " "
                            << h_avg_sgsq1fx[unstag_size] << " " << h_avg_sgsq2fx[unstag_size] << " "
                            << NANval
                            << std::endl;
                } // if good
            } // if (NumDataLogs() > 3)
        } // if IOProcessor
    } // if (NumDataLogs() > 1)
}

writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream &  os )

static

{
    std::string PrettyLine = std::string(78, '=') + "\n";
    std::string OtherLine = std::string(78, '-') + "\n";
    std::string SkipSpace = std::string(8, ' ');
 
    // build information
    os << PrettyLine;
    os << " ERF Build Information\n";
    os << PrettyLine;
 
    os << "build date:    " << buildInfoGetBuildDate() << "\n";
    os << "build machine: " << buildInfoGetBuildMachine() << "\n";
    os << "build dir:     " << buildInfoGetBuildDir() << "\n";
    os << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
 
    os << "\n";
 
    os << "COMP:          " << buildInfoGetComp() << "\n";
    os << "COMP version:  " << buildInfoGetCompVersion() << "\n";
 
    os << "C++ compiler:  " << buildInfoGetCXXName() << "\n";
    os << "C++ flags:     " << buildInfoGetCXXFlags() << "\n";
 
    os << "\n";
 
    os << "Link flags:    " << buildInfoGetLinkFlags() << "\n";
    os << "Libraries:     " << buildInfoGetLibraries() << "\n";
 
    os << "\n";
 
    for (int n = 1; n <= buildInfoGetNumModules(); n++) {
      os << buildInfoGetModuleName(n) << ": "
         << buildInfoGetModuleVal(n) << "\n";
    }
 
    os << "\n";
    const char* githash1 = buildInfoGetGitHash(1);
    const char* githash2 = buildInfoGetGitHash(2);
    if (strlen(githash1) > 0) {
      os << "ERF       git hash: " << githash1 << "\n";
    }
    if (strlen(githash2) > 0) {
      os << "AMReX       git hash: " << githash2 << "\n";
    }
 
    const char* buildgithash = buildInfoGetBuildGitHash();
    const char* buildgitname = buildInfoGetBuildGitName();
    if (strlen(buildgithash) > 0) {
      os << buildgitname << " git hash: " << buildgithash << "\n";
    }
 
    os << "\n";
    os << " ERF Compile time variables: \n";
 
    os << "\n";
    os << " ERF Defines: \n";
#ifdef _OPENMP
    os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
       << std::endl;
#else
    os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
       << std::endl;
#endif
 
#ifdef MPI_VERSION
    os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
       << MPI_VERSION << std::endl;
#else
    os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
       << "UNDEFINED" << std::endl;
#endif
 
#ifdef MPI_SUBVERSION
    os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
       << MPI_SUBVERSION << std::endl;
#else
    os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
       << "UNDEFINED" << std::endl;
#endif
 
    os << "\n\n";
}

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

WriteCheckpointFile()

void ERF::WriteCheckpointFile

(

)

const

ERF function for writing a checkpoint file.

{
    auto dCheckTime0 = amrex::second();
 
    // chk00010            write a checkpoint file with this root directory
    // chk00010/Header     this contains information you need to save (e.g., finest_level, t_new, etc.) and also
    //                     the BoxArrays at each level
    // chk00010/Level_0/
    // chk00010/Level_1/
    // etc.                these subdirectories will hold the MultiFab data at each level of refinement
 
    // checkpoint file name, e.g., chk00010
    const std::string& checkpointname = Concatenate(check_file,istep[0],file_name_digits);
 
    Print() << "Writing native checkpoint " << checkpointname << "\n";
 
    const int nlevels = finest_level+1;
 
    // ---- prebuild a hierarchy of directories
    // ---- dirName is built first.  if dirName exists, it is renamed.  then build
    // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
    // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
    // ---- after all directories are built
    // ---- ParallelDescriptor::IOProcessor() creates the directories
    PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
 
    int ncomp_cons = vars_new[0][Vars::cons].nComp();
 
    // write Header file
    if (ParallelDescriptor::IOProcessor()) {
 
       std::string HeaderFileName(checkpointname + "/Header");
       VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
       std::ofstream HeaderFile;
       HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
       HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                               std::ofstream::trunc |
                                               std::ofstream::binary);
       if(! HeaderFile.good()) {
           FileOpenFailed(HeaderFileName);
       }
 
       HeaderFile.precision(17);
 
       // write out title line
       HeaderFile << "Checkpoint file for ERF\n";
 
       // write out finest_level
       HeaderFile << finest_level << "\n";
 
       // write the number of components
       // for each variable we store
 
       // conservative, cell-centered vars
       HeaderFile << ncomp_cons << "\n";
 
       // x-velocity on faces
       HeaderFile << 1 << "\n";
 
       // y-velocity on faces
       HeaderFile << 1 << "\n";
 
       // z-velocity on faces
       HeaderFile << 1 << "\n";
 
       // write out array of istep
       for (int i = 0; i < istep.size(); ++i) {
           HeaderFile << istep[i] << " ";
       }
       HeaderFile << "\n";
 
       // write out array of dt
       for (int i = 0; i < dt.size(); ++i) {
           HeaderFile << dt[i] << " ";
       }
       HeaderFile << "\n";
 
       // write out array of t_new
       for (int i = 0; i < t_new.size(); ++i) {
           HeaderFile << t_new[i] << " ";
       }
       HeaderFile << "\n";
 
       // write the BoxArray at each level
       for (int lev = 0; lev <= finest_level; ++lev) {
           boxArray(lev).writeOn(HeaderFile);
           HeaderFile << '\n';
       }
 
       // Write separate file that tells how many components we have of the base state
       std::string BaseStateFileName(checkpointname + "/num_base_state_comps");
       std::ofstream BaseStateFile;
       BaseStateFile.open(BaseStateFileName.c_str(), std::ofstream::out   |
                                                     std::ofstream::trunc |
                                                     std::ofstream::binary);
       if(! BaseStateFile.good()) {
           FileOpenFailed(BaseStateFileName);
       } else {
           // write out number of components in base state
           BaseStateFile << BaseState::num_comps      << "\n";
           BaseStateFile << base_state[0].nGrowVect() << "\n";
       }
   }
 
    // write the MultiFab data to, e.g., chk00010/Level_0/
    // Here we make copies of the MultiFab with no ghost cells
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
        MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
        VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
 
        MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
        MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
        VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
 
        MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
        MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
        VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
 
        MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
        MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
        VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
 
        if (solverChoice.anelastic[lev] == 1) {
            MultiFab ppinc(grids[lev],dmap[lev],1,0);
            MultiFab::Copy(ppinc,pp_inc[lev],0,0,1,0);
            VisMF::Write(ppinc, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PP_Inc"));
 
            MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
            MultiFab::Copy(gpx,gradp[lev][GpVars::gpx],0,0,1,0);
            VisMF::Write(gpx, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpx"));
 
            MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
            MultiFab::Copy(gpy,gradp[lev][GpVars::gpy],0,0,1,0);
            VisMF::Write(gpy, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpy"));
 
            MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
            MultiFab::Copy(gpz,gradp[lev][GpVars::gpz],0,0,1,0);
            VisMF::Write(gpz, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpz"));
        }
 
        // Note that we write the ghost cells of the base state (unlike above)
        IntVect ng_base = base_state[lev].nGrowVect();
        int  ncomp_base = base_state[lev].nComp();
        MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
        MultiFab::Copy(base,base_state[lev],0,0,ncomp_base,ng_base);
        VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
 
        if (SolverChoice::mesh_type != MeshType::ConstantDz)  {
            // Note that we also write the ghost cells of z_phys_nd
            IntVect ng = z_phys_nd[lev]->nGrowVect();
            MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
            MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
            VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
        }
 
        // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
        // Write the moisture model restart variables
        std::vector<int> qmoist_indices;
        std::vector<std::string> qmoist_names;
        micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
        int qmoist_nvar = qmoist_indices.size();
        for (int var = 0; var < qmoist_nvar; var++) {
            const int ncomp  = 1;
            IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
            MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
            MultiFab::Copy(moist_vars,*(qmoist[lev][qmoist_indices[var]]),0,0,ncomp,ng_moist);
            VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", qmoist_names[var]));
        }
 
#if defined(ERF_USE_WINDFARM)
        if(solverChoice.windfarm_type == WindFarmType::Fitch or
           solverChoice.windfarm_type == WindFarmType::EWP or
           solverChoice.windfarm_type == WindFarmType::SimpleAD){
            IntVect ng_turb = Nturb[lev].nGrowVect();
            MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng_turb);
            MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng_turb);
            VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
        }
#endif
 
        // Write the LSM data
        if (solverChoice.lsm_type != LandSurfaceType::None) {
            for (int ivar(0); ivar<lsm_data[lev].size(); ++ivar) {
                BoxArray ba = lsm_data[lev][ivar]->boxArray();
                DistributionMapping dm = lsm_data[lev][ivar]->DistributionMap();
                IntVect ng = lsm_data[lev][ivar]->nGrowVect();
                int nvar   = lsm_data[lev][ivar]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                MultiFab::Copy(lsm_vars,*(lsm_data[lev][ivar]),0,0,nvar,ng);
                VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmData" + std::to_string(ivar)));
            }
            for (int iflux(0); iflux<lsm_flux[lev].size(); ++iflux) {
                BoxArray ba = lsm_flux[lev][iflux]->boxArray();
                DistributionMapping dm = lsm_flux[lev][iflux]->DistributionMap();
                IntVect ng = lsm_flux[lev][iflux]->nGrowVect();
                int nvar   = lsm_flux[lev][iflux]->nComp();
                MultiFab lsm_vars(ba,dm,nvar,ng);
                MultiFab::Copy(lsm_vars,*(lsm_flux[lev][iflux]),0,0,nvar,ng);
                VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmFlux" + std::to_string(iflux)));
            }
        }
 
        IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
        MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
        MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_x],0,0,1,ng);
        VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_mx"));
 
#if 0
        if (MapFacType::m_x != MapFacType::m_y) {
            MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_y],0,0,1,ng);
            VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_my"));
        }
#endif
 
        ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
        MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
        MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_x],0,0,1,ng);
        VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_ux"));
 
#if 0
        if (MapFacType::m_x != MapFacType::m_y) {
            MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_y],0,0,1,ng);
            VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_uy"));
        }
#endif
 
        ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
        MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
        MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_x],0,0,1,ng);
        VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vx"));
 
#if 0
        if (MapFacType::m_x != MapFacType::m_y) {
            MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_y],0,0,1,ng);
            VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vy"));
        }
#endif
 
        if (m_SurfaceLayer)  {
            amrex::Print() << "Writing SurfaceLayer variables at level " << lev << std::endl;
            ng = IntVect(1,1,0);
            MultiFab   m_var(ba2d[lev],dmap[lev],1,ng);
            MultiFab* src = nullptr;
 
            // U*
            src = m_SurfaceLayer->get_u_star(lev);
            MultiFab::Copy(m_var,*src,0,0,1,ng);
            VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Ustar"));
 
            // W*
            src = m_SurfaceLayer->get_w_star(lev);
            MultiFab::Copy(m_var,*src,0,0,1,ng);
            VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Wstar"));
 
            // T*
            src = m_SurfaceLayer->get_t_star(lev);
            MultiFab::Copy(m_var,*src,0,0,1,ng);
            VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Tstar"));
 
            // Q*
            src = m_SurfaceLayer->get_q_star(lev);
            MultiFab::Copy(m_var,*src,0,0,1,ng);
            VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qstar"));
 
            // Olen
            src = m_SurfaceLayer->get_olen(lev);
            MultiFab::Copy(m_var,*src,0,0,1,ng);
            VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Olen"));
 
            // Qsurf
            src = m_SurfaceLayer->get_q_surf(lev);
            MultiFab::Copy(m_var,*src,0,0,1,ng);
            VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qsurf"));
 
            // PBLH
            src = m_SurfaceLayer->get_pblh(lev);
            MultiFab::Copy(m_var,*src,0,0,1,ng);
            VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PBLH"));
 
            // Z0
            src = m_SurfaceLayer->get_z0(lev);
            MultiFab::Copy(m_var,*src,0,0,1,ng);
            VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z0"));
        }
 
        if (sst_lev[lev][0]) {
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                MultiFab::Copy(sst_at_t,*sst_lev[lev][nt],0,0,1,ng);
                VisMF::Write(sst_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
                                                             "SST_" + std::to_string(nt)));
            }
        }
 
        if (tsk_lev[lev][0]) {
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                MultiFab::Copy(tsk_at_t,*tsk_lev[lev][nt],0,0,1,ng);
                VisMF::Write(tsk_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
                                                             "TSK_" + std::to_string(nt)));
            }
        }
 
        {
            int ntimes = 1;
            ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
            MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
            for (int nt(0); nt<ntimes; ++nt) {
                for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
                    const Box& bx = mfi.growntilebox();
                    Array4<int>  const& src_arr = lmask_lev[lev][nt]->array(mfi);
                    Array4<Real> const& dst_arr = lmask_at_t.array(mfi);
                    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                    {
                        dst_arr(i,j,k) = Real(src_arr(i,j,k));
                    });
                }
                VisMF::Write(lmask_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
                                                              "LMASK_" + std::to_string(nt)));
            }
        }
 
        IntVect ngv = ng; ngv[2] = 0;
 
        // Write lat/lon if it exists
        if (lat_m[lev] && lon_m[lev] && solverChoice.has_lat_lon) {
            amrex::Print() << "Writing Lat/Lon variables at level " << lev << std::endl;
            MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
            MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
            MultiFab::Copy(lat,*lat_m[lev],0,0,1,ngv);
            MultiFab::Copy(lon,*lon_m[lev],0,0,1,ngv);
            VisMF::Write(lat, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LAT"));
            VisMF::Write(lon, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LON"));
        }
 
 
#ifdef ERF_USE_NETCDF
        // Write sinPhi and cosPhi if it exists
        if (cosPhi_m[lev] && sinPhi_m[lev] && solverChoice.variable_coriolis) {
            amrex::Print() << "Writing Coriolis factors at level " << lev << std::endl;
            MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
            MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
            MultiFab::Copy(sphi,*sinPhi_m[lev],0,0,1,ngv);
            MultiFab::Copy(cphi,*cosPhi_m[lev],0,0,1,ngv);
            VisMF::Write(sphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SinPhi"));
            VisMF::Write(cphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "CosPhi"));
        }
 
        if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
            if (lev == 0) {
                amrex::Print() << "Writing C1H/C2H/MUB variables at level " << lev << std::endl;
                MultiFab tmp1d(ba1d[0],dmap[0],1,0);
 
                MultiFab::Copy(tmp1d,*mf_C1H,0,0,1,0);
                VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C1H"));
 
                MultiFab::Copy(tmp1d,*mf_C2H,0,0,1,0);
                VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C2H"));
 
                MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
 
                MultiFab::Copy(tmp2d,*mf_MUB,0,0,1,mf_MUB->nGrowVect());
                VisMF::Write(tmp2d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MUB"));
            }
        }
#endif
    } // for lev
 
#ifdef ERF_USE_PARTICLES
   particleData.Checkpoint(checkpointname);
#endif
 
#if 0
#ifdef ERF_USE_NETCDF
   // Write bdy_data files
   if ( ParallelDescriptor::IOProcessor() &&
        ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
         solverChoice.use_real_bcs )
   {
       // Vector dimensions
       int num_time = bdy_data_xlo.size();
       int num_var  = bdy_data_xlo[0].size();
 
       // Open header file and write to it
       std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
       bdy_h_file << std::setprecision(1) << std::fixed;
       bdy_h_file << num_time << "\n";
       bdy_h_file << num_var  << "\n";
       bdy_h_file << start_bdy_time << "\n";
       bdy_h_file << bdy_time_interval << "\n";
       bdy_h_file << real_width << "\n";
       for (int ivar(0); ivar<num_var; ++ivar) {
           bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
           bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
           bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
           bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
       }
 
       // Open data file and write to it
       std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
       for (int itime(0); itime<num_time; ++itime) {
           if (bdy_data_xlo[itime].size() > 0) {
               for (int ivar(0); ivar<num_var; ++ivar) {
                   bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
                   bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
                   bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
                   bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
               }
           }
       }
   }
#endif
#endif
 
    if (verbose > 0)
    {
        auto dCheckTime = amrex::second() - dCheckTime0;
        ParallelDescriptor::ReduceRealMax(dCheckTime,ParallelDescriptor::IOProcessorNumber());
        amrex::Print() << "Checkpoint write time = " << dCheckTime << " seconds." << '\n';
    }
}

WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain	(	std::ostream &	HeaderFile,
		int	nlevels,
		const amrex::Vector< amrex::BoxArray > &	bArray,
		const amrex::Vector< std::string > &	varnames,
		const amrex::Vector< amrex::Geometry > &	my_geom,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const amrex::Vector< amrex::IntVect > &	my_ref_ratio,
		const std::string &	versionName,
		const std::string &	levelPrefix,
		const std::string &	mfPrefix
	)		const

{
    AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
    AMREX_ALWAYS_ASSERT(nlevels <= my_ref_ratio.size()+1);
    AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
 
    HeaderFile.precision(17);
 
    // ---- this is the generic plot file type name
    HeaderFile << versionName << '\n';
 
    HeaderFile << varnames.size() << '\n';
 
    for (int ivar = 0; ivar < varnames.size(); ++ivar) {
        HeaderFile << varnames[ivar] << "\n";
    }
    HeaderFile << AMREX_SPACEDIM << '\n';
    HeaderFile << my_time << '\n';
    HeaderFile << finest_level << '\n';
    for (int i = 0; i < AMREX_SPACEDIM; ++i) {
        HeaderFile << my_geom[0].ProbLo(i) << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i < AMREX_SPACEDIM; ++i) {
        HeaderFile << my_geom[0].ProbHi(i) << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i < finest_level; ++i) {
        HeaderFile << my_ref_ratio[i][0] << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i <= finest_level; ++i) {
        HeaderFile << my_geom[i].Domain() << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i <= finest_level; ++i) {
        HeaderFile << level_steps[i] << ' ';
    }
    HeaderFile << '\n';
    for (int i = 0; i <= finest_level; ++i) {
        for (int k = 0; k < AMREX_SPACEDIM; ++k) {
            HeaderFile << my_geom[i].CellSize()[k] << ' ';
        }
        HeaderFile << '\n';
    }
    HeaderFile << (int) my_geom[0].Coord() << '\n';
    HeaderFile << "0\n";
 
    for (int level = 0; level <= finest_level; ++level) {
        HeaderFile << level << ' ' << bArray[level].size() << ' ' << my_time << '\n';
        HeaderFile << level_steps[level] << '\n';
 
        const IntVect& domain_lo = my_geom[level].Domain().smallEnd();
        for (int i = 0; i < bArray[level].size(); ++i)
        {
            // Need to shift because the RealBox ctor we call takes the
            // physical location of index (0,0,0).  This does not affect
            // the usual cases where the domain index starts with 0.
            const Box& b = shift(bArray[level][i], -domain_lo);
            RealBox loc = RealBox(b, my_geom[level].CellSize(), my_geom[level].ProbLo());
            for (int n = 0; n < AMREX_SPACEDIM; ++n) {
                HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
            }
        }
 
        HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
    }
    HeaderFile << "1" << "\n";
    HeaderFile << "3" << "\n";
    HeaderFile << "amrexvec_nu_x" << "\n";
    HeaderFile << "amrexvec_nu_y" << "\n";
    HeaderFile << "amrexvec_nu_z" << "\n";
    std::string mf_nodal_prefix = "Nu_nd";
    for (int level = 0; level <= finest_level; ++level) {
        HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
    }
}

writeJobInfo()

void ERF::writeJobInfo

(

const std::string &

dir

)

const

{
    // job_info file with details about the run
    std::ofstream jobInfoFile;
    std::string FullPathJobInfoFile = dir;
    FullPathJobInfoFile += "/job_info";
    jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out);
 
    std::string PrettyLine = "==================================================="
      "============================\n";
    std::string OtherLine = "----------------------------------------------------"
      "----------------------------\n";
    std::string SkipSpace = "        ";
 
    // job information
    jobInfoFile << PrettyLine;
    jobInfoFile << " ERF Job Information\n";
    jobInfoFile << PrettyLine;
 
    jobInfoFile << "inputs file: " << inputs_name << "\n\n";
 
    jobInfoFile << "number of MPI processes: "
                << ParallelDescriptor::NProcs() << "\n";
#ifdef _OPENMP
    jobInfoFile << "number of threads:       " << omp_get_max_threads() << "\n";
#endif
 
    jobInfoFile << "\n";
    jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
                << getCPUTime() / 3600.0;
 
    jobInfoFile << "\n\n";
 
    if (use_datetime) {
        const std::string dt_format = "%Y-%m-%d %H:%M:%S"; // ISO 8601 standard
        jobInfoFile << "Simulation time: " << getTimestamp(start_time+t_new[0], dt_format) << "\n";
        jobInfoFile << "\n\n";
    }
 
    // plotfile information
    jobInfoFile << PrettyLine;
    jobInfoFile << " Plotfile Information\n";
    jobInfoFile << PrettyLine;
 
    time_t now = time(nullptr);
 
    // Convert now to tm struct for local timezone
    tm* localtm = localtime(&now);
    jobInfoFile << "output data / time: " << asctime(localtm);
 
    std::string currentDir = FileSystem::CurrentPath();
    jobInfoFile << "output dir:         " << currentDir << "\n";
 
    jobInfoFile << "\n\n";
 
    // build information
    jobInfoFile << PrettyLine;
    jobInfoFile << " Build Information\n";
    jobInfoFile << PrettyLine;
 
    jobInfoFile << "build date:    " << buildInfoGetBuildDate() << "\n";
    jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
    jobInfoFile << "build dir:     " << buildInfoGetBuildDir() << "\n";
    jobInfoFile << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
 
    jobInfoFile << "\n";
 
    jobInfoFile << "COMP:          " << buildInfoGetComp() << "\n";
    jobInfoFile << "COMP version:  " << buildInfoGetCompVersion() << "\n";
 
    jobInfoFile << "\n";
 
    for (int n = 1; n <= buildInfoGetNumModules(); n++) {
      jobInfoFile << buildInfoGetModuleName(n) << ": "
                  << buildInfoGetModuleVal(n) << "\n";
    }
 
    jobInfoFile << "\n";
 
    const char* githash1 = buildInfoGetGitHash(1);
    const char* githash2 = buildInfoGetGitHash(2);
    if (strlen(githash1) > 0) {
      jobInfoFile << "ERF       git hash: " << githash1 << "\n";
    }
    if (strlen(githash2) > 0) {
      jobInfoFile << "AMReX       git hash: " << githash2 << "\n";
    }
 
    const char* buildgithash = buildInfoGetBuildGitHash();
    const char* buildgitname = buildInfoGetBuildGitName();
    if (strlen(buildgithash) > 0) {
      jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
    }
 
    jobInfoFile << "\n\n";
 
    // grid information
    jobInfoFile << PrettyLine;
    jobInfoFile << " Grid Information\n";
    jobInfoFile << PrettyLine;
 
    int f_lev = finest_level;
 
    for (int i = 0; i <= f_lev; i++) {
      jobInfoFile << " level: " << i << "\n";
      jobInfoFile << "   number of boxes = " << grids[i].size() << "\n";
      jobInfoFile << "   maximum zones   = ";
      for (int n = 0; n < AMREX_SPACEDIM; n++) {
        jobInfoFile << geom[i].Domain().length(n) << " ";
      }
      jobInfoFile << "\n\n";
    }
 
    jobInfoFile << " Boundary conditions\n";
 
    jobInfoFile << "   -x: " << domain_bc_type[0] << "\n";
    jobInfoFile << "   +x: " << domain_bc_type[3] << "\n";
    jobInfoFile << "   -y: " << domain_bc_type[1] << "\n";
    jobInfoFile << "   +y: " << domain_bc_type[4] << "\n";
    jobInfoFile << "   -z: " << domain_bc_type[2] << "\n";
    jobInfoFile << "   +z: " << domain_bc_type[5] << "\n";
 
    jobInfoFile << "\n\n";
 
    // runtime parameters
    jobInfoFile << PrettyLine;
    jobInfoFile << " Inputs File Parameters\n";
    jobInfoFile << PrettyLine;
 
    ParmParse::dumpTable(jobInfoFile, true);
    jobInfoFile.close();
}

Here is the call graph for this function:

WriteLinePlot()

void ERF::WriteLinePlot	(	const std::string &	filename,
		amrex::Vector< std::array< amrex::Real, 2 >> &	points_xy
	)

{
    std::ofstream ofs(filename);
    if (!ofs.is_open()) {
        amrex::Print() << "Error: Could not open file " << filename << " for writing.\n";
        return;
    }
 
    ofs << std::setprecision(10) << std::scientific;
    ofs << "# x y\n";
 
    for (const auto& p : points_xy) {
        ofs << p[0] << " " << p[1] << "\n";
    }
 
    ofs.close();
 
    amrex::Print() << "Line plot data written to " << filename << "\n";
}

WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain	(	const std::string &	plotfilename,
		int	nlevels,
		const amrex::Vector< const amrex::MultiFab * > &	mf,
		const amrex::Vector< const amrex::MultiFab * > &	mf_nd,
		const amrex::Vector< std::string > &	varnames,
		const amrex::Vector< amrex::Geometry > &	my_geom,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const amrex::Vector< amrex::IntVect > &	my_ref_ratio,
		const std::string &	versionName = "HyperCLaw-V1.1",
		const std::string &	levelPrefix = "Level_",
		const std::string &	mfPrefix = "Cell",
		const amrex::Vector< std::string > &	extra_dirs = amrex::Vector<std::string>()
	)		const

{
    BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
 
    AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
    AMREX_ALWAYS_ASSERT(nlevels <= rr.size()+1);
    AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
    AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
 
    bool callBarrier(false);
    PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
    if (!extra_dirs.empty()) {
        for (const auto& d : extra_dirs) {
            const std::string ed = plotfilename+"/"+d;
            PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
        }
    }
    ParallelDescriptor::Barrier();
 
    if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
        Vector<BoxArray> boxArrays(nlevels);
        for(int level(0); level < boxArrays.size(); ++level) {
            boxArrays[level] = mf[level]->boxArray();
        }
 
        auto f = [=]() {
            VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
            std::string HeaderFileName(plotfilename + "/Header");
            std::ofstream HeaderFile;
            HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
            HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                                    std::ofstream::trunc |
                                                    std::ofstream::binary);
            if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
            WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
                                                  my_geom, time, level_steps, rr, versionName,
                                                  levelPrefix, mfPrefix);
        };
 
        if (AsyncOut::UseAsyncOut()) {
            AsyncOut::Submit(std::move(f));
        } else {
            f();
        }
    }
 
    std::string mf_nodal_prefix = "Nu_nd";
    for (int level = 0; level <= finest_level; ++level)
    {
        if (AsyncOut::UseAsyncOut()) {
            VisMF::AsyncWrite(*mf[level],
                              MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
                              true);
            VisMF::AsyncWrite(*mf_nd[level],
                              MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
                              true);
        } else {
            const MultiFab* data;
            std::unique_ptr<MultiFab> mf_tmp;
            if (mf[level]->nGrowVect() != 0) {
                mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
                                                    mf[level]->DistributionMap(),
                                                    mf[level]->nComp(), 0, MFInfo(),
                                                    mf[level]->Factory());
                MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
                data = mf_tmp.get();
            } else {
                data = mf[level];
            }
            VisMF::Write(*data        , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
            VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
        }
    }
}

WriteMyEBSurface()

void ERF::WriteMyEBSurface

(

)

writeNow()

bool ERF::writeNow	(	const amrex::Real	cur_time,
		const int	nstep,
		const int	plot_int,
		const amrex::Real	plot_per,
		const amrex::Real	dt_0,
		amrex::Real &	last_file_time
	)

{
    bool write_now = false;
 
    if ( plot_int > 0) {
 
        write_now = (nstep % plot_int == 0);
 
    } else if (plot_per > 0.0) {
 
        amrex::Print() << "CUR NEXT PER " << cur_time << " " << next_file_time << " " << plot_per << std::endl;
 
        // Only write now if nstep newly matches the number of elapsed periods
        write_now = (cur_time > (next_file_time - Real(0.1)*dt_0));
    }
 
    return write_now;
}

WriteSubvolume()

void ERF::WriteSubvolume	(	int	isub,
		amrex::Vector< std::string >	subvol_var_names
	)

{
    ParmParse pp("erf.subvol");
 
    Vector<Real> origin;
    Vector< int> ncell;
    Vector<Real> delta;
 
    // **************************************************************
    // Read in the origin, number of cells in each dir, and resolution
    // **************************************************************
 
    int lev_for_sub = 0;
    int offset = isub * AMREX_SPACEDIM;
 
    pp.getarr("origin",origin,offset,AMREX_SPACEDIM);
    pp.getarr("nxnynz", ncell,offset,AMREX_SPACEDIM);
    pp.getarr("dxdydz", delta,offset,AMREX_SPACEDIM);
 
    bool found = false;
    for (int i = 0; i <= finest_level; i++) {
        if (!found) {
            if (almostEqual(delta[offset+0],geom[i].CellSize(0)) &&
                almostEqual(delta[offset+1],geom[i].CellSize(1)) &&
                almostEqual(delta[offset+2],geom[i].CellSize(2)) ) {
 
                amrex::Print() << "WriteSubvolume:Resolution specified matches that of level " << i << std::endl;
                found = true;
                lev_for_sub = i;
            }
        }
    }
 
    if (!found) {
        amrex::Abort("Resolution specified for subvol does not match the resolution of any of the levels.");
    }
 
 
    // **************************************************************
    // Now that we know which level we're at, we can figure out which (i,j,k) the origin corresponds to
    // Note we use 1.0001 as a fudge factor since the division of two reals --> integer will do a floor
    // **************************************************************
    int i0 = static_cast<int>((origin[offset+0] - geom[lev_for_sub].ProbLo(0)) * 1.0001 / delta[offset+0]);
    int j0 = static_cast<int>((origin[offset+1] - geom[lev_for_sub].ProbLo(1)) * 1.0001 / delta[offset+1]);
    int k0 = static_cast<int>((origin[offset+2] - geom[lev_for_sub].ProbLo(2)) * 1.0001 / delta[offset+2]);
 
    found = false;
    if (almostEqual(geom[lev_for_sub].ProbLo(0)+i0*delta[offset+0],origin[offset+0]) &&
        almostEqual(geom[lev_for_sub].ProbLo(1)+j0*delta[offset+1],origin[offset+1]) &&
        almostEqual(geom[lev_for_sub].ProbLo(2)+k0*delta[offset+2],origin[offset+2]) )
    {
        amrex::Print() << "WriteSubvolume:Specified origin is the lower left corner of cell " << IntVect(i0,j0,k0) << std::endl;
        found = true;
    }
 
    if (!found) {
        amrex::Abort("Origin specified does not correspond to a node at this level.");
    }
 
    Box domain(geom[lev_for_sub].Domain());
 
    Box bx(IntVect(i0,j0,k0),IntVect(i0+ncell[offset+0]-1,j0+ncell[offset+1]-1,k0+ncell[offset+2]-1));
    amrex::Print() << "WriteSubvolume:Box requested is " << bx << std::endl;
 
    if (!domain.contains(bx))
    {
        amrex::Abort("WriteSubvolume:Box requested is larger than the existing domain");
    }
 
    Vector<int> cs(AMREX_SPACEDIM);
    int count = pp.countval("chunk_size");
    if (count > 0) {
        pp.queryarr("chunk_size",cs,0,AMREX_SPACEDIM);
    } else {
        cs[0] = max_grid_size[0][0];
        cs[1] = max_grid_size[0][1];
        cs[2] = max_grid_size[0][2];
    }
    IntVect chunk_size(cs[0],cs[1],cs[2]);
 
    BoxArray ba(bx);
    ba.maxSize(chunk_size);
 
    amrex::Print() << "WriteSubvolume:BoxArray is " << ba << std::endl;
 
    Vector<std::string> varnames;
    varnames.insert(varnames.end(), subvol_var_names.begin(), subvol_var_names.end());
 
    int ncomp_mf = subvol_var_names.size();
 
    DistributionMapping dm(ba);
 
    MultiFab mf(ba, dm, ncomp_mf, 0);
 
    int mf_comp = 0;
 
    // *****************************************************************************************
 
    // First, copy any of the conserved state variables into the output plotfile
    for (int i = 0; i < cons_names.size(); ++i) {
        if (containerHasElement(subvol_var_names, cons_names[i])) {
            mf.ParallelCopy(vars_new[lev_for_sub][Vars::cons],i,mf_comp,1,1,0);
            mf_comp++;
        }
    }
 
    // *****************************************************************************************
 
    if (containerHasElement(subvol_var_names, "x_velocity") ||
        containerHasElement(subvol_var_names, "y_velocity") ||
        containerHasElement(subvol_var_names, "z_velocity"))
    {
        MultiFab mf_cc_vel(grids[lev_for_sub], dmap[lev_for_sub], AMREX_SPACEDIM, 0);
        average_face_to_cellcenter(mf_cc_vel,0,
                                   Array<const MultiFab*,3>{&vars_new[lev_for_sub][Vars::xvel],
                                                            &vars_new[lev_for_sub][Vars::yvel],
                                                            &vars_new[lev_for_sub][Vars::zvel]});
        if (containerHasElement(subvol_var_names, "x_velocity")) {
            mf.ParallelCopy(mf_cc_vel,0,mf_comp,1,0,0);
            mf_comp++;
        }
        if (containerHasElement(subvol_var_names, "y_velocity")) {
            mf.ParallelCopy(mf_cc_vel,1,mf_comp,1,0,0);
            mf_comp++;
        }
        if (containerHasElement(subvol_var_names, "z_velocity")) {
            mf.ParallelCopy(mf_cc_vel,2,mf_comp,1,0,0);
            mf_comp++;
        }
    }
 
    // *****************************************************************************************
 
    // Finally, check for any derived quantities and compute them, inserting
    // them into our output multifab
    auto calculate_derived = [&](const std::string& der_name,
                                 MultiFab& src_mf,
                                 decltype(derived::erf_dernull)& der_function)
    {
        if (containerHasElement(subvol_var_names, der_name)) {
            MultiFab dmf(src_mf.boxArray(), src_mf.DistributionMap(), 1, 0);
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
            for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
            {
                const Box& tbx = mfi.tilebox();
                auto& dfab = dmf[mfi];
                auto& sfab = src_mf[mfi];
                der_function(tbx, dfab, 0, 1, sfab, Geom(lev_for_sub), t_new[0], nullptr, lev_for_sub);
            }
            mf.ParallelCopy(dmf,0,mf_comp,1,0,0);
            mf_comp++;
        }
    };
 
    // *****************************************************************************************
    // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
    //       defined in ERF.H
    // *****************************************************************************************
 
    calculate_derived("soundspeed",  vars_new[lev_for_sub][Vars::cons], derived::erf_dersoundspeed);
    if (solverChoice.moisture_type != MoistureType::None) {
        calculate_derived("temp",    vars_new[lev_for_sub][Vars::cons], derived::erf_dermoisttemp);
    } else {
        calculate_derived("temp",    vars_new[lev_for_sub][Vars::cons], derived::erf_dertemp);
    }
    calculate_derived("theta",       vars_new[lev_for_sub][Vars::cons], derived::erf_dertheta);
    calculate_derived("KE",          vars_new[lev_for_sub][Vars::cons], derived::erf_derKE);
    calculate_derived("scalar",      vars_new[lev_for_sub][Vars::cons], derived::erf_derscalar);
 
    // *****************************************************************************************
 
    Real time = t_new[lev_for_sub];
 
    std::string sf = subvol_file + "_" + std::to_string(isub);
    std::string subvol_filename;
 
    if (use_real_time_in_pltname) {
        const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
        subvol_filename = sf + getTimestamp(start_time+time, dt_format);
    } else {
       subvol_filename = Concatenate(sf + "_", istep[0], file_name_digits);
    }
 
    amrex::Print() <<"Writing subvolume into " << subvol_filename << std::endl;
    WriteSingleLevelPlotfile(subvol_filename,mf,varnames,geom[lev_for_sub],time,istep[0]);
 
}

Here is the call graph for this function:

WriteVTKPolyline()

void ERF::WriteVTKPolyline	(	const std::string &	filename,
		amrex::Vector< std::array< amrex::Real, 2 >> &	points_xy
	)

{
    std::ofstream vtkfile(filename);
    if (!vtkfile.is_open()) {
        std::cerr << "Error: Cannot open file " << filename << std::endl;
        return;
    }
 
    int num_points = points_xy.size();
    if (num_points == 0) {
        vtkfile << "# vtk DataFile Version 3.0\n";
        vtkfile << "Storm Track\n";
        vtkfile << "ASCII\n";
        vtkfile << "DATASET POLYDATA\n";
        vtkfile << "POINTS " << num_points << " float\n";
        vtkfile.close();
        return;
    }
    if (num_points < 2) {
        points_xy.push_back(points_xy[0]);
    }
    num_points = points_xy.size();
 
    vtkfile << "# vtk DataFile Version 3.0\n";
    vtkfile << "Storm Track\n";
    vtkfile << "ASCII\n";
    vtkfile << "DATASET POLYDATA\n";
 
    // Write points (Z=0 assumed)
    vtkfile << "POINTS " << num_points << " float\n";
    for (const auto& pt : points_xy) {
        vtkfile << pt[0] << " " << pt[1] << " 10000.0\n";
    }
 
    // Write polyline connectivity
    vtkfile << "LINES 1 " << num_points + 1 << "\n";
    vtkfile << num_points << " ";
    for (int i = 0; i < num_points; ++i) {
        vtkfile << i << " ";
    }
    vtkfile << "\n";
 
    vtkfile.close();
}

Member Data Documentation

advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg

private

Referenced by getAdvFluxReg().

avg_xmom

amrex::Vector<amrex::MultiFab> ERF::avg_xmom

private

avg_ymom

amrex::Vector<amrex::MultiFab> ERF::avg_ymom

private

avg_zmom

amrex::Vector<amrex::MultiFab> ERF::avg_zmom

private

ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax

private

ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src

private

ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay

private

ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src

private

az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az

private

az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src

private

ba1d

amrex::Vector<amrex::BoxArray> ERF::ba1d

private

ba2d

amrex::Vector<amrex::BoxArray> ERF::ba2d

private

base_state

amrex::Vector<amrex::MultiFab> ERF::base_state

private

base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new

private

bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1

staticprivate

bndry_output_planes_per

Real ERF::bndry_output_planes_per = -1.0

staticprivate

bndry_output_planes_start_time

Real ERF::bndry_output_planes_start_time = 0.0

staticprivate

boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level

private

cf_set_width

int ERF::cf_set_width {0}

private

cf_width

int ERF::cf_width {0}

private

cfl

Real ERF::cfl = 0.8

staticprivate

change_max

Real ERF::change_max = 1.1

staticprivate

check_file

std::string ERF::check_file {"chk"}

private

check_for_nans

int ERF::check_for_nans = 0

staticprivate

column_file_name

std::string ERF::column_file_name = "column_data.nc"

staticprivate

column_interval

int ERF::column_interval = -1

staticprivate

column_loc_x

Real ERF::column_loc_x = 0.0

staticprivate

column_loc_y

Real ERF::column_loc_y = 0.0

staticprivate

column_per

Real ERF::column_per = -1.0

staticprivate

cons_names

const amrex::Vector<std::string> ERF::cons_names

private

Initial value:

{"density", "rhotheta", "rhoKE", "rhoadv_0",
                                                     "rhoQ1", "rhoQ2", "rhoQ3",
                                                     "rhoQ4", "rhoQ5", "rhoQ6"}

cosPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::cosPhi_m

private

d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density

private

d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure

private

d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc

private

d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv

private

d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature

private

d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs

private

d_sinesq_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_ptrs

private

d_sinesq_stag_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_stag_ptrs

private

d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs

private

d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos

private

d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos

private

d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid

private

datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog

private

Referenced by DataLog(), NumDataLogs(), and setRecordDataInfo().

datalogname

amrex::Vector<std::string> ERF::datalogname

private

Referenced by DataLogName().

datetime_format

const std::string ERF::datetime_format = "%Y-%m-%d %H:%M:%S"

private

datprecision

const int ERF::datprecision = 6

private

datwidth

const int ERF::datwidth = 14

private

der_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::der_datalog

private

Referenced by DerDataLog(), NumDerDataLogs(), and setRecordDerDataInfo().

der_datalogname

amrex::Vector<std::string> ERF::der_datalogname

private

Referenced by DerDataLogName().

derived_names

const amrex::Vector<std::string> ERF::derived_names

private

derived_names_2d

const amrex::Vector<std::string> ERF::derived_names_2d

private

Initial value:

{
        "z_surf", "landmask", "mapfac", "lat_m", "lon_m",
        "u_star", "w_star", "t_star", "q_star", "Olen", "pblh",
        "t_surf", "q_surf", "z0", "OLR", "sens_flux", "laten_flux",
        "surf_pres", "integrated_qv"
    }

derived_subvol_names

const amrex::Vector<std::string> ERF::derived_subvol_names {"soundspeed", "temp", "theta", "KE", "scalar"}

private

destag_profiles

bool ERF::destag_profiles = true

private

detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc

private

detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new

private

detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src

private

domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type

private

domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type

private

domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d

private

dt

amrex::Vector<amrex::Real> ERF::dt

private

dt_max

Real ERF::dt_max = 1.0e9

staticprivate

dt_max_initial

Real ERF::dt_max_initial = 2.0e100

staticprivate

dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio

private

dz_min

amrex::Vector<amrex::Real> ERF::dz_min

private

eb

amrex::Vector<std::unique_ptr<eb_> > ERF::eb

private

Referenced by EBFactory(), and get_eb().

eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev

private

file_name_digits

int ERF::file_name_digits = 5

private

fine_mask

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::fine_mask

private

finished_wave

bool ERF::finished_wave = false

private

fixed_dt

amrex::Vector<amrex::Real> ERF::fixed_dt

private

fixed_fast_dt

amrex::Vector<amrex::Real> ERF::fixed_fast_dt

private

fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0

staticprivate

forecast_state_1

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_1

forecast_state_2

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_2

forecast_state_interp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_interp

FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c

private

FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u

private

FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v

private

FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w

private

gradp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::gradp

private

h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density

private

h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure

private

h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc

private

h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv

private

h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature

private

h_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs

private

h_sinesq_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_ptrs

private

h_sinesq_stag_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_stag_ptrs

private

h_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs

private

h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos

private

h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos

private

h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid

private

have_read_nc_init_file

Vector< Vector< int > > ERF::have_read_nc_init_file = {{0}}

staticprivate

hurricane_eye_track_latlon

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_latlon

hurricane_eye_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_xy

hurricane_maxvel_vs_time

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_maxvel_vs_time

hurricane_minpressure_vs_time

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_minpressure_vs_time

hurricane_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_track_xy

hurricane_tracker_circle

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_tracker_circle

Hwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave

private

Hwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave_onegrid

private

init_shrink

Real ERF::init_shrink = 1.0

staticprivate

input_bndry_planes

int ERF::input_bndry_planes = 0

staticprivate

input_sounding_data

InputSoundingData ERF::input_sounding_data

private

input_sponge_data

InputSpongeData ERF::input_sponge_data

private

interpolation_type

StateInterpType ERF::interpolation_type

staticprivate

istep

amrex::Vector<int> ERF::istep

private

lagged_delta_rt

amrex::Vector<amrex::MultiFab> ERF::lagged_delta_rt

private

land_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::land_type_lev

private

last_check_file_step

int ERF::last_check_file_step = -1

staticprivate

last_check_file_time

Real ERF::last_check_file_time = 0.0

staticprivate

last_plot2d_file_step_1

int ERF::last_plot2d_file_step_1 = -1

staticprivate

last_plot2d_file_step_2

int ERF::last_plot2d_file_step_2 = -1

staticprivate

last_plot2d_file_time_1

Real ERF::last_plot2d_file_time_1 = 0.0

staticprivate

last_plot2d_file_time_2

Real ERF::last_plot2d_file_time_2 = 0.0

staticprivate

last_plot3d_file_step_1

int ERF::last_plot3d_file_step_1 = -1

staticprivate

last_plot3d_file_step_2

int ERF::last_plot3d_file_step_2 = -1

staticprivate

last_plot3d_file_time_1

Real ERF::last_plot3d_file_time_1 = 0.0

staticprivate

last_plot3d_file_time_2

Real ERF::last_plot3d_file_time_2 = 0.0

staticprivate

last_subvol_step

amrex::Vector<int> ERF::last_subvol_step

private

last_subvol_time

amrex::Vector<amrex::Real> ERF::last_subvol_time

private

lat_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lat_m

private

line_sampler

std::unique_ptr<LineSampler> ERF::line_sampler = nullptr

private

line_sampling_interval

int ERF::line_sampling_interval = -1

private

line_sampling_per

amrex::Real ERF::line_sampling_per = -1.0

private

lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev

private

lon_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lon_m

private

lsm

LandSurface ERF::lsm

private

lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data

private

lsm_data_name

amrex::Vector<std::string> ERF::lsm_data_name

private

lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux

private

lsm_flux_name

amrex::Vector<std::string> ERF::lsm_flux_name

private

Lwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave

private

Lwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave_onegrid

private

m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_extdir_vals

private

m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_neumann_vals

private

m_check_int

int ERF::m_check_int = -1

private

m_check_per

amrex::Real ERF::m_check_per = -1.0

private

m_expand_plotvars_to_unif_rr

bool ERF::m_expand_plotvars_to_unif_rr = false

private

m_forest_drag

amrex::Vector<std::unique_ptr<ForestDrag> > ERF::m_forest_drag

private

m_plot2d_int_1

int ERF::m_plot2d_int_1 = -1

private

m_plot2d_int_2

int ERF::m_plot2d_int_2 = -1

private

m_plot2d_per_1

amrex::Real ERF::m_plot2d_per_1 = -1.0

private

m_plot2d_per_2

amrex::Real ERF::m_plot2d_per_2 = -1.0

private

m_plot3d_int_1

int ERF::m_plot3d_int_1 = -1

private

m_plot3d_int_2

int ERF::m_plot3d_int_2 = -1

private

m_plot3d_per_1

amrex::Real ERF::m_plot3d_per_1 = -1.0

private

m_plot3d_per_2

amrex::Real ERF::m_plot3d_per_2 = -1.0

private

m_plot_face_vels

bool ERF::m_plot_face_vels = false

private

m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr

private

m_subvol_int

amrex::Vector<int> ERF::m_subvol_int

private

m_subvol_per

amrex::Vector<amrex::Real> ERF::m_subvol_per

private

m_SurfaceLayer

std::unique_ptr<SurfaceLayer> ERF::m_SurfaceLayer = nullptr

private

m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr

private

mapfac

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::mapfac

private

max_step

int ERF::max_step = -1

private

metgrid_basic_linear

bool ERF::metgrid_basic_linear {false}

private

metgrid_debug_dry

bool ERF::metgrid_debug_dry {false}

private

metgrid_debug_isothermal

bool ERF::metgrid_debug_isothermal {false}

private

metgrid_debug_msf

bool ERF::metgrid_debug_msf {false}

private

metgrid_debug_psfc

bool ERF::metgrid_debug_psfc {false}

private

metgrid_debug_quiescent

bool ERF::metgrid_debug_quiescent {false}

private

metgrid_force_sfc_k

int ERF::metgrid_force_sfc_k {6}

private

metgrid_interp_theta

bool ERF::metgrid_interp_theta {false}

private

metgrid_order

int ERF::metgrid_order {2}

private

metgrid_proximity

amrex::Real ERF::metgrid_proximity {500.0}

private

metgrid_retain_sfc

bool ERF::metgrid_retain_sfc {false}

private

metgrid_use_below_sfc

bool ERF::metgrid_use_below_sfc {true}

private

metgrid_use_sfc

bool ERF::metgrid_use_sfc {true}

private

mf_C1H

std::unique_ptr<amrex::MultiFab> ERF::mf_C1H

private

mf_C2H

std::unique_ptr<amrex::MultiFab> ERF::mf_C2H

private

mf_MUB

std::unique_ptr<amrex::MultiFab> ERF::mf_MUB

private

mf_PSFC

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_PSFC

private

mg_verbose

int ERF::mg_verbose = 0

staticprivate

micro

std::unique_ptr<Microphysics> ERF::micro

private

mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem

private

nc_bdy_file

std::string ERF::nc_bdy_file

staticprivate

nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}

staticprivate

nc_low_file

std::string ERF::nc_low_file

staticprivate

ng_dens_hse

int ERF::ng_dens_hse

staticprivate

ng_pres_hse

int ERF::ng_pres_hse

staticprivate

nsubsteps

amrex::Vector<int> ERF::nsubsteps

private

num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level

private

num_files_at_level

amrex::Vector<int> ERF::num_files_at_level

private

output_1d_column

int ERF::output_1d_column = 0

staticprivate

output_bndry_planes

int ERF::output_bndry_planes = 0

staticprivate

pert_interval

int ERF::pert_interval = -1

staticprivate

phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type

private

physbcs_base

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_base> > ERF::physbcs_base

private

physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons

private

physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u

private

physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v

private

physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w

private

plane_sampler

std::unique_ptr<PlaneSampler> ERF::plane_sampler = nullptr

private

plane_sampling_interval

int ERF::plane_sampling_interval = -1

private

plane_sampling_per

amrex::Real ERF::plane_sampling_per = -1.0

private

plot2d_file_1

std::string ERF::plot2d_file_1 {"plt2d_1_"}

private

plot2d_file_2

std::string ERF::plot2d_file_2 {"plt2d_2_"}

private

plot2d_var_names_1

amrex::Vector<std::string> ERF::plot2d_var_names_1

private

plot2d_var_names_2

amrex::Vector<std::string> ERF::plot2d_var_names_2

private

plot3d_file_1

std::string ERF::plot3d_file_1 {"plt_1_"}

private

plot3d_file_2

std::string ERF::plot3d_file_2 {"plt_2_"}

private

plot3d_var_names_1

amrex::Vector<std::string> ERF::plot3d_var_names_1

private

plot3d_var_names_2

amrex::Vector<std::string> ERF::plot3d_var_names_2

private

plot_file_on_restart

bool ERF::plot_file_on_restart = true

staticprivate

plot_lsm

bool ERF::plot_lsm = false

private

plot_rad

bool ERF::plot_rad = false

private

plotfile2d_type_1

PlotFileType ERF::plotfile2d_type_1 = PlotFileType::None

staticprivate

plotfile2d_type_2

PlotFileType ERF::plotfile2d_type_2 = PlotFileType::None

staticprivate

plotfile3d_type_1

PlotFileType ERF::plotfile3d_type_1 = PlotFileType::None

staticprivate

plotfile3d_type_2

PlotFileType ERF::plotfile3d_type_2 = PlotFileType::None

staticprivate

pp_inc

amrex::Vector<amrex::MultiFab> ERF::pp_inc

private

pp_prefix

std::string ERF::pp_prefix {"erf"}

previousCPUTimeUsed

Real ERF::previousCPUTimeUsed = 0.0

staticprivate

Referenced by getCPUTime().

prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr

private

profile_int

int ERF::profile_int = -1

private

qheating_rates

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::qheating_rates

private

qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist

private

Qr_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qr_prim

private

Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim

private

rad

amrex::Vector<std::unique_ptr<IRadiation> > ERF::rad

private

rad_datalog_int

int ERF::rad_datalog_int = -1

private

rad_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rad_fluxes

private

real_extrap_w

bool ERF::real_extrap_w {true}

private

real_set_width

int ERF::real_set_width {0}

private

real_width

int ERF::real_width {0}

private

ref_tags

Vector< AMRErrorTag > ERF::ref_tags

staticprivate

regrid_int

int ERF::regrid_int = -1

private

regrid_level_0_on_restart

bool ERF::regrid_level_0_on_restart = false

private

restart_chkfile

std::string ERF::restart_chkfile = ""

private

rhoqt_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rhoqt_src

private

rhotheta_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rhotheta_src

private

rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new

private

rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old

private

rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new

private

rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old

private

rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new

private

rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old

private

sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline

private

Referenced by NumSampleLines(), and SampleLine().

samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog

private

Referenced by NumSampleLineLogs(), SampleLineLog(), and setRecordSampleLineInfo().

samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname

private

Referenced by SampleLineLogName().

samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint

private

Referenced by NumSamplePoints(), and SamplePoint().

sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog

private

Referenced by NumSamplePointLogs(), SamplePointLog(), and setRecordSamplePointInfo().

sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname

private

Referenced by SamplePointLogName().

SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev

private

SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev

private

SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev

private

SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev

private

SFS_q1fx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx1_lev

private

SFS_q1fx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx2_lev

private

SFS_q1fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx3_lev

private

SFS_q2fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q2fx3_lev

private

sinPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sinPhi_m

private

SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev

private

soil_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::soil_type_lev

private

solar_zenith

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::solar_zenith

private

solverChoice

SolverChoice ERF::solverChoice

staticprivate

sponge_type

std::string ERF::sponge_type

staticprivate

sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev

private

start_time

Real ERF::start_time = 0.0

staticprivate

startCPUTime

Real ERF::startCPUTime = 0.0

staticprivate

Referenced by getCPUTime().

stop_time

Real ERF::stop_time = std::numeric_limits<amrex::Real>::max()

staticprivate

stretched_dz_d

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::stretched_dz_d

private

stretched_dz_h

amrex::Vector<amrex::Vector<amrex::Real> > ERF::stretched_dz_h

private

sub_cfl

Real ERF::sub_cfl = 1.0

staticprivate

subdomains

amrex::Vector<amrex::Vector<amrex::BoxArray> > ERF::subdomains

private

subvol3d_var_names

amrex::Vector<std::string> ERF::subvol3d_var_names

private

subvol_file

std::string ERF::subvol_file {"subvol"}

private

sum_interval

int ERF::sum_interval = -1

staticprivate

sum_per

Real ERF::sum_per = -1.0

staticprivate

surface_state_1

amrex::Vector<amrex::MultiFab> ERF::surface_state_1

surface_state_2

amrex::Vector<amrex::MultiFab> ERF::surface_state_2

surface_state_interp

amrex::Vector<amrex::MultiFab> ERF::surface_state_interp

sw_lw_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sw_lw_fluxes

private

t_avg_cnt

amrex::Vector<amrex::Real> ERF::t_avg_cnt

private

t_new

amrex::Vector<amrex::Real> ERF::t_new

private

t_old

amrex::Vector<amrex::Real> ERF::t_old

private

Tau

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau

private

Tau_corr

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau_corr

private

terrain_blanking

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking

private

th_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::th_bc_data

private

Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim

private

thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce

private

thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce

private

thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce

private

timeprecision

const int ERF::timeprecision = 13

private

tot_e_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::tot_e_datalog

private

Referenced by setRecordEnergyDataInfo().

tot_e_datalogname

amrex::Vector<std::string> ERF::tot_e_datalogname

private

tsk_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::tsk_lev

private

turbPert

TurbulentPerturbation ERF::turbPert

private

urb_frac_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::urb_frac_lev

private

use_datetime

bool ERF::use_datetime = false

private

use_fft

bool ERF::use_fft = false

staticprivate

use_real_time_in_pltname

bool ERF::use_real_time_in_pltname = false

private

vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new

private

vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old

private

vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg

private

verbose

int ERF::verbose = 0

staticprivate

walldist

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::walldist

private

weather_forecast_data_1

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_1

weather_forecast_data_2

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_2

xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask

private

xvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::xvel_bc_data

private

yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask

private

yvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::yvel_bc_data

private

z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc

private

z_phys_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc_src

private

z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd

private

z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new

private

z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src

private

z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk

private

zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask

private

zlevels_stag

amrex::Vector<amrex::Vector<amrex::Real> > ERF::zlevels_stag

private

zmom_crse_rhs

amrex::Vector<amrex::MultiFab> ERF::zmom_crse_rhs

private

zvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::zvel_bc_data

private

---

The documentation for this class was generated from the following files:

• Source/ERF.H
• Source/BoundaryConditions/ERF_BoundaryConditionsBndryReg.cpp
• Source/BoundaryConditions/ERF_FillBdyCCVels.cpp
• Source/BoundaryConditions/ERF_FillCoarsePatch.cpp
• Source/BoundaryConditions/ERF_FillIntermediatePatch.cpp
• Source/BoundaryConditions/ERF_FillPatch.cpp
• Source/ERF.cpp
• Source/ERF_MakeNewArrays.cpp
• Source/ERF_MakeNewLevel.cpp
• Source/ERF_Tagging.cpp
• Source/Initialization/ERF_Init1D.cpp
• Source/Initialization/ERF_InitBCs.cpp
• Source/Initialization/ERF_InitCustomPertState.cpp
• Source/Initialization/ERF_InitFromHSE.cpp
• Source/Initialization/ERF_InitFromInputSounding.cpp
• Source/Initialization/ERF_InitGeowind.cpp
• Source/Initialization/ERF_InitImmersedForcing.cpp
• Source/Initialization/ERF_InitRayleigh.cpp
• Source/Initialization/ERF_InitSponge.cpp
• Source/Initialization/ERF_InitTurbPert.cpp
• Source/Initialization/ERF_InitUniform.cpp
• Source/IO/ERF_Checkpoint.cpp
• Source/IO/ERF_ConsoleIO.cpp
• Source/IO/ERF_Plotfile.cpp
• Source/IO/ERF_TrackerOutput.cpp
• Source/IO/ERF_Write1DProfiles.cpp
• Source/IO/ERF_Write1DProfiles_stag.cpp
• Source/IO/ERF_WriteJobInfo.cpp
• Source/IO/ERF_WriteScalarProfiles.cpp
• Source/IO/ERF_WriteSubvolume.cpp
• Source/LinearSolvers/ERF_ComputeDivergence.cpp
• Source/LinearSolvers/ERF_ImposeBCsOnPhi.cpp
• Source/LinearSolvers/ERF_PoissonSolve.cpp
• Source/LinearSolvers/ERF_PoissonSolve_tb.cpp
• Source/LinearSolvers/ERF_PoissonWallDist.cpp
• Source/LinearSolvers/ERF_SolveWithGMRES.cpp
• Source/TimeIntegration/ERF_Advance.cpp
• Source/TimeIntegration/ERF_AdvanceDycore.cpp
• Source/TimeIntegration/ERF_AdvanceLSM.cpp
• Source/TimeIntegration/ERF_AdvanceMicrophysics.cpp
• Source/TimeIntegration/ERF_AdvanceRadiation.cpp
• Source/TimeIntegration/ERF_ComputeTimestep.cpp
• Source/TimeIntegration/ERF_TimeStep.cpp
• Source/Utils/ERF_AverageDown.cpp
• Source/Utils/ERF_MakeSubdomains.cpp
• Source/Utils/ERF_SurfaceDataInterpolation.cpp
• Source/Utils/ERF_VolWgtSum.cpp
• Source/Utils/ERF_WeatherDataInterpolation.cpp