#include <ERF.H>

Inheritance diagram for ERF:

Collaboration diagram for ERF:

[legend]

Public Member Functions
	ERF ()

	~ERF () override

void	ERF_shared ()

	ERF (ERF &&) noexcept=delete

ERF &	operator= (ERF &&other) noexcept=delete

	ERF (const ERF &other)=delete

ERF &	operator= (const ERF &other)=delete

void	Evolve ()

void	ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override

void	HurricaneTracker (int lev, amrex::Real time, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::TagBoxArray *tags=nullptr)

bool	FindInitialEye (int lev, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::Real &eye_x, amrex::Real &eye_y)

std::string	MakeVTKFilename (int nstep)

std::string	MakeVTKFilename_TrackerCircle (int nstep)

std::string	MakeVTKFilename_EyeTracker_xy (int nstep)

std::string	MakeFilename_EyeTracker_latlon (int nstep)

std::string	MakeFilename_EyeTracker_maxvel (int nstep)

std::string	MakeFilename_EyeTracker_minpressure (int nstep)

void	WriteVTKPolyline (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)

void	WriteLinePlot (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)

void	InitData ()

void	InitData_pre ()

void	InitData_post ()

void	Interp2DArrays (int lev, const amrex::BoxArray &my_ba2d, const amrex::DistributionMapping &my_dm)

void	WriteMyEBSurface ()

void	compute_divergence (int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)

void	project_initial_velocity (int lev, amrex::Real time, amrex::Real dt)

void	project_momenta (int lev, amrex::Real l_time, amrex::Real l_dt, amrex::Vector< amrex::MultiFab > &vars)

void	project_velocity_tb (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)

void	poisson_wall_dist (int lev)

void	make_subdomains (const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)

void	solve_with_gmres (int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &az_sub, amrex::MultiFab &, amrex::MultiFab &znd_sub)

void	ImposeBCsOnPhi (int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)

void	init_only (int lev, amrex::Real time)

void	restart ()

void	check_state_for_nans (amrex::MultiFab const &S)

void	check_vels_for_nans (amrex::MultiFab const &xvel, amrex::MultiFab const &yvel, amrex::MultiFab const &zvel)

void	check_for_negative_theta (amrex::MultiFab &S)

void	check_for_low_temp (amrex::MultiFab &S)

bool	writeNow (const amrex::Real cur_time, const int nstep, const int plot_int, const amrex::Real plot_per, const amrex::Real dt_0, amrex::Real &last_file_time)

void	post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev)

void	sum_integrated_quantities (amrex::Real time)

void	sum_derived_quantities (amrex::Real time)

void	sum_energy_quantities (amrex::Real time)

void	write_1D_profiles (amrex::Real time)

void	write_1D_profiles_stag (amrex::Real time)

amrex::Real	cloud_fraction (amrex::Real time)

void	FillBdyCCVels (amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)

void	sample_points (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)

void	sample_lines (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)

void	derive_diag_profiles (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)

void	derive_diag_profiles_stag (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)

void	derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)

void	derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)

amrex::Real	volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, const amrex::MultiFab &dJ, const amrex::MultiFab &mfx, const amrex::MultiFab &mfy, bool finemask, bool local=true)

void	volWgtColumnSum (int lev, const amrex::MultiFab &mf, int comp, amrex::MultiFab &mf_2d, const amrex::MultiFab &dJ)

void	MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override

void	RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override

void	ClearLevel (int lev) override

void	MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override

amrex::Real	estTimeStep (int lev, long &dt_fast_ratio) const

void	advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)

void	advance_microphysics (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)

void	advance_lsm (int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)

void	advance_radiation (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance)

void	build_fine_mask (int lev, amrex::MultiFab &fine_mask)

void	MakeHorizontalAverages ()

void	MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)

void	derive_upwp (amrex::Vector< amrex::Real > &h_havg)

void	Write3DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)

void	Write2DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)

void	WriteSubvolume (int isub, amrex::Vector< std::string > subvol_var_names)

void	WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const

void	WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const

void	erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)

void	init_from_input_sounding (int lev)

void	init_immersed_forcing (int lev)

void	input_sponge (int lev)

void	init_from_hse (int lev)

void	init_thin_body (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)

void	FillForecastStateMultiFabs (const int lev, const std::string &filename, const std::unique_ptr< amrex::MultiFab > &z_phys_nd, amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)

void	FillSurfaceStateMultiFabs (const int lev, const std::string &filename, amrex::Vector< amrex::MultiFab > &surface_state)

void	WeatherDataInterpolation (const int nlevs, const amrex::Real time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)

void	SurfaceDataInterpolation (const int nlevs, const amrex::Real time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)

void	create_random_perturbations (const int lev, amrex::MultiFab &cons_pert, amrex::MultiFab &xvel_pert, amrex::MultiFab &yvel_pert, amrex::MultiFab &zvel_pert)

void	apply_gaussian_smoothing_to_perturbations (const int lev, amrex::MultiFab &cons_pert, amrex::MultiFab &xvel_pert, amrex::MultiFab &yvel_pert, amrex::MultiFab &zvel_pert)

void	fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)

void	MakeEBGeometry ()

void	make_eb_box ()

void	make_eb_regular ()

void	AverageDownTo (int crse_lev, int scomp, int ncomp)

void	WriteCheckpointFile () const

void	ReadCheckpointFile ()

void	ReadCheckpointFileSurfaceLayer ()

void	init_zphys (int lev, amrex::Real elapsed_time)

void	remake_zphys (int lev, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)

void	update_terrain_arrays (int lev)

void	writeJobInfo (const std::string &dir) const

Static Public Member Functions
static bool	is_it_time_for_action (int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)

static void	writeBuildInfo (std::ostream &os)

static void	print_banner (MPI_Comm, std::ostream &)

static void	print_usage (MPI_Comm, std::ostream &)

static void	print_error (MPI_Comm, const std::string &msg)

static void	print_summary (std::ostream &)

static void	print_tpls (std::ostream &)

Public Attributes
amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_track_xy

amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_eye_track_xy

amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_eye_track_latlon

amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_maxvel_vs_time

amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_minpressure_vs_time

amrex::Vector< std::array< amrex::Real, 2 > >	hurricane_tracker_circle

amrex::Vector< amrex::MultiFab >	weather_forecast_data_1

amrex::Vector< amrex::MultiFab >	weather_forecast_data_2

amrex::Vector< amrex::Vector< amrex::MultiFab > >	forecast_state_1

amrex::Vector< amrex::Vector< amrex::MultiFab > >	forecast_state_2

amrex::Vector< amrex::Vector< amrex::MultiFab > >	forecast_state_interp

amrex::Vector< amrex::MultiFab >	surface_state_1

amrex::Vector< amrex::MultiFab >	surface_state_2

amrex::Vector< amrex::MultiFab >	surface_state_interp

std::string	pp_prefix {"erf"}

Private Member Functions
void	ReadParameters ()

void	ParameterSanityChecks ()

void	AverageDown ()

void	update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)

void	Construct_ERFFillPatchers (int lev)

void	Define_ERFFillPatchers (int lev)

void	init1DArrays ()

void	init_bcs ()

void	init_phys_bcs (bool &rho_read, bool &read_prim_theta)

void	init_custom (int lev)

void	init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)

void	turbPert_update (const int lev, const amrex::Real dt)

void	turbPert_amplitude (const int lev)

void	initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)

void	make_physbcs (int lev)

void	initializeMicrophysics (const int &)

void	FillPatchCrseLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)

void	FillPatchFineLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)

void	FillIntermediatePatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)

void	FillCoarsePatch (int lev, amrex::Real time)

void	timeStep (int lev, amrex::Real time, int iteration)

void	Advance (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)

void	initHSE ()
	Initialize HSE. More...

void	initHSE (int lev)

void	initRayleigh ()
	Initialize Rayleigh damping profiles. More...

void	initSponge ()
	Initialize sponge profiles. More...

void	setRayleighRefFromSounding (bool restarting)
	Set Rayleigh mean profiles from input sounding. More...

void	setSpongeRefFromSounding (bool restarting)
	Set sponge mean profiles from input sounding. More...

void	ComputeDt (int step=-1)

std::string	PlotFileName (int lev) const

void	setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)

void	setPlotVariables2D (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)

void	appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)

void	setSubVolVariables (const std::string &pp_subvol_var_names, amrex::Vector< std::string > &subvol_var_names)

void	init_Dirichlet_bc_data (const std::string input_file)

void	InitializeFromFile ()

void	InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)

void	post_update (amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)

void	fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)

void	init_geo_wind_profile (const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)

void	refinement_criteria_setup ()

AMREX_FORCE_INLINE amrex::YAFluxRegister *	getAdvFluxReg (int lev)

AMREX_FORCE_INLINE std::ostream &	DataLog (int i)

AMREX_FORCE_INLINE std::ostream &	DerDataLog (int i)

AMREX_FORCE_INLINE int	NumDataLogs () noexcept

AMREX_FORCE_INLINE int	NumDerDataLogs () noexcept

AMREX_FORCE_INLINE std::ostream &	SamplePointLog (int i)

AMREX_FORCE_INLINE int	NumSamplePointLogs () noexcept

AMREX_FORCE_INLINE std::ostream &	SampleLineLog (int i)

AMREX_FORCE_INLINE int	NumSampleLineLogs () noexcept

amrex::IntVect &	SamplePoint (int i)

AMREX_FORCE_INLINE int	NumSamplePoints () noexcept

amrex::IntVect &	SampleLine (int i)

AMREX_FORCE_INLINE int	NumSampleLines () noexcept

void	setRecordDataInfo (int i, const std::string &filename)

void	setRecordDerDataInfo (int i, const std::string &filename)

void	setRecordEnergyDataInfo (int i, const std::string &filename)

void	setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)

void	setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)

std::string	DataLogName (int i) const noexcept
	The filename of the ith datalog file. More...

std::string	DerDataLogName (int i) const noexcept

std::string	SamplePointLogName (int i) const noexcept
	The filename of the ith sampleptlog file. More...

std::string	SampleLineLogName (int i) const noexcept
	The filename of the ith samplelinelog file. More...

eb_ const &	get_eb (int lev) const noexcept

amrex::EBFArrayBoxFactory const &	EBFactory (int lev) const noexcept

Static Private Member Functions
static amrex::Vector< std::string >	PlotFileVarNames (amrex::Vector< std::string > plot_var_names)

static void	GotoNextLine (std::istream &is)

static AMREX_FORCE_INLINE int	ComputeGhostCells (const SolverChoice &sc)

static amrex::Real	getCPUTime ()

static int	nghost_eb_basic ()

static int	nghost_eb_volume ()

static int	nghost_eb_full ()

Private Attributes
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	lat_m

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	lon_m

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	sinPhi_m

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	cosPhi_m

InputSoundingData	input_sounding_data

InputSpongeData	input_sponge_data

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	xvel_bc_data

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	yvel_bc_data

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	zvel_bc_data

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	th_bc_data

std::unique_ptr< ProblemBase >	prob = nullptr

amrex::Vector< int >	num_boxes_at_level

amrex::Vector< int >	num_files_at_level

amrex::Vector< amrex::Vector< amrex::Box > >	boxes_at_level

amrex::Vector< int >	istep

amrex::Vector< int >	nsubsteps

amrex::Vector< amrex::Real >	t_new

amrex::Vector< amrex::Real >	t_old

amrex::Vector< amrex::Real >	dt

amrex::Vector< long >	dt_mri_ratio

amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_new

amrex::Vector< amrex::Vector< amrex::MultiFab > >	vars_old

amrex::Vector< amrex::Vector< amrex::MultiFab > >	gradp

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vel_t_avg

amrex::Vector< amrex::Real >	t_avg_cnt

amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > >	mri_integrator_mem

amrex::Vector< amrex::MultiFab >	pp_inc

amrex::Vector< amrex::MultiFab >	lagged_delta_rt

amrex::Vector< amrex::MultiFab >	avg_xmom

amrex::Vector< amrex::MultiFab >	avg_ymom

amrex::Vector< amrex::MultiFab >	avg_zmom

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > >	physbcs_cons

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > >	physbcs_u

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > >	physbcs_v

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > >	physbcs_w

amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > >	physbcs_base

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Theta_prim

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Qv_prim

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Qr_prim

amrex::Vector< amrex::MultiFab >	rU_old

amrex::Vector< amrex::MultiFab >	rU_new

amrex::Vector< amrex::MultiFab >	rV_old

amrex::Vector< amrex::MultiFab >	rV_new

amrex::Vector< amrex::MultiFab >	rW_old

amrex::Vector< amrex::MultiFab >	rW_new

amrex::Vector< amrex::MultiFab >	zmom_crse_rhs

std::unique_ptr< Microphysics >	micro

amrex::Vector< amrex::Vector< amrex::MultiFab * > >	qmoist

LandSurface	lsm

amrex::Vector< std::string >	lsm_data_name

amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_data

amrex::Vector< std::string >	lsm_flux_name

amrex::Vector< amrex::Vector< amrex::MultiFab * > >	lsm_flux

amrex::Vector< std::unique_ptr< IRadiation > >	rad

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	qheating_rates

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	rad_fluxes

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	sw_lw_fluxes

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	solar_zenith

bool	plot_rad = false

int	rad_datalog_int = -1

int	cf_width {0}

int	cf_set_width {0}

amrex::Vector< ERFFillPatcher >	FPr_c

amrex::Vector< ERFFillPatcher >	FPr_u

amrex::Vector< ERFFillPatcher >	FPr_v

amrex::Vector< ERFFillPatcher >	FPr_w

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	Tau

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	Tau_corr

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	eddyDiffs_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SmnSmn_lev

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	sst_lev

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	tsk_lev

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	lmask_lev

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	land_type_lev

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > >	soil_type_lev

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	urb_frac_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx1_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx2_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_hfx3_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_diss_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx1_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx2_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q1fx3_lev

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	SFS_q2fx3_lev

amrex::Vector< amrex::Vector< amrex::Real > >	zlevels_stag

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_cc

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_cc_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ax_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	ay_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	az_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_phys_nd_new

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	detJ_cc_new

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	z_t_rk

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	terrain_blanking

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	walldist

amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > >	mapfac

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	fine_mask

amrex::Vector< amrex::Vector< amrex::Real > >	stretched_dz_h

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	stretched_dz_d

amrex::Vector< amrex::MultiFab >	base_state

amrex::Vector< amrex::MultiFab >	base_state_new

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Hwave

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Lwave

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Hwave_onegrid

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	Lwave_onegrid

bool	finished_wave = false

amrex::Vector< amrex::YAFluxRegister * >	advflux_reg

amrex::Vector< amrex::BCRec >	domain_bcs_type

amrex::Gpu::DeviceVector< amrex::BCRec >	domain_bcs_type_d

amrex::Array< std::string, 2 *AMREX_SPACEDIM >	domain_bc_type

amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max >	m_bc_extdir_vals

amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max >	m_bc_neumann_vals

amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 >	phys_bc_type

amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	xflux_imask

amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	yflux_imask

amrex::Vector< std::unique_ptr< amrex::iMultiFab > >	zflux_imask

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_xforce

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_yforce

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	thin_zforce

amrex::Vector< int >	last_subvol_step

amrex::Vector< amrex::Real >	last_subvol_time

const int	datwidth = 14

const int	datprecision = 6

const int	timeprecision = 13

int	max_step = -1

bool	use_datetime = false

const std::string	datetime_format = "%Y-%m-%d %H:%M:%S"

std::string	restart_chkfile = ""

amrex::Vector< amrex::Real >	fixed_dt

amrex::Vector< amrex::Real >	fixed_fast_dt

int	regrid_int = -1

bool	regrid_level_0_on_restart = false

std::string	plot3d_file_1 {"plt_1_"}

std::string	plot3d_file_2 {"plt_2_"}

std::string	plot2d_file_1 {"plt2d_1_"}

std::string	plot2d_file_2 {"plt2d_2_"}

std::string	subvol_file {"subvol"}

bool	m_expand_plotvars_to_unif_rr = false

int	m_plot3d_int_1 = -1

int	m_plot3d_int_2 = -1

int	m_plot2d_int_1 = -1

int	m_plot2d_int_2 = -1

amrex::Vector< int >	m_subvol_int

amrex::Vector< amrex::Real >	m_subvol_per

amrex::Real	m_plot3d_per_1 = -1.0

amrex::Real	m_plot3d_per_2 = -1.0

amrex::Real	m_plot2d_per_1 = -1.0

amrex::Real	m_plot2d_per_2 = -1.0

bool	m_plot_face_vels = false

bool	plot_lsm = false

int	profile_int = -1

bool	destag_profiles = true

std::string	check_file {"chk"}

int	m_check_int = -1

amrex::Real	m_check_per = -1.0

amrex::Vector< std::string >	subvol3d_var_names

amrex::Vector< std::string >	plot3d_var_names_1

amrex::Vector< std::string >	plot3d_var_names_2

amrex::Vector< std::string >	plot2d_var_names_1

amrex::Vector< std::string >	plot2d_var_names_2

const amrex::Vector< std::string >	cons_names

const amrex::Vector< std::string >	derived_names

const amrex::Vector< std::string >	derived_names_2d

const amrex::Vector< std::string >	derived_subvol_names {"soundspeed", "temp", "theta", "KE", "scalar"}

TurbulentPerturbation	turbPert

int	file_name_digits = 5

bool	use_real_time_in_pltname = false

int	real_width {0}

bool	real_extrap_w {true}

bool	metgrid_debug_quiescent {false}

bool	metgrid_debug_isothermal {false}

bool	metgrid_debug_dry {false}

bool	metgrid_debug_psfc {false}

bool	metgrid_debug_msf {false}

bool	metgrid_interp_theta {false}

bool	metgrid_basic_linear {false}

bool	metgrid_use_below_sfc {true}

bool	metgrid_use_sfc {true}

bool	metgrid_retain_sfc {false}

amrex::Real	metgrid_proximity {500.0}

int	metgrid_order {2}

int	metgrid_force_sfc_k {6}

amrex::Vector< amrex::BoxArray >	ba1d

amrex::Vector< amrex::BoxArray >	ba2d

std::unique_ptr< amrex::MultiFab >	mf_C1H

std::unique_ptr< amrex::MultiFab >	mf_C2H

std::unique_ptr< amrex::MultiFab >	mf_MUB

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mf_PSFC

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	rhotheta_src

amrex::Vector< std::unique_ptr< amrex::MultiFab > >	rhoqt_src

amrex::Vector< amrex::Vector< amrex::Real > >	h_w_subsid

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_w_subsid

amrex::Vector< amrex::Vector< amrex::Real > >	h_u_geos

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_u_geos

amrex::Vector< amrex::Vector< amrex::Real > >	h_v_geos

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_v_geos

amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_rayleigh_ptrs

amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > >	h_sponge_ptrs

amrex::Vector< amrex::Vector< amrex::Real > >	h_sinesq_ptrs

amrex::Vector< amrex::Vector< amrex::Real > >	h_sinesq_stag_ptrs

amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_rayleigh_ptrs

amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > >	d_sponge_ptrs

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_sinesq_ptrs

amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > >	d_sinesq_stag_ptrs

amrex::Vector< amrex::Real >	h_havg_density

amrex::Vector< amrex::Real >	h_havg_temperature

amrex::Vector< amrex::Real >	h_havg_pressure

amrex::Vector< amrex::Real >	h_havg_qv

amrex::Vector< amrex::Real >	h_havg_qc

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_density

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_temperature

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_pressure

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qv

amrex::Gpu::DeviceVector< amrex::Real >	d_havg_qc

std::unique_ptr< WriteBndryPlanes >	m_w2d = nullptr

std::unique_ptr< ReadBndryPlanes >	m_r2d = nullptr

std::unique_ptr< SurfaceLayer >	m_SurfaceLayer = nullptr

amrex::Vector< std::unique_ptr< ForestDrag > >	m_forest_drag

amrex::Vector< amrex::Vector< amrex::BoxArray > >	subdomains

amrex::Vector< amrex::Real >	dz_min

int	line_sampling_interval = -1

int	plane_sampling_interval = -1

amrex::Real	line_sampling_per = -1.0

amrex::Real	plane_sampling_per = -1.0

std::unique_ptr< LineSampler >	line_sampler = nullptr

std::unique_ptr< PlaneSampler >	plane_sampler = nullptr

amrex::Vector< std::unique_ptr< std::fstream > >	datalog

amrex::Vector< std::unique_ptr< std::fstream > >	der_datalog

amrex::Vector< std::unique_ptr< std::fstream > >	tot_e_datalog

amrex::Vector< std::string >	datalogname

amrex::Vector< std::string >	der_datalogname

amrex::Vector< std::string >	tot_e_datalogname

amrex::Vector< std::unique_ptr< std::fstream > >	sampleptlog

amrex::Vector< std::string >	sampleptlogname

amrex::Vector< amrex::IntVect >	samplepoint

amrex::Vector< std::unique_ptr< std::fstream > >	samplelinelog

amrex::Vector< std::string >	samplelinelogname

amrex::Vector< amrex::IntVect >	sampleline

amrex::Vector< std::unique_ptr< eb_ > >	eb

Static Private Attributes
static int	last_plot3d_file_step_1 = -1

static int	last_plot3d_file_step_2 = -1

static int	last_plot2d_file_step_1 = -1

static int	last_plot2d_file_step_2 = -1

static int	last_check_file_step = -1

static amrex::Real	last_plot3d_file_time_1 = 0.0

static amrex::Real	last_plot3d_file_time_2 = 0.0

static amrex::Real	last_plot2d_file_time_1 = 0.0

static amrex::Real	last_plot2d_file_time_2 = 0.0

static amrex::Real	last_check_file_time = 0.0

static bool	plot_file_on_restart = true

static amrex::Real	start_time = 0.0

static amrex::Real	stop_time = std::numeric_limits<amrex::Real>::max()

static amrex::Real	cfl = 0.8

static amrex::Real	sub_cfl = 1.0

static amrex::Real	init_shrink = 1.0

static amrex::Real	change_max = 1.1

static amrex::Real	dt_max_initial = 2.0e100

static amrex::Real	dt_max = 1.0e9

static int	fixed_mri_dt_ratio = 0

static SolverChoice	solverChoice

static int	verbose = 0

static int	mg_verbose = 0

static bool	use_fft = false

static int	check_for_nans = 0

static int	sum_interval = -1

static int	pert_interval = -1

static amrex::Real	sum_per = -1.0

static PlotFileType	plotfile3d_type_1 = PlotFileType::None

static PlotFileType	plotfile3d_type_2 = PlotFileType::None

static PlotFileType	plotfile2d_type_1 = PlotFileType::None

static PlotFileType	plotfile2d_type_2 = PlotFileType::None

static StateInterpType	interpolation_type

static std::string	sponge_type

static amrex::Vector< amrex::Vector< std::string > >	nc_init_file = {{""}}

static amrex::Vector< amrex::Vector< int > >	have_read_nc_init_file = {{0}}

static std::string	nc_bdy_file

static std::string	nc_low_file

static int	output_1d_column = 0

static int	column_interval = -1

static amrex::Real	column_per = -1.0

static amrex::Real	column_loc_x = 0.0

static amrex::Real	column_loc_y = 0.0

static std::string	column_file_name = "column_data.nc"

static int	output_bndry_planes = 0

static int	bndry_output_planes_interval = -1

static amrex::Real	bndry_output_planes_per = -1.0

static amrex::Real	bndry_output_planes_start_time = 0.0

static int	input_bndry_planes = 0

static int	ng_dens_hse

static int	ng_pres_hse

static amrex::Vector< amrex::AMRErrorTag >	ref_tags

static amrex::Real	startCPUTime = 0.0

static amrex::Real	previousCPUTimeUsed = 0.0

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

◆ ERF() [1/3]

ERF::ERF ( )

 {
     int fix_random_seed = 0;
     ParmParse pp("erf"); pp.query("fix_random_seed", fix_random_seed);
     // Note that the value of 1024UL is not significant -- the point here is just to set the
     // same seed for all MPI processes for the purpose of regression testing
     if (fix_random_seed) {
         Print() << "Fixing the random seed" << std::endl;
         InitRandom(1024UL, ParallelDescriptor::NProcs(), 1024UL);
     }
  
     ERF_shared();
 }

Here is the call graph for this function:

◆ ~ERF()

ERF::~ERF ( )

overridedefault

◆ ERF() [2/3]

ERF::ERF ( ERF && )

deletenoexcept

◆ ERF() [3/3]

ERF::ERF ( const ERF & other )

delete

Member Function Documentation

◆ Advance()

void ERF::Advance	(	int	lev,
		amrex::Real	time,
		amrex::Real	dt_lev,
		int	iteration,
		int	ncycle
	)

private

Function that advances the solution at one level for a single time step – this does some preliminaries then calls erf_advance

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	dt_lev	time step for this time advance

 {
     BL_PROFILE("ERF::Advance()");
  
     // We must swap the pointers so the previous step's "new" is now this step's "old"
     std::swap(vars_old[lev], vars_new[lev]);
  
     MultiFab& S_old = vars_old[lev][Vars::cons];
     MultiFab& S_new = vars_new[lev][Vars::cons];
  
     MultiFab& U_old = vars_old[lev][Vars::xvel];
     MultiFab& V_old = vars_old[lev][Vars::yvel];
     MultiFab& W_old = vars_old[lev][Vars::zvel];
  
     MultiFab& U_new = vars_new[lev][Vars::xvel];
     MultiFab& V_new = vars_new[lev][Vars::yvel];
     MultiFab& W_new = vars_new[lev][Vars::zvel];
  
     // We need to set these because otherwise in the first call to erf_advance we may
     //    read uninitialized data on ghost values in setting the bc's on the velocities
     U_new.setVal(1.e34,U_new.nGrowVect());
     V_new.setVal(1.e34,V_new.nGrowVect());
     W_new.setVal(1.e34,W_new.nGrowVect());
  
     //
     // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
     // If lev == 0 we have already FillPatched this in ERF::TimeStep
     //
     if (lev > 0) {
         // Set ghost cells to bogus values so they aren't uninitialized
         W_old.setBndry(1.234e20);
         FillPatchFineLevel(lev, time, {&S_old, &U_old, &V_old, &W_old},
                            {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]},
                            base_state[lev], base_state[lev]);
     }
  
     //
     // So we must convert the fillpatched to momenta, including the ghost values
     //
     const MultiFab* c_vfrac = nullptr;
     if (solverChoice.terrain_type == TerrainType::EB) {
         c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
     }
  
     VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
                        V_old, rV_old[lev].nGrowVect(),
                        W_old, rW_old[lev].nGrowVect(),
                        S_old, rU_old[lev], rV_old[lev], rW_old[lev],
                        Geom(lev).Domain(),
                        domain_bcs_type, c_vfrac);
  
     // Update the inflow perturbation update time and amplitude
     if (solverChoice.pert_type == PerturbationType::Source ||
         solverChoice.pert_type == PerturbationType::Direct ||
         solverChoice.pert_type == PerturbationType::CPM)
     {
         turbPert.calc_tpi_update(lev, dt_lev, U_old, V_old, S_old);
     }
  
     // If PerturbationType::Direct or CPM is selected, directly add the computed perturbation
     // on the conserved field
     if (solverChoice.pert_type == PerturbationType::Direct ||
         solverChoice.pert_type == PerturbationType::CPM)
     {
         auto m_ixtype = S_old.boxArray().ixType(); // Conserved term
         for (MFIter mfi(S_old,TileNoZ()); mfi.isValid(); ++mfi) {
             Box bx  = mfi.tilebox();
             const Array4<Real> &cell_data  = S_old.array(mfi);
             const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
             turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cell_data, pert_cell);
         }
     }
  
     // configure SurfaceLayer params if needed
     if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
         if (m_SurfaceLayer) {
             IntVect ng = Theta_prim[lev]->nGrowVect();
             MultiFab::Copy(  *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
             MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp     , 0, 1, ng);
             if (solverChoice.moisture_type != MoistureType::None) {
                 ng = Qv_prim[lev]->nGrowVect();
  
                 MultiFab::Copy(  *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
                 MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp  , 0, 1, ng);
  
                 if (solverChoice.moisture_indices.qr > -1) {
                     MultiFab::Copy(  *Qr_prim[lev], S_old, solverChoice.moisture_indices.qr, 0, 1, ng);
                     MultiFab::Divide(*Qr_prim[lev], S_old, Rho_comp  , 0, 1, ng);
                 } else {
                     Qr_prim[lev]->setVal(0.0);
                 }
             }
             // NOTE: std::swap above causes the field ptrs to be out of date.
             //       Reassign the field ptrs for MAC avg computation.
             m_SurfaceLayer->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim, Qr_prim);
             m_SurfaceLayer->update_pblh(lev, vars_old, z_phys_cc[lev].get(),
                                         solverChoice.moisture_indices);
  
 #ifdef ERF_USE_NETCDF
             Real elapsed_time_since_start_low = time + (start_time - start_low_time);
 #else
             Real elapsed_time_since_start_low = time;
 #endif
             m_SurfaceLayer->update_fluxes(lev, elapsed_time_since_start_low,
                                           S_old, z_phys_nd[lev], walldist[lev]);
         }
     }
  
 #if defined(ERF_USE_WINDFARM)
     // **************************************************************************************
     // Update the windfarm sources
     // **************************************************************************************
     if (solverChoice.windfarm_type != WindFarmType::None) {
         advance_windfarm(Geom(lev), dt_lev, S_old,
                          U_old, V_old, W_old, vars_windfarm[lev],
                          Nturb[lev], SMark[lev], time);
     }
  
 #endif
  
     // **************************************************************************************
     // Update the radiation sources with the "old" state
     // **************************************************************************************
     advance_radiation(lev, S_old, dt_lev);
  
 #ifdef ERF_USE_SHOC
     // **************************************************************************************
     // Update the "old" state using SHOC
     // **************************************************************************************
     if (solverChoice.use_shoc) {
         // Get SFC fluxes from SurfaceLayer
         if (m_SurfaceLayer) {
             Vector<const MultiFab*> mfs = {&S_old, &U_old, &V_old, &W_old};
             m_SurfaceLayer->impose_SurfaceLayer_bcs(lev, mfs, Tau[lev],
                                                     SFS_hfx1_lev[lev].get() , SFS_hfx2_lev[lev].get() , SFS_hfx3_lev[lev].get(),
                                                     SFS_q1fx1_lev[lev].get(), SFS_q1fx2_lev[lev].get(), SFS_q1fx3_lev[lev].get(),
                                                     z_phys_nd[lev].get());
         }
  
         // Get Shoc tendencies and update the state
         Real* w_sub = (solverChoice.custom_w_subsidence) ? d_w_subsid[lev].data() : nullptr;
         compute_shoc_tendencies(lev, &S_old, &U_old, &V_old, &W_old, w_sub,
                                 Tau[lev][TauType::tau13].get(), Tau[lev][TauType::tau23].get(),
                                 SFS_hfx3_lev[lev].get()       , SFS_q1fx3_lev[lev].get()      ,
                                 eddyDiffs_lev[lev].get()      , z_phys_nd[lev].get()          ,
                                 dt_lev);
     }
 #endif
  
     const BoxArray&            ba = S_old.boxArray();
     const DistributionMapping& dm = S_old.DistributionMap();
  
     int nvars = S_old.nComp();
  
     // Source array for conserved cell-centered quantities -- this will be filled
     //     in the call to make_sources in ERF_TI_slow_rhs_pre.H
     MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0.0);
  
     // Source arrays for momenta -- these will be filled
     //     in the call to make_mom_sources in ERF_TI_slow_rhs_pre.H
     BoxArray ba_x(ba); ba_x.surroundingNodes(0);
     MultiFab xmom_source(ba_x,dm,1,1); xmom_source.setVal(0.0);
  
     BoxArray ba_y(ba); ba_y.surroundingNodes(1);
     MultiFab ymom_source(ba_y,dm,1,1); ymom_source.setVal(0.0);
  
     BoxArray ba_z(ba); ba_z.surroundingNodes(2);
     MultiFab zmom_source(ba_z,dm,1,1); zmom_source.setVal(0.0);
     MultiFab    buoyancy(ba_z,dm,1,1); buoyancy.setVal(0.0);
  
     amrex::Vector<MultiFab> state_old;
     amrex::Vector<MultiFab> state_new;
  
     // **************************************************************************************
     // Here we define state_old and state_new which are to be advanced
     // **************************************************************************************
     // Initial solution
     // Note that "old" and "new" here are relative to each RK stage.
     state_old.push_back(MultiFab(S_old      , amrex::make_alias, 0, nvars)); // cons
     state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0,     1)); // xmom
     state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0,     1)); // ymom
     state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0,     1)); // zmom
  
     // Final solution
     // state_new at the end of the last RK stage holds the t^{n+1} data
     state_new.push_back(MultiFab(S_new      , amrex::make_alias, 0, nvars)); // cons
     state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0,     1)); // xmom
     state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0,     1)); // ymom
     state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0,     1)); // zmom
  
     // **************************************************************************************
     // Tests on the reasonableness of the solution before the dycore
     // **************************************************************************************
     // Test for NaNs after dycore
     if (check_for_nans > 1) {
         if (verbose > 1) {
             amrex::Print() << "Testing old state and vels for NaNs before dycore" << std::endl;
         }
         check_state_for_nans(S_old);
         check_vels_for_nans(rU_old[lev],rV_old[lev],rW_old[lev]);
     }
  
     // We only test on low temp if we have a moisture model because we are protecting against
     //    the test on low temp inside the moisture models
     if (solverChoice.moisture_type != MoistureType::None) {
         if (verbose > 1) {
             amrex::Print() << "Testing on low temperature before dycore" << std::endl;
         }
         check_for_low_temp(S_old);
     } else {
         if (verbose > 1) {
             amrex::Print() << "Testing on negative temperature before dycore" << std::endl;
         }
         check_for_negative_theta(S_old);
     }
  
     // **************************************************************************************
     // Update the dycore
     // **************************************************************************************
     advance_dycore(lev, state_old, state_new,
                    U_old, V_old, W_old,
                    U_new, V_new, W_new,
                    cc_source, xmom_source, ymom_source, zmom_source, buoyancy,
                    Geom(lev), dt_lev, time);
  
     // **************************************************************************************
     // Tests on the reasonableness of the solution after the dycore
     // **************************************************************************************
     // Test for NaNs after dycore
     if (check_for_nans > 0) {
         if (verbose > 1) {
             amrex::Print() << "Testing new state and vels for NaNs after dycore" << std::endl;
         }
         check_state_for_nans(S_new);
         check_vels_for_nans(rU_new[lev],rV_new[lev],rW_new[lev]);
     }
  
     // We only test on low temp if we have a moisture model because we are protecting against
     //    the test on low temp inside the moisture models
     if (solverChoice.moisture_type != MoistureType::None) {
         if (verbose > 1) {
             amrex::Print() << "Testing on low temperature after dycore" << std::endl;
         }
         check_for_low_temp(S_new);
     } else {
         // Otherwise we will test on negative (rhotheta) coming out of the dycore
         if (verbose > 1) {
             amrex::Print() << "Testing on negative temperature after dycore" << std::endl;
         }
         check_for_negative_theta(S_new);
     }
  
     // **************************************************************************************
     // Update the microphysics (moisture)
     // **************************************************************************************
     if (!solverChoice.moisture_tight_coupling)
     {
         advance_microphysics(lev, S_new, dt_lev, iteration, time);
  
         // Test for NaNs after microphysics
         if (check_for_nans > 0) {
             amrex::Print() << "Testing new state for NaNs after advance_microphysics" << std::endl;
             check_state_for_nans(S_new);
         }
     }
  
     // **************************************************************************************
     // Update the land surface model
     // **************************************************************************************
     advance_lsm(lev, S_new, U_new, V_new, dt_lev);
  
 #ifdef ERF_USE_PARTICLES
     // **************************************************************************************
     // Update the particle positions
     // **************************************************************************************
    evolveTracers( lev, dt_lev, vars_new, z_phys_nd );
 #endif
  
     // ***********************************************************************************************
     // Impose domain boundary conditions here so that in FillPatching the fine data we won't
     // need to re-fill these
     // ***********************************************************************************************
     if (lev < finest_level) {
          IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
          (*physbcs_cons[lev])(vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                               0,vars_new[lev][Vars::cons].nComp(),
                               vars_new[lev][Vars::cons].nGrowVect(),time,BCVars::cons_bc,true);
             (*physbcs_u[lev])(vars_new[lev][Vars::xvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                               ngvect_vels,time,BCVars::xvel_bc,true);
             (*physbcs_v[lev])(vars_new[lev][Vars::yvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                               ngvect_vels,time,BCVars::yvel_bc,true);
             (*physbcs_w[lev])(vars_new[lev][Vars::zvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
                               ngvect_vels,time,BCVars::zvel_bc,true);
     }
  
     // **************************************************************************************
     // Register old and new coarse data if we are at a level less than the finest level
     // **************************************************************************************
     if (lev < finest_level) {
         if (cf_width > 0) {
             // We must fill the ghost cells of these so that the parallel copy works correctly
             state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
             state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
             FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
                                           {time, time+dt_lev});
         }
  
         if (cf_width >= 0) {
             // We must fill the ghost cells of these so that the parallel copy works correctly
             state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
             state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
             FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
                                           {time, time+dt_lev});
  
             state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
             state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
             FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
                                           {time, time+dt_lev});
  
             state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
             state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
             FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
                                           {time, time+dt_lev});
         }
  
             //
             // Now create a MultiFab that holds (S_new - S_old) / dt from the coarse level interpolated
             //     on to the coarse/fine boundary at the fine resolution
             //
             Interpolater* mapper_f = &face_cons_linear_interp;
  
             // PhysBCFunctNoOp null_bc;
             // MultiFab tempx(vars_new[lev+1][Vars::xvel].boxArray(),vars_new[lev+1][Vars::xvel].DistributionMap(),1,0);
             // tempx.setVal(0.0);
             // xmom_crse_rhs[lev+1].setVal(0.0);
             // FPr_u[lev].FillSet(tempx               , time       , null_bc, domain_bcs_type);
             // FPr_u[lev].FillSet(xmom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
             // MultiFab::Subtract(xmom_crse_rhs[lev+1],tempx,0,0,1,IntVect{0});
             // xmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
  
             // MultiFab tempy(vars_new[lev+1][Vars::yvel].boxArray(),vars_new[lev+1][Vars::yvel].DistributionMap(),1,0);
             // tempy.setVal(0.0);
             // ymom_crse_rhs[lev+1].setVal(0.0);
             // FPr_v[lev].FillSet(tempy               , time       , null_bc, domain_bcs_type);
             // FPr_v[lev].FillSet(ymom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
             // MultiFab::Subtract(ymom_crse_rhs[lev+1],tempy,0,0,1,IntVect{0});
             // ymom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
  
             MultiFab temp_state(zmom_crse_rhs[lev+1].boxArray(),zmom_crse_rhs[lev+1].DistributionMap(),1,0);
             InterpFromCoarseLevel(temp_state,            IntVect{0}, IntVect{0}, state_old[IntVars::zmom], 0, 0, 1,
                                   geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
             InterpFromCoarseLevel(zmom_crse_rhs[lev+1],  IntVect{0}, IntVect{0}, state_new[IntVars::zmom], 0, 0, 1,
                                   geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
             MultiFab::Subtract(zmom_crse_rhs[lev+1],temp_state,0,0,1,IntVect{0});
             zmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
     }
  
     // ***********************************************************************************************
     // Update the time averaged velocities if they are requested
     // ***********************************************************************************************
     if (solverChoice.time_avg_vel) {
         Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
     }
 }

Here is the call graph for this function:

◆ advance_dycore()

void ERF::advance_dycore	(	int	level,
		amrex::Vector< amrex::MultiFab > &	state_old,
		amrex::Vector< amrex::MultiFab > &	state_new,
		amrex::MultiFab &	xvel_old,
		amrex::MultiFab &	yvel_old,
		amrex::MultiFab &	zvel_old,
		amrex::MultiFab &	xvel_new,
		amrex::MultiFab &	yvel_new,
		amrex::MultiFab &	zvel_new,
		amrex::MultiFab &	source,
		amrex::MultiFab &	xmom_src,
		amrex::MultiFab &	ymom_src,
		amrex::MultiFab &	zmom_src,
		amrex::MultiFab &	buoyancy,
		amrex::Geometry	fine_geom,
		amrex::Real	dt,
		amrex::Real	time
	)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters

[in]	level	level of refinement (coarsest level is 0)
[in]	state_old	old-time conserved variables
[in]	state_new	new-time conserved variables
[in]	xvel_old	old-time x-component of velocity
[in]	yvel_old	old-time y-component of velocity
[in]	zvel_old	old-time z-component of velocity
[in]	xvel_new	new-time x-component of velocity
[in]	yvel_new	new-time y-component of velocity
[in]	zvel_new	new-time z-component of velocity
[in]	cc_src	source term for conserved variables
[in]	xmom_src	source term for x-momenta
[in]	ymom_src	source term for y-momenta
[in]	zmom_src	source term for z-momenta
[in]	fine_geom	container for geometry information at current level
[in]	dt_advance	time step for this time advance
[in]	old_time	old time for this time advance

 {
     BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
  
     const Box& domain = fine_geom.Domain();
  
     DiffChoice dc    = solverChoice.diffChoice;
     TurbChoice tc    = solverChoice.turbChoice[level];
     SpongeChoice sc  = solverChoice.spongeChoice;
  
     MultiFab r_hse (base_state[level], make_alias, BaseState::r0_comp , 1);
     MultiFab p_hse (base_state[level], make_alias, BaseState::p0_comp , 1);
     MultiFab pi_hse(base_state[level], make_alias, BaseState::pi0_comp, 1);
  
     // These pointers are used in the MRI utility functions
     MultiFab* r0  = &r_hse;
     MultiFab* p0  = &p_hse;
     MultiFab* pi0 = &pi_hse;
  
     MultiFab* rhotheta_src_ptr = solverChoice.custom_rhotheta_forcing ? rhotheta_src[level].get() : nullptr;
     MultiFab* rhoqt_src_ptr    = solverChoice.custom_moisture_forcing ? rhoqt_src[level].get()   : nullptr;
     Real* dptr_wbar_sub        = solverChoice.custom_w_subsidence     ? d_w_subsid[level].data()     : nullptr;
  
     // Turbulent Perturbation Pointer
     //Real* dptr_rhotheta_src = solverChoice.pert_type ? d_rhotheta_src[level].data() : nullptr;
  
     Vector<Real*> d_rayleigh_ptrs_at_lev;
     d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
     d_rayleigh_ptrs_at_lev[Rayleigh::ubar]     = solverChoice.dampingChoice.rayleigh_damp_U   ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
     d_rayleigh_ptrs_at_lev[Rayleigh::vbar]     = solverChoice.dampingChoice.rayleigh_damp_V   ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
     d_rayleigh_ptrs_at_lev[Rayleigh::wbar]     = solverChoice.dampingChoice.rayleigh_damp_W   ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
     d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.dampingChoice.rayleigh_damp_T   ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
  
     bool use_rayleigh =
        (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
         solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T);
     Real* d_sinesq_at_lev      = (use_rayleigh)  ? d_sinesq_ptrs[level].data() : nullptr;
     Real* d_sinesq_stag_at_lev = (use_rayleigh)  ? d_sinesq_stag_ptrs[level].data() : nullptr;
  
     Vector<Real*> d_sponge_ptrs_at_lev;
     if(sc.sponge_type=="input_sponge")
     {
         d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
         d_sponge_ptrs_at_lev[Sponge::ubar_sponge]  =  d_sponge_ptrs[level][Sponge::ubar_sponge].data();
         d_sponge_ptrs_at_lev[Sponge::vbar_sponge]  =  d_sponge_ptrs[level][Sponge::vbar_sponge].data();
     }
  
     bool l_use_terrain_fitted_coords = (solverChoice.mesh_type != MeshType::ConstantDz);
     bool l_use_kturb   = tc.use_kturb;
     bool l_use_diff    = ( (dc.molec_diff_type != MolecDiffType::None) ||
                            l_use_kturb );
  
     const bool use_SurfLayer = (m_SurfaceLayer != nullptr);
     const MultiFab* z_0     = (use_SurfLayer) ? m_SurfaceLayer->get_z0(level) : nullptr;
  
     const BoxArray& ba            = state_old[IntVars::cons].boxArray();
     const BoxArray& ba_z          = zvel_old.boxArray();
     const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
  
     int num_prim = state_old[IntVars::cons].nComp() - 1;
  
     MultiFab    S_prim  (ba  , dm, num_prim,          state_old[IntVars::cons].nGrowVect());
     MultiFab  pi_stage  (ba  , dm,        1,          1);
     MultiFab fast_coeffs(ba_z, dm,        5,          0);
  
     MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
     MultiFab* SmnSmn    = SmnSmn_lev[level].get();
  
     // **************************************************************************************
     // Compute strain for use in slow RHS and Smagorinsky model
     // **************************************************************************************
     {
     BL_PROFILE("erf_advance_strain");
     if (l_use_diff) {
  
         const BCRec* bc_ptr_h = domain_bcs_type.data();
         const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
  
 #ifdef _OPENMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
         for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
         {
             Box bxcc  = mfi.growntilebox(IntVect(1,1,0));
             Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
             Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
             Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
  
             if (bxcc.smallEnd(2) != domain.smallEnd(2)) {
                  bxcc.growLo(2,1);
                 tbxxy.growLo(2,1);
                 tbxxz.growLo(2,1);
                 tbxyz.growLo(2,1);
             }
  
             if (bxcc.bigEnd(2) != domain.bigEnd(2)) {
                  bxcc.growHi(2,1);
                 tbxxy.growHi(2,1);
                 tbxxz.growHi(2,1);
                 tbxyz.growHi(2,1);
             }
  
             const Array4<const Real> & u = xvel_old.array(mfi);
             const Array4<const Real> & v = yvel_old.array(mfi);
             const Array4<const Real> & w = zvel_old.array(mfi);
  
             Array4<Real> tau11 = Tau[level][TauType::tau11].get()->array(mfi);
             Array4<Real> tau22 = Tau[level][TauType::tau22].get()->array(mfi);
             Array4<Real> tau33 = Tau[level][TauType::tau33].get()->array(mfi);
             Array4<Real> tau12 = Tau[level][TauType::tau12].get()->array(mfi);
             Array4<Real> tau13 = Tau[level][TauType::tau13].get()->array(mfi);
             Array4<Real> tau23 = Tau[level][TauType::tau23].get()->array(mfi);
  
             Array4<Real> tau21 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau21].get()->array(mfi) : Array4<Real>{};
             Array4<Real> tau31 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau31].get()->array(mfi) : Array4<Real>{};
             Array4<Real> tau32 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau32].get()->array(mfi) : Array4<Real>{};
             const Array4<const Real>& z_nd = z_phys_nd[level]->const_array(mfi);
  
             const Array4<const Real> mf_mx = mapfac[level][MapFacType::m_x]->const_array(mfi);
             const Array4<const Real> mf_ux = mapfac[level][MapFacType::u_x]->const_array(mfi);
             const Array4<const Real> mf_vx = mapfac[level][MapFacType::v_x]->const_array(mfi);
             const Array4<const Real> mf_my = mapfac[level][MapFacType::m_y]->const_array(mfi);
             const Array4<const Real> mf_uy = mapfac[level][MapFacType::u_y]->const_array(mfi);
             const Array4<const Real> mf_vy = mapfac[level][MapFacType::v_y]->const_array(mfi);
  
             // We update Tau_corr[level] in erf_make_tau_terms, not here
             Array4<Real> no_tau_corr_update_here{};
  
             if (solverChoice.mesh_type == MeshType::StretchedDz) {
                 ComputeStrain_S(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                 u, v, w,
                                 tau11, tau22, tau33,
                                 tau12, tau21,
                                 tau13, tau31,
                                 tau23, tau32,
                                 stretched_dz_d[level], dxInv,
                                 mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
                                 no_tau_corr_update_here, no_tau_corr_update_here);
             } else if (l_use_terrain_fitted_coords) {
                 ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                 u, v, w,
                                 tau11, tau22, tau33,
                                 tau12, tau21,
                                 tau13, tau31,
                                 tau23, tau32,
                                 z_nd, detJ_cc[level]->const_array(mfi), dxInv,
                                 mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
                                 no_tau_corr_update_here, no_tau_corr_update_here);
             } else {
                 ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
                                 u, v, w,
                                 tau11, tau22, tau33,
                                 tau12, tau13, tau23,
                                 dxInv,
                                 mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
                                 no_tau_corr_update_here, no_tau_corr_update_here);
             }
         } // mfi
     } // l_use_diff
     } // profile
  
 #include "ERF_TI_utils.H"
  
     // Additional SFS quantities, calculated once per timestep
     MultiFab* Hfx1  = SFS_hfx1_lev[level].get();
     MultiFab* Hfx2  = SFS_hfx2_lev[level].get();
     MultiFab* Hfx3  = SFS_hfx3_lev[level].get();
     MultiFab* Q1fx1 = SFS_q1fx1_lev[level].get();
     MultiFab* Q1fx2 = SFS_q1fx2_lev[level].get();
     MultiFab* Q1fx3 = SFS_q1fx3_lev[level].get();
     MultiFab* Q2fx3 = SFS_q2fx3_lev[level].get();
     MultiFab* Diss  = SFS_diss_lev[level].get();
  
     // *************************************************************************
     // Calculate cell-centered eddy viscosity & diffusivities
     //
     // Notes -- we fill all the data in ghost cells before calling this so
     //    that we can fill the eddy viscosity in the ghost regions and
     //    not have to call a boundary filler on this data itself
     //
     // LES - updates both horizontal and vertical eddy viscosity components
     // PBL - only updates vertical eddy viscosity components so horizontal
     //       components come from the LES model or are left as zero.
     // *************************************************************************
     if (l_use_kturb)
     {
         // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
         bool l_use_moisture = ( solverChoice.moisture_type != MoistureType::None );
         const BCRec* bc_ptr_h = domain_bcs_type.data();
         ComputeTurbulentViscosity(dt_advance, xvel_old, yvel_old,Tau[level],
                                   state_old[IntVars::cons],
                                   *walldist[level].get(),
                                   *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
                                   fine_geom, mapfac[level],
                                   z_phys_nd[level], solverChoice,
                                   m_SurfaceLayer, z_0, l_use_terrain_fitted_coords,
                                   l_use_moisture, level,
                                   bc_ptr_h);
     }
  
     // ***********************************************************************************************
     // Update user-defined source terms -- these are defined once per time step (not per RK stage)
     // ***********************************************************************************************
     if (solverChoice.custom_rhotheta_forcing) {
         prob->update_rhotheta_sources(old_time,
                                       rhotheta_src_ptr,
                                       fine_geom, z_phys_cc[level]);
     }
  
     if (solverChoice.custom_moisture_forcing) {
         prob->update_rhoqt_sources(old_time,
                                    rhoqt_src_ptr,
                                    fine_geom, z_phys_cc[level]);
     }
  
     if (solverChoice.custom_geostrophic_profile) {
         prob->update_geostrophic_profile(old_time,
                                    h_u_geos[level], d_u_geos[level],
                                    h_v_geos[level], d_v_geos[level],
                                    fine_geom, z_phys_cc[level]);
     }
  
     if (solverChoice.custom_w_subsidence) {
         prob->update_w_subsidence(old_time,
                                   h_w_subsid[level], d_w_subsid[level],base_state[level],
                                   fine_geom, z_phys_nd[level]);
     }
  
     // ***********************************************************************************************
     // Convert old velocity available on faces to old momentum on faces to be used in time integration
     // ***********************************************************************************************
     MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
  
     //
     // This is an optimization since we won't need more than one ghost
     // cell of momentum in the integrator if not using numerical diffusion
     //
     IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
     IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
     IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
  
     const MultiFab* c_vfrac = nullptr;
     if (solverChoice.terrain_type == TerrainType::EB) {
         c_vfrac = &((get_eb(level).get_const_factory())->getVolFrac());
     }
  
     VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
                        state_old[IntVars::xmom],
                        state_old[IntVars::ymom],
                        state_old[IntVars::zmom],
                        domain, domain_bcs_type, c_vfrac);
  
     MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
     MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
     MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
  
     bool fast_only = false;
     bool vel_and_mom_synced = true;
  
     apply_bcs(state_old, old_time,
               state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
               fast_only, vel_and_mom_synced);
     cons_to_prim(state_old[IntVars::cons], state_old[IntVars::cons].nGrow());
  
     // ***********************************************************************************************
     // Define a new MultiFab that holds q_total and fill it by summing the moisture components --
     //      to be used in buoyancy calculation and as part of the inertial weighting in the
     // ***********************************************************************************************
  
     const bool l_eb_terrain = (solverChoice.terrain_type == TerrainType::EB);
     MultiFab qt(grids[level], dmap[level], 1, (l_eb_terrain) ? 2 : 1);
     qt.setVal(0.0);
  
 #include "ERF_TI_no_substep_fun.H"
 #include "ERF_TI_substep_fun.H"
 #include "ERF_TI_slow_rhs_pre.H"
 #include "ERF_TI_slow_rhs_post.H"
  
     // ***************************************************************************************
     // Setup the integrator and integrate for a single timestep
     // **************************************************************************************
     MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
  
     // Define rhs and 'post update' utility function that is called after calculating
     // any state data (e.g. at RK stages or at the end of a timestep)
     mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
     mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
  
     mri_integrator.set_acoustic_substepping(acoustic_substepping_fun);
     mri_integrator.set_slow_fast_timestep_ratio(fixed_mri_dt_ratio > 0 ? fixed_mri_dt_ratio : dt_mri_ratio[level]);
     mri_integrator.set_no_substep(no_substep_fun);
  
     mri_integrator.advance(state_old, state_new, old_time, dt_advance);
  
     if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
 }

Here is the call graph for this function:

◆ advance_lsm()

void ERF::advance_lsm	(	int	lev,
		amrex::MultiFab &	cons_in,
		amrex::MultiFab &	xvel_in,
		amrex::MultiFab &	yvel_in,
		const amrex::Real &	dt_advance
	)

 {
     if (solverChoice.lsm_type != LandSurfaceType::None) {
         if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
             lsm.Advance(lev, cons_in, xvel_in, yvel_in, SFS_hfx3_lev[lev].get(), SFS_q1fx3_lev[lev].get(), dt_advance, istep[0]);
         } else {
             lsm.Advance(lev, dt_advance);
         }
     }
 }

Here is the call graph for this function:

◆ advance_microphysics()

void ERF::advance_microphysics	(	int	lev,
		amrex::MultiFab &	cons_in,
		const amrex::Real &	dt_advance,
		const int &	iteration,
		const amrex::Real &	time
	)

 {
     if (solverChoice.moisture_type != MoistureType::None) {
         micro->Set_RealWidth(lev, real_width);
         micro->Update_Micro_Vars_Lev(lev, cons);
         micro->Advance(lev, dt_advance, iteration, time, solverChoice, vars_new, z_phys_nd, phys_bc_type);
         micro->Update_State_Vars_Lev(lev, cons);
     }
 }

◆ advance_radiation()

void ERF::advance_radiation	(	int	lev,
		amrex::MultiFab &	cons_in,
		const amrex::Real &	dt_advance
	)

 {
     if (solverChoice.rad_type != RadiationType::None) {
 #ifdef ERF_USE_NETCDF
         MultiFab *lat_ptr = lat_m[lev].get();
         MultiFab *lon_ptr = lon_m[lev].get();
 #else
         MultiFab *lat_ptr = nullptr;
         MultiFab *lon_ptr = nullptr;
 #endif
         // T surf from SurfaceLayer if we have it
         MultiFab* t_surf = (m_SurfaceLayer) ? m_SurfaceLayer->get_t_surf(lev) : nullptr;
  
         // RRTMGP inputs names and pointers
         Vector<std::string> lsm_input_names = rad[lev]->get_lsm_input_varnames();
         Vector<MultiFab*> lsm_input_ptrs(lsm_input_names.size(),nullptr);
         for (int i(0); i<lsm_input_ptrs.size(); ++i) {
             int varIdx = lsm.Get_DataIdx(lev,lsm_input_names[i]);
             lsm_input_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
         }
  
         // RRTMGP output names and pointers
         Vector<std::string> lsm_output_names = rad[lev]->get_lsm_output_varnames();
         Vector<MultiFab*> lsm_output_ptrs(lsm_output_names.size(),nullptr);
         for (int i(0); i<lsm_output_ptrs.size(); ++i) {
             int varIdx = lsm.Get_DataIdx(lev,lsm_output_names[i]);
             lsm_output_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
         }
  
         // Enter radiation class driver
         amrex::Real time_for_rad = t_new[lev] + start_time;
         rad[lev]->Run(lev, istep[lev], time_for_rad, dt_advance,
                       cons.boxArray(), geom[lev], &(cons),
                       lmask_lev[lev][0].get(), t_surf,
                       sw_lw_fluxes[lev].get(), solar_zenith[lev].get(),
                       lsm_input_ptrs, lsm_output_ptrs,
                       qheating_rates[lev].get(), rad_fluxes[lev].get(),
                       z_phys_nd[lev].get()     , lat_ptr, lon_ptr);
     }
 }

◆ appendPlotVariables()

void ERF::appendPlotVariables	(	const std::string &	pp_plot_var_names,
		amrex::Vector< std::string > &	plot_var_names
	)

private

 {
     ParmParse pp(pp_prefix);
  
     Vector<std::string> plot_var_names(0);
     if (pp.contains(pp_plot_var_names.c_str())) {
         std::string nm;
         int nPltVars = pp.countval(pp_plot_var_names.c_str());
         for (int i = 0; i < nPltVars; i++) {
             pp.get(pp_plot_var_names.c_str(), nm, i);
             // Add the named variable to our list of plot variables
             // if it is not already in the list
             if (!containerHasElement(plot_var_names, nm)) {
                 plot_var_names.push_back(nm);
             }
         }
     }
  
     Vector<std::string> tmp_plot_names(0);
 #ifdef ERF_USE_PARTICLES
     Vector<std::string> particle_mesh_plot_names;
     particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
     if (particle_mesh_plot_names.size() > 0) {
         static bool first_call = true;
         if (first_call) {
             Print() << "ParticleData: the following additional Eulerian variables are available to plot:\n";
             for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
                 Print() << "    " << particle_mesh_plot_names[i] << "\n";
             }
             first_call = false;
         }
         for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
             std::string tmp(particle_mesh_plot_names[i]);
             if (containerHasElement(plot_var_names, tmp) ) {
                 tmp_plot_names.push_back(tmp);
             }
         }
     }
 #endif
  
     {
         Vector<std::string> microphysics_plot_names;
         micro->GetPlotVarNames(microphysics_plot_names);
         if (microphysics_plot_names.size() > 0) {
             static bool first_call = true;
             if (first_call) {
                 Print() << getEnumNameString(solverChoice.moisture_type)
                         << ": the following additional variables are available to plot:\n";
                 for (int i = 0; i < microphysics_plot_names.size(); i++) {
                     Print() << "    " << microphysics_plot_names[i] << "\n";
                 }
                 first_call = false;
             }
             for (auto& plot_name : microphysics_plot_names) {
                 if (containerHasElement(plot_var_names, plot_name)) {
                     tmp_plot_names.push_back(plot_name);
                 }
             }
         }
     }
  
     for (int i = 0; i < tmp_plot_names.size(); i++) {
         a_plot_var_names.push_back( tmp_plot_names[i] );
     }
  
     // Finally, check to see if we found all the requested variables
     for (const auto& plot_name : plot_var_names) {
         if (!containerHasElement(a_plot_var_names, plot_name)) {
              if (amrex::ParallelDescriptor::IOProcessor()) {
                  Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
              }
         }
     }
 }

Here is the call graph for this function:

◆ apply_gaussian_smoothing_to_perturbations()

void ERF::apply_gaussian_smoothing_to_perturbations	(	const int	lev,
		amrex::MultiFab &	cons_pert,
		amrex::MultiFab &	xvel_pert,
		amrex::MultiFab &	yvel_pert,
		amrex::MultiFab &	zvel_pert
	)

 {
     ignore_unused(cons_pert);
     ignore_unused(yvel_pert);
     ignore_unused(zvel_pert);
  
     const Geometry& gm = geom[lev];
     const Real dx = gm.CellSize(0);
     const Real dy = gm.CellSize(1);
  
     const Real dmesh = std::min(dx, dy);
     // ---- User choices ----
     const Real sigma = solverChoice.pert_correlated_radius; // e.g. 2 km correlation length
     const int  r     = static_cast<int>(3.0 * sigma / dmesh);  // stencil radius
  
     // ---- Precompute Gaussian weights on host ----
     const int wsize = 2*r + 1;
     Vector<Real> w_host(wsize * wsize);
  
     Real Z = 0.0;
     for (int m = -r; m <= r; ++m) {
         for (int n = -r; n <= r; ++n) {
             Real val = std::exp(-(m*m*dx*dx + n*n*dy*dy)/(2.0*sigma*sigma));
             w_host[(m+r)*wsize + (n+r)] = val;
             Z += val;
         }
     }
     for (auto& v : w_host) {
         v = v/Z;
     }
  
     Gpu::DeviceVector<Real> w_dev;
     w_dev.resize(w_host.size());
     Gpu::copy(Gpu::hostToDevice, w_host.begin(), w_host.end(), w_dev.begin());
  
     Real const* w = w_dev.data();
  
     // 1. Define ngrow_big using the actual dimension macro
     IntVect ngrow_big(AMREX_D_DECL(r, r, 0));
  
     // 2. Create the copy
     MultiFab xvel_pert_copy(xvel_pert.boxArray(),
                         xvel_pert.DistributionMap(),
                         1, ngrow_big);
     //MultiFab::Copy(xvel_pert_copy, xvel_pert, 0, 0, 1, 0);
  
     // 3. Use the built-in copy that includes ghost cell logic
     // Copy(dst, src, src_comp, dst_comp, num_comp, ngrow)
     // Setting ngrow to 0 ensures we only take valid data from the original
     xvel_pert_copy.ParallelCopy(xvel_pert, 0, 0, 1, IntVect(0), ngrow_big, gm.periodicity());
  
     for (MFIter mfi(xvel_pert, TileNoZ()); mfi.isValid(); ++mfi)
     {
         const Box& tbx = mfi.tilebox();
  
         auto const& in  = xvel_pert_copy.array(mfi);
         auto const& out = xvel_pert.array(mfi);
  
         ParallelFor(tbx,
         [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
         {
             Real sum = 0.0;
             for (int m = -r; m <= r; ++m) {
                 for (int n = -r; n <= r; ++n) {
                     Real wij = w[(m+r)*wsize + (n+r)];
                     sum += wij * in(i+m, j+n, k);
                 }
             }
             out(i,j,k) = sum;
         });
     }
 }

Here is the call graph for this function:

◆ AverageDown()

void ERF::AverageDown ( )

private

 {
     AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
  
     int src_comp, num_comp;
     for (int lev = finest_level-1; lev >= 0; --lev)
     {
         // If anelastic we don't average down rho because rho == rho0.
         if (solverChoice.anelastic[lev]) {
             src_comp = 1;
         } else {
             src_comp = 0;
         }
         num_comp = vars_new[0][Vars::cons].nComp() - src_comp;
         AverageDownTo(lev,src_comp,num_comp);
     }
 }

Here is the call graph for this function:

◆ AverageDownTo()

void ERF::AverageDownTo	(	int	crse_lev,
		int	scomp,
		int	ncomp
	)

 {
     if (solverChoice.anelastic[crse_lev]) {
         AMREX_ALWAYS_ASSERT(scomp == 1);
     } else {
         AMREX_ALWAYS_ASSERT(scomp == 0);
     }
  
     AMREX_ALWAYS_ASSERT(ncomp == vars_new[crse_lev][Vars::cons].nComp() - scomp);
     AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
  
     // ******************************************************************************************
     // First do cell-centered quantities
     // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
     // m is the map scale factor at cell centers
     // Here we multiply (rho S) by detJ and divide (rho S) by m^2 before average down
     // ******************************************************************************************
     for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
       for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.tilebox();
         const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
         const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
         const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
         if (SolverChoice::mesh_type != MeshType::ConstantDz) {
             const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
             ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
             {
                 cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
             });
         } else {
             ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
             {
                 cons_arr(i,j,k,scomp+n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
             });
         }
       } // mfi
     } // lev
  
     int fine_lev = crse_lev+1;
  
     if (interpolation_type == StateInterpType::Perturbational) {
         // Make the fine rho and (rho theta) be perturbational
         MultiFab::Divide(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
                          Rho_comp,RhoTheta_comp,1,IntVect{0});
         MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                            BaseState::r0_comp,Rho_comp,1,IntVect{0});
         MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                            BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
  
         // Make the crse rho and (rho theta) be perturbational
         MultiFab::Divide(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                          Rho_comp,RhoTheta_comp,1,IntVect{0});
         MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                            BaseState::r0_comp,Rho_comp,1,IntVect{0});
         MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                            BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
     }
  
     if (SolverChoice::terrain_type != TerrainType::EB) {
         average_down(vars_new[crse_lev+1][Vars::cons],vars_new[crse_lev  ][Vars::cons],
                     scomp, ncomp, refRatio(crse_lev));
     } else {
         // const auto dx = geom[fine_lev].CellSize();
         // Setting cell_vol to the exact value may cause round-off errors in volume average.
         // const Real cell_vol = dx[0]*dx[1]*dx[2];
         constexpr Real cell_vol = 1.0;
         const BoxArray& ba = vars_new[fine_lev][IntVars::cons].boxArray();
         const DistributionMapping& dm = vars_new[fine_lev][IntVars::cons].DistributionMap();
         MultiFab vol_fine(ba, dm, 1, 0);
         vol_fine.setVal(cell_vol);
         EB_average_down(vars_new[fine_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                     vol_fine, *detJ_cc[fine_lev],
                     scomp, ncomp, refRatio(crse_lev));
     }
  
     if (interpolation_type == StateInterpType::Perturbational) {
         // Restore the fine data to what it was
         MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                       BaseState::r0_comp,Rho_comp,1,IntVect{0});
         MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
                       BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
         MultiFab::Multiply(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
                            Rho_comp,RhoTheta_comp,1,IntVect{0});
  
         // Make the crse data be full values not perturbational
         MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                       BaseState::r0_comp,Rho_comp,1,IntVect{0});
         MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
                       BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
         MultiFab::Multiply(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
                            Rho_comp,RhoTheta_comp,1,IntVect{0});
     }
  
     vars_new[crse_lev][Vars::cons].FillBoundary(geom[crse_lev].periodicity());
  
     // ******************************************************************************************
     // Here we multiply (rho S) by m^2 and divide by detJ after average down
     // ******************************************************************************************
     for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
       for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.tilebox();
         const Array4<      Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
         const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
         const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
         if (SolverChoice::mesh_type != MeshType::ConstantDz) {
             const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
             ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
             {
                 cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
             });
         } else { // MeshType::ConstantDz
             ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
             {
                 cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
             });
         }
       } // mfi
     } // lev
  
     // Fill EB covered cells by old values
     // (This won't be needed because EB_average_down copies the covered value.)
     if (SolverChoice::terrain_type == TerrainType::EB) {
         for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
             for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.tilebox();
                 const Array4<      Real> cons_new = vars_new[lev][Vars::cons].array(mfi);
                 const Array4<const Real> cons_old = vars_old[lev][Vars::cons].array(mfi);
                 const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
                 ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                 {
                     if (detJ_arr(i,j,k) == 0.0) {
                         cons_new(i,j,k,scomp+n) = cons_old(i,j,k,scomp+n);
                     }
                 });
             } // mfi
         } // lev
     }
  
     // ******************************************************************************************
     // Now average down momenta.
     // Note that vars_new holds velocities not momenta, but we want to do conservative
     //    averaging so we first convert to momentum, then average down, then convert
     //    back to velocities -- only on the valid region
     // ******************************************************************************************
     for (int lev = crse_lev; lev <= crse_lev+1; lev++)
     {
         // FillBoundary for density so we can go back and forth between velocity and momentum
         vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
  
         const MultiFab* c_vfrac = nullptr;
         if (SolverChoice::terrain_type == TerrainType::EB) {
             c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
         }
  
         VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
                         vars_new[lev][Vars::yvel], IntVect(0,0,0),
                         vars_new[lev][Vars::zvel], IntVect(0,0,0),
                         vars_new[lev][Vars::cons],
                             rU_new[lev],
                             rV_new[lev],
                             rW_new[lev],
                         Geom(lev).Domain(),
                         domain_bcs_type,
                         c_vfrac);
     }
  
     if (SolverChoice::terrain_type != TerrainType::EB) {
         average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
         average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
         average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
     } else {
         EB_average_down_faces({&rU_new[crse_lev+1], &rV_new[crse_lev+1], &rW_new[crse_lev+1]},
                             {&rU_new[crse_lev], &rV_new[crse_lev], &rW_new[crse_lev]},
                             refRatio(crse_lev), 0);
     }
  
     for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
  
         const MultiFab* c_vfrac = nullptr;
         if (SolverChoice::terrain_type == TerrainType::EB) {
             c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
         }
  
         MomentumToVelocity(vars_new[lev][Vars::xvel],
                         vars_new[lev][Vars::yvel],
                         vars_new[lev][Vars::zvel],
                         vars_new[lev][Vars::cons],
                             rU_new[lev],
                             rV_new[lev],
                             rW_new[lev],
                         Geom(lev).Domain(),
                         domain_bcs_type,
                         c_vfrac);
     }
 }

Here is the call graph for this function:

◆ build_fine_mask()

void ERF::build_fine_mask	(	int	lev,
		amrex::MultiFab &	fine_mask
	)

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters

level Fine level index which masks underlying coarser data

 {
     // Mask for zeroing covered cells
     AMREX_ASSERT(level > 0);
  
     BoxArray cba            = grids[level-1];
     DistributionMapping cdm = dmap[level-1];
  
     BoxArray fba            = fine_mask_lev.boxArray();
  
     iMultiFab ifine_mask_lev = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
  
     const auto  fma =  fine_mask_lev.arrays();
     const auto ifma = ifine_mask_lev.arrays();
     ParallelFor(fine_mask_lev, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
     {
        fma[bno](i,j,k) = ifma[bno](i,j,k);
     });
 }

Here is the call graph for this function:

◆ check_for_low_temp()

void ERF::check_for_low_temp ( amrex::MultiFab & S )

 {
     // *****************************************************************************
     // Test for low temp (low is defined as beyond the microphysics range of validity)
     // *****************************************************************************
     //
     // This value is defined in erf_dtesati in Source/Utils/ERF_MicrophysicsUtils.H
     Real t_low = 273.16 - 85.;
     //
     for (MFIter mfi(S); mfi.isValid(); ++mfi)
     {
         Box bx = mfi.tilebox();
         const Array4<Real> &s_arr  = S.array(mfi);
         ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
         {
             const Real rho      = s_arr(i, j, k, Rho_comp);
             const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
             const Real qv       = s_arr(i, j, k, RhoQ1_comp) / rho;
  
             Real temp = getTgivenRandRTh(rho, rhotheta, qv);
  
             if (temp < t_low) {
 #ifdef AMREX_USE_GPU
                 AMREX_DEVICE_PRINTF("Temperature too low in cell: %d %d %d %e \n", i,j,k,temp);
 #else
                 printf("Temperature too low in cell: %d %d %d \n", i,j,k);
                 printf("Based on temp / rhotheta / rho / qv %e %e %e %e \n", temp,rhotheta,rho,qv);
 #endif
                 Abort();
             }
         });
     }
 }

Here is the call graph for this function:

◆ check_for_negative_theta()

void ERF::check_for_negative_theta ( amrex::MultiFab & S )

 {
     // *****************************************************************************
     // Test for negative (rho theta)
     // *****************************************************************************
     for (MFIter mfi(S); mfi.isValid(); ++mfi)
     {
         Box bx = mfi.tilebox();
         const Array4<Real> &s_arr  = S.array(mfi);
         ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
         {
             const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
             if (rhotheta <= 0.) {
 #ifdef AMREX_USE_GPU
                 AMREX_DEVICE_PRINTF("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
 #else
                 printf("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
                 Abort("Bad theta in check_for_negative_theta");
 #endif
             }
             });
     } // mfi
 }

Here is the call graph for this function:

◆ check_state_for_nans()

void ERF::check_state_for_nans ( amrex::MultiFab const & S )

 {
     int ncomp = S.nComp();
     for (int lev = 0; lev <= finest_level; lev++)
     {
         //
         // Test at the end of every full timestep whether the solution data contains NaNs
         //
         bool any_have_nans = false;
         for (int i = 0; i < ncomp; i++) {
             if (S.contains_nan(i,1,0))
             {
                 amrex::Print() << "Component " << i << " of conserved variables contains NaNs" << '\n';
                 any_have_nans = true;
             }
         }
         if (any_have_nans) {
             exit(0);
         }
     }
 }

◆ check_vels_for_nans()

void ERF::check_vels_for_nans	(	amrex::MultiFab const &	xvel,
		amrex::MultiFab const &	yvel,
		amrex::MultiFab const &	zvel
	)

 {
     //
     // Test at the end of every full timestep whether the solution data contains NaNs
     //
     bool any_have_nans = false;
     if (xvel.contains_nan(0,1,0))
     {
         amrex::Print() << "x-velocity contains NaNs " << '\n';
         any_have_nans = true;
     }
     if (yvel.contains_nan(0,1,0))
     {
         amrex::Print() << "y-velocity contains NaNs" << '\n';
         any_have_nans = true;
     }
     if (zvel.contains_nan(0,1,0))
     {
         amrex::Print() << "z-velocity contains NaNs" << '\n';
         any_have_nans = true;
     }
     if (any_have_nans) {
         exit(0);
     }
 }

◆ ClearLevel()

void ERF::ClearLevel ( int lev )

override

 {
     for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
         vars_new[lev][var_idx].clear();
         vars_old[lev][var_idx].clear();
     }
  
     base_state[lev].clear();
  
     rU_new[lev].clear();
     rU_old[lev].clear();
     rV_new[lev].clear();
     rV_old[lev].clear();
     rW_new[lev].clear();
     rW_old[lev].clear();
  
     if (lev > 0) {
         zmom_crse_rhs[lev].clear();
     }
  
     if ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) {
         pp_inc[lev].clear();
     }
     if (solverChoice.anelastic[lev] == 0) {
         lagged_delta_rt[lev].clear();
     }
     avg_xmom[lev].clear();
     avg_ymom[lev].clear();
     avg_zmom[lev].clear();
  
     // Clears the integrator memory
     mri_integrator_mem[lev].reset();
  
     // Clears the physical boundary condition routines
     physbcs_cons[lev].reset();
     physbcs_u[lev].reset();
     physbcs_v[lev].reset();
     physbcs_w[lev].reset();
     physbcs_base[lev].reset();
  
     // Clears the flux register array
     advflux_reg[lev]->reset();
  
     // Clears the 2D arrays
     if (sst_lev[lev][0]) {
         for (int n = 0; n < sst_lev[lev].size(); n++) {
             sst_lev[lev][n].reset();
         }
     }
     if (tsk_lev[lev][0]) {
         for (int n = 0; n < tsk_lev[lev].size(); n++) {
             tsk_lev[lev][n].reset();
         }
     }
     if (lat_m[lev]) {
         lat_m[lev].reset();
     }
     if (lon_m[lev]) {
         lon_m[lev].reset();
     }
     if (sinPhi_m[lev]) {
         sinPhi_m[lev].reset();
     }
     if (cosPhi_m[lev]) {
         cosPhi_m[lev].reset();
     }
 }

◆ cloud_fraction()

Real ERF::cloud_fraction ( amrex::Real time )

 {
     BL_PROFILE("ERF::cloud_fraction()");
  
     int lev = 0;
     // This holds all of qc
     MultiFab qc(vars_new[lev][Vars::cons],make_alias,RhoQ2_comp,1);
  
     int direction = 2; // z-direction
     Box const& domain = geom[lev].Domain();
  
     auto const& qc_arr = qc.const_arrays();
  
     // qc_2d is an BaseFab<int> holding the max value over the column
     auto qc_2d = ReduceToPlane<ReduceOpMax,int>(direction, domain, qc,
          [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) -> int
          {
              if (qc_arr[box_no](i,j,k) > 0) {
                  return 1;
              } else {
                  return 0;
              }
          });
  
     auto* p = qc_2d.dataPtr();
  
     Long numpts = qc_2d.numPts();
  
     AMREX_ASSERT(numpts < Long(std::numeric_limits<int>::max));
  
 #if 1
     if (ParallelDescriptor::UseGpuAwareMpi()) {
         ParallelDescriptor::ReduceIntMax(p,static_cast<int>(numpts));
     } else {
         Gpu::PinnedVector<int> hv(numpts);
         Gpu::copyAsync(Gpu::deviceToHost, p, p+numpts, hv.data());
         Gpu::streamSynchronize();
         ParallelDescriptor::ReduceIntMax(hv.data(),static_cast<int>(numpts));
         Gpu::copyAsync(Gpu::hostToDevice, hv.data(), hv.data()+numpts, p);
     }
  
     // Sum over component 0
     Long num_cloudy = qc_2d.template sum<RunOn::Device>(0);
  
 #else
     //
     // We need this if we allow domain decomposition in the vertical
     //    but for now we leave it commented out
     //
     Long num_cloudy = Reduce::Sum<Long>(numpts,
          [=] AMREX_GPU_DEVICE (Long i) -> Long {
              if (p[i] == 1) {
                  return 1;
              } else {
                  return 0;
              }
          });
     ParallelDescriptor::ReduceLongSum(num_cloudy);
 #endif
  
     Real num_total = qc_2d.box().d_numPts();
  
     Real cloud_frac = num_cloudy / num_total;
  
     return cloud_frac;
 }

◆ compute_divergence()

void ERF::compute_divergence	(	int	lev,
		amrex::MultiFab &	rhs,
		amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM >	rho0_u_const,
		amrex::Geometry const &	geom_at_lev
	)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level 0.

 {
     BL_PROFILE("ERF::compute_divergence()");
  
     auto dxInv = geom_at_lev.InvCellSizeArray();
  
     // ****************************************************************************
     // Compute divergence which will form RHS
     // Note that we replace "rho0w" with the contravariant momentum, Omega
     // ****************************************************************************
     if (solverChoice.terrain_type == TerrainType::EB)
     {
         bool already_on_centroids = true;
         EB_computeDivergence(rhs, rho0_u_const, geom_at_lev, already_on_centroids);
     }
     else if (SolverChoice::mesh_type == MeshType::ConstantDz)
     {
         computeDivergence(rhs, rho0_u_const, geom_at_lev);
     }
     else
     {
         for ( MFIter mfi(rhs,TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             Box bx = mfi.tilebox();
             const Array4<Real const>& rho0u_arr = rho0_u_const[0]->const_array(mfi);
             const Array4<Real const>& rho0v_arr = rho0_u_const[1]->const_array(mfi);
             const Array4<Real const>& rho0w_arr = rho0_u_const[2]->const_array(mfi);
             const Array4<Real      >&   rhs_arr = rhs.array(mfi);
  
             const Array4<Real const>&      mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
             const Array4<Real const>&      mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
             const Array4<Real const>&      mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
             const Array4<Real const>&      mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
  
             if (SolverChoice::mesh_type == MeshType::StretchedDz)
             {
                 Real* stretched_dz_d_ptr = stretched_dz_d[lev].data();
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     Real inv_dz = 1.0/stretched_dz_d_ptr[k];
                     Real mfsq   = mf_mx(i,j,0) * mf_my(i,j,0);
                     rhs_arr(i,j,k) = (  (rho0u_arr(i+1,j  ,k  )/mf_uy(i+1,j,0) - rho0u_arr(i,j,k)/mf_uy(i,j,0)) * dxInv[0]
                                        +(rho0v_arr(i  ,j+1,k  )/mf_vx(i,j+1,0) - rho0v_arr(i,j,k)/mf_vx(i,j,0)) * dxInv[1]
                                        +(rho0w_arr(i  ,j  ,k+1)/mfsq           - rho0w_arr(i,j,k)/mfsq        ) * inv_dz  ) * mfsq;
                 });
             }
             else
             {
                 //
                 // Note we compute the divergence using "rho0w" == Omega
                 //
                 const Array4<Real const>& ax_arr = ax[lev]->const_array(mfi);
                 const Array4<Real const>& ay_arr = ay[lev]->const_array(mfi);
                 const Array4<Real const>& dJ_arr = detJ_cc[lev]->const_array(mfi);
  
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
                     rhs_arr(i,j,k) = ( ( ax_arr(i+1,j,k)*rho0u_arr(i+1,j,k)/mf_uy(i+1,j,0)
                                         -ax_arr(i  ,j,k)*rho0u_arr(i  ,j,k)/mf_uy(i  ,j,0)  ) * dxInv[0]
                                      + ( ay_arr(i,j+1,k)*rho0v_arr(i,j+1,k)/mf_vx(i,j+1,0)
                                         -ay_arr(i,j  ,k)*rho0v_arr(i,j  ,k)/mf_vx(i,j  ,0)  ) * dxInv[1]
                                       +(                 rho0w_arr(i,j,k+1)/mfsq
                                         -                rho0w_arr(i,j,k  )/mfsq            ) * dxInv[2] ) * mfsq / dJ_arr(i,j,k);
                 });
             }
         } // mfi
     }
 }

Here is the call graph for this function:

◆ ComputeDt()

void ERF::ComputeDt ( int step = -1 )

private

Function that calls estTimeStep for each level

 {
     Vector<Real> dt_tmp(finest_level+1);
  
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
     }
  
     ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
  
     Real dt_0 = dt_tmp[0];
     int n_factor = 1;
     for (int lev = 0; lev <= finest_level; ++lev) {
         dt_tmp[lev] = amrex::min(dt_tmp[lev], change_max*dt[lev]);
         n_factor *= nsubsteps[lev];
         dt_0 = amrex::min(dt_0, n_factor*dt_tmp[lev]);
         if (step == 0){
             dt_0 *= init_shrink;
             if (verbose && init_shrink != 1.0) {
                 Print() << "Timestep 0: shrink initial dt at level " << lev << " by " << init_shrink << std::endl;
             }
         }
     }
     //
     // Limit dt by the value of stop_time.
     // Recall that stop_time is total time, but t_new is elapsed time,
     //     so we must add start_time to t_new
     //
     const Real eps = 1.e-3*dt_0;
     if (t_new[0] + dt_0 > (stop_time - start_time) - eps) {
         dt_0 = (stop_time - start_time) - t_new[0];
     }
  
     dt[0] = dt_0;
     for (int lev = 1; lev <= finest_level; ++lev) {
         dt[lev] = dt[lev-1] / nsubsteps[lev];
     }
 }

◆ ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells ( const SolverChoice & sc )

inlinestaticprivate

     {
         int ngrow = 0;
  
         if (sc.use_num_diff)
         {
             ngrow = 3;
         } else {
             if (
                  (sc.advChoice.dycore_horiz_adv_type    == AdvType::Centered_6th)
               || (sc.advChoice.dycore_vert_adv_type     == AdvType::Centered_6th)
               || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
               || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
               || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Centered_6th)
               || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Centered_6th) )
             { ngrow = 3; }
             else if (
                  (sc.advChoice.dycore_horiz_adv_type    == AdvType::Upwind_5th)
               || (sc.advChoice.dycore_vert_adv_type     == AdvType::Upwind_5th)
               || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
               || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
               || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Upwind_5th)
               || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Upwind_5th) )
             { ngrow = 3; }
             else if (
                   (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5)
                || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_5)
                || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_5)
                || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_5)
                || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_5Z)
                || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_5Z)
                || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_5Z)
                || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_5Z) )
             { ngrow = 3; }
             else if (
                   (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_7)
                || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_7)
                || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_7)
                || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_7)
                || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_7Z)
                || (sc.advChoice.moistscal_vert_adv_type  == AdvType::Weno_7Z)
                || (sc.advChoice.dryscal_horiz_adv_type   == AdvType::Weno_7Z)
                || (sc.advChoice.dryscal_vert_adv_type    == AdvType::Weno_7Z) )
             { ngrow = 4; }
             else
             {
                 if (sc.terrain_type == TerrainType::EB){
                     ngrow = 4;
                 } else {
                     ngrow = 2;
                 }
             }
         }
  
         return ngrow;
     }

◆ Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int lev )

private

 {
     auto& fine_new = vars_new[lev];
     auto& crse_new = vars_new[lev-1];
     auto& ba_fine  = fine_new[Vars::cons].boxArray();
     auto& ba_crse  = crse_new[Vars::cons].boxArray();
     auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
     auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
  
     int ncomp = vars_new[lev][Vars::cons].nComp();
  
     FPr_c.emplace_back(ba_fine, dm_fine, geom[lev]  ,
                        ba_crse, dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
     FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
     FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
     FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                        convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                        -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
 }

◆ create_random_perturbations()

void ERF::create_random_perturbations	(	const int	lev,
		amrex::MultiFab &	cons_pert,
		amrex::MultiFab &	xvel_pert,
		amrex::MultiFab &	yvel_pert,
		amrex::MultiFab &	zvel_pert
	)

 {
     ignore_unused(cons_pert);
     ignore_unused(yvel_pert);
     ignore_unused(zvel_pert);
  
     auto& lev_new = vars_new[lev];
     for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
         const auto &xvel_pert_arr = xvel_pert.array(mfi);
         const Box &xbx = mfi.tilebox(IntVect(1,0,0));
         ParallelForRNG(xbx, [=] AMREX_GPU_DEVICE(int i, int j, int k, const amrex::RandomEngine& engine) noexcept
         {
             xvel_pert_arr(i, j, k) = amrex::Random(engine);
         });
     }
 }

Here is the call graph for this function:

◆ DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int i )

inlineprivate

     {
         return *datalog[i];
     }

◆ DataLogName()

std::string ERF::DataLogName ( int i ) const

inlineprivatenoexcept

The filename of the ith datalog file.

1616 { return datalogname[i]; }

ERF::datalogname

amrex::Vector< std::string > datalogname

Definition: ERF.H:1603

◆ Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int lev )

private

 {
     auto& fine_new = vars_new[lev];
     auto& crse_new = vars_new[lev-1];
     auto& ba_fine  = fine_new[Vars::cons].boxArray();
     auto& ba_crse  = crse_new[Vars::cons].boxArray();
     auto& dm_fine  = fine_new[Vars::cons].DistributionMap();
     auto& dm_crse  = crse_new[Vars::cons].DistributionMap();
  
     int ncomp = fine_new[Vars::cons].nComp();
  
     FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev]  ,
                         ba_crse, dm_crse, geom[lev-1],
                         -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
     FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev]  ,
                         convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
                         -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
     FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev]  ,
                         convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
                         -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
     FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev]  ,
                         convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
                         -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
 }

◆ DerDataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DerDataLog ( int i )

inlineprivate

     {
         return *der_datalog[i];
     }

◆ DerDataLogName()

std::string ERF::DerDataLogName ( int i ) const

inlineprivatenoexcept

1617 { return der_datalogname[i]; }

ERF::der_datalogname

amrex::Vector< std::string > der_datalogname

Definition: ERF.H:1604

◆ derive_diag_profiles()

void ERF::derive_diag_profiles	(	amrex::Real	time,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Kmv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Khv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qi,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qg,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wthv
	)

Computes the profiles for diagnostic quantities.

Parameters

h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_uiuiu	Profile for u_iu_iu triple product on Host
h_avg_uiuiv	Profile for u_iu_iv triple product on Host
h_avg_uiuiw	Profile for u_iu_iw triple product on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host

 {
     // We assume that this is always called at level 0
     int lev = 0;
  
     bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
     bool l_use_KE    = solverChoice.turbChoice[lev].use_tke;
     // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, uw, vv, vw, ww, uth, vth, wth,
     //                  0      1     2    3    4   5   6   7   8   9  10   11   12   13
     //                thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr,
     //                  14     15     16     17 18  19  20  21  22  23  24   25   26   27
     //                  qi, qs, qg, wthv
     //                  28  29  30    31
     MultiFab mf_out(grids[lev], dmap[lev], 32, 0);
  
     MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
  
     MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
     MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
     MultiFab  w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
  
     average_face_to_cellcenter(mf_vels,0,
         Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
  
     int zdir = 2;
     auto domain = geom[0].Domain();
  
     // Sum in the horizontal plane
     h_avg_u  = sumToLine(mf_vels ,0,1,domain,zdir);
     h_avg_v  = sumToLine(mf_vels ,1,1,domain,zdir);
     h_avg_w  = sumToLine(mf_vels ,2,1,domain,zdir);
  
     int hu_size =  h_avg_u.size();
  
     // Divide by the total number of cells we are averaging over
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     for (int k = 0; k < hu_size; ++k) {
         h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
     }
  
     Gpu::DeviceVector<Real> d_avg_u(hu_size, Real(0.0));
     Gpu::DeviceVector<Real> d_avg_v(hu_size, Real(0.0));
     Gpu::DeviceVector<Real> d_avg_w(hu_size, Real(0.0));
  
 #if 0
     auto* avg_u_ptr = d_avg_u.data();
     auto* avg_v_ptr = d_avg_v.data();
     auto* avg_w_ptr = d_avg_w.data();
 #endif
  
     Gpu::copy(Gpu::hostToDevice, h_avg_u.begin(), h_avg_u.end(), d_avg_u.begin());
     Gpu::copy(Gpu::hostToDevice, h_avg_v.begin(), h_avg_v.end(), d_avg_v.begin());
     Gpu::copy(Gpu::hostToDevice, h_avg_w.begin(), h_avg_w.end(), d_avg_w.begin());
  
     int nvars = vars_new[lev][Vars::cons].nComp();
     MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
  
     MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
  
     bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
  
     for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
         const Array4<Real>& fab_arr = mf_out.array(mfi);
         const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
         const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
         const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
         const Array4<Real>& cons_arr = mf_cons.array(mfi);
         const Array4<Real>&   p0_arr = p_hse.array(mfi);
         const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
                                                             Array4<const Real>{};
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 1) = theta;
             Real ksgs = 0.0;
             if (l_use_KE) {
                 ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
             }
             fab_arr(i, j, k, 2) = ksgs;
 #if 1
             if (l_use_kturb) {
                 fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
                 fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
             } else {
                 fab_arr(i, j, k, 3) = 0.0;
                 fab_arr(i, j, k, 4) = 0.0;
             }
 #else
             // Here we hijack the "Kturb" variable name to print out the resolved kinetic energy
             Real upert = u_cc_arr(i,j,k) - avg_u_ptr[k];
             Real vpert = v_cc_arr(i,j,k) - avg_v_ptr[k];
             Real wpert = w_cc_arr(i,j,k) - avg_w_ptr[k];
             fab_arr(i, j, k, 3) = 0.5 * (upert*upert + vpert*vpert + wpert*wpert);
 #endif
             fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
             fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
             fab_arr(i, j, k, 7) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // u*w
             fab_arr(i, j, k, 8) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
             fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // v*w
             fab_arr(i, j, k,10) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k);   // w*w
             fab_arr(i, j, k,11) = u_cc_arr(i,j,k) * theta; // u*th
             fab_arr(i, j, k,12) = v_cc_arr(i,j,k) * theta; // v*th
             fab_arr(i, j, k,13) = w_cc_arr(i,j,k) * theta; // w*th
             fab_arr(i, j, k,14) = theta * theta;           // th*th
  
             // if the number of fields is changed above, then be sure to update
             // the following def!
             Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,8) + fab_arr(i,j,k,10);
             fab_arr(i, j, k,15) = uiui * u_cc_arr(i,j,k);   // (ui*ui)*u
             fab_arr(i, j, k,16) = uiui * v_cc_arr(i,j,k);   // (ui*ui)*v
             fab_arr(i, j, k,17) = uiui * w_cc_arr(i,j,k);   // (ui*ui)*w
  
             if (!use_moisture) {
                 Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                 p -= p0_arr(i,j,k);
                 fab_arr(i, j, k,18) = p;                       // p
                 fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k);     // p*u
                 fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k);     // p*v
                 fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k);     // p*w
                 fab_arr(i, j, k,22) = 0.;  // qv
                 fab_arr(i, j, k,23) = 0.;  // qc
                 fab_arr(i, j, k,24) = 0.;  // qr
                 fab_arr(i, j, k,25) = 0.;  // w*qv
                 fab_arr(i, j, k,26) = 0.;  // w*qc
                 fab_arr(i, j, k,27) = 0.;  // w*qr
                 fab_arr(i, j, k,28) = 0.;  // qi
                 fab_arr(i, j, k,29) = 0.;  // qs
                 fab_arr(i, j, k,30) = 0.;  // qg
                 fab_arr(i, j, k,31) = 0.;  // w*thv
             }
         });
     } // mfi
  
     if (use_moisture)
     {
         int n_qstate_moist = micro->Get_Qstate_Moist_Size();
  
         for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             const Box& bx = mfi.tilebox();
             const Array4<Real>& fab_arr  = mf_out.array(mfi);
             const Array4<Real>& cons_arr = mf_cons.array(mfi);
             const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
             const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
             const Array4<Real>& w_cc_arr =  w_cc.array(mfi);
             const Array4<Real>&   p0_arr = p_hse.array(mfi);
  
             int rhoqr_comp = solverChoice.moisture_indices.qr;
  
             ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
             {
                 Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
                 Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
                 Real qr = (rhoqr_comp > -1) ?  cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
                                                Real(0.0);
                 Real p  = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
  
                 p -= p0_arr(i,j,k);
                 fab_arr(i, j, k,18) = p;                       // p
                 fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k);     // p*u
                 fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k);     // p*v
                 fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k);     // p*w
                 fab_arr(i, j, k,22) = qv;  // qv
                 fab_arr(i, j, k,23) = qc;  // qc
                 fab_arr(i, j, k,24) = qr;  // qr
                 fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * qv;  // w*qv
                 fab_arr(i, j, k,26) = w_cc_arr(i,j,k) * qc;  // w*qc
                 fab_arr(i, j, k,27) = w_cc_arr(i,j,k) * qr;  // w*qr
                 if (n_qstate_moist > 3) {
                     fab_arr(i, j, k,28) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp);  // qi
                     fab_arr(i, j, k,29) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp);  // qs
                     fab_arr(i, j, k,30) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp);  // qg
                 } else {
                     fab_arr(i, j, k,28) = 0.0;  // qi
                     fab_arr(i, j, k,29) = 0.0;  // qs
                     fab_arr(i, j, k,30) = 0.0;  // qg
                 }
                 Real ql    = qc + qr;
                 Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
                 Real thv   = theta * (1 + 0.61*qv - ql);
                 fab_arr(i, j, k,31) = w_cc_arr(i,j,k) * thv; // w*thv
             });
         } // mfi
     } // use_moisture
  
     h_avg_rho   = sumToLine(mf_out, 0,1,domain,zdir);
     h_avg_th    = sumToLine(mf_out, 1,1,domain,zdir);
     h_avg_ksgs  = sumToLine(mf_out, 2,1,domain,zdir);
     h_avg_Kmv   = sumToLine(mf_out, 3,1,domain,zdir);
     h_avg_Khv   = sumToLine(mf_out, 4,1,domain,zdir);
     h_avg_uu    = sumToLine(mf_out, 5,1,domain,zdir);
     h_avg_uv    = sumToLine(mf_out, 6,1,domain,zdir);
     h_avg_uw    = sumToLine(mf_out, 7,1,domain,zdir);
     h_avg_vv    = sumToLine(mf_out, 8,1,domain,zdir);
     h_avg_vw    = sumToLine(mf_out, 9,1,domain,zdir);
     h_avg_ww    = sumToLine(mf_out,10,1,domain,zdir);
     h_avg_uth   = sumToLine(mf_out,11,1,domain,zdir);
     h_avg_vth   = sumToLine(mf_out,12,1,domain,zdir);
     h_avg_wth   = sumToLine(mf_out,13,1,domain,zdir);
     h_avg_thth  = sumToLine(mf_out,14,1,domain,zdir);
     h_avg_uiuiu = sumToLine(mf_out,15,1,domain,zdir);
     h_avg_uiuiv = sumToLine(mf_out,16,1,domain,zdir);
     h_avg_uiuiw = sumToLine(mf_out,17,1,domain,zdir);
     h_avg_p     = sumToLine(mf_out,18,1,domain,zdir);
     h_avg_pu    = sumToLine(mf_out,19,1,domain,zdir);
     h_avg_pv    = sumToLine(mf_out,20,1,domain,zdir);
     h_avg_pw    = sumToLine(mf_out,21,1,domain,zdir);
     h_avg_qv    = sumToLine(mf_out,22,1,domain,zdir);
     h_avg_qc    = sumToLine(mf_out,23,1,domain,zdir);
     h_avg_qr    = sumToLine(mf_out,24,1,domain,zdir);
     h_avg_wqv   = sumToLine(mf_out,25,1,domain,zdir);
     h_avg_wqc   = sumToLine(mf_out,26,1,domain,zdir);
     h_avg_wqr   = sumToLine(mf_out,27,1,domain,zdir);
     h_avg_qi    = sumToLine(mf_out,28,1,domain,zdir);
     h_avg_qs    = sumToLine(mf_out,29,1,domain,zdir);
     h_avg_qg    = sumToLine(mf_out,30,1,domain,zdir);
     h_avg_wthv  = sumToLine(mf_out,31,1,domain,zdir);
  
     // Divide by the total number of cells we are averaging over
     int h_avg_u_size = static_cast<int>(h_avg_u.size());
     for (int k = 0; k < h_avg_u_size; ++k) {
         h_avg_rho[k] /= area_z;
         h_avg_ksgs[k]  /= area_z;
         h_avg_Kmv[k]   /= area_z;
         h_avg_Khv[k]   /= area_z;
         h_avg_th[k]    /= area_z;
         h_avg_thth[k]  /= area_z;
         h_avg_uu[k]    /= area_z;
         h_avg_uv[k]    /= area_z;
         h_avg_uw[k]    /= area_z;
         h_avg_vv[k]    /= area_z;
         h_avg_vw[k]    /= area_z;
         h_avg_ww[k]    /= area_z;
         h_avg_uth[k]   /= area_z;
         h_avg_vth[k]   /= area_z;
         h_avg_wth[k]   /= area_z;
         h_avg_uiuiu[k] /= area_z;
         h_avg_uiuiv[k] /= area_z;
         h_avg_uiuiw[k] /= area_z;
         h_avg_p[k]     /= area_z;
         h_avg_pu[k]    /= area_z;
         h_avg_pv[k]    /= area_z;
         h_avg_pw[k]    /= area_z;
         h_avg_qv[k]    /= area_z;
         h_avg_qc[k]    /= area_z;
         h_avg_qr[k]    /= area_z;
         h_avg_wqv[k]   /= area_z;
         h_avg_wqc[k]   /= area_z;
         h_avg_wqr[k]   /= area_z;
         h_avg_qi[k]    /= area_z;
         h_avg_qs[k]    /= area_z;
         h_avg_qg[k]    /= area_z;
         h_avg_wthv[k]  /= area_z;
     }
  
 #if 0
     // Here we print the integrated total kinetic energy as computed in the 1D profile above
     Real sum = 0.;
     Real dz = geom[0].ProbHi(2) / static_cast<Real>(h_avg_u_size);
     for (int k = 0; k < h_avg_u_size; ++k) {
         sum += h_avg_kturb[k] * h_avg_rho[k] * dz;
     }
     amrex::Print() << "ITKE " << time << " " << sum << " using " << h_avg_u_size << " " << dz << std::endl;
 #endif
 }

Here is the call graph for this function:

◆ derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag	(	amrex::Real	time,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_u,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_v,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_w,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_rho,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_th,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ksgs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Kmv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_Khv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqc,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wqr,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qi,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qs,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_qg,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ww,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_uth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_vth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_thth,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_ku,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_kw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_p,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pu,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pv,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_pw,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_wthv
	)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters

h_avg_u	Profile for x-velocity on Host
h_avg_v	Profile for y-velocity on Host
h_avg_w	Profile for z-velocity on Host
h_avg_rho	Profile for density on Host
h_avg_th	Profile for potential temperature on Host
h_avg_ksgs	Profile for Kinetic Energy on Host
h_avg_uu	Profile for x-velocity squared on Host
h_avg_uv	Profile for x-velocity * y-velocity on Host
h_avg_uw	Profile for x-velocity * z-velocity on Host
h_avg_vv	Profile for y-velocity squared on Host
h_avg_vw	Profile for y-velocity * z-velocity on Host
h_avg_ww	Profile for z-velocity squared on Host
h_avg_uth	Profile for x-velocity * potential temperature on Host
h_avg_uiuiu	Profile for u_iu_iu triple product on Host
h_avg_uiuiv	Profile for u_iu_iv triple product on Host
h_avg_uiuiw	Profile for u_iu_iw triple product on Host
h_avg_p	Profile for pressure perturbation on Host
h_avg_pu	Profile for pressure perturbation * x-velocity on Host
h_avg_pv	Profile for pressure perturbation * y-velocity on Host
h_avg_pw	Profile for pressure perturbation * z-velocity on Host

 {
     // We assume that this is always called at level 0
     int lev = 0;
  
     bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
     bool l_use_KE    = solverChoice.turbChoice[lev].use_tke;
     // Note: "uiui" == u_i*u_i = u*u + v*v + w*w
     // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, vv, uth, vth,
     //       indices:   0      1     2    3    4   5   6   7    8    9
     //                thth, uiuiu, uiuiv, p, pu, pv, qv, qc, qr, qi, qs, qg
     //                  10     11     12 13  14  15  16  17  18  19  20  21
     MultiFab mf_out(grids[lev], dmap[lev], 22, 0);
  
     // This will hold uw, vw, ww, wth, uiuiw, pw, wqv, wqc, wqr, wthv
     //       indices:  0   1   2    3      4   5    6    7    8     9
     MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 10, 0);
  
     // This is only used to average u and v; w is not averaged to cell centers
     MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
  
     MultiFab  u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
     MultiFab  v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
     MultiFab  w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers (staggered)
  
     int zdir = 2;
     auto domain = geom[0].Domain();
     Box stag_domain = domain;
     stag_domain.convert(IntVect(0,0,1));
  
     int nvars = vars_new[lev][Vars::cons].nComp();
     MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
  
     MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
  
     bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
  
     for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
         const Array4<Real>& fab_arr = mf_out.array(mfi);
         const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
         const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
         const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
         const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
         const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
         const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
         const Array4<Real>& cons_arr = mf_cons.array(mfi);
         const Array4<Real>&   p0_arr = p_hse.array(mfi);
         const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
                                                             Array4<const Real>{};
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             u_cc_arr(i,j,k) = 0.5 * (u_arr(i,j,k) + u_arr(i+1,j  ,k));
             v_cc_arr(i,j,k) = 0.5 * (v_arr(i,j,k) + v_arr(i  ,j+1,k));
  
             Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
             fab_arr(i, j, k, 1) = theta;
             Real ksgs = 0.0;
             if (l_use_KE) {
                 ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
             }
             fab_arr(i, j, k, 2) = ksgs;
             if (l_use_kturb) {
                 fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
                 fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
             } else {
                 fab_arr(i, j, k, 3) = 0.0;
                 fab_arr(i, j, k, 4) = 0.0;
             }
             fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k);   // u*u
             fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // u*v
             fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k);   // v*v
             fab_arr(i, j, k, 8) = u_cc_arr(i,j,k) * theta;             // u*th
             fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * theta;             // v*th
             fab_arr(i, j, k,10) = theta * theta;                       // th*th
  
             Real wcc = 0.5 * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
  
             // if the number of fields is changed above, then be sure to update
             // the following def!
             Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,7) + wcc*wcc;
             fab_arr(i, j, k,11) = uiui * u_cc_arr(i,j,k);           // (ui*ui)*u
             fab_arr(i, j, k,12) = uiui * v_cc_arr(i,j,k);           // (ui*ui)*v
  
             if (!use_moisture) {
                 Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
                 p -= p0_arr(i,j,k);
                 fab_arr(i, j, k,13) = p;                       // p
                 fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k);     // p*u
                 fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k);     // p*v
                 fab_arr(i, j, k,16) = 0.;  // qv
                 fab_arr(i, j, k,17) = 0.;  // qc
                 fab_arr(i, j, k,18) = 0.;  // qr
                 fab_arr(i, j, k,19) = 0.;  // qi
                 fab_arr(i, j, k,20) = 0.;  // qs
                 fab_arr(i, j, k,21) = 0.;  // qg
             }
         });
  
         const Box& zbx = mfi.tilebox(IntVect(0,0,1));
         ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             // average to z faces (first to cell centers, then in z)
             Real uface = 0.25 * ( u_arr(i  ,j,k) + u_arr(i  ,j,k-1)
                                 + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
             Real vface = 0.25 * ( v_arr(i,j  ,k) + v_arr(i,j  ,k-1)
                                 + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
             Real theta0 = cons_arr(i,j,k  ,RhoTheta_comp) / cons_arr(i,j,k  ,Rho_comp);
             Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
             Real thface = 0.5*(theta0 + theta1);
             fab_arr_stag(i,j,k,0) =           uface * w_fc_arr(i,j,k); // u*w
             fab_arr_stag(i,j,k,1) =           vface * w_fc_arr(i,j,k); // v*w
             fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
             fab_arr_stag(i,j,k,3) =          thface * w_fc_arr(i,j,k); // th*w
             Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
             fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
             if (!use_moisture) {
                 Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp)) - p0_arr(i,j,k  );
                 Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
                 Real pface = 0.5 * (p0 + p1);
                 fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k);       // p*w
                 fab_arr_stag(i,j,k,6) = 0.;  // w*qv
                 fab_arr_stag(i,j,k,7) = 0.;  // w*qc
                 fab_arr_stag(i,j,k,8) = 0.;  // w*qr
                 fab_arr_stag(i,j,k,9) = 0.;  // w*thv
             }
         });
  
     } // mfi
  
     if (use_moisture)
     {
         int n_qstate_moist = micro->Get_Qstate_Moist_Size();
  
         for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             const Box& bx = mfi.tilebox();
             const Array4<Real>& fab_arr  = mf_out.array(mfi);
             const Array4<Real>& fab_arr_stag  = mf_out_stag.array(mfi);
             const Array4<Real>& cons_arr = mf_cons.array(mfi);
             const Array4<Real>& u_cc_arr =  u_cc.array(mfi);
             const Array4<Real>& v_cc_arr =  v_cc.array(mfi);
             const Array4<Real>& w_fc_arr =  w_fc.array(mfi);
             const Array4<Real>&   p0_arr = p_hse.array(mfi);
  
             int rhoqr_comp = solverChoice.moisture_indices.qr;
  
             ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
             {
                 Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
                 Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
                 Real qr = (rhoqr_comp > -1) ?  cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
                                                Real(0.0);
                 Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
  
                 p -= p0_arr(i,j,k);
                 fab_arr(i, j, k,13) = p;                       // p
                 fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k);     // p*u
                 fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k);     // p*v
                 fab_arr(i, j, k,16) = qv;  // qv
                 fab_arr(i, j, k,17) = qc;  // qc
                 fab_arr(i, j, k,18) = qr;  // qr
                 if (n_qstate_moist > 3) { // SAM model
                     fab_arr(i, j, k,19) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp);  // qi
                     fab_arr(i, j, k,20) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp);  // qs
                     fab_arr(i, j, k,21) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp);  // qg
                 } else {
                     fab_arr(i, j, k,19) = 0.0;  // qi
                     fab_arr(i, j, k,20) = 0.0;  // qs
                     fab_arr(i, j, k,21) = 0.0;  // qg
                 }
             });
  
             const Box& zbx = mfi.tilebox(IntVect(0,0,1));
             ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
             {
                 Real qv0 = cons_arr(i,j,k  ,RhoQ1_comp) / cons_arr(i,j,k  ,Rho_comp);
                 Real qv1 = cons_arr(i,j,k-1,RhoQ1_comp) / cons_arr(i,j,k-1,Rho_comp);
                 Real qc0 = cons_arr(i,j,k  ,RhoQ2_comp) / cons_arr(i,j,k  ,Rho_comp);
                 Real qc1 = cons_arr(i,j,k-1,RhoQ2_comp) / cons_arr(i,j,k-1,Rho_comp);
                 Real qr0 = (rhoqr_comp > -1) ? cons_arr(i,j,k  ,RhoQ3_comp) / cons_arr(i,j,k  ,Rho_comp) :
                                                Real(0.0);
                 Real qr1 = (rhoqr_comp > -1) ? cons_arr(i,j,k-1,RhoQ3_comp) / cons_arr(i,j,k-1,Rho_comp) :
                                                Real(0.0);
                 Real qvface = 0.5 * (qv0 + qv1);
                 Real qcface = 0.5 * (qc0 + qc1);
                 Real qrface = 0.5 * (qr0 + qr1);
  
                 Real p0 = getPgivenRTh(cons_arr(i, j, k  , RhoTheta_comp), qv0) - p0_arr(i,j,k  );
                 Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv1) - p0_arr(i,j,k-1);
                 Real pface = 0.5 * (p0 + p1);
  
                 Real theta0 = cons_arr(i,j,k  ,RhoTheta_comp) / cons_arr(i,j,k  ,Rho_comp);
                 Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
                 Real thface = 0.5*(theta0 + theta1);
                 Real ql = qcface + qrface;
                 Real thv = thface * (1 + 0.61*qvface - ql);
  
                 fab_arr_stag(i,j,k,5) = pface  * w_fc_arr(i,j,k); // p*w
                 fab_arr_stag(i,j,k,6) = qvface * w_fc_arr(i,j,k); // w*qv
                 fab_arr_stag(i,j,k,7) = qcface * w_fc_arr(i,j,k); // w*qc
                 fab_arr_stag(i,j,k,8) = qrface * w_fc_arr(i,j,k); // w*qr
                 fab_arr_stag(i,j,k,9) = thv    * w_fc_arr(i,j,k); // w*thv
             });
         } // mfi
     } // use_moisture
  
     // Sum in the horizontal plane
     h_avg_u = sumToLine(u_cc,0,1,     domain,zdir);
     h_avg_v = sumToLine(v_cc,0,1,     domain,zdir);
     h_avg_w = sumToLine(w_fc,0,1,stag_domain,zdir);
  
     h_avg_rho   = sumToLine(mf_out, 0,1,domain,zdir);
     h_avg_th    = sumToLine(mf_out, 1,1,domain,zdir);
     h_avg_ksgs  = sumToLine(mf_out, 2,1,domain,zdir);
     h_avg_Kmv   = sumToLine(mf_out, 3,1,domain,zdir);
     h_avg_Khv   = sumToLine(mf_out, 4,1,domain,zdir);
     h_avg_uu    = sumToLine(mf_out, 5,1,domain,zdir);
     h_avg_uv    = sumToLine(mf_out, 6,1,domain,zdir);
     h_avg_vv    = sumToLine(mf_out, 7,1,domain,zdir);
     h_avg_uth   = sumToLine(mf_out, 8,1,domain,zdir);
     h_avg_vth   = sumToLine(mf_out, 9,1,domain,zdir);
     h_avg_thth  = sumToLine(mf_out,10,1,domain,zdir);
     h_avg_uiuiu = sumToLine(mf_out,11,1,domain,zdir);
     h_avg_uiuiv = sumToLine(mf_out,12,1,domain,zdir);
     h_avg_p     = sumToLine(mf_out,13,1,domain,zdir);
     h_avg_pu    = sumToLine(mf_out,14,1,domain,zdir);
     h_avg_pv    = sumToLine(mf_out,15,1,domain,zdir);
     h_avg_qv    = sumToLine(mf_out,16,1,domain,zdir);
     h_avg_qc    = sumToLine(mf_out,17,1,domain,zdir);
     h_avg_qr    = sumToLine(mf_out,18,1,domain,zdir);
     h_avg_qi    = sumToLine(mf_out,19,1,domain,zdir);
     h_avg_qs    = sumToLine(mf_out,20,1,domain,zdir);
     h_avg_qg    = sumToLine(mf_out,21,1,domain,zdir);
  
     h_avg_uw    = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
     h_avg_vw    = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
     h_avg_ww    = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
     h_avg_wth   = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
     h_avg_uiuiw = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
     h_avg_pw    = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
     h_avg_wqv   = sumToLine(mf_out_stag,6,1,stag_domain,zdir);
     h_avg_wqc   = sumToLine(mf_out_stag,7,1,stag_domain,zdir);
     h_avg_wqr   = sumToLine(mf_out_stag,8,1,stag_domain,zdir);
     h_avg_wthv  = sumToLine(mf_out_stag,9,1,stag_domain,zdir);
  
     // Divide by the total number of cells we are averaging over
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
     for (int k = 0; k < unstag_size; ++k) {
         h_avg_u[k]     /= area_z;
         h_avg_v[k]     /= area_z;
         h_avg_rho[k]   /= area_z;
         h_avg_ksgs[k]  /= area_z;
         h_avg_Kmv[k]   /= area_z;
         h_avg_Khv[k]   /= area_z;
         h_avg_th[k]    /= area_z;
         h_avg_thth[k]  /= area_z;
         h_avg_uu[k]    /= area_z;
         h_avg_uv[k]    /= area_z;
         h_avg_vv[k]    /= area_z;
         h_avg_uth[k]   /= area_z;
         h_avg_vth[k]   /= area_z;
         h_avg_uiuiu[k] /= area_z;
         h_avg_uiuiv[k] /= area_z;
         h_avg_p[k]     /= area_z;
         h_avg_pu[k]    /= area_z;
         h_avg_pv[k]    /= area_z;
         h_avg_qv[k]    /= area_z;
         h_avg_qc[k]    /= area_z;
         h_avg_qr[k]    /= area_z;
         h_avg_qi[k]    /= area_z;
         h_avg_qs[k]    /= area_z;
         h_avg_qg[k]    /= area_z;
     }
  
     for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
         h_avg_w[k]     /= area_z;
         h_avg_uw[k]    /= area_z;
         h_avg_vw[k]    /= area_z;
         h_avg_ww[k]    /= area_z;
         h_avg_wth[k]   /= area_z;
         h_avg_uiuiw[k] /= area_z;
         h_avg_pw[k]    /= area_z;
         h_avg_wqv[k]   /= area_z;
         h_avg_wqc[k]   /= area_z;
         h_avg_wqr[k]   /= area_z;
         h_avg_wthv[k]  /= area_z;
     }
 }

Here is the call graph for this function:

◆ derive_stress_profiles()

void ERF::derive_stress_profiles	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q1fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q2fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

 {
     int lev = 0;
  
     // This will hold the stress tensor components
     MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
  
     MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
  
     bool l_use_moist   = ( solverChoice.moisture_type != MoistureType::None );
  
     for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
         const Array4<Real>& fab_arr = mf_out.array(mfi);
  
         const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
  
         // NOTE: These are from the last RK stage...
         const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
         const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
         const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
         const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
         const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
         const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
  
         // These should be re-calculated during ERF_slow_rhs_post
         // -- just vertical SFS kinematic heat flux for now
         //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
         //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
         const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
         const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
                                                               Array4<const Real>{};
         const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
                                                               Array4<const Real>{};
         const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
             fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
             fab_arr(i, j, k, 1) = ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                   + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
                                 / (   rho_arr(i,j  ,k) +   rho_arr(i+1,j  ,k)
                                   +   rho_arr(i,j+1,k) +   rho_arr(i+1,j+1,k) );
             fab_arr(i, j, k, 2) = ( tau13_arr(i,j,k  ) + tau13_arr(i+1,j,k  )
                                   + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) )
                                 / (   rho_arr(i,j,k  ) +   rho_arr(i+1,j,k  )
                                   +   rho_arr(i,j,k+1) +   rho_arr(i+1,j,k+1) );
             fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
             fab_arr(i, j, k, 4) = ( tau23_arr(i,j,k  ) + tau23_arr(i,j+1,k  )
                                   + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) )
                                 / (   rho_arr(i,j,k  ) +   rho_arr(i,j+1,k  )
                                   +   rho_arr(i,j,k+1) +   rho_arr(i,j+1,k+1) );
             fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
             fab_arr(i, j, k, 6) = 0.5 * ( hfx3_arr(i,j,k) + hfx3_arr(i,j,k+1) ) / rho_arr(i,j,k);
             fab_arr(i, j, k, 7) = (l_use_moist) ? 0.5 * ( q1fx3_arr(i,j,k) + q1fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
             fab_arr(i, j, k, 8) = (l_use_moist) ? 0.5 * ( q2fx3_arr(i,j,k) + q2fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
             fab_arr(i, j, k, 9) =  diss_arr(i,j,k) / rho_arr(i,j,k);
         });
     }
  
     int zdir = 2;
     auto domain = geom[0].Domain();
  
     h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
     h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
     h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
     h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
     h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
     h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
     h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
     h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
     h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
     h_avg_diss  = sumToLine(mf_out,9,1,domain,zdir);
  
     int ht_size =  h_avg_tau11.size();
  
     // Divide by the total number of cells we are averaging over
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     for (int k = 0; k < ht_size; ++k) {
         h_avg_tau11[k] /= area_z;
         h_avg_tau12[k] /= area_z;
         h_avg_tau13[k] /= area_z;
         h_avg_tau22[k] /= area_z;
         h_avg_tau23[k] /= area_z;
         h_avg_tau33[k] /= area_z;
         h_avg_hfx3[k] /= area_z;
         h_avg_q1fx3[k] /= area_z;
         h_avg_q2fx3[k] /= area_z;
         h_avg_diss[k] /= area_z;
     }
 }

Here is the call graph for this function:

◆ derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag	(	amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau11,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau12,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau13,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau22,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau23,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_tau33,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_hfx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q1fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_q2fx3,
		amrex::Gpu::HostVector< amrex::Real > &	h_avg_diss
	)

 {
     int lev = 0;
  
     // This will hold the stress tensor components
     MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
  
     // This will hold Tau13 and Tau23
     MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 5, 0);
  
     MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
  
     bool l_use_moist   = ( solverChoice.moisture_type != MoistureType::None );
  
     for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
         const Array4<Real>& fab_arr = mf_out.array(mfi);
         const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
  
         const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
  
         // NOTE: These are from the last RK stage...
         const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
         const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
         const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
         const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
         const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
         const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
  
         // These should be re-calculated during ERF_slow_rhs_post
         // -- just vertical SFS kinematic heat flux for now
         //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
         //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
         const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
         const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
                                                               Array4<const Real>{};
         const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
                                                               Array4<const Real>{};
         const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
             fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
             fab_arr(i, j, k, 1) = ( tau12_arr(i,j  ,k) + tau12_arr(i+1,j  ,k)
                                   + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
                                 / (   rho_arr(i,j  ,k) +   rho_arr(i+1,j  ,k)
                                   +   rho_arr(i,j+1,k) +   rho_arr(i+1,j+1,k) );
             fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
             fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
             fab_arr(i, j, k, 9) =  diss_arr(i,j,k) / rho_arr(i,j,k);
         });
  
         const Box& zbx = mfi.tilebox(IntVect(0,0,1));
         ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             Real rho_face = 0.5 * (rho_arr(i,j,k-1) + rho_arr(i,j,k));
             // average from edge to face center
             fab_arr_stag(i,j,k,0) = 0.5*(tau13_arr(i,j,k) + tau13_arr(i+1,j  ,k)) / rho_face;
             fab_arr_stag(i,j,k,1) = 0.5*(tau23_arr(i,j,k) + tau23_arr(i  ,j+1,k)) / rho_face;
  
             fab_arr_stag(i,j,k,2) = hfx3_arr(i,j,k) / rho_face;
             fab_arr_stag(i,j,k,3) = (l_use_moist) ? q1fx3_arr(i,j,k) / rho_face : 0.0;
             fab_arr_stag(i,j,k,4) = (l_use_moist) ? q2fx3_arr(i,j,k) / rho_face : 0.0;
         });
     }
  
     int zdir = 2;
     auto domain = geom[0].Domain();
     Box stag_domain = domain;
     stag_domain.convert(IntVect(0,0,1));
  
     h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
     h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
 //  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
     h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
 //  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
     h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
 //  h_avg_hfx3  = sumToLine(mf_out,6,1,domain,zdir);
 //  h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
 //  h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
     h_avg_diss  = sumToLine(mf_out,9,1,domain,zdir);
  
     h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
     h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
     h_avg_hfx3  = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
     h_avg_q1fx3 = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
     h_avg_q2fx3 = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
  
     int ht_size =  h_avg_tau11.size(); // _un_staggered
  
     // Divide by the total number of cells we are averaging over
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     for (int k = 0; k < ht_size; ++k) {
         h_avg_tau11[k] /= area_z;
         h_avg_tau12[k] /= area_z;
         h_avg_tau13[k] /= area_z;
         h_avg_tau22[k] /= area_z;
         h_avg_tau23[k] /= area_z;
         h_avg_tau33[k] /= area_z;
         h_avg_hfx3[k] /= area_z;
         h_avg_q1fx3[k] /= area_z;
         h_avg_q2fx3[k] /= area_z;
         h_avg_diss[k] /= area_z;
     }
     // staggered heights
     h_avg_tau13[ht_size] /= area_z;
     h_avg_tau23[ht_size] /= area_z;
     h_avg_hfx3[ht_size]  /= area_z;
     h_avg_q1fx3[ht_size] /= area_z;
     h_avg_q2fx3[ht_size] /= area_z;
 }

Here is the call graph for this function:

◆ derive_upwp()

void ERF::derive_upwp ( amrex::Vector< amrex::Real > & h_havg )

◆ EBFactory()

amrex::EBFArrayBoxFactory const& ERF::EBFactory ( int lev ) const

inlineprivatenoexcept

                                        {
         return *(eb[lev]->get_const_factory());
     }

◆ erf_enforce_hse()

void ERF::erf_enforce_hse	(	int	lev,
		amrex::MultiFab &	dens,
		amrex::MultiFab &	pres,
		amrex::MultiFab &	pi,
		amrex::MultiFab &	th,
		amrex::MultiFab &	qv,
		std::unique_ptr< amrex::MultiFab > &	z_cc
	)

Enforces hydrostatic equilibrium when using terrain.

Parameters

[in]	lev	Integer specifying the current level
[out]	dens	MultiFab storing base state density
[out]	pres	MultiFab storing base state pressure
[out]	pi	MultiFab storing base state Exner function
[in]	z_cc	Pointer to MultiFab storing cell centered z-coordinates

 {
     Real l_gravity = solverChoice.gravity;
     bool l_use_terrain = (solverChoice.mesh_type != MeshType::ConstantDz);
  
     const auto geomdata = geom[lev].data();
     const Real dz = geomdata.CellSize(2);
  
     for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
     {
         // Create a flat box with same horizontal extent but only one cell in vertical
         const Box& tbz = mfi.nodaltilebox(2);
         int klo = tbz.smallEnd(2);
         int khi = tbz.bigEnd(2);
  
         // Note we only grow by 1 because that is how big z_cc is.
         Box b2d = tbz; // Copy constructor
         b2d.grow(0,1);
         b2d.grow(1,1);
         b2d.setRange(2,0);
  
         // Intersect this box with the domain
         Box zdomain = convert(geom[lev].Domain(),tbz.ixType());
         b2d &= zdomain;
  
         // We integrate to the first cell (and below) by using rho in this cell
         // If gravity == 0 this is constant pressure
         // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
         // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
         // Note ng_pres_hse = 1
  
        // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
        // Note that gravity is positive
  
         Array4<Real>  rho_arr = dens.array(mfi);
         Array4<Real> pres_arr = pres.array(mfi);
         Array4<Real>   pi_arr =   pi.array(mfi);
         Array4<Real>   th_arr = theta.array(mfi);
         Array4<Real> zcc_arr;
         if (l_use_terrain) {
            zcc_arr = z_cc->array(mfi);
         }
  
         const Real rdOcp = solverChoice.rdOcp;
  
         ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
         {
             // Set value at surface from Newton iteration for rho
             if (klo == 0)
             {
                 // Physical height of the terrain at cell center
                 Real hz;
                 if (l_use_terrain) {
                     hz = zcc_arr(i,j,klo);
                 } else {
                     hz = 0.5*dz;
                 }
  
                 pres_arr(i,j,klo) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
                   pi_arr(i,j,klo) = getExnergivenP(pres_arr(i,j,klo), rdOcp);
                   th_arr(i,j,klo) = getRhoThetagivenP(pres_arr(i,j,klo)) / rho_arr(i,j,klo);
  
                 //
                 // Set ghost cell with dz and rho at boundary
                 // (We will set the rest of the ghost cells in the boundary condition routine)
                 //
                 pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
                   pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
                   th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
  
             } else {
  
                 // If level > 0 and klo > 0, we need to use the value of pres_arr(i,j,klo-1) which was
                 //    filled from FillPatch-ing it.
                 Real dz_loc;
                 if (l_use_terrain) {
                     dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
                 } else {
                     dz_loc = dz;
                 }
  
                 Real dens_interp = 0.5*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
                 pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
  
                 pi_arr(i,j,klo  ) = getExnergivenP(pres_arr(i,j,klo  ), rdOcp);
                 th_arr(i,j,klo  ) = getRhoThetagivenP(pres_arr(i,j,klo  )) / rho_arr(i,j,klo  );
  
                 pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
                 th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
             }
  
             Real dens_interp;
             if (l_use_terrain) {
                 for (int k = klo+1; k <= khi; k++) {
                     Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
                     dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                     pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
                     pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                     th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
                 }
             } else {
                 for (int k = klo+1; k <= khi; k++) {
                     dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
                     pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
                     pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
                     th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
                 }
             }
         });
  
     } // mfi
  
      dens.FillBoundary(geom[lev].periodicity());
      pres.FillBoundary(geom[lev].periodicity());
        pi.FillBoundary(geom[lev].periodicity());
     theta.FillBoundary(geom[lev].periodicity());
        qv.FillBoundary(geom[lev].periodicity());
 }

Here is the call graph for this function:

◆ ERF_shared()

void ERF::ERF_shared ( )

 {
     if (ParallelDescriptor::IOProcessor()) {
         const char* erf_hash = buildInfoGetGitHash(1);
         const char* amrex_hash = buildInfoGetGitHash(2);
         const char* buildgithash = buildInfoGetBuildGitHash();
         const char* buildgitname = buildInfoGetBuildGitName();
  
         if (strlen(erf_hash) > 0) {
           Print() << "\n"
                          << "ERF git hash: " << erf_hash << "\n";
         }
         if (strlen(amrex_hash) > 0) {
           Print() << "AMReX git hash: " << amrex_hash << "\n";
         }
         if (strlen(buildgithash) > 0) {
           Print() << buildgitname << " git hash: " << buildgithash << "\n";
         }
  
         Print() << "\n";
     }
  
     int nlevs_max = max_level + 1;
  
 #ifdef ERF_USE_WINDFARM
     Nturb.resize(nlevs_max);
     vars_windfarm.resize(nlevs_max);
     SMark.resize(nlevs_max);
 #endif
  
     qheating_rates.resize(nlevs_max);
     rad_fluxes.resize(nlevs_max);
     sw_lw_fluxes.resize(nlevs_max);
     solar_zenith.resize(nlevs_max);
  
     // NOTE: size lsm before readparams (chooses the model at all levels)
     lsm.ReSize(nlevs_max);
     lsm_data.resize(nlevs_max);
     lsm_flux.resize(nlevs_max);
  
     rhotheta_src.resize(nlevs_max);
     rhoqt_src.resize(nlevs_max);
  
     // NOTE: size canopy model before readparams (if file exists, we construct)
     m_forest_drag.resize(nlevs_max);
     for (int lev = 0; lev <= max_level; ++lev) { m_forest_drag[lev] = nullptr;}
  
     ReadParameters();
     initializeMicrophysics(nlevs_max);
  
 #ifdef ERF_USE_WINDFARM
     initializeWindFarm(nlevs_max);
 #endif
  
 #ifdef ERF_USE_SHOC
     shoc_interface.resize(nlevs_max);
     if (solverChoice.use_shoc) {
         for (int lev = 0; lev <= max_level; ++lev) {
             shoc_interface[lev] = std::make_unique<SHOCInterface>(lev, solverChoice);
         }
     }
 #endif
  
     rad.resize(nlevs_max);
     for (int lev = 0; lev <= max_level; ++lev) {
         if (solverChoice.rad_type == RadiationType::RRTMGP) {
 #ifdef ERF_USE_RRTMGP
             rad[lev] = std::make_unique<Radiation>(lev, solverChoice);
             // pass radiation datalog frequency to model - RRTMGP needs to know when to save data for profiles
             rad[lev]->setDataLogFrequency(rad_datalog_int);
 #endif
         } else if (solverChoice.rad_type != RadiationType::None) {
             Abort("Don't know this radiation model!");
         }
     }
  
     const std::string& pv3d_1 = "plot_vars_1"  ; setPlotVariables(pv3d_1,plot3d_var_names_1);
     const std::string& pv3d_2 = "plot_vars_2"  ; setPlotVariables(pv3d_2,plot3d_var_names_2);
     const std::string& pv2d_1 = "plot2d_vars_1"; setPlotVariables2D(pv2d_1,plot2d_var_names_1);
     const std::string& pv2d_2 = "plot2d_vars_2"; setPlotVariables2D(pv2d_2,plot2d_var_names_2);
  
     // This is only used when we have mesh_type == MeshType::StretchedDz
     stretched_dz_h.resize(nlevs_max);
     stretched_dz_d.resize(nlevs_max);
  
     // Initialize staggered vertical levels for grid stretching or terrain, and
     // to simplify Rayleigh damping layer calculations.
     zlevels_stag.resize(max_level+1);
     init_zlevels(zlevels_stag,
                  stretched_dz_h,
                  stretched_dz_d,
                  geom,
                  refRatio(),
                  solverChoice.grid_stretching_ratio,
                  solverChoice.zsurf,
                  solverChoice.dz0);
  
     if (SolverChoice::mesh_type == MeshType::StretchedDz ||
         SolverChoice::mesh_type == MeshType::VariableDz) {
         int nz = geom[0].Domain().length(2) + 1; // staggered
         if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
             Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
                 << " does not match highest requested z level " << zlevels_stag[0][nz-1]
                 << std::endl;
         }
         if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > 1.0e-4) {
             Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
                 << " does not match lowest requested level " << zlevels_stag[0][0]
                 << std::endl;
         }
  
         // Redefine the problem domain here?
     }
  
     // Get lo/hi indices for massflux calc
     if ((solverChoice.const_massflux_u != 0) || (solverChoice.const_massflux_v != 0)) {
         if (solverChoice.mesh_type == MeshType::ConstantDz) {
             const Real massflux_zlo = solverChoice.const_massflux_layer_lo - geom[0].ProbLo(2);
             const Real massflux_zhi = solverChoice.const_massflux_layer_hi - geom[0].ProbLo(2);
             const Real dz = geom[0].CellSize(2);
             if (massflux_zlo == -1e34) {
                 solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
             } else {
                 solverChoice.massflux_klo = static_cast<int>(std::ceil(massflux_zlo / dz - 0.5));
             }
             if (massflux_zhi ==  1e34) {
                 solverChoice.massflux_khi = geom[0].Domain().bigEnd(2);
             } else {
                 solverChoice.massflux_khi = static_cast<int>(std::floor(massflux_zhi / dz - 0.5));
             }
         } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
             const Real massflux_zlo = solverChoice.const_massflux_layer_lo;
             const Real massflux_zhi = solverChoice.const_massflux_layer_hi;
             solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
             solverChoice.massflux_khi = geom[0].Domain().bigEnd(2) + 1;
             for (int k=0; k <= geom[0].Domain().bigEnd(2)+1; ++k) {
                 if (zlevels_stag[0][k] <= massflux_zlo) solverChoice.massflux_klo = k;
                 if (zlevels_stag[0][k] <= massflux_zhi) solverChoice.massflux_khi = k;
             }
         } else { // solverChoice.mesh_type == MeshType::VariableDz
             Error("Const massflux with variable dz not supported -- planar averages are on k rather than constant-z planes");
         }
  
         Print() << "Constant mass flux based on k in ["
             << solverChoice.massflux_klo << ", " << solverChoice.massflux_khi << "]" << std::endl;
     }
  
     prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
  
     // Geometry on all levels has been defined already.
  
     // No valid BoxArray and DistributionMapping have been defined.
     // But the arrays for them have been resized.
  
     t_new.resize(nlevs_max, 0.0);
     t_old.resize(nlevs_max, -1.e100);
     dt.resize(nlevs_max, std::min(1.e100,dt_max_initial));
     dt_mri_ratio.resize(nlevs_max, 1);
  
     vars_new.resize(nlevs_max);
     vars_old.resize(nlevs_max);
     gradp.resize(nlevs_max);
  
     // We resize this regardless in order to pass it without error
     pp_inc.resize(nlevs_max);
  
     // Used in the fast substepping only
     lagged_delta_rt.resize(nlevs_max);
     avg_xmom.resize(nlevs_max);
     avg_ymom.resize(nlevs_max);
     avg_zmom.resize(nlevs_max);
  
     rU_new.resize(nlevs_max);
     rV_new.resize(nlevs_max);
     rW_new.resize(nlevs_max);
  
     rU_old.resize(nlevs_max);
     rV_old.resize(nlevs_max);
     rW_old.resize(nlevs_max);
  
     // xmom_crse_rhs.resize(nlevs_max);
     // ymom_crse_rhs.resize(nlevs_max);
     zmom_crse_rhs.resize(nlevs_max);
  
     for (int lev = 0; lev < nlevs_max; ++lev) {
         vars_new[lev].resize(Vars::NumTypes);
         vars_old[lev].resize(Vars::NumTypes);
         gradp[lev].resize(AMREX_SPACEDIM);
     }
  
     // Time integrator
     mri_integrator_mem.resize(nlevs_max);
  
     // Physical boundary conditions
     physbcs_cons.resize(nlevs_max);
     physbcs_u.resize(nlevs_max);
     physbcs_v.resize(nlevs_max);
     physbcs_w.resize(nlevs_max);
     physbcs_base.resize(nlevs_max);
  
     // Planes to hold Dirichlet values at boundaries
     xvel_bc_data.resize(nlevs_max);
     yvel_bc_data.resize(nlevs_max);
     zvel_bc_data.resize(nlevs_max);
     th_bc_data.resize(nlevs_max);
  
     advflux_reg.resize(nlevs_max);
  
     // Stresses
     Tau.resize(nlevs_max);
     Tau_corr.resize(nlevs_max);
     SFS_hfx1_lev.resize(nlevs_max); SFS_hfx2_lev.resize(nlevs_max); SFS_hfx3_lev.resize(nlevs_max);
     SFS_diss_lev.resize(nlevs_max);
     SFS_q1fx1_lev.resize(nlevs_max); SFS_q1fx2_lev.resize(nlevs_max); SFS_q1fx3_lev.resize(nlevs_max);
     SFS_q2fx3_lev.resize(nlevs_max);
     eddyDiffs_lev.resize(nlevs_max);
     SmnSmn_lev.resize(nlevs_max);
  
     // Sea surface temps
     sst_lev.resize(nlevs_max);
     tsk_lev.resize(nlevs_max);
     lmask_lev.resize(nlevs_max);
  
     // Land and soil grid type and urban fractions
     land_type_lev.resize(nlevs_max);
     soil_type_lev.resize(nlevs_max);
     urb_frac_lev.resize(nlevs_max);
  
     // Metric terms
     z_phys_nd.resize(nlevs_max);
     z_phys_cc.resize(nlevs_max);
     detJ_cc.resize(nlevs_max);
     ax.resize(nlevs_max);
     ay.resize(nlevs_max);
     az.resize(nlevs_max);
  
     z_phys_nd_new.resize(nlevs_max);
     detJ_cc_new.resize(nlevs_max);
  
     z_phys_nd_src.resize(nlevs_max);
     z_phys_cc_src.resize(nlevs_max);
     detJ_cc_src.resize(nlevs_max);
     ax_src.resize(nlevs_max);
     ay_src.resize(nlevs_max);
     az_src.resize(nlevs_max);
  
     z_t_rk.resize(nlevs_max);
  
     terrain_blanking.resize(nlevs_max);
  
     // Wall distance
     walldist.resize(nlevs_max);
  
     // BoxArrays to make MultiFabs needed to convert WRFBdy data
     ba1d.resize(nlevs_max);
     ba2d.resize(nlevs_max);
  
     // MultiFabs needed to convert WRFBdy data
     mf_PSFC.resize(nlevs_max);
  
     // Map factors
     mapfac.resize(nlevs_max);
  
     // Fine mask
     fine_mask.resize(nlevs_max);
  
     // Thin immersed body
     xflux_imask.resize(nlevs_max);
     yflux_imask.resize(nlevs_max);
     zflux_imask.resize(nlevs_max);
     //overset_imask.resize(nlevs_max);
     thin_xforce.resize(nlevs_max);
     thin_yforce.resize(nlevs_max);
     thin_zforce.resize(nlevs_max);
  
     // Base state
     base_state.resize(nlevs_max);
     base_state_new.resize(nlevs_max);
  
     // Wave coupling data
     Hwave.resize(nlevs_max);
     Lwave.resize(nlevs_max);
     for (int lev = 0; lev < max_level; ++lev)
     {
         Hwave[lev] = nullptr;
         Lwave[lev] = nullptr;
     }
     Hwave_onegrid.resize(nlevs_max);
     Lwave_onegrid.resize(nlevs_max);
     for (int lev = 0; lev < max_level; ++lev)
     {
         Hwave_onegrid[lev] = nullptr;
         Lwave_onegrid[lev] = nullptr;
     }
  
     // Theta prim for MOST
     Theta_prim.resize(nlevs_max);
  
     // Qv prim for MOST
     Qv_prim.resize(nlevs_max);
  
     // Qr prim for MOST
     Qr_prim.resize(nlevs_max);
  
     // Time averaged velocity field
     vel_t_avg.resize(nlevs_max);
     t_avg_cnt.resize(nlevs_max);
  
     // Size lat long arrays and default to null pointers
     lat_m.resize(nlevs_max);
     lon_m.resize(nlevs_max);
     for (int lev = 0; lev < max_level; ++lev) {
         lat_m[lev] = nullptr;
         lon_m[lev] = nullptr;
     }
  
     // Variable coriolis
     sinPhi_m.resize(nlevs_max);
     cosPhi_m.resize(nlevs_max);
     for (int lev = 0; lev < max_level; ++lev) {
         sinPhi_m[lev] = nullptr;
         cosPhi_m[lev] = nullptr;
     }
  
     // Initialize tagging criteria for mesh refinement
     refinement_criteria_setup();
  
     for (int lev = 0; lev < max_level; ++lev)
     {
        Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
           ref_ratio[lev][0]  << " " << ref_ratio[lev][1]  <<  " " << ref_ratio[lev][2] << std::endl;
     }
  
     // We will create each of these in MakeNewLevelFromScratch
     eb.resize(max_level+1);
     for (int lev = 0; lev < max_level + 1; lev++){
         eb[lev] = std::make_unique<eb_>();
     }
  
     //
     // Construct the EB data structures and store in a separate class
     //
     // This is needed before initializing level MultiFabs
     if ( solverChoice.terrain_type == TerrainType::EB ||
          solverChoice.terrain_type == TerrainType::ImmersedForcing)
     {
         std::string geometry ="terrain";
         ParmParse pp("eb2");
         pp.queryAdd("geometry", geometry);
  
         constexpr int ngrow_for_eb = 4;  // This is the default in amrex but we need to explicitly pass it here since
                                // we want to also pass the build_coarse_level_by_coarsening argument
         const bool build_eb_for_multigrid = (solverChoice.terrain_type == TerrainType::EB &&
                                             ((solverChoice.project_initial_velocity[0] == 1) ||
                                             solverChoice.anelastic[0] == 1));
         // Note this just needs to be an integer > number of V-cycles one might use
         const int max_coarsening_level = (build_eb_for_multigrid) ? 100 : 0;
         const bool build_coarse_level_by_coarsening(false);
  
         // Define GeometryShop using the implicit function
         if (geometry == "terrain") {
             Box terrain_bx(surroundingNodes(geom[max_level].Domain())); terrain_bx.grow(3);
             FArrayBox terrain_fab(makeSlab(terrain_bx,2,0),1);
             Real dummy_time = 0.0;
             prob->init_terrain_surface(geom[max_level], terrain_fab, dummy_time);
             TerrainIF implicit_fun(terrain_fab, geom[max_level], stretched_dz_d[max_level]);
             auto gshop = EB2::makeShop(implicit_fun);
             if (build_eb_for_multigrid) {
                 EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                             ngrow_for_eb, build_coarse_level_by_coarsening);
             } else {
                 EB2::Build(gshop, this->Geom(), ngrow_for_eb);
             }
         } else if (geometry == "box") {
             RealArray box_lo{0.0, 0.0, 0.0};
             RealArray box_hi{0.0, 0.0, 0.0};
             pp.query("box_lo", box_lo);
             pp.query("box_hi", box_hi);
             EB2::BoxIF implicit_fun(box_lo, box_hi, false);
             auto gshop = EB2::makeShop(implicit_fun);
             if (build_eb_for_multigrid) {
                 EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                             ngrow_for_eb, build_coarse_level_by_coarsening);
             } else {
                 EB2::Build(gshop, this->Geom(), ngrow_for_eb);
             }
         } else if (geometry == "sphere") {
             auto ProbLoArr = geom[max_level].ProbLoArray();
             auto ProbHiArr = geom[max_level].ProbHiArray();
             const Real xcen = 0.5 * (ProbLoArr[0] + ProbHiArr[0]);
             const Real ycen = 0.5 * (ProbLoArr[1] + ProbHiArr[1]);
             RealArray sphere_center = {xcen, ycen, 0.0};
             EB2::SphereIF implicit_fun(0.5, sphere_center, false);
             auto gshop = EB2::makeShop(implicit_fun);
             if (build_eb_for_multigrid) {
                 EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
                             ngrow_for_eb, build_coarse_level_by_coarsening);
             } else {
                 EB2::Build(gshop, this->Geom(), ngrow_for_eb);
             }
         }
     }
  
     if ( solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
         constexpr int ngrow_for_eb = 4;
         Box buildings_bx(surroundingNodes(geom[max_level].Domain())); buildings_bx.grow(3);
         FArrayBox buildings_fab(makeSlab(buildings_bx,2,0),1);
         Real dummy_time = 0.0;
         prob->init_buildings_surface(geom[max_level], buildings_fab, dummy_time);
         TerrainIF implicit_fun(buildings_fab, geom[max_level], stretched_dz_d[max_level]);
         auto gshop = EB2::makeShop(implicit_fun);
         EB2::Build(gshop, this->Geom(), ngrow_for_eb);
     }
  
     forecast_state_1.resize(nlevs_max);
     forecast_state_2.resize(nlevs_max);
     forecast_state_interp.resize(nlevs_max);
  
     surface_state_1.resize(nlevs_max);
     surface_state_2.resize(nlevs_max);
     surface_state_interp.resize(nlevs_max);
 }

Here is the call graph for this function:

◆ ErrorEst()

void ERF::ErrorEst	(	int	lev,
		amrex::TagBoxArray &	tags,
		amrex::Real	time,
		int	ngrow
	)

override

 {
     const int clearval = TagBox::CLEAR;
     const int   tagval = TagBox::SET;
  
 #ifdef ERF_USE_NETCDF
     if (solverChoice.init_type == InitType::WRFInput) {
         int ratio;
         Box subdomain;
  
         if (!nc_init_file[levc+1].empty())
         {
             Real levc_start_time = read_start_time_from_wrfinput(levc  , nc_init_file[levc  ][0]);
             amrex::Print() << " WRFInput       time at level " << levc << " is " << levc_start_time << std::endl;
  
             for (int isub = 0; isub < nc_init_file[levc+1].size(); isub++) {
                 if (!have_read_nc_init_file[levc+1][isub])
                 {
                     Real levf_start_time = read_start_time_from_wrfinput(levc+1, nc_init_file[levc+1][isub]);
                     amrex::Print() << " WRFInput start_time at level " << levc+1 << " is " << levf_start_time << std::endl;
  
                     // We assume there is only one subdomain at levc; otherwise we don't know
                     //     which one is the parent of the fine region we are trying to create
                     AMREX_ALWAYS_ASSERT(subdomains[levc].size() == 1);
  
                     if ( (ref_ratio[levc][2]) != 1) {
                         amrex::Abort("The ref_ratio specified in the inputs file must have 1 in the z direction; please use ref_ratio_vect rather than ref_ratio");
                     }
  
                     if ( levf_start_time <= (levc_start_time + t_new[levc]) ) {
                         amrex::Print() << " WRFInput file to read: " << nc_init_file[levc+1][isub] << std::endl;
                         subdomain = read_subdomain_from_wrfinput(levc, nc_init_file[levc+1][isub], ratio);
                         amrex::Print() << " WRFInput subdomain " << isub << " at level " << levc+1 << " is " << subdomain << std::endl;
  
                         if ( (ratio != ref_ratio[levc][0]) || (ratio != ref_ratio[levc][1]) ) {
                             amrex::Print() << "File " << nc_init_file[levc+1][0] << " has refinement ratio = " << ratio << std::endl;
                             amrex::Print() << "The inputs file has refinement ratio = " << ref_ratio[levc] << std::endl;
                             amrex::Abort("These must be the same -- please edit your inputs file and try again.");
                         }
  
                         subdomain.coarsen(IntVect(ratio,ratio,1));
  
                         Box coarser_level(subdomains[levc][isub].minimalBox());
                         subdomain.shift(coarser_level.smallEnd());
  
                         if (verbose > 0) {
                             amrex::Print() << " Crse subdomain to be tagged is" << subdomain << std::endl;
                         }
  
                         Box new_fine(subdomain); new_fine.refine(IntVect(ratio,ratio,1));
                         num_boxes_at_level[levc+1] = 1;
                         boxes_at_level[levc+1].push_back(new_fine);
  
                         for (MFIter mfi(tags); mfi.isValid(); ++mfi) {
                             auto tag_arr = tags.array(mfi);  // Get device-accessible array
  
                             Box bx = mfi.validbox(); bx &= subdomain;
  
                             if (!bx.isEmpty()) {
                                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                                     tag_arr(i,j,k) = TagBox::SET;
                                 });
                             }
                         }
                     } // time is right
                 } else {
                     // Re-tag this region
                     for (MFIter mfi(tags); mfi.isValid(); ++mfi)
                     {
                         auto tag_arr = tags.array(mfi);  // Get device-accessible array
  
                         Box existing_bx_coarsened(boxes_at_level[levc+1][isub]);
                         existing_bx_coarsened.coarsen(ref_ratio[levc]);
  
                         Box bx = mfi.validbox(); bx &= existing_bx_coarsened;
  
                         if (!bx.isEmpty()) {
                             ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                                 tag_arr(i,j,k) = TagBox::SET;
                             });
                         }
                     }
                 } // has file been read?
             } // isub
             return;
         } // file not empty
     }
 #endif
  
     //
     // Make sure the ghost cells of the level we are tagging at are filled
     //    in case we take differences that require them
     // NOTE: We are Fillpatching only the cell-centered variables here
     //
     MultiFab& S_new = vars_new[levc][Vars::cons];
     MultiFab& U_new = vars_new[levc][Vars::xvel];
     MultiFab& V_new = vars_new[levc][Vars::yvel];
     MultiFab& W_new = vars_new[levc][Vars::zvel];
     //
     if (levc == 0) {
         FillPatchCrseLevel(levc, time, {&S_new, &U_new, &V_new, &W_new});
     } else {
         FillPatchFineLevel(levc, time, {&S_new, &U_new, &V_new, &W_new},
                            {&S_new, &rU_new[levc], &rV_new[levc], &rW_new[levc]},
                            base_state[levc], base_state[levc],
                            false, true);
     }
  
     for (int j=0; j < ref_tags.size(); ++j)
     {
         //
         // This mf must have ghost cells because we may take differences between adjacent values
         //
         std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 1);
         mf->setVal(0.0);
  
         // This allows dynamic refinement based on the value of the density
         if (ref_tags[j].Field() == "density")
         {
             MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,1);
  
         // This allows dynamic refinement based on the value of qv
         } else if ( ref_tags[j].Field() == "qv" ) {
             MultiFab::Copy(  *mf, vars_new[levc][Vars::cons], RhoQ1_comp, 0, 1, 1);
             MultiFab::Divide(*mf, vars_new[levc][Vars::cons],   Rho_comp, 0, 1, 1);
  
  
         // This allows dynamic refinement based on the value of qc
         } else if (ref_tags[j].Field() == "qc" ) {
             MultiFab::Copy(  *mf, vars_new[levc][Vars::cons], RhoQ2_comp, 0, 1, 1);
             MultiFab::Divide(*mf, vars_new[levc][Vars::cons],   Rho_comp, 0, 1, 1);
  
         // This allows dynamic refinement based on the value of the z-component of vorticity
         } else if (ref_tags[j].Field() == "vorticity" ) {
             Vector<MultiFab> mf_cc_vel(1);
             mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
             average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
  
             // Impose bc's at domain boundaries at all levels
             FillBdyCCVels(mf_cc_vel,levc);
  
             mf->setVal(0.);
  
             for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 auto& dfab = (*mf)[mfi];
                 auto& sfab = mf_cc_vel[0][mfi];
                 derived::erf_dervortz(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
             }
  
         // This allows dynamic refinement based on the value of the scalar/theta
         } else if ( (ref_tags[j].Field() == "scalar"  ) ||
                     (ref_tags[j].Field() == "theta"   ) )
         {
             for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.growntilebox();
                 auto& dfab = (*mf)[mfi];
                 auto& sfab = vars_new[levc][Vars::cons][mfi];
                 if (ref_tags[j].Field() == "scalar") {
                     derived::erf_derscalar(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                 } else if (ref_tags[j].Field() == "theta") {
                     derived::erf_dertheta(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                 }
             } // mfi
         // This allows dynamic refinement based on the value of the density
         } else if ( (SolverChoice::terrain_type == TerrainType::ImmersedForcing) &&
                     (ref_tags[j].Field() == "terrain_blanking") )
         {
             MultiFab::Copy(*mf,*terrain_blanking[levc],0,0,1,1);
         }
         else if (ref_tags[j].Field() == "velmag")
         {
             ParmParse pp(pp_prefix);
             Vector<std::string> refinement_indicators;
             pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
             Real velmag_threshold;
             bool is_hurricane_tracker = false;
             for (int i=0; i<refinement_indicators.size(); ++i)
             {
                 if (refinement_indicators[i]=="hurricane_tracker") {
                     is_hurricane_tracker = true;
                     std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
                     ParmParse ppr(ref_prefix);
                     ppr.get("value_greater", velmag_threshold);
                     break;
                 }
             }
  
             Vector<MultiFab> mf_cc_vel(1);
             mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
             average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
  
             if (is_hurricane_tracker) {
                 HurricaneTracker(levc, time, mf_cc_vel[0], velmag_threshold, &tags);
             } else {
                 for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     auto& dfab = (*mf)[mfi];
                     auto& sfab = mf_cc_vel[0][mfi];
                     derived::erf_dermagvel(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
                 }
             }
  
 #ifdef ERF_USE_PARTICLES
         } else {
             //
             // This allows dynamic refinement based on the number of particles per cell
             //
             // Note that we must count all the particles in levels both at and above the current,
             //      since otherwise, e.g., if the particles are all at level 1, counting particles at
             //      level 0 will not trigger refinement when regridding so level 1 will disappear,
             //      then come back at the next regridding
             //
             const auto& particles_namelist( particleData.getNames() );
             mf->setVal(0.0);
             for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
             {
                 std::string tmp_string(particles_namelist[i]+"_count");
                 IntVect rr = IntVect::TheUnitVector();
                 if (ref_tags[j].Field() == tmp_string) {
                     for (int lev = levc; lev <= finest_level; lev++)
                     {
                         MultiFab temp_dat(grids[lev], dmap[lev], 1, 0); temp_dat.setVal(0);
                         particleData[particles_namelist[i]]->IncrementWithTotal(temp_dat, lev);
  
                         MultiFab temp_dat_crse(grids[levc], dmap[levc], 1, 0); temp_dat_crse.setVal(0);
  
                         if (lev == levc) {
                             MultiFab::Copy(*mf, temp_dat, 0, 0, 1, 0);
                         } else {
                             for (int d = 0; d < AMREX_SPACEDIM; d++) {
                                 rr[d] *= ref_ratio[levc][d];
                             }
                             average_down(temp_dat, temp_dat_crse, 0, 1, rr);
                             MultiFab::Add(*mf, temp_dat_crse, 0, 0, 1, 0);
                         }
                     }
                 }
             }
 #endif
         }
  
         ref_tags[j](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
     } // loop over j
  
     // ********************************************************************************************
     // Refinement based on 2d distance from the "eye" which is defined here as the (x,y) location of
     //    the integrated qv
     // ********************************************************************************************
     ParmParse pp(pp_prefix);
     Vector<std::string> refinement_indicators;
     pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
     for (int i=0; i<refinement_indicators.size(); ++i)
     {
         if ( (refinement_indicators[i]=="storm_tracker") && (solverChoice.moisture_type != MoistureType::None) )
         {
             std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
             ParmParse ppr(ref_prefix);
  
             Real ref_start_time = -1.0;
             ppr.query("start_time",ref_start_time);
  
             if (time >= ref_start_time) {
  
                 Real max_radius = -1.0;
                 ppr.get("max_radius", max_radius);
  
                 // Create the volume-weighted sum of (rho qv) in each column
                 MultiFab mf_qv_int(ba2d[levc], dmap[levc], 1, 0); mf_qv_int.setVal(0.);
  
                 // Define the 2D MultiFab holding the column-integrated (rho qv)
                 volWgtColumnSum(levc, S_new, RhoQ1_comp, mf_qv_int, *detJ_cc[levc]);
  
                 // Find the max value in the domain
                 IntVect eye = mf_qv_int.maxIndex(0);
  
                 const auto dx      = geom[levc].CellSizeArray();
                 const auto prob_lo = geom[levc].ProbLoArray();
  
                 Real eye_x = prob_lo[0] + (eye[0] + 0.5) * dx[0];
                 Real eye_y = prob_lo[1] + (eye[1] + 0.5) * dx[1];
  
                 tag_on_distance_from_eye(geom[levc], &tags, eye_x, eye_y, max_radius);
             }
         }
     }
 }

Here is the call graph for this function:

◆ estTimeStep()

Real ERF::estTimeStep	(	int	level,
		long &	dt_fast_ratio
	)		const

Function that calls estTimeStep for each level

Parameters

[in]	level	level of refinement (coarsest level i 0)
[out]	dt_fast_ratio	ratio of slow to fast time step

 {
     BL_PROFILE("ERF::estTimeStep()");
  
     Real estdt_comp = 1.e20;
     Real estdt_lowM = 1.e20;
  
     // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation
     const int nxc = geom[0].Domain().length(0);
     const int nyc = geom[0].Domain().length(1);
  
     auto const dxinv = geom[level].InvCellSizeArray();
     auto const dzinv = 1.0 / dz_min[level];
  
     MultiFab const& S_new = vars_new[level][Vars::cons];
  
     MultiFab ccvel(grids[level],dmap[level],3,0);
  
     average_face_to_cellcenter(ccvel,0,
                                Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
                                                         &vars_new[level][Vars::yvel],
                                                         &vars_new[level][Vars::zvel]});
  
     bool l_substepping = (solverChoice.substepping_type[level] == SubsteppingType::Implicit);
     int  l_anelastic   = solverChoice.anelastic[level];
  
     bool l_comp_substepping_diag = (verbose && l_substepping && !l_anelastic && solverChoice.substepping_diag);
  
     Real estdt_comp_inv;
     Real estdt_vert_comp_inv;
     Real estdt_vert_lowM_inv;
  
     if (l_substepping && (nxc==1) && (nyc==1)) {
         // SCM -- should not depend on dx or dy; force minimum number of substeps
         estdt_comp_inv = std::numeric_limits<Real>::min();
     }
     else if (solverChoice.terrain_type == TerrainType::EB)
     {
         const eb_& eb_lev = get_eb(level);
         const MultiFab& detJ = (eb_lev.get_const_factory())->getVolFrac();
  
         estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
         [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                    Array4<Real const> const& s,
                                    Array4<Real const> const& u,
                                    Array4<Real const> const& vf) -> Real
         {
            Real new_comp_dt = -1.e100;
            amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
            {
                if (vf(i,j,k) > 0.)
                {
                    const Real rho      = s(i, j, k, Rho_comp);
                    const Real rhotheta = s(i, j, k, RhoTheta_comp);
  
                    // NOTE: even when moisture is present,
                    //       we only use the partial pressure of the dry air
                    //       to compute the soundspeed
                    Real pressure = getPgivenRTh(rhotheta);
                    Real c = std::sqrt(Gamma * pressure / rho);
  
                    // If we are doing implicit acoustic substepping, then the z-direction does not contribute
                    //    to the computation of the time step
                    if (l_substepping) {
                        if ((nxc > 1) && (nyc==1)) {
                            // 2-D in x-z
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
                        } else if ((nyc > 1) && (nxc==1)) {
                            // 2-D in y-z
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                        } else {
                            // 3-D
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                     ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                        }
  
                    // If we are not doing implicit acoustic substepping, then the z-direction contributes
                    //    to the computation of the time step
                    } else {
                        if (nxc > 1 && nyc > 1) {
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                     ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                     ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                        } else if (nxc > 1) {
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                     ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                        } else if (nyc > 1) {
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                     ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                        } else {
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                        }
  
                    }
                }
            });
            return new_comp_dt;
        });
  
     } else {
        estdt_comp_inv = ReduceMax(S_new, ccvel, 0,
        [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                   Array4<Real const> const& s,
                                   Array4<Real const> const& u) -> Real
        {
            Real new_comp_dt = -1.e100;
            amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
            {
                {
                    const Real rho      = s(i, j, k, Rho_comp);
                    const Real rhotheta = s(i, j, k, RhoTheta_comp);
  
                    // NOTE: even when moisture is present,
                    //       we only use the partial pressure of the dry air
                    //       to compute the soundspeed
                    Real pressure = getPgivenRTh(rhotheta);
                    Real c = std::sqrt(Gamma * pressure / rho);
  
                    // If we are doing implicit acoustic substepping, then the z-direction does not contribute
                    //    to the computation of the time step
                    if (l_substepping) {
                        if ((nxc > 1) && (nyc==1)) {
                            // 2-D in x-z
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
                        } else if ((nyc > 1) && (nxc==1)) {
                            // 2-D in y-z
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                        } else {
                            // 3-D
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                     ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
                        }
  
                    // If we are not doing implicit acoustic substepping, then the z-direction contributes
                    //    to the computation of the time step
                    } else {
                        if (nxc > 1 && nyc > 1) {
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                     ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                     ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                        } else if (nxc > 1) {
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
                                                     ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                        } else if (nyc > 1) {
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
                                                     ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                        } else {
                            new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv   ), new_comp_dt);
                        }
  
                    }
                }
            });
            return new_comp_dt;
        });
     } // not EB
  
     ParallelDescriptor::ReduceRealMax(estdt_comp_inv);
     estdt_comp = cfl / estdt_comp_inv;
  
      Real estdt_lowM_inv = ReduceMax(ccvel, 0,
        [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                   Array4<Real const> const& u) -> Real
        {
            Real new_lm_dt = -1.e100;
            Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
            {
                new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
                                       ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
                                       ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
            });
            return new_lm_dt;
        });
  
      ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
      if (estdt_lowM_inv > 0.0_rt)
          estdt_lowM = cfl / estdt_lowM_inv;
  
      // Additional vertical diagnostics
      if (l_comp_substepping_diag) {
          estdt_vert_comp_inv = ReduceMax(S_new, ccvel, 0,
          [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                     Array4<Real const> const& s,
                                     Array4<Real const> const& u) -> Real
          {
              Real new_comp_dt = -1.e100;
              amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
              {
                  {
                      const Real rho      = s(i, j, k, Rho_comp);
                      const Real rhotheta = s(i, j, k, RhoTheta_comp);
  
                      // NOTE: even when moisture is present,
                      //       we only use the partial pressure of the dry air
                      //       to compute the soundspeed
                      Real pressure = getPgivenRTh(rhotheta);
                      Real c = std::sqrt(Gamma * pressure / rho);
  
                      // Look at z-direction only
                      new_comp_dt = amrex::max((amrex::Math::abs(u(i,j,k,2)) + c) * dzinv, new_comp_dt);
                  }
              });
              return new_comp_dt;
          });
  
          estdt_vert_lowM_inv = ReduceMax(ccvel, 0,
          [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                     Array4<Real const> const& u) -> Real
          {
              Real new_lowM_dt = -1.e100;
              amrex::Loop(b, [=,&new_lowM_dt] (int i, int j, int k) noexcept
              {
                  new_lowM_dt = amrex::max((amrex::Math::abs(u(i,j,k,2))) * dzinv, new_lowM_dt);
              });
              return new_lowM_dt;
          });
  
          ParallelDescriptor::ReduceRealMax(estdt_vert_comp_inv);
          ParallelDescriptor::ReduceRealMax(estdt_vert_lowM_inv);
      }
  
      if (verbose) {
          if (fixed_dt[level] <= 0.0) {
              Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " <<
                1.0/dxinv[0] << " " << 1.0/dxinv[1] << " " << dz_min[level] << std::endl;
              Print() << "Compressible dt at level " << level << ":  " << estdt_comp << std::endl;
              if (estdt_lowM_inv > 0.0_rt) {
                  Print() << "Anelastic dt at level " << level << ":  " << estdt_lowM << std::endl;
              } else {
                  Print() << "Anelastic dt at level " << level << ": undefined " << std::endl;
              }
          }
  
          if (fixed_dt[level] > 0.0) {
              Print() << "Based on cfl of 1.0 " << std::endl;
              Print() << "Compressible dt at level " << level << " would be:  " << estdt_comp/cfl << std::endl;
              if (estdt_lowM_inv > 0.0_rt) {
                  Print() << "Anelastic dt at level " << level << " would be:  " << estdt_lowM/cfl << std::endl;
              } else {
                  Print() << "Anelastic dt at level " << level << " would be undefined " << std::endl;
              }
              Print() << "Fixed dt at level " << level << "       is:  " << fixed_dt[level] << std::endl;
              if (fixed_fast_dt[level] > 0.0) {
                  Print() << "Fixed fast dt at level " << level << "       is:  " << fixed_fast_dt[level] << std::endl;
              }
          }
      }
  
      if (solverChoice.substepping_type[level] != SubsteppingType::None) {
          if (fixed_dt[level] > 0. && fixed_fast_dt[level] > 0.) {
              dt_fast_ratio = static_cast<long>( fixed_dt[level] / fixed_fast_dt[level] );
          } else if (fixed_dt[level] > 0.) {
              // Max CFL_c = 1.0 for substeps by default, but we enforce a min of 4 substeps
              auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
              dt_fast_ratio = static_cast<long>( std::max(fixed_dt[level]/dt_sub_max,4.) );
          } else {
              // auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
              // dt_fast_ratio = static_cast<long>( std::max(estdt_comp/dt_sub_max,4.) );
              dt_fast_ratio = static_cast<long>( std::max(cfl / sub_cfl, 4.) );
          }
  
          // Force time step ratio to be an even value
          if (solverChoice.force_stage1_single_substep) {
              if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
          } else {
              if ( dt_fast_ratio%6 != 0) {
                  Print() << "mri_dt_ratio = " << dt_fast_ratio
                          << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
                  dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/6.0) * 6);
              }
          }
  
          if (verbose) {
              Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
          }
      } // if substepping
  
      // Print out some extra diagnostics -- dt calcs are repeated so as to not
      // disrupt the overall code flow...
      if (l_comp_substepping_diag) {
          Real dt_diag = (fixed_dt[level] > 0.0) ? fixed_dt[level] : estdt_comp;
          int  ns      = (fixed_mri_dt_ratio > 0.0) ? fixed_mri_dt_ratio : dt_fast_ratio;
  
          // horizontal acoustic CFL must be < 1 (fully explicit)
          // vertical   acoustic CFL may  be > 1
          Print() << "effective horiz,vert acoustic CFL with " << ns << " substeps : "
             << (dt_diag / ns) * estdt_comp_inv << " "
             << (dt_diag / ns) * estdt_vert_comp_inv << std::endl;
  
          // vertical advective CFL should be < 1, otherwise w-damping may be needed
          Print() << "effective vert advective CFL : "
             << dt_diag * estdt_vert_lowM_inv << std::endl;
      }
  
      if (fixed_dt[level] > 0.0) {
          return fixed_dt[level];
      } else {
          // Anelastic (substepping is not allowed)
          if (l_anelastic) {
  
             // Make sure that timestep is less than the dt_max
             estdt_lowM = amrex::min(estdt_lowM, dt_max);
  
             // On the first timestep enforce dt_max_initial
             if (istep[level] == 0) {
                 return amrex::min(dt_max_initial, estdt_lowM);
             } else {
                 return estdt_lowM;
             }
  
  
          // Compressible with or without substepping
          } else {
              return estdt_comp;
          }
      }
 }

Here is the call graph for this function:

◆ Evolve()

void ERF::Evolve ( )

 {
     BL_PROFILE_VAR("ERF::Evolve()", evolve);
  
     //
     // cur_time = t_new is elapsed time, not total time
     // stop_time is total time
     //
     Real cur_time = t_new[0];
  
     // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
     //      for finer levels (with or without subcycling)
     for (int step = istep[0]; (step < max_step) && (start_time+cur_time < stop_time); ++step)
     {
         if (use_datetime) {
             Print() << "\n" << getTimestamp(start_time+cur_time, datetime_format)
                     << " (" << cur_time << " s elapsed)" << std::endl;
         }
         Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
  
         ComputeDt(step);
  
         // Make sure we have read enough of the boundary plane data to make it through this timestep
         if (input_bndry_planes)
         {
             m_r2d->read_input_files(cur_time,dt[0],m_bc_extdir_vals);
         }
  
 #ifdef ERF_USE_PARTICLES
         // We call this every time step with the knowledge that the particles may be
         //    initialized at a later time than the simulation start time.
         // The ParticleContainer carries a "start time" so the initialization will happen
         //    only when a) time > start_time, and b) particles have not yet been initialized
         initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,cur_time);
 #endif
  
         if(solverChoice.init_type == InitType::HindCast and
           solverChoice.hindcast_lateral_forcing) {
             for(int lev=0;lev<finest_level+1;lev++){
                 WeatherDataInterpolation(lev,cur_time,z_phys_nd,false);
             }
         }
  
         if(solverChoice.init_type == InitType::HindCast and
           solverChoice.hindcast_surface_bcs) {
             for(int lev=0;lev<finest_level+1;lev++){
                 SurfaceDataInterpolation(lev,cur_time,z_phys_nd,false);
             }
         }
  
         auto dEvolveTime0 = amrex::second();
  
         int iteration = 1;
         timeStep(0, cur_time, iteration);
  
         cur_time  += dt[0];
  
         Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
                 << " DT = " << dt[0]  << std::endl;
  
         if (check_for_nans > 0) {
             amrex::Print() << "Testing new state and vels for NaNs at end of timestep" << std::endl;
             for (int lev = 0; lev <= finest_level; ++lev) {
                 check_state_for_nans(vars_new[lev][IntVars::cons]);
                 check_vels_for_nans(vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],vars_new[lev][Vars::zvel]);
             }
         }
  
         if (verbose > 0)
         {
             auto dEvolveTime = amrex::second() - dEvolveTime0;
             ParallelDescriptor::ReduceRealMax(dEvolveTime,ParallelDescriptor::IOProcessorNumber());
             amrex::Print() << "Timestep time = " << dEvolveTime << " seconds." << '\n';
         }
  
         post_timestep(step, cur_time, dt[0]);
  
         if (writeNow(cur_time, step+1, m_plot3d_int_1, m_plot3d_per_1, dt[0], last_plot3d_file_time_1)) {
             last_plot3d_file_step_1 = step+1;
             Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
             for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
             if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 += m_plot3d_per_1;}
         }
         if (writeNow(cur_time, step+1, m_plot3d_int_2, m_plot3d_per_2, dt[0], last_plot3d_file_time_2)) {
             last_plot3d_file_step_2 = step+1;
             Write3DPlotFile(2,plotfile3d_type_2,plot3d_var_names_2);
             for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
             if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 += m_plot3d_per_2;}
         }
  
         if (writeNow(cur_time, step+1, m_plot2d_int_1, m_plot2d_per_1, dt[0], last_plot2d_file_time_1)) {
             last_plot2d_file_step_1 = step+1;
             Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
             if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 += m_plot2d_per_1;}
         }
  
         if (writeNow(cur_time, step+1, m_plot2d_int_2, m_plot2d_per_2, dt[0], last_plot2d_file_time_2)) {
             last_plot2d_file_step_2 = step+1;
             Write2DPlotFile(2,plotfile2d_type_2,plot2d_var_names_2);
             if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 += m_plot2d_per_2;}
         }
  
         for (int i = 0; i < m_subvol_int.size(); i++) {
             if (writeNow(cur_time, step+1, m_subvol_int[i], m_subvol_per[i], dt[0], last_subvol_time[i])) {
                 last_subvol_step[i] = step+1;
                 WriteSubvolume(i,subvol3d_var_names);
                 if (m_subvol_per[i] > 0.) {last_subvol_time[i] += m_subvol_per[i];}
             }
         }
  
         if (writeNow(cur_time, step+1, m_check_int, m_check_per, dt[0], last_check_file_time)) {
             last_check_file_step = step+1;
             WriteCheckpointFile();
             if (m_check_per > 0.) {last_check_file_time += m_check_per;}
         }
  
 #ifdef AMREX_MEM_PROFILING
         {
             std::ostringstream ss;
             ss << "[STEP " << step+1 << "]";
             MemProfiler::report(ss.str());
         }
 #endif
  
         if (start_time+cur_time >= stop_time - 1.e-6*dt[0]) break;
     }
  
     // Write plotfiles at final time
     if ( (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.) && istep[0] > last_plot3d_file_step_1 ) {
         Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
         if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 += m_plot3d_per_1;}
     }
     if ( (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.) && istep[0] > last_plot3d_file_step_2) {
         Write3DPlotFile(2,plotfile3d_type_1,plot3d_var_names_2);
         if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 += m_plot3d_per_2;}
     }
     if ( (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.) && istep[0] > last_plot2d_file_step_1 ) {
         Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
         if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 += m_plot2d_per_1;}
     }
     if ( (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.) && istep[0] > last_plot2d_file_step_2) {
         Write2DPlotFile(2,plotfile2d_type_1,plot2d_var_names_2);
         if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 += m_plot2d_per_2;}
     }
  
     for (int i = 0; i < m_subvol_int.size(); i++) {
         if ( (m_subvol_int[i] > 0 || m_subvol_per[i] > 0.) && istep[0] > last_subvol_step[i]) {
             WriteSubvolume(i,subvol3d_var_names);
             if (m_subvol_per[i] > 0.) {last_subvol_time[i] += m_subvol_per[i];}
         }
     }
  
     if ( (m_check_int > 0 || m_check_per > 0.) && istep[0] > last_check_file_step) {
         WriteCheckpointFile();
         if (m_check_per > 0.) {last_check_file_time += m_check_per;}
     }
  
     BL_PROFILE_VAR_STOP(evolve);
 }

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fill_from_bndryregs()

void ERF::fill_from_bndryregs	(	const amrex::Vector< amrex::MultiFab * > &	mfs,
		amrex::Real	time
	)

 {
     //
     // We now assume that if we read in on one face, we read in on all faces
     //
     AMREX_ALWAYS_ASSERT(m_r2d);
  
     int lev = 0;
     const Box& domain = geom[lev].Domain();
  
     const auto& dom_lo = lbound(domain);
     const auto& dom_hi = ubound(domain);
  
     Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time);
  
     const BCRec* bc_ptr = domain_bcs_type_d.data();
  
     // xlo: ori = 0
     // ylo: ori = 1
     // zlo: ori = 2
     // xhi: ori = 3
     // yhi: ori = 4
     // zhi: ori = 5
     const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
     const auto& bdatylo = (*bndry_data[1])[lev].const_array();
     const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
     const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
  
     int bccomp;
  
     for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
     {
         MultiFab& mf = *mfs[var_idx];
         const int icomp = 0;
         const int ncomp = mf.nComp();
  
         if (var_idx == Vars::xvel) {
            bccomp = BCVars::xvel_bc;
         } else if (var_idx == Vars::yvel) {
            bccomp = BCVars::yvel_bc;
         } else if (var_idx == Vars::zvel) {
            bccomp = BCVars::zvel_bc;
         } else  if (var_idx == Vars::cons) {
            bccomp = BCVars::cons_bc;
         }
  
 #ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
         for (MFIter mfi(mf); mfi.isValid(); ++mfi)
         {
             const Array4<Real>& dest_arr = mf.array(mfi);
             Box bx = mfi.growntilebox();
  
             // x-faces
             {
             Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
             if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
  
             Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
             if (var_idx == Vars::xvel) bx_xhi.setSmall(0,dom_hi.x);
  
             ParallelFor(
                 bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                     int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                    BCVars::RhoScalar_bc_comp : icomp+n;
                     if (bc_ptr[bc_comp].lo(0) == ERFBCType::ext_dir_ingested) {
                         int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                         int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                         dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
                     }
                 },
                 bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                     int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                    BCVars::RhoScalar_bc_comp : icomp+n;
                     if (bc_ptr[bc_comp].hi(0) == ERFBCType::ext_dir_ingested) {
                         int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
                         int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                         dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
                     }
                 }
             );
             } // x-faces
  
             // y-faces
             {
             Box bx_ylo(bx); bx_ylo.setBig  (1,dom_lo.y-1);
             if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
  
             Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
             if (var_idx == Vars::yvel) bx_yhi.setSmall(1,dom_hi.y);
  
             ParallelFor(
                bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                     int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                    BCVars::RhoScalar_bc_comp : icomp+n;
                     if (bc_ptr[bc_comp].lo(1) == ERFBCType::ext_dir_ingested) {
                         int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                         int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                         dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
                     }
                 },
                 bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
                     int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
                                    BCVars::RhoScalar_bc_comp : icomp+n;
                     if (bc_ptr[bc_comp].hi(1) == ERFBCType::ext_dir_ingested) {
                         int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
                         int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
                         dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
                     }
                 }
             );
             } // y-faces
         } // mf
     } // var_idx
 }

Here is the call graph for this function:

◆ fill_rhs()

void ERF::fill_rhs	(	amrex::MultiFab &	rhs_mf,
		const amrex::MultiFab &	state_mf,
		amrex::Real	time,
		const amrex::Geometry &	geom
	)

private

◆ FillBdyCCVels()

void ERF::FillBdyCCVels	(	amrex::Vector< amrex::MultiFab > &	mf_cc_vel,
		int	levc = `0`
	)

 {
     // Impose bc's at domain boundaries
     for (int ilev(0); ilev < mf_cc_vel.size(); ++ilev)
     {
         int lev = ilev + levc;
         Box domain(Geom(lev).Domain());
  
         int ihi = domain.bigEnd(0);
         int jhi = domain.bigEnd(1);
         int khi = domain.bigEnd(2);
  
         // Impose periodicity first
         mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
  
         int jper = (Geom(lev).isPeriodic(1));
         int kper = (Geom(lev).isPeriodic(2));
  
         for (MFIter mfi(mf_cc_vel[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             const Box& bx = mfi.tilebox();
             const Array4<Real>& vel_arr = mf_cc_vel[lev].array(mfi);
  
             if (!Geom(lev).isPeriodic(0)) {
                 // Low-x side
                 if (bx.smallEnd(0) <= domain.smallEnd(0)) {
                     Real multn = ( (phys_bc_type[0] == ERF_BC::slip_wall   ) ||
                                    (phys_bc_type[0] == ERF_BC::no_slip_wall) ||
                                    (phys_bc_type[0] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                     Real multt = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
                     ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                     {
                         vel_arr(-1,j,k,0) = multn*vel_arr(0,j,k,0); // u
                         vel_arr(-1,j,k,1) = multt*vel_arr(0,j,k,1); // v
                         vel_arr(-1,j,k,2) = multt*vel_arr(0,j,k,2); // w
                     });
                 }
  
                 // High-x side
                 if (bx.bigEnd(0) >= domain.bigEnd(0)) {
                     Real multn = ( (phys_bc_type[3] == ERF_BC::slip_wall   ) ||
                                    (phys_bc_type[3] == ERF_BC::no_slip_wall) ||
                                    (phys_bc_type[3] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                     Real multt = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
                     ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
                     {
                         vel_arr(ihi+1,j,k,0) = multn*vel_arr(ihi,j,k,0); // u
                         vel_arr(ihi+1,j,k,1) = multt*vel_arr(ihi,j,k,1); // v
                         vel_arr(ihi+1,j,k,2) = multt*vel_arr(ihi,j,k,2); // w
                     });
                 }
             } // !periodic
  
             if (!Geom(lev).isPeriodic(1)) {
                 // Low-y side
                 if (bx.smallEnd(1) <= domain.smallEnd(1)) {
                     Real multn = ( (phys_bc_type[1] == ERF_BC::slip_wall   ) ||
                                    (phys_bc_type[1] == ERF_BC::no_slip_wall) ||
                                    (phys_bc_type[1] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                     Real multt = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
                     ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int  , int k) noexcept
                     {
                         vel_arr(i,-1,k,0) = multt*vel_arr(i,0,k,0); // u
                         vel_arr(i,-1,k,1) = multn*vel_arr(i,0,k,1); // u
                         vel_arr(i,-1,k,2) = multt*vel_arr(i,0,k,2); // w
                     });
                 }
  
                 // High-y side
                 if (bx.bigEnd(1) >= domain.bigEnd(1)) {
                     Real multn = ( (phys_bc_type[4] == ERF_BC::slip_wall   ) ||
                                    (phys_bc_type[4] == ERF_BC::no_slip_wall) ||
                                    (phys_bc_type[4] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                     Real multt = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
                     ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
                     {
                         vel_arr(i,jhi+1,k,0) = multt*vel_arr(i,jhi,k,0); // u
                         vel_arr(i,jhi+1,k,1) = multn*vel_arr(i,jhi,k,1); // v
                         vel_arr(i,jhi+1,k,2) = multt*vel_arr(i,jhi,k,2); // w
                     });
                 }
             } // !periodic
  
             if (!Geom(lev).isPeriodic(2)) {
                 // Low-z side
                 if (bx.smallEnd(2) <= domain.smallEnd(2)) {
                     Real multn = ( (phys_bc_type[2] == ERF_BC::slip_wall   ) ||
                                    (phys_bc_type[2] == ERF_BC::no_slip_wall) ||
                                    (phys_bc_type[2] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                     Real multt = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
                     ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                     {
                         vel_arr(i,j,-1,0) = multt*vel_arr(i,j,0,0); // u
                         vel_arr(i,j,-1,1) = multt*vel_arr(i,j,0,1); // v
                         vel_arr(i,j,-1,2) = multn*vel_arr(i,j,0,2); // w
                     });
                 }
  
                 // High-z side
                 if (bx.bigEnd(2) >= domain.bigEnd(2)) {
                     Real multn = ( (phys_bc_type[5] == ERF_BC::slip_wall   ) ||
                                    (phys_bc_type[5] == ERF_BC::no_slip_wall) ||
                                    (phys_bc_type[5] == ERF_BC::symmetry    ) ) ? -1. : 1.;
                     Real multt = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -1. : 1.;
                     Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
                     ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
                     {
                         vel_arr(i,j,khi+1,0) = multt*vel_arr(i,j,khi,0); // u
                         vel_arr(i,j,khi+1,1) = multt*vel_arr(i,j,khi,1); // v
                         vel_arr(i,j,khi+1,2) = multn*vel_arr(i,j,khi,2); // w
                     });
                 }
             } // !periodic
         } // MFIter
  
         // Impose periodicity again
         mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
     } // lev
 }

Here is the call graph for this function:

◆ FillCoarsePatch()

void ERF::FillCoarsePatch	(	int	lev,
		amrex::Real	time
	)

private

 {
     BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
     AMREX_ASSERT(lev > 0);
  
     //
     //****************************************************************************************************************
     // First fill velocities and density at the COARSE level so we can convert velocity to momenta at the COARSE level
     //****************************************************************************************************************
     //
     bool cons_only = false;
     if (lev == 1) {
         FillPatchCrseLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
                            &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
                            cons_only);
     } else {
         FillPatchFineLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
                            &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
                           {&vars_new[lev-1][Vars::cons],
                            &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
                            base_state[lev-1], base_state[lev-1],
                            false, cons_only);
     }
  
     //
     // ************************************************
     // Convert velocity to momentum at the COARSE level
     // ************************************************
     //
     const MultiFab* c_vfrac = nullptr;
     if (solverChoice.terrain_type == TerrainType::EB) {
         c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
     }
  
     VelocityToMomentum(vars_new[lev-1][Vars::xvel], IntVect{0},
                        vars_new[lev-1][Vars::yvel], IntVect{0},
                        vars_new[lev-1][Vars::zvel], IntVect{0},
                        vars_new[lev-1][Vars::cons],
                          rU_new[lev-1],
                          rV_new[lev-1],
                          rW_new[lev-1],
                        Geom(lev).Domain(),
                        domain_bcs_type, c_vfrac);
     //
     // *****************************************************************
     // Interpolate all cell-centered variables from coarse to fine level
     // *****************************************************************
     //
     Interpolater* mapper_c = &cell_cons_interp;
     Interpolater* mapper_f = &face_cons_linear_interp;
  
     //
     //************************************************************************************************
     // Interpolate cell-centered data from coarse to fine level
     // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
     // ************************************************************************************************
     IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
     int      ncomp_cons = vars_new[lev][Vars::cons].nComp();
  
     InterpFromCoarseLevel(vars_new[lev  ][Vars::cons], ngvect_cons, IntVect(0,0,0),
                           vars_new[lev-1][Vars::cons], 0, 0, ncomp_cons,
                           geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper_c, domain_bcs_type, BCVars::cons_bc);
  
     // ***************************************************************************
     // Physical bc's for cell centered variables at domain boundary
     // ***************************************************************************
     (*physbcs_cons[lev])(vars_new[lev][Vars::cons],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                          0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc,true);
  
     //
     //************************************************************************************************
     // Interpolate x-momentum from coarse to fine level
     // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
     // ************************************************************************************************
     //
     InterpFromCoarseLevel(rU_new[lev], IntVect{0}, IntVect{0}, rU_new[lev-1], 0, 0, 1,
                           geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::xvel_bc);
  
     //
     //************************************************************************************************
     // Interpolate y-momentum from coarse to fine level
     // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
     // ************************************************************************************************
     //
     InterpFromCoarseLevel(rV_new[lev], IntVect{0}, IntVect{0}, rV_new[lev-1], 0, 0, 1,
                           geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::yvel_bc);
  
     //************************************************************************************************
     // Interpolate z-momentum from coarse to fine level
     // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
     // ************************************************************************************************
     InterpFromCoarseLevel(rW_new[lev],  IntVect{0}, IntVect{0}, rW_new[lev-1], 0, 0, 1,
                           geom[lev-1], geom[lev],
                           refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::zvel_bc);
     //
     // *********************************************************
     // After interpolation of momentum, convert back to velocity
     // *********************************************************
     //
     for (int which_lev = lev-1; which_lev <= lev; which_lev++)
     {
         c_vfrac = nullptr;
         if (solverChoice.terrain_type == TerrainType::EB) {
             c_vfrac = &((get_eb(which_lev).get_const_factory())->getVolFrac());
         }
  
         MomentumToVelocity(vars_new[which_lev][Vars::xvel],
                            vars_new[which_lev][Vars::yvel],
                            vars_new[which_lev][Vars::zvel],
                            vars_new[which_lev][Vars::cons],
                              rU_new[which_lev],
                              rV_new[which_lev],
                              rW_new[which_lev],
                            Geom(which_lev).Domain(),
                            domain_bcs_type, c_vfrac);
     }
  
     // ***************************************************************************
     // Physical bc's at domain boundary
     // ***************************************************************************
     IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
  
     (*physbcs_u[lev])(vars_new[lev][Vars::xvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                       ngvect_vels,time,BCVars::xvel_bc,true);
     (*physbcs_v[lev])(vars_new[lev][Vars::yvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                       ngvect_vels,time,BCVars::yvel_bc,true);
     (*physbcs_w[lev])(vars_new[lev][Vars::zvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                       ngvect_vels,time,BCVars::zvel_bc,true);
  
     // ***************************************************************************
     // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
     // ***************************************************************************
 }

Here is the call graph for this function:

◆ FillForecastStateMultiFabs()

void ERF::FillForecastStateMultiFabs	(	const int	lev,
		const std::string &	filename,
		const std::unique_ptr< amrex::MultiFab > &	z_phys_nd,
		amrex::Vector< amrex::Vector< amrex::MultiFab >> &	forecast_state
	)

 {
  
     Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
     Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
  
     ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
                        xvec_h, yvec_h, zvec_h, rho_h,
                        uvel_h, vvel_h, wvel_h,
                        theta_h, qv_h, qc_h, qr_h);
  
     Real zmax = *std::max_element(zvec_h.begin(), zvec_h.end());
  
     const auto prob_lo_erf  = geom[lev].ProbLoArray();
     const auto prob_hi_erf  = geom[lev].ProbHiArray();
     const auto dx_erf       = geom[lev].CellSizeArray();
  
     if (prob_hi_erf[2] >= zmax) {
         Abort("ERROR: the maximum z of the domain (" + std::to_string(prob_hi_erf[2]) +
         ") should be less than the maximum z in the forecast data (" + std::to_string(zmax) +
         "). Change geometry.prob_hi[2] in the inputs to be less than " + std::to_string(zmax) + "."
         );
     }
  
     if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
         amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
     }
     if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
         amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
     }
     if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
         amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
     }
     if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
         amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
     }
  
  
     int nx = xvec_h.size();
     int ny = yvec_h.size();
     int nz = zvec_h.size();
  
     amrex::Real dxvec = (xvec_h[nx-1]-xvec_h[0])/(nx-1);
     amrex::Real dyvec = (yvec_h[ny-1]-yvec_h[0])/(ny-1);
  
     amrex::Gpu::DeviceVector<Real> latvec_d(nx*ny), lonvec_d(nx*ny), zvec_d(nz);
     amrex::Gpu::DeviceVector<Real> xvec_d(nx*ny*nz), yvec_d(nx*ny*nz);
     amrex::Gpu::DeviceVector<Real> rho_d(nx*ny*nz), uvel_d(nx*ny*nz), vvel_d(nx*ny*nz), wvel_d(nx*ny*nz),
                                    theta_d(nx*ny*nz), qv_d(nx*ny*nz), qc_d(nx*ny*nz), qr_d(nx*ny*nz);
  
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, latvec_h.begin(), latvec_h.end(), latvec_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, lonvec_h.begin(), lonvec_h.end(), lonvec_d.begin());
  
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, xvec_h.begin(), xvec_h.end(), xvec_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, yvec_h.begin(), yvec_h.end(), yvec_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, zvec_h.begin(), zvec_h.end(), zvec_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, rho_h.begin(), rho_h.end(), rho_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, theta_h.begin(), theta_h.end(), theta_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, uvel_h.begin(), uvel_h.end(), uvel_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, vvel_h.begin(), vvel_h.end(), vvel_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, wvel_h.begin(), wvel_h.end(), wvel_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qv_h.begin(), qv_h.end(), qv_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qc_h.begin(), qc_h.end(), qc_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qr_h.begin(), qr_h.end(), qr_d.begin());
  
     amrex::Gpu::streamSynchronize();
  
     Real* latvec_d_ptr = latvec_d.data();
     Real* lonvec_d_ptr = lonvec_d.data();
     Real* xvec_d_ptr = xvec_d.data();
     Real* yvec_d_ptr = yvec_d.data();
     Real* zvec_d_ptr = zvec_d.data();
     Real* rho_d_ptr   = rho_d.data();
     Real* uvel_d_ptr  = uvel_d.data();
     Real* vvel_d_ptr  = vvel_d.data();
     Real* wvel_d_ptr  = wvel_d.data();
     Real* theta_d_ptr = theta_d.data();
     Real* qv_d_ptr = qv_d.data();
     Real* qc_d_ptr = qc_d.data();
     Real* qr_d_ptr = qr_d.data();
  
     MultiFab& erf_mf_cons   = forecast_state[lev][Vars::cons];
     MultiFab& erf_mf_xvel   = forecast_state[lev][Vars::xvel];
     MultiFab& erf_mf_yvel   = forecast_state[lev][Vars::yvel];
     MultiFab& erf_mf_zvel   = forecast_state[lev][Vars::zvel];
     MultiFab& erf_mf_latlon = forecast_state[lev][4];
  
     erf_mf_cons.setVal(0.0);
     erf_mf_xvel.setVal(0.0);
     erf_mf_yvel.setVal(0.0);
     erf_mf_zvel.setVal(0.0);
     erf_mf_latlon.setVal(0.0);
  
     // Interpolate the data on to the ERF mesh
  
      for (MFIter mfi(erf_mf_cons); mfi.isValid(); ++mfi) {
         const auto z_arr    = (a_z_phys_nd) ? a_z_phys_nd->const_array(mfi) :
                                             Array4<const Real> {};
         const Array4<Real> &fine_cons_arr = erf_mf_cons.array(mfi);
         const Array4<Real> &fine_xvel_arr = erf_mf_xvel.array(mfi);
         const Array4<Real> &fine_yvel_arr = erf_mf_yvel.array(mfi);
         const Array4<Real> &fine_zvel_arr = erf_mf_zvel.array(mfi);
         const Array4<Real> &fine_latlon_arr = erf_mf_latlon.array(mfi);
  
  
         const Box& gbx = mfi.growntilebox(); // tilebox + ghost cells
  
         const Box &gtbx = mfi.tilebox(IntVect(1,0,0));
         const Box &gtby = mfi.tilebox(IntVect(0,1,0));
         const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
         const auto prob_lo  = geom[lev].ProbLoArray();
         const auto dx       = geom[lev].CellSizeArray();
        //const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
  
         ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
             // Geometry (note we must include these here to get the data on device)
             const Real x        = prob_lo[0] + (i + 0.5) * dx[0];
             const Real y        = prob_lo[1] + (j + 0.5) * dx[1];
             //const Real z        = prob_lo[2] + (k + 0.5) * dx[2];
             const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/2.0;
  
             // First interpolate where the weather data is available from
             Real tmp_rho, tmp_theta, tmp_qv, tmp_qc, tmp_qr, tmp_lat, tmp_lon;
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, nz,
                                    x, y, z,
                                    rho_d_ptr, tmp_rho);
  
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, nz,
                                    x, y, z,
                                    theta_d_ptr, tmp_theta);
  
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, nz,
                                    x, y, z,
                                    qv_d_ptr, tmp_qv);
  
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, nz,
                                    x, y, z,
                                    qc_d_ptr, tmp_qc);
  
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, nz,
                                    x, y, z,
                                    qr_d_ptr, tmp_qr);
  
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, 1,
                                    x, y, 0.0,
                                    latvec_d_ptr, tmp_lat);
  
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, 1,
                                    x, y, 0.0,
                                    lonvec_d_ptr, tmp_lon);
  
             fine_cons_arr(i,j,k,Rho_comp) = tmp_rho;
             fine_latlon_arr(i,j,k,0) = tmp_lat;
             fine_latlon_arr(i,j,k,1) = tmp_lon;
         });
  
         ParallelFor(gtbx, gtby, gtbz,
         [=] AMREX_GPU_DEVICE(int i, int j, int k) {
              // Physical location of the fine node
             Real x = prob_lo_erf[0] + i       * dx_erf[0];
             Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
             //Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
             const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/2.0;
  
             Real tmp_uvel;
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, nz,
                                    x, y, z,
                                    uvel_d_ptr, tmp_uvel);
  
             fine_xvel_arr(i, j, k, 0) = tmp_uvel;
         },
         [=] AMREX_GPU_DEVICE(int i, int j, int k) {
              // Physical location of the fine node
             Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
             Real y = prob_lo_erf[1] + j       * dx_erf[1];
             //Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
             const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/2.0;
  
             Real tmp_vvel;
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, nz,
                                    x, y, z,
                                    vvel_d_ptr, tmp_vvel);
  
             fine_yvel_arr(i, j, k, 0) = tmp_vvel;
         },
         [=] AMREX_GPU_DEVICE(int i, int j, int k) {
              // Physical location of the fine node
             Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
             Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
             Real z = prob_lo_erf[2] + k       * dx_erf[2];
             //const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/2.0;
  
             Real tmp_wvel;
             bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
                                    dxvec, dyvec,
                                    nx, ny, nz,
                                    x, y, z,
                                    wvel_d_ptr, tmp_wvel);
  
             fine_zvel_arr(i, j, k, 0) = tmp_wvel;
         });
     }
  
     /*Vector<std::string> varnames = {
     "rho", "uvel", "vvel", "wvel", "theta", "qv", "qc", "qr"
     }; // Customize variable names
  
      Vector<std::string> varnames_cons = {
     "rho", "rhotheta", "ke", "sc", "rhoqv", "rhoqc", "rhoqr"
     }; // Customize variable names
  
     Vector<std::string> varnames_plot_mf = {
     "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
     }; // Customize variable names
  
     const Real time = 0.0;
  
     std::string pltname = "plt_interp";
  
     MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
                      10, 0);
  
     plot_mf.setVal(0.0);
  
     for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
         const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
         const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
         const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
         const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
         const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
         const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
  
         const Box& bx = mfi.validbox();
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
             plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
             plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
             plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
             plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
             plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
  
             plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/2.0;
             plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/2.0;
             plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/2.0;
  
             plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
             plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
         });
     }
  
  
     WriteSingleLevelPlotfile(
             pltname,
             plot_mf,
             varnames_plot_mf,
             geom[0],
             time,
             0 // level
         );*/
 }

Here is the call graph for this function:

◆ FillIntermediatePatch()

void ERF::FillIntermediatePatch	(	int	lev,
		amrex::Real	time,
		const amrex::Vector< amrex::MultiFab * > &	mfs_vel,
		const amrex::Vector< amrex::MultiFab * > &	mfs_mom,
		int	ng_cons,
		int	ng_vel,
		bool	cons_only,
		int	icomp_cons,
		int	ncomp_cons
	)

private

 {
     BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
     Interpolater* mapper;
  
     PhysBCFunctNoOp null_bc;
  
     //
     // ***************************************************************************
     // The first thing we do is interpolate the momenta on the "valid" faces of
     // the fine grids (where the interface is coarse/fine not fine/fine) -- this
     // will not be over-written by interpolation below because the FillPatch
     // operators see these as valid faces.  But we must have these interpolated
     // values in the fine data before we call FillPatchTwoLevels.
     //
     // Also -- note that we might be filling values by interpolation at physical boundaries
     //         here but that's ok because we will overwrite those values when we impose
     //         the physical bc's below
     // ***************************************************************************
     if (lev>0) {
         if (cf_set_width > 0) {
             // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
             FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
         }
         if ( !cons_only && (cf_set_width >= 0) ) {
             FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
             FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
             FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
         }
     }
  
     // amrex::Print() << "LEVEL " << lev << " CONS ONLY   " << cons_only <<
     //                   " ICOMP NCOMP " << icomp_cons << " " << ncomp_cons << " NGHOST " << ng_cons << std::endl;
  
     if (!cons_only) {
         AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
         AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
     }
  
     // Enforce no penetration for thin immersed body
     if (!cons_only) {
         // Enforce no penetration for thin immersed body
         if (xflux_imask[lev]) {
             ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
         }
         if (yflux_imask[lev]) {
             ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
         }
         if (zflux_imask[lev]) {
             ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
         }
     }
  
     //
     // We now start working on conserved quantities + VELOCITY
     //
     if (lev == 0)
     {
         // We don't do anything here because we will call the physbcs routines below,
         // which calls FillBoundary and fills other domain boundary conditions
         // Physical boundaries will be filled below
  
         if (!cons_only)
         {
             // ***************************************************************************
             // We always come in to this call with updated momenta but we need to create updated velocity
             //    in order to impose the rest of the bc's
             // ***************************************************************************
             const MultiFab* c_vfrac = nullptr;
             if (solverChoice.terrain_type == TerrainType::EB) {
                 c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
             }
  
             // This only fills VALID region of velocity
             MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                                *mfs_vel[Vars::cons],
                                *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                                 Geom(lev).Domain(), domain_bcs_type, c_vfrac);
         }
     }
     else
     {
         //
         // We must fill a temporary then copy it back so we don't double add/subtract
         //
         MultiFab mf(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
                     mfs_vel[Vars::cons]->nComp()   ,mfs_vel[Vars::cons]->nGrowVect());
         //
         // Set all components to 1.789e19, then copy just the density from *mfs_vel[Vars::cons]
         //
         mf.setVal(1.789e19);
         MultiFab::Copy(mf,*mfs_vel[Vars::cons],Rho_comp,Rho_comp,1,mf.nGrowVect());
  
         Vector<MultiFab*> fmf = {mfs_vel[Vars::cons],mfs_vel[Vars::cons]};
         Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
         Vector<Real> ctime    = {t_old[lev-1], t_new[lev-1]};
         Vector<Real> ftime    = {time,time};
  
         if (interpolation_type == StateInterpType::Perturbational)
         {
             if (icomp_cons+ncomp_cons > 1)
             {
                 // Divide (rho theta) by rho to get theta
                 MultiFab::Divide(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
  
                 // Subtract theta_0 from theta
                 MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
  
                 if (!amrex::almostEqual(time,ctime[1])) {
                     MultiFab::Divide(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                                      Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                     MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                        BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                 }
                 if (!amrex::almostEqual(time,ctime[0])) {
                     MultiFab::Divide(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                                      Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                     MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                        BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                 }
             }
  
             // Subtract rho_0 from rho before we interpolate -- note we only subtract
             //    on valid region of mf since the ghost cells will be filled below
             if (icomp_cons == 0)
             {
                 MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
  
                 if (!amrex::almostEqual(time,ctime[1])) {
                     MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                        BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                 }
                 if (!amrex::almostEqual(time,ctime[0])) {
                     MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                        BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                 }
             }
         } // interpolation_type == StateInterpType::Perturbational
  
         // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
         mapper = &cell_cons_interp;
         FillPatchTwoLevels(mf, IntVect{ng_cons}, IntVect(0,0,0),
                            time, cmf, ctime, fmf, ftime,
                            icomp_cons, icomp_cons, ncomp_cons, geom[lev-1], geom[lev],
                            refRatio(lev-1), mapper, domain_bcs_type,
                            icomp_cons);
  
         if (interpolation_type == StateInterpType::Perturbational)
         {
             if (icomp_cons == 0)
             {
                 // Restore the coarse values to what they were
                 if (!amrex::almostEqual(time,ctime[1])) {
                     MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                   BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                 }
                 if (!amrex::almostEqual(time,ctime[0])) {
                     MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                   BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                 }
  
                 // Set values in the cells outside the domain boundary so that we can do the Add
                 //     without worrying about uninitialized values outside the domain -- these
                 //     will be filled in the physbcs call
                 mf.setDomainBndry(1.234e20,Rho_comp,1,geom[lev]);
  
                 // Add rho_0 back to rho after we interpolate -- on all the valid + ghost region
                 MultiFab::Add(mf, base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{ng_cons});
             }
  
             if (icomp_cons+ncomp_cons > 1)
             {
                 // Add theta_0 to theta
                 if (!amrex::almostEqual(time,ctime[1])) {
                     MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                                   BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                     MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                                        Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
                 }
                 if (!amrex::almostEqual(time,ctime[0])) {
                     MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                                   BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                     MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                                        Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
                 }
  
                 // Multiply theta by rho to get (rho theta)
                 MultiFab::Multiply(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
  
                 // Add theta_0 to theta
                 MultiFab::Add(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
  
                 // Add theta_0 back to theta
                 MultiFab::Add(mf,base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{ng_cons});
  
                 // Multiply (theta) by rho to get (rho theta)
                 MultiFab::Multiply(mf,mf,Rho_comp,RhoTheta_comp,1,IntVect{ng_cons});
             }
         } // interpolation_type == StateInterpType::Perturbational
  
         // Impose physical bc's on fine data (note time and 0 are not used)
         // Note that we do this after the FillPatch because imposing physical bc's on fine ghost
         //      cells that need to be filled from coarse requires that we have done the interpolation first
         bool do_fb = true; bool do_terrain_adjustment = false;
         (*physbcs_cons[lev])(mf,*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                              icomp_cons,ncomp_cons,IntVect{ng_cons},time,BCVars::cons_bc,
                              do_fb, do_terrain_adjustment);
  
         // Make sure to only copy back the components we worked on
         MultiFab::Copy(*mfs_vel[Vars::cons],mf,icomp_cons,icomp_cons,ncomp_cons,IntVect{ng_cons});
  
         // *****************************************************************************************
  
         if (!cons_only)
         {
             // ***************************************************************************
             // We always come in to this call with updated momenta but we need to create updated velocity
             //    in order to impose the rest of the bc's
             // ***************************************************************************
             const MultiFab* c_vfrac = nullptr;
             if (solverChoice.terrain_type == TerrainType::EB) {
                 c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
             }
  
             // This only fills VALID region of velocity
             MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                                *mfs_vel[Vars::cons],
                                *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                                 Geom(lev).Domain(), domain_bcs_type, c_vfrac);
  
             mapper = &face_cons_linear_interp;
  
             //
             // NOTE: All interpolation here happens on velocities not momenta;
             //       note we only do the interpolation and FillBoundary here,
             //       physical bc's are imposed later
             //
             // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
             //       unlike the FillSet calls above which filled momenta on the coarse/fine bdy
             //
  
             MultiFab& mfu = *mfs_vel[Vars::xvel];
  
             fmf = {&mfu,&mfu};
             cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
  
             // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
             FillPatchTwoLevels(mfu, IntVect{ng_vel}, IntVect(0,0,0),
                                time, cmf, ctime, fmf, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                refRatio(lev-1), mapper, domain_bcs_type,
                                BCVars::xvel_bc);
  
             // *****************************************************************************************
  
             MultiFab& mfv = *mfs_vel[Vars::yvel];
  
             fmf = {&mfv,&mfv};
             cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
  
             // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
             FillPatchTwoLevels(mfv, IntVect{ng_vel}, IntVect(0,0,0),
                                time, cmf, ctime, fmf, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                refRatio(lev-1), mapper, domain_bcs_type,
                                BCVars::yvel_bc);
  
             // *****************************************************************************************
  
             MultiFab& mfw = *mfs_vel[Vars::zvel];
  
             fmf = {&mfw,&mfw};
             cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
  
             // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
             FillPatchTwoLevels(mfw, IntVect{ng_vel}, IntVect(0,0,0),
                                time, cmf, ctime, fmf, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                refRatio(lev-1), mapper, domain_bcs_type,
                                BCVars::zvel_bc);
         } // !cons_only
     } // lev > 0
  
     // ***************************************************************************
     // Physical bc's at domain boundary
     // ***************************************************************************
     IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
     IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
  
     bool do_fb = true;
  
     if (m_r2d) fill_from_bndryregs(mfs_vel,time);
  
     // We call this even if use_real_bcs is true because these will fill the vertical bcs
     (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
     if (!cons_only) {
         (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                           ngvect_vels,time,BCVars::xvel_bc, do_fb);
         (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                           ngvect_vels,time,BCVars::yvel_bc, do_fb);
         (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                           ngvect_vels,time,BCVars::zvel_bc, do_fb);
     }
     // ***************************************************************************
  
     // We always come in to this call with momenta so we need to leave with momenta!
     // We need to make sure we convert back on all ghost cells/faces because this is
     // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
     if (!cons_only)
     {
         IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
         IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
         IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
  
         const MultiFab* c_vfrac = nullptr;
         if (solverChoice.terrain_type == TerrainType::EB) {
             c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
         }
  
         VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
                            *mfs_vel[Vars::yvel], ngv,
                            *mfs_vel[Vars::zvel], ngw,
                            *mfs_vel[Vars::cons],
                            *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
                            Geom(lev).Domain(),
                            domain_bcs_type, c_vfrac);
     }
  
     // NOTE: There are not FillBoundary calls here for the following reasons:
     // Removal of the FillBoundary (FB) calls has bee completed for the following reasons:
     //
     // 1. physbc_cons is called before VelocityToMomentum and a FB is completed in that functor.
     //    Therefore, the conserved CC vars have their inter-rank ghost cells filled and then their
     //    domain ghost cells filled from the BC operations. We should not call FB on this MF again.
     //
     // 2. physbc_u/v/w is also called before VelocityToMomentum and a FB is completed those functors.
     //    Furthermore, VelocityToMomentum operates on a growntilebox so we exit that routine with momentum
     //    filled everywhere---i.e., physbc_u/v/w fills velocity ghost cells (inter-rank and domain)
     //    and then V2M does the conversion to momenta everywhere; so there is again no need to do a FB on momenta.
 }

Here is the call graph for this function:

◆ FillPatchCrseLevel()

void ERF::FillPatchCrseLevel	(	int	lev,
		amrex::Real	time,
		const amrex::Vector< amrex::MultiFab * > &	mfs_vel,
		bool	cons_only = `false`
	)

private

 {
     BL_PROFILE_VAR("ERF::FillPatchCrseLevel()",ERF_FillPatchCrseLevel);
  
     AMREX_ALWAYS_ASSERT(lev == 0);
  
     IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
     IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
  
     Vector<Real> ftime    = {t_old[lev], t_new[lev]};
  
     //
     // Below we call FillPatchSingleLevel which does NOT fill ghost cells outside the domain
     //
  
     Vector<MultiFab*> fmf;
     Vector<MultiFab*> fmf_u;
     Vector<MultiFab*> fmf_v;
     Vector<MultiFab*> fmf_w;
  
     if (amrex::almostEqual(time,ftime[0])) {
         fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
     } else if (amrex::almostEqual(time,ftime[1])) {
         fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
     } else {
         fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
     }
  
     const int  ncomp = mfs_vel[Vars::cons]->nComp();
  
     FillPatchSingleLevel(*mfs_vel[Vars::cons], ngvect_cons, time, fmf, IntVect(0,0,0), ftime,
                          0, 0, ncomp, geom[lev]);
  
     if (!cons_only) {
         if (amrex::almostEqual(time,ftime[0])) {
             fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
             fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
             fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
         } else if (amrex::almostEqual(time,ftime[1])) {
             fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
             fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
             fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
         } else {
             fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
             fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
             fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
         }
         FillPatchSingleLevel(*mfs_vel[Vars::xvel], ngvect_vels, time, fmf_u,
                              IntVect(0,0,0), ftime,  0, 0, 1, geom[lev]);
  
         FillPatchSingleLevel(*mfs_vel[Vars::yvel], ngvect_vels, time, fmf_v,
                              IntVect(0,0,0), ftime,  0, 0, 1, geom[lev]);
  
         FillPatchSingleLevel(*mfs_vel[Vars::zvel], ngvect_vels, time, fmf_w,
                              IntVect(0,0,0), ftime,  0, 0, 1, geom[lev]);
     } // !cons_only
  
     // ***************************************************************************
     // Physical bc's at domain boundary
     // ***************************************************************************
     int icomp_cons = 0;
     int ncomp_cons = mfs_vel[Vars::cons]->nComp();
  
     bool do_fb = true;
  
     if (m_r2d) fill_from_bndryregs(mfs_vel,time);
  
     // We call this even if use_real_bcs is true because these will fill the vertical bcs
     // Note that we call FillBoundary inside the physbcs call
     (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                           icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
     if (!cons_only) {
         (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                           ngvect_vels,time,BCVars::xvel_bc, do_fb);
         (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                           ngvect_vels,time,BCVars::yvel_bc, do_fb);
         (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                           ngvect_vels,time,BCVars::zvel_bc, do_fb);
     }
 }

Here is the call graph for this function:

◆ FillPatchFineLevel()

void ERF::FillPatchFineLevel	(	int	lev,
		amrex::Real	time,
		const amrex::Vector< amrex::MultiFab * > &	mfs_vel,
		const amrex::Vector< amrex::MultiFab * > &	mfs_mom,
		const amrex::MultiFab &	old_base_state,
		const amrex::MultiFab &	new_base_state,
		bool	fillset = `true`,
		bool	cons_only = `false`
	)

private

 {
     BL_PROFILE_VAR("ERF::FillPatchFineLevel()",ERF_FillPatchFineLevel);
  
     AMREX_ALWAYS_ASSERT(lev > 0);
  
     Interpolater* mapper = nullptr;
  
     PhysBCFunctNoOp null_bc;
  
     //
     // ***************************************************************************
     // The first thing we do is interpolate the momenta on the "valid" faces of
     // the fine grids (where the interface is coarse/fine not fine/fine) -- this
     // will not be over-written below because the FillPatch operators see these as
     // valid faces.
     //
     // Note that we interpolate momentum not velocity, but all the other boundary
     // conditions are imposed on velocity, so we convert to momentum here then
     // convert back.
     // ***************************************************************************
     if (fillset) {
         if (cf_set_width > 0) {
             FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
         }
         if (cf_set_width >= 0 && !cons_only) {
  
             const MultiFab* c_vfrac = nullptr;
             if (solverChoice.terrain_type == TerrainType::EB) {
                 c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
             }
  
             VelocityToMomentum(*mfs_vel[Vars::xvel], IntVect{0},
                                *mfs_vel[Vars::yvel], IntVect{0},
                                *mfs_vel[Vars::zvel], IntVect{0},
                                *mfs_vel[Vars::cons],
                                *mfs_mom[IntVars::xmom],
                                *mfs_mom[IntVars::ymom],
                                *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(),
                                domain_bcs_type, c_vfrac);
  
             FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
             FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
             FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
  
             MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
                                *mfs_vel[Vars::cons],
                                *mfs_mom[IntVars::xmom],
                                *mfs_mom[IntVars::ymom],
                                *mfs_mom[IntVars::zmom],
                                Geom(lev).Domain(),
                                domain_bcs_type, c_vfrac);
         }
     }
  
     IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
     IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
  
     Vector<Real> ftime    = {t_old[lev  ], t_new[lev  ]};
     Vector<Real> ctime    = {t_old[lev-1], t_new[lev-1]};
  
     amrex::Real small_dt = 1.e-8 * (ftime[1] - ftime[0]);
  
     Vector<MultiFab*> fmf;
     if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
         fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
     } else if (amrex::almostEqual(time,ftime[1])) {
         fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
     } else {
         fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
     }
     Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
  
     // We must fill a temporary then copy it back so we don't double add/subtract
     MultiFab mf_c(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
                   mfs_vel[Vars::cons]->nComp()   ,mfs_vel[Vars::cons]->nGrowVect());
  
     mapper = &cell_cons_interp;
  
     if (interpolation_type == StateInterpType::Perturbational)
     {
         // Divide (rho theta) by rho to get theta (before we subtract rho0 from rho!)
         if (!amrex::almostEqual(time,ctime[1])) {
             MultiFab::Divide(vars_old[lev-1][Vars::cons],vars_old[lev-1][Vars::cons],
                              Rho_comp,RhoTheta_comp,1,ngvect_cons);
             MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
                                BaseState::r0_comp,Rho_comp,1,ngvect_cons);
             MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
                                BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
         }
         if (!amrex::almostEqual(time,ctime[0])) {
             MultiFab::Divide(vars_new[lev-1][Vars::cons],vars_new[lev-1][Vars::cons],
                              Rho_comp,RhoTheta_comp,1,ngvect_cons);
             MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
                                BaseState::r0_comp,Rho_comp,1,ngvect_cons);
             MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
                                BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
         }
  
         if (!amrex::almostEqual(time,ftime[1])) {
             MultiFab::Divide(vars_old[lev  ][Vars::cons],vars_old[lev  ][Vars::cons],
                              Rho_comp,RhoTheta_comp,1,IntVect{0});
             MultiFab::Subtract(vars_old[lev  ][Vars::cons],old_base_state,
                                BaseState::r0_comp,Rho_comp,1,IntVect{0});
             MultiFab::Subtract(vars_old[lev  ][Vars::cons],old_base_state,
                                BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
         }
         if (!amrex::almostEqual(time,ftime[0])) {
             MultiFab::Divide(vars_new[lev  ][Vars::cons],vars_new[lev  ][Vars::cons],
                              Rho_comp,RhoTheta_comp,1,IntVect{0});
             MultiFab::Subtract(vars_new[lev  ][Vars::cons],old_base_state,
                                BaseState::r0_comp,Rho_comp,1,IntVect{0});
             MultiFab::Subtract(vars_new[lev  ][Vars::cons],old_base_state,
                                BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
         }
     }
  
     // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
     FillPatchTwoLevels(mf_c, ngvect_cons, IntVect(0,0,0),
                        time, cmf, ctime, fmf, ftime,
                        0, 0, mf_c.nComp(), geom[lev-1], geom[lev],
                        refRatio(lev-1), mapper, domain_bcs_type,
                        BCVars::cons_bc);
  
     if (interpolation_type == StateInterpType::Perturbational)
     {
         // Restore the coarse values to what they were
         if (!amrex::almostEqual(time,ctime[1])) {
             MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                           BaseState::r0_comp,Rho_comp,1,ngvect_cons);
             MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
                           BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
             MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
                                Rho_comp,RhoTheta_comp,1,ngvect_cons);
         }
         if (!amrex::almostEqual(time,ctime[0])) {
             MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                           BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
             MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
                           BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
             MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
                                Rho_comp,RhoTheta_comp,1,ngvect_cons);
         }
  
         if (!amrex::almostEqual(time,ftime[1])) {
             MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev  ],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
             MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev  ],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
             MultiFab::Multiply(vars_old[lev][Vars::cons], vars_old[lev][Vars::cons],
                                    Rho_comp,RhoTheta_comp,1,ngvect_cons);
         }
         if (!amrex::almostEqual(time,ftime[0])) {
             MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
             MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
             MultiFab::Multiply(vars_new[lev][Vars::cons], vars_new[lev][Vars::cons],
                                Rho_comp,RhoTheta_comp,1,ngvect_cons);
         }
  
         // Set values in the cells outside the domain boundary so that we can do the Add
         //     without worrying about uninitialized values outside the domain -- these
         //     will be filled in the physbcs call
         mf_c.setDomainBndry(1.234e20,0,2,geom[lev]); // Do both rho and (rho theta) together
  
         // Add rho_0 back to rho and theta_0 back to theta
         MultiFab::Add(mf_c, new_base_state,BaseState::r0_comp,Rho_comp,1,ngvect_cons);
         MultiFab::Add(mf_c, new_base_state,BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
  
         // Multiply (theta) by rho to get (rho theta)
         MultiFab::Multiply(mf_c,mf_c,Rho_comp,RhoTheta_comp,1,ngvect_cons);
     }
  
     MultiFab::Copy(*mfs_vel[Vars::cons],mf_c,0,0,mf_c.nComp(),mf_c.nGrowVect());
  
     // ***************************************************************************************
  
     if (!cons_only)
     {
             mapper = &face_cons_linear_interp;
  
             MultiFab& mf_u = *mfs_vel[Vars::xvel];
             MultiFab& mf_v = *mfs_vel[Vars::yvel];
             MultiFab& mf_w = *mfs_vel[Vars::zvel];
  
             Vector<MultiFab*> fmf_u; Vector<MultiFab*> fmf_v; Vector<MultiFab*> fmf_w;
             Vector<MultiFab*> cmf_u; Vector<MultiFab*> cmf_v; Vector<MultiFab*> cmf_w;
  
             // **********************************************************************
  
             if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
                 fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
                 fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
                 fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
             } else if ( amrex::almostEqual(time,ftime[1]) ) {
                 fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
                 fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
                 fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
             } else {
                 fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
                 fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
                 fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
             }
             cmf_u = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
             cmf_v = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
             cmf_w = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
  
             // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
             FillPatchTwoLevels(mf_u, ngvect_vels, IntVect(0,0,0),
                                time, cmf_u, ctime, fmf_u, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                refRatio(lev-1), mapper, domain_bcs_type,
                                BCVars::xvel_bc);
  
             FillPatchTwoLevels(mf_v, ngvect_vels, IntVect(0,0,0),
                                time, cmf_v, ctime, fmf_v, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                refRatio(lev-1), mapper, domain_bcs_type,
                                BCVars::yvel_bc);
  
             // We put these here because these may be used in constructing omega outside the
             // domain when fillpatching w
             bool do_fb = true;
             (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                               ngvect_vels,time,BCVars::xvel_bc, do_fb);
             (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                               ngvect_vels,time,BCVars::yvel_bc, do_fb);
  
             // **********************************************************************
  
             // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
             FillPatchTwoLevels(mf_w, ngvect_vels, IntVect(0,0,0),
                                time, cmf_w, ctime, fmf_w, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                refRatio(lev-1), mapper, domain_bcs_type,
                                BCVars::zvel_bc);
     } // !cons_only
  
     // ***************************************************************************
     // Physical bc's at domain boundary
     // ***************************************************************************
     int icomp_cons = 0;
     int ncomp_cons = mfs_vel[Vars::cons]->nComp();
  
     bool do_fb = true;
  
     if (m_r2d) fill_from_bndryregs(mfs_vel,time);
  
     // We call this even if use_real_bcs is true because these will fill the vertical bcs
     // Note that we call FillBoundary inside the physbcs call
     (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                          icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
     if (!cons_only) {
         // Note that we need to fill u and v in the case of terrain because we will use
         //      these in the call of WFromOmega in lateral ghost cells of the fine grid
         // (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
         //                   ngvect_vels,time,BCVars::xvel_bc, do_fb);
         // (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
         //                   ngvect_vels,time,BCVars::yvel_bc, do_fb);
         (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
                           ngvect_vels,time,BCVars::zvel_bc, do_fb);
     }
 }

Here is the call graph for this function:

◆ FillSurfaceStateMultiFabs()

void ERF::FillSurfaceStateMultiFabs	(	const int	lev,
		const std::string &	filename,
		amrex::Vector< amrex::MultiFab > &	surface_state
	)

 {
        // Open the binary file in input mode
     std::ifstream infile(filename, std::ios::binary);
     if (!infile) {
         std::cerr << "Error: Could not open file " << filename << std::endl;
     }
     Vector<Real> xvec_h, yvec_h, zvec_h;
     Vector<Real> sst_h, q_star_h, t_star_h, u_star_h, ls_mask_h;
  
     int nx, ny, nz, ndata;
     float value;
  
     // Read the four integers
     infile.read(reinterpret_cast<char*>(&nx), sizeof(int));
     infile.read(reinterpret_cast<char*>(&ny), sizeof(int));
     infile.read(reinterpret_cast<char*>(&nz), sizeof(int));
     infile.read(reinterpret_cast<char*>(&ndata), sizeof(int));
  
     amrex::Gpu::DeviceVector<Real> xvec_d(nx*ny*nz), yvec_d(nx*ny*nz), zvec_d(nz);
     for(int i=0; i<nx; i++) {
         infile.read(reinterpret_cast<char*>(&value), sizeof(float));
         xvec_h.emplace_back(value);
     }
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, xvec_h.begin(), xvec_h.end(), xvec_d.begin());
  
     for(int j=0; j<ny; j++) {
         infile.read(reinterpret_cast<char*>(&value), sizeof(float));
         yvec_h.emplace_back(value);
     }
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, yvec_h.begin(), yvec_h.end(), yvec_d.begin());
  
     for(int k=0; k<nz; k++) {
         infile.read(reinterpret_cast<char*>(&value), sizeof(float));
         zvec_h.emplace_back(value);
     }
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, zvec_h.begin(), zvec_h.end(), zvec_d.begin());
  
          // Vector to store the data
  
     Vector<Real>* data_h = nullptr; // Declare pointer outside the loop
  
     Real* xvec_d_ptr = xvec_d.data();
     Real* yvec_d_ptr = yvec_d.data();
  
     Real dxvec = (xvec_h[nx-1]-xvec_h[0])/(nx-1);
     Real dyvec = (yvec_h[ny-1]-yvec_h[0])/(ny-1);
  
     // Read the file
     for(int idx=0; idx<ndata; idx++){
         if(idx == 0){
             data_h = &sst_h;
         } else if (idx==1) {
             data_h = &q_star_h;
         } else if (idx==2) {
             data_h = &t_star_h;
         } else if (idx==3) {
             data_h = &u_star_h;
         } else if(idx==4) {
             data_h = &ls_mask_h;
         }
         for(int k=0; k<nz; k++) {
             for(int j=0; j<ny; j++) {
                 for(int i=0; i<nx; i++) {
                     infile.read(reinterpret_cast<char*>(&value), sizeof(float));
                     //if(idx == 3) {
                         //printf("theta is %0.15g, %0.15g, %0.15g %0.15g\n", xvec_h[i], yvec_h[j], zvec_h[k], value);
                     //}
                     data_h->emplace_back(value);
                 }
             }
         }
     }
  
     infile.close();
  
     amrex::Gpu::DeviceVector<Real> ls_mask_d(nx*ny*nz), sst_d(nx*ny*nz);
  
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, ls_mask_h.begin(), ls_mask_h.end(), ls_mask_d.begin());
     amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, sst_h.begin(), sst_h.end(), sst_d.begin());
  
     Real* ls_mask_d_ptr = ls_mask_d.data();
     Real* sst_d_ptr   = sst_d.data();
  
     const auto prob_lo  = geom[lev].ProbLo();
     const auto dx       = geom[lev].CellSize();
  
     for (amrex::MFIter mfi(surface_state[lev]); mfi.isValid(); ++mfi) {
         const Box gbx = mfi.growntilebox();
         const Array4<Real>& surf_arr = surface_state[lev].array(mfi);
  
         ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
  
             if(k == 0) {
                 const Real x        = prob_lo[0] + (i + 0.5) * dx[0];
                 const Real y        = prob_lo[1] + (j + 0.5) * dx[1];
  
                 // First interpolate where the weather data is available from
                 Real tmp_ls_mask, tmp_sst;
  
                 bilinear_interpolation_2d(xvec_d_ptr, yvec_d_ptr,
                                           dxvec, dyvec,
                                           nx, ny,
                                           x, y,
                                           ls_mask_d_ptr, tmp_ls_mask);
  
                 bilinear_interpolation_2d(xvec_d_ptr, yvec_d_ptr,
                                           dxvec, dyvec,
                                           nx, ny,
                                           x, y,
                                           sst_d_ptr, tmp_sst);
  
                 surf_arr(i, j, 0) = std::min(tmp_ls_mask, 1.0);
                 surf_arr(i, j, 1) = tmp_sst;
             }
         });
     }
  
 }

Here is the call graph for this function:

◆ FindInitialEye()

bool ERF::FindInitialEye	(	int	lev,
		const amrex::MultiFab &	cc_vel,
		const amrex::Real	velmag_threshold,
		amrex::Real &	eye_x,
		amrex::Real &	eye_y
	)

 {
     const auto dx = geom[levc].CellSizeArray();
     const auto prob_lo = geom[levc].ProbLoArray();
  
     Gpu::DeviceVector<Real> d_coords(2, 0.0);
     Gpu::DeviceVector<int>  d_found(1,0);
  
     Real* d_coords_ptr = d_coords.data();
     int*   d_found_ptr = d_found.data();
  
     for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi)
     {
         const Box& box = mfi.validbox();
         const Array4<const Real>& vel_arr = mf_cc_vel.const_array(mfi);
  
         ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k)
         {
             Real magnitude = std::sqrt(vel_arr(i,j,k,0) * vel_arr(i,j,k,0) +
                                        vel_arr(i,j,k,1) * vel_arr(i,j,k,1) +
                                        vel_arr(i,j,k,2) * vel_arr(i,j,k,2));
  
             magnitude *= 3.6;
  
             Real z = prob_lo[2] + (k + 0.5) * dx[2];
  
             // Check if magnitude exceeds threshold
             if (z < 2000. && magnitude > velmag_threshold) {
                 // Use atomic operations to set found flag and store coordinates
                 Gpu::Atomic::Add(&d_found_ptr[0], 1); // Mark as found
  
                 Real x = prob_lo[0] + (i + 0.5) * dx[0];
                 Real y = prob_lo[1] + (j + 0.5) * dx[1];
  
                 // Store coordinates
                 Gpu::Atomic::Add(&d_coords_ptr[0],x); // Store x index
                 Gpu::Atomic::Add(&d_coords_ptr[1],y); // Store x index
             }
         });
     }
  
     // Synchronize to ensure all threads complete their execution
     amrex::Gpu::streamSynchronize(); // Wait for all GPU threads to finish
  
     Vector<int> h_found(1,0);
     Gpu::copy(Gpu::deviceToHost, d_found.begin(), d_found.end(), h_found.begin());
     ParallelAllReduce::Sum(h_found.data(), h_found.size(), ParallelContext::CommunicatorAll());
  
     // Broadcast coordinates if found
     if (h_found[0] > 0) {
         Vector<Real> h_coords(2,-1e10);
         Gpu::copy(Gpu::deviceToHost, d_coords.begin(), d_coords.end(), h_coords.begin());
  
         ParallelAllReduce::Sum(h_coords.data(), h_coords.size(), ParallelContext::CommunicatorAll());
  
         eye_x = h_coords[0]/h_found[0];
         eye_y = h_coords[1]/h_found[0];
  
     } else {
         // Random large negative numbers so we don't trigger refinement in this case
         eye_x = -1.e20;
         eye_y = -1.e20;
     }
  
     return (h_found[0] > 0);
 }

Here is the call graph for this function:

◆ get_eb()

eb_ const& ERF::get_eb ( int lev ) const

inlineprivatenoexcept

                                                              {
         AMREX_ASSERT(lev >= 0 && lev < eb.size() && eb[lev] != nullptr);
         return *eb[lev];
     }

◆ getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int lev )

inlineprivate

     {
         return advflux_reg[lev];
     }

◆ getCPUTime()

static amrex::Real ERF::getCPUTime ( )

inlinestaticprivate

     {
       int numCores = amrex::ParallelDescriptor::NProcs();
 #ifdef _OPENMP
       numCores = numCores * omp_get_max_threads();
 #endif
  
       amrex::Real T =
         numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
         previousCPUTimeUsed;
  
       return T;
     }

◆ GotoNextLine()

void ERF::GotoNextLine ( std::istream & is )

staticprivate

Utility to skip to next line in Header file input stream.

 {
     constexpr std::streamsize bl_ignore_max { 100000 };
     is.ignore(bl_ignore_max, '\n');
 }

◆ HurricaneTracker()

void ERF::HurricaneTracker	(	int	lev,
		amrex::Real	time,
		const amrex::MultiFab &	cc_vel,
		const amrex::Real	velmag_threshold,
		amrex::TagBoxArray *	tags = `nullptr`
	)

 {
     bool is_found;
  
     Real eye_x, eye_y;
  
     if (time==0.0) {
         is_found = FindInitialEye(levc, mf_cc_vel, velmag_threshold, eye_x, eye_y);
     } else {
         is_found = true;
         const auto& last = hurricane_eye_track_xy.back();
         eye_x = last[0];
         eye_y = last[1];
     }
  
     if (is_found) {
         Real rad_tag = 4.e5 * std::pow(2, max_level-1-levc);
         tag_on_distance_from_eye(geom[levc], tags, eye_x, eye_y, rad_tag);
     }
 }

Here is the call graph for this function:

◆ ImposeBCsOnPhi()

void ERF::ImposeBCsOnPhi	(	int	lev,
		amrex::MultiFab &	phi,
		const amrex::Box &	subdomain
	)

Impose bc's on the pressure that comes out of the solve

 {
     BL_PROFILE("ERF::ImposeBCsOnPhi()");
  
     auto const sub_lo = lbound(subdomain);
     auto const sub_hi = ubound(subdomain);
  
     auto const dom_lo = lbound(geom[lev].Domain());
     auto const dom_hi = ubound(geom[lev].Domain());
  
     phi.setBndry(1.e25);
     phi.FillBoundary(geom[lev].periodicity());
  
     // ****************************************************************************
     // Impose bc's on pprime
     // ****************************************************************************
 #ifdef _OPENMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         Array4<Real> const& pp_arr  = phi.array(mfi);
         Box const& bx    = mfi.tilebox();
         auto const bx_lo = lbound(bx);
         auto const bx_hi = ubound(bx);
  
         auto bc_type_xlo = domain_bc_type[Orientation(0,Orientation::low)];
         auto bc_type_xhi = domain_bc_type[Orientation(0,Orientation::high)];
         auto bc_type_ylo = domain_bc_type[Orientation(1,Orientation::low)];
         auto bc_type_yhi = domain_bc_type[Orientation(1,Orientation::high)];
         auto bc_type_zhi = domain_bc_type[Orientation(2,Orientation::high)];
  
         if ( (bx_lo.x == dom_lo.x) && (bc_type_xlo == "Outflow" || bc_type_xlo == "Open") && !solverChoice.use_real_bcs) {
             ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i-1,j,k) = -pp_arr(i,j,k);
             });
         } else if (bx_lo.x == sub_lo.x) {
             ParallelFor(makeSlab(bx,0,sub_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i-1,j,k) = pp_arr(i,j,k);
             });
         }
  
         if ( (bx_hi.x == dom_hi.x) && (bc_type_xhi == "Outflow" || bc_type_xhi == "Open") && !solverChoice.use_real_bcs) {
             ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i+1,j,k) = -pp_arr(i,j,k);
             });
         } else if (bx_hi.x == sub_hi.x) {
             ParallelFor(makeSlab(bx,0,sub_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i+1,j,k) = pp_arr(i,j,k);
             });
         }
  
         if ( (bx_lo.y == dom_lo.y) && (bc_type_ylo == "Outflow" || bc_type_ylo == "Open") && !solverChoice.use_real_bcs) {
             ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i,j-1,k) = -pp_arr(i,j,k);
             });
         } else if (bx_lo.y == sub_lo.y) {
             ParallelFor(makeSlab(bx,1,sub_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i,j-1,k) = pp_arr(i,j,k);
             });
         }
  
         if ( (bx_hi.y == dom_hi.y) && (bc_type_yhi == "Outflow" || bc_type_yhi == "Open") && !solverChoice.use_real_bcs) {
             ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i,j+1,k) = -pp_arr(i,j,k);
             });
         } else if (bx_hi.y == sub_hi.y) {
             ParallelFor(makeSlab(bx,1,sub_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i,j+1,k) = pp_arr(i,j,k);
             });
         }
  
         // At low z we are always Neumann whether the box touches the bottom boundary or not
         Box zbx(bx); zbx.grow(0,1); zbx.grow(1,1); // Grow in x-dir and y-dir because we have filled that above
         if (bx_lo.z == sub_lo.z) {
             ParallelFor(makeSlab(zbx,2,dom_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i,j,k-1) = pp_arr(i,j,k);
             });
         }
  
         if ( (bx_hi.z == dom_hi.z) && (bc_type_zhi == "Outflow" || bc_type_zhi == "Open") ) {
             ParallelFor(makeSlab(bx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i,j,k+1) = -pp_arr(i,j,k);
             });
         } else if (bx_hi.z == sub_hi.z) {
             ParallelFor(makeSlab(bx,2,sub_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 pp_arr(i,j,k+1) = pp_arr(i,j,k);
             });
         }
     } // mfi
  
     // Now overwrite with periodic fill outside domain and fine-fine fill inside
     phi.FillBoundary(geom[lev].periodicity());
 }

Here is the call graph for this function:

◆ init1DArrays()

void ERF::init1DArrays ( )

private

◆ init_bcs()

void ERF::init_bcs ( )

private

 {
     bool rho_read = false;
     bool read_prim_theta = true;
  
     init_phys_bcs(rho_read, read_prim_theta);
  
     Vector<Real> cons_dir_init(NBCVAR_max,0.0);
     cons_dir_init[BCVars::Rho_bc_comp] = 1.0;
     cons_dir_init[BCVars::RhoTheta_bc_comp] = -1.0;
  
     bool keqn_dir = (solverChoice.turbChoice[max_level].rans_type == RANSType::kEqn &&
                      solverChoice.turbChoice[max_level].dirichlet_k == true);
     if (keqn_dir) {
         // Need to change wall BC type, assume for now that all levels are RANS
         for (int lev = 0; lev < max_level; ++lev) {
             if (solverChoice.turbChoice[lev].rans_type != RANSType::kEqn) {
                 Error("If using one-eqn RANS, all levels must be RANS for now");
             }
         }
         Print() << "Using dirichlet BC for k equation" << std::endl;
     }
  
     // *****************************************************************************
     //
     // Here we translate the physical boundary conditions -- one type per face --
     //     into logical boundary conditions for each velocity component
     //
     // *****************************************************************************
     {
         domain_bcs_type.resize(AMREX_SPACEDIM+NBCVAR_max);
         domain_bcs_type_d.resize(AMREX_SPACEDIM+NBCVAR_max);
  
         for (OrientationIter oit; oit; ++oit) {
             Orientation ori = oit();
             int dir = ori.coordDir();
             Orientation::Side side = ori.faceDir();
             auto const bct = phys_bc_type[ori];
             if ( bct == ERF_BC::symmetry )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::reflect_even);
                     }
                     domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::reflect_odd);
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::reflect_even);
                     }
                     domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::reflect_odd);
                 }
             }
             else if (bct == ERF_BC::outflow or bct == ERF_BC::ho_outflow )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                     }
                     if (!solverChoice.anelastic[0]) {
                         domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::neumann_int);
                     }
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                     }
                     if (!solverChoice.anelastic[0]) {
                         domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::neumann_int);
                     }
                 }
             }
             else if (bct == ERF_BC::open)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++)
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::open);
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++)
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::open);
                 }
             }
             else if (bct == ERF_BC::inflow)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                         if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                             domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                         }
                     }
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                         if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
                             domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                         }
                     }
                 }
             }
             else if (bct == ERF_BC::inflow_outflow)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_upwind);
                     }
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_upwind);
                     }
                 }
             }
             else if (bct == ERF_BC::no_slip_wall)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
                     }
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
                     }
                 }
             }
             else if (bct == ERF_BC::slip_wall)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
                     }
                     // Only normal direction has ext_dir
                     domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::ext_dir);
  
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
                     }
                     // Only normal direction has ext_dir
                     domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::ext_dir);
                 }
             }
             else if (bct == ERF_BC::periodic)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::int_dir);
                     }
                 } else {
                     for (int i = 0; i < AMREX_SPACEDIM; i++) {
                         domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::int_dir);
                     }
                 }
             }
             else if ( bct == ERF_BC::surface_layer )
             {
                 AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
                 domain_bcs_type[BCVars::xvel_bc+0].setLo(dir, ERFBCType::hoextrap);
                 domain_bcs_type[BCVars::xvel_bc+1].setLo(dir, ERFBCType::hoextrap);
                 domain_bcs_type[BCVars::xvel_bc+2].setLo(dir, ERFBCType::ext_dir);
             }
         }
     }
  
     // *****************************************************************************
     //
     // Here we translate the physical boundary conditions -- one type per face --
     //     into logical boundary conditions for each cell-centered variable
     //     (including the base state variables)
     // NOTE: all "scalars" share the same type of boundary condition
     //
     // *****************************************************************************
     {
         for (OrientationIter oit; oit; ++oit) {
             Orientation ori = oit();
             int dir = ori.coordDir();
             Orientation::Side side = ori.faceDir();
             auto const bct = phys_bc_type[ori];
             if ( bct == ERF_BC::symmetry )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::reflect_even);
                     }
                 } else {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::reflect_even);
                     }
                 }
             }
             else if ( bct == ERF_BC::outflow )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                     }
                 } else {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                     }
                 }
             }
             else if ( bct == ERF_BC::ho_outflow )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::hoextrap);
                     }
                 } else {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::hoextrap);
                     }
                 }
             }
             else if ( bct == ERF_BC::open )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NBCVAR_max; i++)
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::open);
                 } else {
                     for (int i = 0; i < NBCVAR_max; i++)
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::open);
                 }
             }
             else if ( bct == ERF_BC::no_slip_wall )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                         if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                             if (rho_read) {
                                 domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                             } else {
                                 domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
                             }
                         }
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
                     }
                 } else {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                         if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                             if (rho_read) {
                                 domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                             } else {
                                 domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
                             }
                         }
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
                     }
                 }
             }
             else if (bct == ERF_BC::slip_wall)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                         if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                             if (rho_read) {
                                 domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                             } else {
                                 domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
                             }
                         }
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::Rho_bc_comp].setLo(dir, ERFBCType::neumann);
                     }
                 } else {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                         if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
                             if (rho_read) {
                                 domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                             } else {
                                 domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
                             }
                         }
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
                     }
                     if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
                         domain_bcs_type[BCVars::Rho_bc_comp].setHi(dir, ERFBCType::neumann);
                     }
                 }
             }
             else if (bct == ERF_BC::inflow)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
                         if ((BCVars::cons_bc+i == RhoTheta_comp) &&
                             (th_bc_data[0].data() != nullptr))
                         {
                             if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
                         }
                         else if (input_bndry_planes && dir < 2 && (
                            ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )) )
                         {
                             domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
                         }
                         else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                             domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                         }
                     }
                 } else {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
                         if ((BCVars::cons_bc+i == RhoTheta_comp) &&
                             (th_bc_data[0].data() != nullptr))
                         {
                             if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
                         }
                         else if (input_bndry_planes && dir < 2 && (
                            ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp)       && m_r2d->ingested_density()) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp)  && m_r2d->ingested_theta()  ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp)     && m_r2d->ingested_KE()     ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp)     && m_r2d->ingested_q1()     ) ||
                            ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp)     && m_r2d->ingested_q2()     )
                            ) )
                         {
                             domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
                         }
                         else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                             domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                         }
                     }
                 }
             }
             else if (bct == ERF_BC::inflow_outflow )
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_upwind);
                         if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                             domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
                         }
                     }
                 } else {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_upwind);
                         if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
                             domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
                         }
                     }
                 }
             }
             else if (bct == ERF_BC::periodic)
             {
                 if (side == Orientation::low) {
                     for (int i = 0; i < NBCVAR_max; i++) {
                         domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::int_dir);
                     }
                 } else {
                     for (int i = 0; i < NBCVAR_max; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::int_dir);
                     }
                 }
             }
             else if ( bct == ERF_BC::surface_layer )
             {
                 AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
                 for (int i = 0; i < NBCVAR_max; i++) {
                     domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::hoextrap);
                 }
                 if (keqn_dir) {
                     Print() << "Setting surface layer logical BC to dirichlet for RANS with k model" << std::endl;
                     domain_bcs_type[BCVars::RhoKE_bc_comp].setLo(dir, ERFBCType::ext_dir);
                 }
             }
         }
     }
  
     // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
     Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
 }

Here is the call graph for this function:

◆ init_custom()

void ERF::init_custom ( int lev )

private

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters

lev	Integer specifying the current level

 {
     auto& lev_new = vars_new[lev];
  
     MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
     MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
  
     MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
                        lev_new[Vars::cons].nComp()   , lev_new[Vars::cons].nGrow());
     MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
     MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
     MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
  
     // Default all perturbations to zero
     cons_pert.setVal(0.);
     xvel_pert.setVal(0.);
     yvel_pert.setVal(0.);
     zvel_pert.setVal(0.);
  
     // If the initial condition needs to have spatially correlated perturbations superimposed onto
     // the base state, then populate the "pert" multifabs with random perturbations,
     // and then apply a Gaussian smoothing to make the perturbations spatially correlated
     if(solverChoice.is_init_with_correlated_pert) {
         create_random_perturbations(lev, cons_pert, xvel_pert, yvel_pert, zvel_pert);
         apply_gaussian_smoothing_to_perturbations(lev, cons_pert, xvel_pert, yvel_pert, zvel_pert);
     }
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
     {
         const Box &bx  = mfi.tilebox();
         const Box &xbx = mfi.tilebox(IntVect(1,0,0));
         const Box &ybx = mfi.tilebox(IntVect(0,1,0));
         const Box &zbx = mfi.tilebox(IntVect(0,0,1));
  
         const auto &cons_pert_arr = cons_pert.array(mfi);
         const auto &xvel_pert_arr = xvel_pert.array(mfi);
         const auto &yvel_pert_arr = yvel_pert.array(mfi);
         const auto &zvel_pert_arr = zvel_pert.array(mfi);
  
         Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
         Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
         Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
  
         // Here we arbitrarily choose the x-oriented map factor -- this should be generalized
         Array4<Real const> mf_m     = mapfac[lev][MapFacType::m_x]->const_array(mfi);
         Array4<Real const> mf_u     = mapfac[lev][MapFacType::u_x]->const_array(mfi);
         Array4<Real const> mf_v     = mapfac[lev][MapFacType::v_y]->const_array(mfi);
  
         Array4<Real> r_hse_arr = r_hse.array(mfi);
         Array4<Real> p_hse_arr = p_hse.array(mfi);
  
         prob->init_custom_pert(bx, cons_arr, cons_pert_arr,
                                r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
                                geom[lev].data(), mf_m, solverChoice, lev);
         prob->init_custom_pert_vels(xbx, ybx, zbx,
                                     xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
                                     z_nd_arr, geom[lev].data(), mf_u, mf_v,
                                     solverChoice, lev);
     } //mfi
  
     // Add problem-specific perturbation to background flow if not doing anelastic with fixed-in-time density
     if (!solverChoice.fixed_density[lev]) {
         MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp,      Rho_comp,             1, cons_pert.nGrow());
     }
     MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp,        1, cons_pert.nGrow());
     MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,NSCALARS, cons_pert.nGrow());
  
     // RhoKE is relevant if using Deardorff with LES, k-equation for RANS, or MYNN with PBL
     if (solverChoice.turbChoice[lev].use_tke) {
         MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp,    RhoKE_comp,    1, cons_pert.nGrow());
     }
  
     if (solverChoice.moisture_type != MoistureType::None) {
         int qstate_size = micro->Get_Qstate_Size();
         for (int q_offset(0); q_offset<qstate_size; ++q_offset) {
             int q_idx = RhoQ1_comp+q_offset;
             MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
         }
     }
  
     MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0,             0,             1, xvel_pert.nGrowVect());
     MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0,             0,             1, yvel_pert.nGrowVect());
     MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0,             0,             1, zvel_pert.nGrowVect());
 }

Here is the call graph for this function:

◆ init_Dirichlet_bc_data()

void ERF::init_Dirichlet_bc_data ( const std::string input_file )

private

 {
     // Read the dirichlet_input file
     Print() << "dirichlet_input file location : " << input_file << std::endl;
     std::ifstream input_reader(input_file);
     if (!input_reader.is_open()) {
         amrex::Abort("Error opening the dirichlet_input file.\n");
     }
  
     Print() << "Successfully opened the dirichlet_input file. Now reading... " << std::endl;
     std::string line;
  
     // Size of Ninp (number of z points in input file)
     Vector<Real> z_inp_tmp, u_inp_tmp, v_inp_tmp, w_inp_tmp, th_inp_tmp;
  
     // Top and bot for domain
     const int  klo  = geom[0].Domain().smallEnd()[2];
     const int  khi  = geom[0].Domain().bigEnd()[2];
     const Real zbot = zlevels_stag[0][klo];
     const Real ztop = zlevels_stag[0][khi+1];
  
     // Flag if theta input
     Real th_init = -300.0;
     bool th_read{false};
  
     // Add surface
     z_inp_tmp.push_back(zbot); // height above sea level [m]
     u_inp_tmp.push_back(0.);
     v_inp_tmp.push_back(0.);
     w_inp_tmp.push_back(0.);
     th_inp_tmp.push_back(th_init);
  
     // Read the vertical profile at each given height
     Real z, u, v, w, th;
     while(std::getline(input_reader, line)) {
         std::istringstream iss_z(line);
  
         Vector<Real> rval_v;
         Real rval;
         while (iss_z >> rval) {
             rval_v.push_back(rval);
         }
         z = rval_v[0];
         u = rval_v[1];
         v = rval_v[2];
         w = rval_v[3];
  
         // Format without theta
         if (rval_v.size() == 4) {
             if (z == zbot) {
                 u_inp_tmp[0] = u;
                 v_inp_tmp[0] = v;
                 w_inp_tmp[0] = w;
             } else {
                 AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
                 z_inp_tmp.push_back(z);
                 u_inp_tmp.push_back(u);
                 v_inp_tmp.push_back(v);
                 w_inp_tmp.push_back(w);
                 if (z >= ztop) break;
             }
         } else if (rval_v.size() == 5) {
             th_read = true;
             th = rval_v[4];
             if (z == zbot) {
                 u_inp_tmp[0]  = u;
                 v_inp_tmp[0]  = v;
                 w_inp_tmp[0]  = w;
                 th_inp_tmp[0] = th;
             } else {
                 AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
                 z_inp_tmp.push_back(z);
                 u_inp_tmp.push_back(u);
                 v_inp_tmp.push_back(v);
                 w_inp_tmp.push_back(w);
                 th_inp_tmp.push_back(th);
                 if (z >= ztop) break;
             }
         } else {
             Abort("Unknown inflow file format!");
         }
     }
  
     // Ensure we set a reasonable theta surface
     if (th_read) {
         if (th_inp_tmp[0] == th_init) {
             Real slope = (th_inp_tmp[2] - th_inp_tmp[1]) / (z_inp_tmp[2] - z_inp_tmp[1]);
             Real dz    = z_inp_tmp[0] - z_inp_tmp[1];
             th_inp_tmp[0] = slope * dz + th_inp_tmp[1];
         }
     }
  
     amrex::Print() << "Successfully read and interpolated the dirichlet_input file..." << std::endl;
     input_reader.close();
  
     for (int lev = 0; lev <= max_level; lev++) {
  
         const int Nz  = geom[lev].Domain().size()[2];
  
         // Size of Nz (domain grid)
         Vector<Real> zcc_inp(Nz  );
         Vector<Real> znd_inp(Nz+1);
         Vector<Real> u_inp(Nz  ); xvel_bc_data[lev].resize(Nz  ,0.0);
         Vector<Real> v_inp(Nz  ); yvel_bc_data[lev].resize(Nz  ,0.0);
         Vector<Real> w_inp(Nz+1); zvel_bc_data[lev].resize(Nz+1,0.0);
         Vector<Real> th_inp;
         if (th_read) {
             th_inp.resize(Nz);
             th_bc_data[lev].resize(Nz, 0.0);
         }
  
         // At this point, we have an input from zbot up to
         // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights
         const int Ninp = z_inp_tmp.size();
         for (int k(0); k<Nz; ++k) {
             zcc_inp[k] = 0.5 * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
             znd_inp[k] = zlevels_stag[lev][k+1];
             u_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), u_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
             v_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), v_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
             w_inp[k]   = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), znd_inp[k], Ninp);
             if (th_read) {
                 th_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), th_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
             }
         }
         znd_inp[Nz] = ztop;
         w_inp[Nz]   = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), ztop, Ninp);
  
         // Copy host data to the device
         Gpu::copy(Gpu::hostToDevice, u_inp.begin(), u_inp.end(), xvel_bc_data[lev].begin());
         Gpu::copy(Gpu::hostToDevice, v_inp.begin(), v_inp.end(), yvel_bc_data[lev].begin());
         Gpu::copy(Gpu::hostToDevice, w_inp.begin(), w_inp.end(), zvel_bc_data[lev].begin());
         if (th_read) {
            Gpu::copy(Gpu::hostToDevice, th_inp.begin(), th_inp.end(), th_bc_data[lev].begin());
         }
  
         // NOTE: These device vectors are passed to the PhysBC constructors when that
         //       class is instantiated in ERF_MakeNewArrays.cpp.
     } // lev
 }

Here is the call graph for this function:

◆ init_from_hse()

void ERF::init_from_hse ( int lev )

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

call prob->erf_init_dens_hse(...)
- call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
- save r_hse
call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse and th_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation

Parameters

lev	Integer specifying the current level

 {
     auto& lev_new = vars_new[lev];
  
     MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
     MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
     {
         const Box &gbx = mfi.growntilebox(1);
         const Array4<Real      >& cons_arr  = lev_new[Vars::cons].array(mfi);
         const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
         const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
  
         ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             cons_arr(i,j,k,Rho_comp)      = r_hse_arr(i,j,k);
             cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
         });
     } //mfi
 }

Here is the call graph for this function:

◆ init_from_input_sounding()

void ERF::init_from_input_sounding ( int lev )

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters

lev	Integer specifying the current level

 {
     // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
     if (lev == 0) {
         if (input_sounding_data.input_sounding_file.empty()) {
             Error("input_sounding file name must be provided via input");
         }
  
         input_sounding_data.resize_arrays();
  
         // this will interpolate the input profiles to the nominal height levels
         // (ranging from 0 to the domain top)
         for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
             input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n);
         }
  
         // this will calculate the hydrostatically balanced density and pressure
         // profiles following WRF ideal.exe
         if (solverChoice.sounding_type == SoundingType::Ideal) {
             input_sounding_data.calc_rho_p(0);
         } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
                    solverChoice.sounding_type == SoundingType::DryIsentropic) {
             input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
             input_sounding_data.calc_rho_p_isentropic(0);
         }
  
     } else {
         //
         // We need to do this interp from coarse level in order to set the values of
         // the base state inside the domain but outside of the fine region
         //
         base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
         //
         // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
         //       have been pre-filled - this includes ghost cells both inside and outside
         //       the domain
         //
         InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                               IntVect(0,0,0), // do not fill ghost cells outside the domain
                               base_state[lev-1], 0, 0, base_state[lev].nComp(),
                               geom[lev-1], geom[lev],
                               refRatio(lev-1), &cell_cons_interp,
                               domain_bcs_type, BaseBCVars::rho0_bc_comp);
  
          // We need to do this here because the interpolation above may leave corners unfilled
          //    when the corners need to be filled by, for example, reflection of the fine ghost
          //    cell outside the fine region but inide the domain.
          (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
     }
  
     auto& lev_new = vars_new[lev];
  
     // updated if sounding is ideal (following WRF) or isentropic
     const bool l_isentropic = (solverChoice.sounding_type == SoundingType::Isentropic ||
                                solverChoice.sounding_type == SoundingType::DryIsentropic);
     const bool sounding_ideal_or_isentropic = (solverChoice.sounding_type == SoundingType::Ideal ||
                                                l_isentropic);
     MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
     MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
     MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
     MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
     MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
  
     const Real l_gravity = solverChoice.gravity;
     const Real l_rdOcp   = solverChoice.rdOcp;
     const bool l_moist   = (solverChoice.moisture_type != MoistureType::None);
  
     int ngz = r_hse.nGrow(2);
  
 #ifdef _OPENMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box &bx = mfi.tilebox();
         const auto &cons_arr = lev_new[Vars::cons].array(mfi);
         const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
         const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
         const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
         Array4<Real>  r_hse_arr =  r_hse.array(mfi);
         Array4<Real>  p_hse_arr =  p_hse.array(mfi);
         Array4<Real> pi_hse_arr = pi_hse.array(mfi);
         Array4<Real> th_hse_arr = th_hse.array(mfi);
         Array4<Real> qv_hse_arr = qv_hse.array(mfi);
  
         Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
         Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
  
         if (sounding_ideal_or_isentropic)
         {
             // HSE will be initialized here, interpolated from values previously
             // calculated by calc_rho_p or calc_rho_p_isentropic
             init_bx_scalars_from_input_sounding_hse(
                 bx, cons_arr,
                 r_hse_arr, p_hse_arr, pi_hse_arr, th_hse_arr, qv_hse_arr,
                 geom[lev].data(), z_cc_arr,
                 l_gravity, l_rdOcp, l_moist, input_sounding_data,
                 l_isentropic, ngz);
         }
         else
         {
             // This assumes rho_0 = 1.0
             // HSE will be calculated later with call to initHSE
             init_bx_scalars_from_input_sounding(
                 bx, cons_arr,
                 geom[lev].data(), z_cc_arr,
                 l_moist, input_sounding_data);
         }
  
         init_bx_velocities_from_input_sounding(
             bx, xvel_arr, yvel_arr, zvel_arr,
             geom[lev].data(), z_nd_arr,
             input_sounding_data);
  
     } //mfi
 }

Here is the call graph for this function:

◆ init_geo_wind_profile()

void ERF::init_geo_wind_profile	(	const std::string	input_file,
		amrex::Vector< amrex::Real > &	u_geos,
		amrex::Gpu::DeviceVector< amrex::Real > &	u_geos_d,
		amrex::Vector< amrex::Real > &	v_geos,
		amrex::Gpu::DeviceVector< amrex::Real > &	v_geos_d,
		const amrex::Geometry &	lgeom,
		const amrex::Vector< amrex::Real > &	zlev_stag
	)

private

 {
     const int klo = 0;
     const int khi = lgeom.Domain().bigEnd()[AMREX_SPACEDIM-1];
     const amrex::Real dz = lgeom.CellSize()[AMREX_SPACEDIM-1];
  
     const bool grid_stretch = (zlev_stag.size() > 0);
     const Real zbot = (grid_stretch) ? zlev_stag[klo]   : lgeom.ProbLo(AMREX_SPACEDIM-1);
     const Real ztop = (grid_stretch) ? zlev_stag[khi+1] : lgeom.ProbHi(AMREX_SPACEDIM-1);
  
     amrex::Print() << "Reading geostrophic wind profile from " << input_file << std::endl;
     std::ifstream profile_reader(input_file);
     if(!profile_reader.is_open()) {
         amrex::Error("Error opening the abl_geo_wind_table\n");
     }
  
     // First, read the input data into temp vectors
     std::string line;
     Vector<Real> z_inp, Ug_inp, Vg_inp;
     Real z, Ug, Vg;
     amrex::Print() << "z  Ug  Vg" << std::endl;
     while(std::getline(profile_reader, line)) {
         std::istringstream iss(line);
         iss >> z >> Ug >> Vg;
         amrex::Print() << z << " " << Ug << " " << Vg << std::endl;
         z_inp.push_back(z);
         Ug_inp.push_back(Ug);
         Vg_inp.push_back(Vg);
         if (z >= ztop) break;
     }
  
     const int Ninp = z_inp.size();
     AMREX_ALWAYS_ASSERT(z_inp[0] <= zbot);
     AMREX_ALWAYS_ASSERT(z_inp[Ninp-1] >= ztop);
  
     // Now, interpolate vectors to the cell centers
     for (int k = 0; k <= khi; k++) {
         z = (grid_stretch) ? 0.5 * (zlev_stag[k] + zlev_stag[k+1])
                            : zbot + (k + 0.5) * dz;
         u_geos[k] = interpolate_1d(z_inp.dataPtr(), Ug_inp.dataPtr(), z, Ninp);
         v_geos[k] = interpolate_1d(z_inp.dataPtr(), Vg_inp.dataPtr(), z, Ninp);
     }
  
     // Copy from host version to device version
     Gpu::copy(Gpu::hostToDevice, u_geos.begin(), u_geos.end(), u_geos_d.begin());
     Gpu::copy(Gpu::hostToDevice, v_geos.begin(), v_geos.end(), v_geos_d.begin());
  
     profile_reader.close();
 }

Here is the call graph for this function:

◆ init_immersed_forcing()

void ERF::init_immersed_forcing ( int lev )

Set velocities in cells that are immersed to be 0 (or a very small number)

Parameters

lev	Integer specifying the current level

 {
     auto& lev_new = vars_new[lev];
     MultiFab* terrain_blank = terrain_blanking[lev].get();
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
     {
         const Box &xbx = mfi.tilebox(IntVect(1,0,0));
         const Box &ybx = mfi.tilebox(IntVect(0,1,0));
         const Box &zbx = mfi.tilebox(IntVect(0,0,1));
         const Real epsilon = 1e-2;
  
         const Array4<const Real>& t_blank_arr = terrain_blank->const_array(mfi);
  
         const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
         const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
         const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
  
         // Set the x,y,z-velocities
         ParallelFor(xbx, ybx, zbx,
         [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
             const Real t_blank = 0.5 * (t_blank_arr(i, j, k) + t_blank_arr(i-1, j, k));
             if (t_blank == 1.0) { xvel_arr(i, j, k) = epsilon; }
         },
         [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
             const Real t_blank = 0.5 * (t_blank_arr(i, j, k) + t_blank_arr(i, j-1, k));
             if (t_blank == 1.0) { yvel_arr(i, j, k) = epsilon; }
         },
         [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
             const Real t_blank = 0.5 * (t_blank_arr(i, j, k) + t_blank_arr(i, j, k-1));
             if (t_blank == 1.0) { zvel_arr(i, j, k) = epsilon; }
         });
     } //mfi
 }

Here is the call graph for this function:

◆ init_only()

void ERF::init_only	(	int	lev,
		amrex::Real	time
	)

 {
     t_new[lev] = elapsed_time;
     t_old[lev] = elapsed_time - 1.e200;
  
     auto& lev_new = vars_new[lev];
     auto& lev_old = vars_old[lev];
  
     // Loop over grids at this level to initialize our grid data
     lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
     lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
     lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
     lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
  
     // Initialize background flow (optional)
     if (solverChoice.init_type == InitType::Input_Sounding) {
         // The physbc's need the terrain but are needed for initHSE
         // We have already made the terrain in the call to init_zphys
         //    in MakeNewLevelFromScratch
         make_physbcs(lev);
  
         // Now init the base state and the data itself
         init_from_input_sounding(lev);
  
         // The base state has been initialized by integrating vertically
         // through the sounding for ideal (like WRF) or isentropic approaches
         if (solverChoice.sounding_type == SoundingType::Ideal ||
             solverChoice.sounding_type == SoundingType::Isentropic ||
             solverChoice.sounding_type == SoundingType::DryIsentropic) {
             AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
                 "Gravity should be on to be consistent with sounding initialization.");
         } else { // SoundingType::ConstantDensity
             AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!solverChoice.use_gravity || (solverChoice.anelastic[lev] == 1),
                 "Constant density probably doesn't make sense for compressible flow with gravity");
             initHSE();
         }
  
 #ifdef ERF_USE_NETCDF
     }
     else if (solverChoice.init_type == InitType::WRFInput && !nc_init_file[lev].empty())
     {
         // The base state is initialized from WRF wrfinput data, output by
         // ideal.exe or real.exe
  
         init_from_wrfinput(lev, *mf_C1H, *mf_C2H, *mf_MUB, *mf_PSFC[lev]);
  
         // The physbc's need the terrain but are needed for initHSE
         if (!solverChoice.use_real_bcs) {
             make_physbcs(lev);
         }
     }
     else if (solverChoice.init_type == InitType::WRFInput && nc_init_file[lev].empty())
     {
         amrex::Abort("This pathway is not quite implemented yet");
     }
     else if (solverChoice.init_type == InitType::NCFile)
     {
         // The state is initialized by reading from a Netcdf file
         init_from_ncfile(lev);
  
         // The physbc's need the terrain but are needed for initHSE
         make_physbcs(lev);
     }
     else if (solverChoice.init_type == InitType::Metgrid)
     {
         // The base state is initialized from data output by WPS metgrid;
         // we will rebalance after interpolation
         init_from_metgrid(lev);
 #endif
     } else if ( (solverChoice.init_type == InitType::Uniform        ) ||
                 (solverChoice.init_type == InitType::ConstantDensity) ||
                 (solverChoice.init_type == InitType::Isentropic     ) ||
                 (solverChoice.init_type == InitType::HindCast       ) ||
                 (solverChoice.init_type == InitType::MoistBaseState ) ) {
         // Initialize a uniform density/entropy background field and base state
         // based on the problem-specified reference density and temperature
  
         // The physbc's need the terrain but are needed for initHSE
         make_physbcs(lev);
  
         // We will initialize the state from the background state so must set that first
         // The choice between constant rho and constant theta will be made inside initHSE
         initHSE(lev);
  
         // Copy rho and rhotheta from rho_hse and p_hse
         init_from_hse(lev);
  
     } else {
         Abort("Unknown init_type!");
     }
  
     // Add problem-specific flow features
     //
     // Notes:
     // - This calls init_custom_pert that is defined for each problem
     // - This may modify the base state
     // - The fields set by init_custom_pert are **perturbations** to the
     //   background flow set based on init_type
     if (solverChoice.init_type != InitType::NCFile) {
         init_custom(lev);
     }
  
     // Ensure that the face-based data are the same on both sides of a periodic domain.
     // The data associated with the lower grid ID is considered the correct value.
     lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
     lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
     lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
  
    if(solverChoice.spongeChoice.sponge_type == "input_sponge"){
         input_sponge(lev);
    }
  
     // Initialize turbulent perturbation
     if (solverChoice.pert_type == PerturbationType::Source ||
         solverChoice.pert_type == PerturbationType::Direct ||
         solverChoice.pert_type == PerturbationType::CPM) {
         turbPert_update(lev, 0.);
         turbPert_amplitude(lev);
     }
  
     // Set initial velocity field for immersed cells to be close to 0
     if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
         init_immersed_forcing(lev);
     }
 }

◆ init_phys_bcs()

void ERF::init_phys_bcs	(	bool &	rho_read,
		bool &	read_prim_theta
	)

private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, InflowOutflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

 {
     auto f = [this,&rho_read,&read_prim_theta] (std::string const& bcid, Orientation ori)
     {
         // These are simply defaults for Dirichlet faces -- they should be over-written below
         m_bc_extdir_vals[BCVars::Rho_bc_comp][ori]       =  1.0;
         m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori]  = -1.0; // It is important to set this negative
                                                                  // because the sign is tested on below
         for (int n = BCVars::RhoKE_bc_comp; n < BCVars::xvel_bc; n++) {
             m_bc_extdir_vals[n][ori]                     = 0.0;
         }
  
         m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.0; // default
         m_bc_extdir_vals[BCVars::yvel_bc][ori] = 0.0;
         m_bc_extdir_vals[BCVars::zvel_bc][ori] = 0.0;
  
         // These are simply defaults for Neumann gradients -- they should be over-written below
         m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]       = 0.0;
         m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]  = 0.0;
  
         m_bc_neumann_vals[BCVars::RhoKE_bc_comp][ori]     = 0.0;
         m_bc_neumann_vals[BCVars::RhoScalar_bc_comp][ori] = 0.0;
         m_bc_neumann_vals[BCVars::RhoQ1_bc_comp][ori]     = 0.0;
         m_bc_neumann_vals[BCVars::RhoQ2_bc_comp][ori]     = 0.0;
         m_bc_neumann_vals[BCVars::RhoQ3_bc_comp][ori]     = 0.0;
         m_bc_neumann_vals[BCVars::RhoQ4_bc_comp][ori]     = 0.0;
         m_bc_neumann_vals[BCVars::RhoQ5_bc_comp][ori]     = 0.0;
         m_bc_neumann_vals[BCVars::RhoQ6_bc_comp][ori]     = 0.0;
  
         m_bc_neumann_vals[BCVars::xvel_bc][ori] = 0.0;
         m_bc_neumann_vals[BCVars::yvel_bc][ori] = 0.0;
         m_bc_neumann_vals[BCVars::zvel_bc][ori] = 0.0;
  
         std::string pp_text = pp_prefix + "." + bcid;
         ParmParse pp(pp_text);
  
         std::string bc_type_in;
         if (pp.query("type", bc_type_in) <= 0)
         {
             pp_text = bcid;
             pp = ParmParse(pp_text);
             pp.query("type", bc_type_in);
         }
  
         std::string bc_type = amrex::toLower(bc_type_in);
  
         if (bc_type == "symmetry")
         {
             // Print() << bcid << " set to symmetry.\n";
               phys_bc_type[ori] = ERF_BC::symmetry;
             domain_bc_type[ori] = "Symmetry";
         }
         else if (bc_type == "outflow")
         {
             // Print() << bcid << " set to outflow.\n";
               phys_bc_type[ori] = ERF_BC::outflow;
             domain_bc_type[ori] = "Outflow";
         }
         else if (bc_type == "open")
         {
             // Print() << bcid << " set to open.\n";
             AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
               phys_bc_type[ori] = ERF_BC::open;
             domain_bc_type[ori] = "Open";
         }
         else if (bc_type == "ho_outflow")
         {
             phys_bc_type[ori] = ERF_BC::ho_outflow;
             domain_bc_type[ori] = "HO_Outflow";
         }
  
         else if (bc_type == "inflow" || bc_type == "inflow_outflow")
         {
             if (bc_type == "inflow") {
                 // Print() << bcid << " set to inflow.\n";
                   phys_bc_type[ori] = ERF_BC::inflow;
                 domain_bc_type[ori] = "Inflow";
             } else {
                 // Print() << bcid << " set to inflow_outflow.\n";
                   phys_bc_type[ori] = ERF_BC::inflow_outflow;
                 domain_bc_type[ori] = "InflowOutflow";
             }
  
             std::vector<Real> v;
             if (input_bndry_planes && m_r2d->ingested_velocity()) {
                 m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.;
                 m_bc_extdir_vals[BCVars::yvel_bc][ori] = 0.;
                 m_bc_extdir_vals[BCVars::zvel_bc][ori] = 0.;
             } else {
                 // Test for input data file if at xlo face
                 std::string dirichlet_file;
                 auto file_exists = pp.query("dirichlet_file", dirichlet_file);
                 if (file_exists) {
                     pp.query("read_prim_theta", read_prim_theta);
                     init_Dirichlet_bc_data(dirichlet_file);
                 } else {
                     pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
                     m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                     m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                     m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
                 }
             }
  
             Real rho_in = 0.;
             if (input_bndry_planes && m_r2d->ingested_density()) {
                 m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = 0.;
             } else {
                 if (!pp.query("density", rho_in)) {
                     amrex::Print() << "Using interior values to set conserved vars" << std::endl;
                 }
                 m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
             }
  
             bool th_read  = (th_bc_data[0].data()!=nullptr);
             Real theta_in = 0.;
             if (input_bndry_planes && m_r2d->ingested_theta()) {
                 m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = 0.;
             } else if (!th_read) {
                 if (rho_in > 0) {
                     pp.get("theta", theta_in);
                 }
                 m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
             }
  
             Real scalar_in = 0.;
             if (input_bndry_planes && m_r2d->ingested_scalar()) {
                 m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = 0.;
             } else {
                 if (pp.query("scalar", scalar_in))
                 m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
             }
  
             if (solverChoice.moisture_type != MoistureType::None) {
                 Real qv_in = 0.;
                 if (input_bndry_planes && m_r2d->ingested_q1()) {
                     m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = 0.;
                 } else {
                     if (pp.query("qv", qv_in))
                     m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
                 }
                 Real qc_in = 0.;
                 if (input_bndry_planes && m_r2d->ingested_q2()) {
                     m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = 0.;
                 } else {
                     if (pp.query("qc", qc_in))
                     m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
                 }
             }
  
             Real KE_in = 0.;
             if (input_bndry_planes && m_r2d->ingested_KE()) {
                 m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = 0.;
             } else {
                 if (pp.query("KE", KE_in))
                 m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
             }
         }
         else if (bc_type == "noslipwall")
         {
             // Print() << bcid <<" set to no-slip wall.\n";
               phys_bc_type[ori] = ERF_BC::no_slip_wall;
             domain_bc_type[ori] = "NoSlipWall";
  
             std::vector<Real> v;
  
             // The values of m_bc_extdir_vals default to 0.
             // But if we find "velocity" in the inputs file, use those values instead.
             if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
             {
                 v[ori.coordDir()] = 0.0;
                 m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                 m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                 m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
             }
  
             Real rho_in;
             rho_read = pp.query("density", rho_in);
             if (rho_read)
             {
                 m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
             }
  
             Real theta_in;
             if (pp.query("theta", theta_in))
             {
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
             }
  
             Real theta_grad_in;
             if (pp.query("theta_grad", theta_grad_in))
             {
                 m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
             }
  
             Real qv_in;
             if (pp.query("qv", qv_in))
             {
                 m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = qv_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
             }
         }
         else if (bc_type == "slipwall")
         {
             // Print() << bcid <<" set to slip wall.\n";
  
               phys_bc_type[ori] = ERF_BC::slip_wall;
             domain_bc_type[ori] = "SlipWall";
  
             Real rho_in;
             rho_read = pp.query("density", rho_in);
             if (rho_read)
             {
                 m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
             }
  
             Real theta_in;
             if (pp.query("theta", theta_in))
             {
                m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
             }
  
             Real rho_grad_in;
             if (pp.query("density_grad", rho_grad_in))
             {
                m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = rho_grad_in;
             }
  
             Real theta_grad_in;
             if (pp.query("theta_grad", theta_grad_in))
             {
                m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
             }
         }
         else if (bc_type == "surface_layer")
         {
               phys_bc_type[ori] = ERF_BC::surface_layer;
             domain_bc_type[ori] = "surface_layer";
         }
         else
         {
             phys_bc_type[ori] = ERF_BC::undefined;
         }
  
         if (geom[0].isPeriodic(ori.coordDir())) {
             domain_bc_type[ori] = "Periodic";
             if (phys_bc_type[ori] == ERF_BC::undefined)
             {
                 phys_bc_type[ori] = ERF_BC::periodic;
             } else {
                 Abort("Wrong BC type for periodic boundary");
             }
         }
  
         if (phys_bc_type[ori] == ERF_BC::undefined)
         {
              Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
              Abort("This BC type is unknown");
         }
     };
  
     f("xlo", Orientation(Direction::x,Orientation::low));
     f("xhi", Orientation(Direction::x,Orientation::high));
     f("ylo", Orientation(Direction::y,Orientation::low));
     f("yhi", Orientation(Direction::y,Orientation::high));
     f("zlo", Orientation(Direction::z,Orientation::low));
     f("zhi", Orientation(Direction::z,Orientation::high));
 }

Here is the call graph for this function:

◆ init_stuff()

void ERF::init_stuff	(	int	lev,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm,
		amrex::Vector< amrex::MultiFab > &	lev_new,
		amrex::Vector< amrex::MultiFab > &	lev_old,
		amrex::MultiFab &	tmp_base_state,
		std::unique_ptr< amrex::MultiFab > &	tmp_zphys_nd
	)

private

 {
     // ********************************************************************************************
     // Base state holds r_0, pres_0, pi_0, th_0 (in that order)
     //
     // Here is where we set the number of ghost cells for the base state!
     // ********************************************************************************************
     int ngb = (solverChoice.terrain_type == TerrainType::EB) ? ComputeGhostCells(solverChoice)+1 : 3;
     tmp_base_state.define(ba,dm,BaseState::num_comps,ngb);
     tmp_base_state.setVal(0.);
  
     if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
         base_state_new[lev].define(ba,dm,BaseState::num_comps,base_state[lev].nGrowVect());
         base_state_new[lev].setVal(0.);
     }
  
     // ********************************************************************************************
     // Allocate terrain arrays
     // ********************************************************************************************
  
     BoxArray ba_nd(ba);
     ba_nd.surroundingNodes();
  
     // NOTE: this is where we actually allocate z_phys_nd -- but here it's called "tmp_zphys_nd"
     // We need this to be one greater than the ghost cells to handle levels > 0
  
     int ngrow = ComputeGhostCells(solverChoice) + 2;
     tmp_zphys_nd = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
  
     z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,2);
     init_default_zphys(lev, geom[lev], *tmp_zphys_nd, *z_phys_cc[lev]);
  
     if (solverChoice.terrain_type == TerrainType::MovingFittedMesh)
     {
         detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
         detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
  
         ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
         ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
         az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
  
         z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
  
         z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
         z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
         z_phys_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
     }
     else
     {
         z_phys_nd_new[lev] = nullptr;
           detJ_cc_new[lev] = nullptr;
  
         z_phys_nd_src[lev] = nullptr;
         z_phys_cc_src[lev] = nullptr;
           detJ_cc_src[lev] = nullptr;
  
                z_t_rk[lev] = nullptr;
     }
  
     if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing)
     {
         terrain_blanking[lev] = std::make_unique<MultiFab>(ba,dm,1,ngrow);
         terrain_blanking[lev]->setVal(1.0);
     }
  
     // We use these area arrays regardless of terrain, EB or none of the above
     detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
          ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
          ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
          az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
  
     detJ_cc[lev]->setVal(1.0);
          ax[lev]->setVal(1.0);
          ay[lev]->setVal(1.0);
          az[lev]->setVal(1.0);
  
     // ********************************************************************************************
     // Create wall distance array for RANS modeling
     // ********************************************************************************************
     if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
         walldist[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
         walldist[lev]->setVal(1e23);
     } else {
         walldist[lev] = nullptr;
     }
  
     // ********************************************************************************************
     // These are the persistent containers for the old and new data
     // ********************************************************************************************
     int ncomp;
     if (lev > 0) {
         ncomp = vars_new[lev-1][Vars::cons].nComp();
     } else {
         int n_qstate   = micro->Get_Qstate_Size();
         ncomp = NDRY + NSCALARS + n_qstate;
     }
  
     // ********************************************************************************************
     // The number of ghost cells for density must be 1 greater than that for velocity
     //     so that we can go back in forth between velocity and momentum on all faces
     // ********************************************************************************************
     int ngrow_state = ComputeGhostCells(solverChoice) + 1;
     int ngrow_vels  = ComputeGhostCells(solverChoice);
  
     // ********************************************************************************************
     // New solution data containers
     // ********************************************************************************************
     if (solverChoice.terrain_type != TerrainType::EB) {
         lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
         lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
     } else {
         // EB: Define the MultiFabs with the EBFactory
         lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
         lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
     }
  
     // Initialize all components to zero so we don't need to explicitly set
     //     scalars / moisture variables to zero in the initialization
     lev_new[Vars::cons].setVal(0.0);
     lev_old[Vars::cons].setVal(0.0);
  
     lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
     lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
  
     lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
     lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
  
     // Set these to avoid operations on uninitialized data
     lev_new[Vars::xvel].setVal(1.234e20);
     lev_old[Vars::xvel].setVal(1.234e20);
     lev_new[Vars::yvel].setVal(1.234e20);
     lev_old[Vars::yvel].setVal(1.234e20);
  
     // Note that we need the ghost cells in the z-direction if we are doing any
     // kind of domain decomposition in the vertical (at level 0 or above)
     lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
     lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
  
     gradp[lev][GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp[lev][GpVars::gpx].setVal(0.);
     gradp[lev][GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp[lev][GpVars::gpy].setVal(0.);
     gradp[lev][GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp[lev][GpVars::gpz].setVal(0.);
  
     if ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) {
         pp_inc[lev].define(ba, dm, 1, 1);
         pp_inc[lev].setVal(0.0);
     }
  
     // We use this in the fast substepping only
     if (solverChoice.anelastic[lev] == 0) {
         lagged_delta_rt[lev].define(ba, dm, 1, 1);
         lagged_delta_rt[lev].setVal(0.0);
     }
  
     // We use these for advecting the slow variables, whether anelastic or compressible
     avg_xmom[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, 1);
     avg_ymom[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, 1);
     avg_zmom[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, 1);
     avg_xmom[lev].setVal(0.0); avg_ymom[lev].setVal(0.0); avg_zmom[lev].setVal(0.0);
  
     // ********************************************************************************************
     // These are just used for scratch in the time integrator but we might as well define them here
     // ********************************************************************************************
     if (solverChoice.terrain_type != TerrainType::EB) {
         rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
         rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
  
         rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
         rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
  
         rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
         rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
     } else {
         // EB: Define the MultiFabs with the EBFactory
         rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
         rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
  
         rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
         rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
  
         rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
         rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
     }
  
     if (lev > 0) {
         //xmom_crse_rhs[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, IntVect{0});
         //ymom_crse_rhs[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, IntVect{0});
         zmom_crse_rhs[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect{0});
     }
  
     // We do this here just so they won't be undefined in the initial FillPatch
     rU_old[lev].setVal(1.2e21);
     rV_old[lev].setVal(3.4e22);
     rW_old[lev].setVal(5.6e23);
     rU_new[lev].setVal(1.2e21);
     rV_new[lev].setVal(3.4e22);
     rW_new[lev].setVal(5.6e23);
  
     // ********************************************************************************************
     // These are just time averaged fields for diagnostics
     // ********************************************************************************************
  
     // NOTE: We are not completing a fillpatch call on the time averaged data;
     //       which would copy on intersection and interpolate from coarse.
     //       Therefore, we are restarting the averaging when the ba changes,
     //       this may give poor statistics for dynamic mesh refinement.
     vel_t_avg[lev] = nullptr;
     if (solverChoice.time_avg_vel) {
         vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
         vel_t_avg[lev]->setVal(0.0);
         t_avg_cnt[lev] = 0.0;
     }
  
     // ********************************************************************************************
     // Initialize flux registers whenever we create/re-create a level
     // ********************************************************************************************
     if (solverChoice.coupling_type == CouplingType::TwoWay) {
         if (lev == 0) {
             advflux_reg[0] = nullptr;
         } else {
             int ncomp_reflux = vars_new[0][Vars::cons].nComp();
             advflux_reg[lev] = new YAFluxRegister(ba       , grids[lev-1],
                                                   dm       ,  dmap[lev-1],
                                                   geom[lev],  geom[lev-1],
                                                   ref_ratio[lev-1], lev, ncomp_reflux);
         }
     }
  
     // ********************************************************************************************
     // Define Theta_prim storage if using surface_layer BC
     // ********************************************************************************************
     if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
         Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
         if (solverChoice.moisture_type != MoistureType::None) {
             Qv_prim[lev]    = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
             Qr_prim[lev]    = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
         } else {
             Qv_prim[lev]    = nullptr;
             Qr_prim[lev]    = nullptr;
         }
     } else {
         Theta_prim[lev] = nullptr;
         Qv_prim[lev]    = nullptr;
         Qr_prim[lev]    = nullptr;
     }
  
     // ********************************************************************************************
     // Map factors
     // ********************************************************************************************
     BoxList bl2d_mf = ba.boxList();
     for (auto& b : bl2d_mf) {
         b.setRange(2,0);
     }
     BoxArray ba2d_mf(std::move(bl2d_mf));
  
     mapfac[lev].resize(MapFacType::num);
     mapfac[lev][MapFacType::m_x] = std::make_unique<MultiFab>(                        ba2d_mf,dm,1,IntVect(3,3,0));
     mapfac[lev][MapFacType::u_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,IntVect(3,3,0));
     mapfac[lev][MapFacType::v_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,IntVect(3,3,0));
  
 #if 0
     // For now we comment this out to avoid CI failures but we will need to re-enable
     //     this if using non-conformal mappings
     if (MapFacType::m_y != MapFacType::m_x) {
         mapfac[lev][MapFacType::m_y] = std::make_unique<MultiFab>(ba2d_mf,dm,1,IntVect(3,3,0));
     }
     if (MapFacType::u_y != MapFacType::u_x) {
         mapfac[lev][MapFacType::u_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,IntVect(3,3,0));
     }
     if (MapFacType::v_y != MapFacType::v_x) {
         mapfac[lev][MapFacType::v_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,IntVect(3,3,0));
     }
 #endif
  
     if (solverChoice.test_mapfactor) {
         for (int i = 0; i < 3; i++) {
             mapfac[lev][i]->setVal(0.5);
         }
         for (int i = 3; i < mapfac[lev].size(); i++) {
             mapfac[lev][i]->setVal(0.25);
         }
     } else {
         for (int i = 0; i < mapfac[lev].size(); i++) {
             mapfac[lev][i]->setVal(1.0);
         }
     }
  
     // ********************************************************************************************
     // Build 1D BA and 2D BA
     // ********************************************************************************************
     BoxList bl1d = ba.boxList();
     for (auto& b : bl1d) {
         b.setRange(0,0);
         b.setRange(1,0);
     }
     ba1d[lev]  = BoxArray(std::move(bl1d));
  
     // Build 2D BA
     BoxList bl2d = ba.boxList();
     for (auto& b : bl2d) {
         b.setRange(2,0);
     }
     ba2d[lev]  = BoxArray(std::move(bl2d));
  
     IntVect ng  = vars_new[lev][Vars::cons].nGrowVect();
  
     if (lev == 0) {
         mf_C1H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
         mf_C2H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
         mf_MUB = std::make_unique<MultiFab>(ba2d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
     }
  
     mf_PSFC[lev] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ng);
  
     //*********************************************************
     // Variables for Fitch model for windfarm parametrization
     //*********************************************************
 #if defined(ERF_USE_WINDFARM)
     if (solverChoice.windfarm_type == WindFarmType::Fitch){
         vars_windfarm[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
     }
     if (solverChoice.windfarm_type == WindFarmType::EWP){
         vars_windfarm[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
     }
     if (solverChoice.windfarm_type == WindFarmType::SimpleAD) {
         vars_windfarm[lev].define(ba, dm, 2, ngrow_state);// dudt, dvdt
     }
     if (solverChoice.windfarm_type == WindFarmType::GeneralAD) {
         vars_windfarm[lev].define(ba, dm, 3, ngrow_state);// dudt, dvdt, dwdt
     }
         Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
         SMark[lev].define(ba, dm, 2, 1); // Free stream velocity/source term
                                                    // sampling marker in a cell - 2 components
 #endif
  
     if(solverChoice.init_type == InitType::HindCast and
         solverChoice.hindcast_lateral_forcing) {
  
         int ncomp_extra = 2;
         int nvars = vars_new[lev].size();
  
         // Resize all containers
         forecast_state_1[lev].resize(nvars + 1);
         forecast_state_2[lev].resize(nvars + 1);
         forecast_state_interp[lev].resize(nvars + 1);
  
         // Define the "normal" components
         for (int comp = 0; comp < nvars; ++comp) {
             const MultiFab& src = vars_new[lev][comp];
             ncomp = src.nComp();
             ngrow = src.nGrow();
  
             forecast_state_1[lev][comp].define(ba, dm, ncomp, ng);
             forecast_state_2[lev][comp].define(ba, dm, ncomp, ng);
             forecast_state_interp[lev][comp].define(ba, dm, ncomp, ng);
         }
  
         // Define the "extra" component (last slot)
         {
             const MultiFab& src0 = vars_new[lev][0];
             ngrow = src0.nGrow();
             int idx = nvars;
  
             forecast_state_1[lev][idx].define(ba, dm, ncomp_extra, ngrow);
             forecast_state_2[lev][idx].define(ba, dm, ncomp_extra, ngrow);
             forecast_state_interp[lev][idx].define(ba, dm, ncomp_extra, ngrow);
         }
         bool regrid_forces_file_read = true;
         WeatherDataInterpolation(lev, t_new[0],z_phys_nd, regrid_forces_file_read);
     }
  
  
     if(solverChoice.init_type == InitType::HindCast and
         solverChoice.hindcast_surface_bcs)
  
     {
         const MultiFab& src = vars_new[lev][0];
         BoxArray ba_hc = src.boxArray();
         BoxList bl2d_hc = ba_hc.boxList();
         for (auto& b : bl2d_hc) {
             b.setRange(2, 0);
         }
         BoxArray ba2d_hc(std::move(bl2d_hc));
         const amrex::DistributionMapping& dm_hc = src.DistributionMap();
  
         surface_state_1[lev].define(ba2d_hc, dm_hc, 2, src.nGrow());
         surface_state_2[lev].define(ba2d_hc, dm_hc, 2, src.nGrow());
         surface_state_interp[lev].define(ba2d_hc, dm_hc, 2, src.nGrow());
  
         bool regrid_forces_file_read = true;
         SurfaceDataInterpolation(lev, t_new[0], z_phys_nd, regrid_forces_file_read);
     }
  
 #ifdef ERF_USE_WW3_COUPLING
     // create a new BoxArray and DistributionMapping for a MultiFab with 1 box
     BoxArray ba_onegrid(geom[lev].Domain());
     BoxList bl2d_onegrid = ba_onegrid.boxList();
     for (auto& b : bl2d_onegrid) {
         b.setRange(2,0);
     }
     BoxArray ba2d_onegrid(std::move(bl2d_onegrid));
     Vector<int> pmap;
     pmap.resize(1);
     pmap[0]=0;
     DistributionMapping dm_onegrid(ba2d_onegrid);
     dm_onegrid.define(pmap);
  
     Hwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
     Lwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
  
     BoxList bl2d_wave = ba.boxList();
     for (auto& b : bl2d_wave) {
         b.setRange(2,0);
     }
     BoxArray ba2d_wave(std::move(bl2d_wave));
  
     Hwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
     Lwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
  
     std::cout<<ba_onegrid<<std::endl;
     std::cout<<ba2d_onegrid<<std::endl;
     std::cout<<dm_onegrid<<std::endl;
 #endif
  
  
     //*********************************************************
     // Radiation heating source terms
     //*********************************************************
     if (solverChoice.rad_type != RadiationType::None)
     {
         qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, 0);
         rad_fluxes[lev]     = std::make_unique<MultiFab>(ba, dm, 4, 0);
         qheating_rates[lev]->setVal(0.);
         rad_fluxes[lev]->setVal(0.);
     }
  
     //*********************************************************
     // Radiation fluxes for coupling to LSM
     //*********************************************************
  
     // NOTE: Finer levels do not need to coincide with the bottom domain boundary
     //       at k=0. We make slabs here with the kmin for a given box. Therefore,
     //       care must be taken before applying these fluxes to an LSM model. For
  
     // Radiative fluxes for LSM
     if (solverChoice.lsm_type != LandSurfaceType::None &&
         solverChoice.rad_type != RadiationType::None)
     {
         BoxList m_bl = ba.boxList();
         for (auto& b : m_bl) {
             int kmin = b.smallEnd(2);
             b.setRange(2,kmin);
         }
         BoxArray m_ba(std::move(m_bl));
  
         sw_lw_fluxes[lev] = std::make_unique<MultiFab>(m_ba, dm, 6, 0); // DIR/DIF VIS/NIR (4), NET SW (1), LW (1)
         solar_zenith[lev] = std::make_unique<MultiFab>(m_ba, dm, 1, 0);
  
         sw_lw_fluxes[lev]->setVal(0.);
         solar_zenith[lev]->setVal(0.);
     }
  
     //*********************************************************
     // Turbulent perturbation region initialization
     //*********************************************************
     if (solverChoice.pert_type == PerturbationType::Source ||
         solverChoice.pert_type == PerturbationType::Direct ||
         solverChoice.pert_type == PerturbationType::CPM)
     {
         amrex::Box bnd_bx = ba.minimalBox();
         turbPert.init_tpi_type(solverChoice.pert_type);
         turbPert.init_tpi(lev, bnd_bx.smallEnd(), bnd_bx.bigEnd(), geom[lev].CellSizeArray(),
                           ba, dm, ngrow_state, pp_prefix, refRatio(), max_level);
     }
  
     //
     // Define the land mask here and set it to all land by default
     // NOTE: the logic below will BREAK if we have any grids not touching the bottom boundary
     //
     {
     lmask_lev[lev].resize(1);
     auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0;
     BoxList bl2d_mask = ba.boxList();
     for (auto& b : bl2d_mask) {
         b.setRange(2,0);
     }
     BoxArray ba2d_mask(std::move(bl2d_mask));
     lmask_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
     lmask_lev[lev][0]->setVal(1);
     lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
  
     land_type_lev[lev].resize(1);
     land_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
     land_type_lev[lev][0]->setVal(0);
     land_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
  
     soil_type_lev[lev].resize(1);
     soil_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
     soil_type_lev[lev][0]->setVal(0);
     soil_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
  
     urb_frac_lev[lev].resize(1);
     urb_frac_lev[lev][0] = std::make_unique<MultiFab>(ba2d_mask,dm,1,ngv);
     urb_frac_lev[lev][0]->setVal(1.0);
     urb_frac_lev[lev][0]->FillBoundary(geom[lev].periodicity());
     }
  
     // Read in tables needed for windfarm simulations
     // fill in Nturb multifab - number of turbines in each mesh cell
     // write out the vtk files for wind turbine location and/or
     // actuator disks
     #ifdef ERF_USE_WINDFARM
         //init_windfarm(lev);
     #endif
  
     if (lev > 0) {
         fine_mask[lev] = std::make_unique<MultiFab>(grids[lev-1], dmap[lev-1], 1, 0);
         build_fine_mask(lev, *fine_mask[lev].get());
     }
 }

Here is the call graph for this function:

◆ init_thin_body()

void ERF::init_thin_body	(	int	lev,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

 {
     //********************************************************************************************
     // Thin immersed body
     // *******************************************************************************************
 #if 0
     if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
         (solverChoice.advChoice.zero_yflux.size() > 0) ||
         (solverChoice.advChoice.zero_zflux.size() > 0))
     {
         overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
         overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
     }
 #endif
  
     if (solverChoice.advChoice.zero_xflux.size() > 0) {
         amrex::Print() << "Setting up thin immersed body for "
             << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
         BoxArray ba_xf(ba);
         ba_xf.surroundingNodes(0);
         thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
         thin_xforce[lev]->setVal(0.0);
         xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
         xflux_imask[lev]->setVal(1);
         for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
         {
             Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
             //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
             Box xbx = mfi.nodaltilebox(0);
             for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
                 const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
                 if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
                     (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
                     (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
                 {
                     imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
                     amrex::AllPrint() << "  mask xface at " << faceidx << std::endl;
                 }
             }
         }
     } else {
         thin_xforce[lev] = nullptr;
         xflux_imask[lev] = nullptr;
     }
  
     if (solverChoice.advChoice.zero_yflux.size() > 0) {
         amrex::Print() << "Setting up thin immersed body for "
             << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
         BoxArray ba_yf(ba);
         ba_yf.surroundingNodes(1);
         thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
         thin_yforce[lev]->setVal(0.0);
         yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
         yflux_imask[lev]->setVal(1);
         for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
         {
             Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
             //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
             Box ybx = mfi.nodaltilebox(1);
             for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
                 const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
                 if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
                     (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
                     (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
                 {
                     imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
                     amrex::AllPrint() << "  mask yface at " << faceidx << std::endl;
                 }
             }
         }
     } else {
         thin_yforce[lev] = nullptr;
         yflux_imask[lev] = nullptr;
     }
  
     if (solverChoice.advChoice.zero_zflux.size() > 0) {
         amrex::Print() << "Setting up thin immersed body for "
             << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
         BoxArray ba_zf(ba);
         ba_zf.surroundingNodes(2);
         thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
         thin_zforce[lev]->setVal(0.0);
         zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
         zflux_imask[lev]->setVal(1);
         for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
         {
             Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
             //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
             Box zbx = mfi.nodaltilebox(2);
             for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
                 const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
                 if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
                     (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
                     (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
                 {
                     imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
                     //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
                     amrex::AllPrint() << "  mask zface at " << faceidx << std::endl;
                 }
             }
         }
     } else {
         thin_zforce[lev] = nullptr;
         zflux_imask[lev] = nullptr;
     }
 }

◆ init_zphys()

void ERF::init_zphys	(	int	lev,
		amrex::Real	elapsed_time
	)

 {
     if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
     {
         if (lev > 0) {
             //
             // First interpolate from coarser level if there is one
             // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
             //       have been pre-filled - this includes ghost cells both inside and outside
             //       the domain
             //
             InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
                                   IntVect(0,0,0), // do NOT fill ghost cells outside the domain
                                   *z_phys_nd[lev-1], 0, 0, 1,
                                   geom[lev-1], geom[lev],
                                   refRatio(lev-1), &node_bilinear_interp,
                                   domain_bcs_type, BCVars::cons_bc);
         }
  
         int ngrow = ComputeGhostCells(solverChoice) + 2;
         Box bx(surroundingNodes(Geom(lev).Domain())); bx.grow(ngrow);
         FArrayBox terrain_fab(makeSlab(bx,2,0),1);
  
         //
         // If we are using fitted mesh then we use the surface as defined above
         // If we are not using fitted mesh but are using z_levels, we still need z_phys (for now)
         //    but we need to use a flat terrain for the mesh itself (the EB data has already been made
         //    from the correct terrain)
         //
         if (solverChoice.terrain_type != TerrainType::StaticFittedMesh &&
             solverChoice.terrain_type != TerrainType::MovingFittedMesh) {
                 terrain_fab.template setVal<RunOn::Device>(0.0);
         } else {
             //
             // Fill the values of the terrain height at k=0 only
             //
             prob->init_terrain_surface(geom[lev],terrain_fab,elapsed_time);
         }
  
         for (MFIter mfi(*z_phys_nd[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             Box isect = terrain_fab.box() & (*z_phys_nd[lev])[mfi].box();
             if (!isect.isEmpty()) {
                 (*z_phys_nd[lev])[mfi].template copy<RunOn::Device>(terrain_fab,isect,0,isect,0,1);
             }
         }
  
         make_terrain_fitted_coords(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type);
  
         z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
  
         if (lev == 0) {
             Real zmax = z_phys_nd[0]->max(0,0,false);
             Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
             if (rel_diff < 1.e-8) {
                 amrex::Print() << "max of zphys_nd " << zmax << std::endl;
                 amrex::Print() << "max of zlevels  " << zlevels_stag[0][zlevels_stag[0].size()-1] << std::endl;
                 AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < 1.e-8, "Terrain is taller than domain top!");
             }
         } // lev == 0
  
     } else {
         // NOTE: If a WRFInput file is NOT provided for a finer level,
         //       we simply interpolate from the coarse. This is necessary
         //       since we average_down the terrain (ERF_MakeNewLevel.cpp L351).
         //       If a WRFInput file IS present, it overwrites the terrain data.
         if (lev > 0) {
             //
             // First interpolate from coarser level if there is one
             // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
             //       have been pre-filled - this includes ghost cells both inside and outside
             //       the domain
             //
             InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
                                   z_phys_nd[lev]->nGrowVect(), // DO fill ghost cells outside the domain
                                   *z_phys_nd[lev-1], 0, 0, 1,
                                   geom[lev-1], geom[lev],
                                   refRatio(lev-1), &node_bilinear_interp,
                                   domain_bcs_type, BCVars::cons_bc);
         }
     } // init_type
  
     if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
         terrain_blanking[lev]->setVal(1.0);
         MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ComputeGhostCells(solverChoice) + 2);
         terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
         init_immersed_forcing(lev); // needed for real cases
     }
  
     // Compute the min dz and pass to the micro model
     Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
     micro->Set_dzmin(lev, dzmin);
 }

Here is the call graph for this function:

◆ InitData()

void ERF::InitData ( )

 {
     BL_PROFILE_VAR("ERF::InitData()", InitData);
     InitData_pre();
     InitData_post();
     BL_PROFILE_VAR_STOP(InitData);
 }

Referenced by main().

Here is the caller graph for this function:

◆ InitData_post()

void ERF::InitData_post ( )

 {
     if (solverChoice.advChoice.have_zero_flux_faces)
     {
         AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
             "Thin immersed body with refinement not currently supported.");
         if (SolverChoice::mesh_type != MeshType::ConstantDz) {
             amrex::Print() << "NOTE: Thin immersed body with non-constant dz has not been tested." << std::endl;
         }
     }
  
     if (!restart_chkfile.empty()) {
         restart();
     }
  
     //
     // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one and if two way coupling
     //
     if (SolverChoice::mesh_type != MeshType::ConstantDz) {
         if (solverChoice.coupling_type == CouplingType::TwoWay) {
             for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
                 average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
                 average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
             }
         }
         for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
               detJ_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
             z_phys_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
         }
     }
  
 #ifdef ERF_IMPLICIT_W
     if (SolverChoice::mesh_type == MeshType::VariableDz &&
         (solverChoice.vert_implicit_fac[0] > 0 ||
          solverChoice.vert_implicit_fac[1] > 0 ||
          solverChoice.vert_implicit_fac[2] > 0  )       &&
         solverChoice.implicit_momentum_diffusion)
     {
         Warning("Doing implicit solve for u, v, and w with terrain -- this has not been tested");
     }
 #endif
  
     //
     // Copy vars_new into vars_old, then use vars_old to fill covered cells in vars_new during AverageDown
     //
     if (SolverChoice::terrain_type == TerrainType::EB) {
         for (int lev = 0; lev <= finest_level; lev++) {
             int ncomp_cons = vars_new[lev][Vars::cons].nComp();
             MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,ncomp_cons,vars_new[lev][Vars::cons].nGrowVect());
         }
     }
  
     if (restart_chkfile.empty()) {
         if (solverChoice.coupling_type == CouplingType::TwoWay) {
             AverageDown();
         }
     }
  
 #ifdef ERF_USE_PARTICLES
     if (restart_chkfile.empty()) {
         if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
             if (solverChoice.moisture_tight_coupling) {
                 Warning("Tight coupling has not been tested with Lagrangian microphysics");
             }
  
             for (int lev = 0; lev <= finest_level; lev++) {
                 dynamic_cast<LagrangianMicrophysics&>(*micro).initParticles(z_phys_nd[lev]);
             }
         }
     }
 #endif
  
     if (!restart_chkfile.empty()) { // Restart from a checkpoint
  
         // Create the physbc objects for {cons, u, v, w, base state}
         // We fill the additional base state ghost cells just in case we have read the old format
         for (int lev(0); lev <= finest_level; ++lev) {
             make_physbcs(lev);
             (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
         }
  
         if (solverChoice.do_forest_drag) {
             for (int lev(0); lev <= finest_level; ++lev) {
                 m_forest_drag[lev]->define_drag_field(grids[lev], dmap[lev], geom[lev],
                                                       z_phys_cc[lev].get(), z_phys_nd[lev].get());
             }
         }
  
 #ifdef ERF_USE_NETCDF
         //
         // Create the needed bdy_data_xlo etc ... since we don't read it in from checkpoint any more
         // This follows init_from_wrfinput()
         //
         bool use_moist = (solverChoice.moisture_type != MoistureType::None);
         if (solverChoice.use_real_bcs) {
  
             if ( geom[0].isPeriodic(0) || geom[0].isPeriodic(1) ) {
                  amrex::Error("Cannot set periodic lateral boundary conditions when reading in real boundary values");
             }
  
             bdy_time_interval = read_times_from_wrfbdy(nc_bdy_file,
                                                        bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                                        start_bdy_time, final_bdy_time);
  
             Real time_since_start_bdy = t_new[0] + start_time - start_bdy_time;
             int n_time_old = static_cast<int>(time_since_start_bdy /  bdy_time_interval);
  
             int lev = 0;
  
             int ntimes = std::min(n_time_old+3, static_cast<int>(bdy_data_xlo.size()));
  
             for (int itime = n_time_old; itime < ntimes; itime++)
             {
                 amrex::Print() << "READING IN BDY " << itime << std::endl;
                 read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
                                  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                  real_width);
                 convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
                                         *mf_MUB, *mf_C1H, *mf_C2H,
                                         vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel], vars_new[lev][Vars::cons],
                                         geom[lev], use_moist);
             } // itime
         } // use_real_bcs
  
         if (!nc_low_file.empty())
         {
             low_time_interval = read_times_from_wrflow(nc_low_file, low_data_zlo, start_low_time, final_low_time);
  
             int lev = 0;
             sst_lev[lev].resize(low_data_zlo.size());
             tsk_lev[lev].resize(low_data_zlo.size());
  
             Real time_since_start_low = t_new[0] + start_time - start_low_time;
             int n_time_old = static_cast<int>(time_since_start_low /  low_time_interval);
  
             int ntimes = std::min(n_time_old+3, static_cast<int>(low_data_zlo.size()));
  
             for (int itime = n_time_old; itime < ntimes; itime++)
             {
                 amrex::Print() << "READING IN LOW " << itime << std::endl;
                 read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
  
                 // Need to read PSFC
                 FArrayBox NC_fab_var_file;
                 for (int idx = 0; idx < num_boxes_at_level[lev]; idx++) {
                     int success, use_theta_m;
                     read_from_wrfinput(lev, boxes_at_level[lev][idx], nc_init_file[lev][0],
                                        NC_fab_var_file, "PSFC", geom[lev],
                                        use_theta_m, success);
                     auto& var_fab = NC_fab_var_file;
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                     for ( MFIter mfi(*mf_PSFC[lev], false); mfi.isValid(); ++mfi )
                     {
                         FArrayBox &cur_fab = (*mf_PSFC[lev])[mfi];
                         cur_fab.template copy<RunOn::Device>(var_fab, 0, 0, 1);
                     }
                     var_fab.clear();
                 }
  
                 update_sst_tsk(itime, geom[lev], ba2d[lev],
                                sst_lev[lev], tsk_lev[lev],
                                m_SurfaceLayer, low_data_zlo,
                                vars_new[lev][Vars::cons], *mf_PSFC[lev],
                                solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
             } // itime
         }
 #endif
     } // end restart
  
 #ifdef ERF_USE_PARTICLES
     /* If using a Lagrangian microphysics model, its particle container has now been
        constructed and initialized (calls to micro->Init). So, add its pointer to
        ERF::particleData and remove its name from list of unallocated particle containers. */
     if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
         const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
         const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
         particleData.pushBack(pc_name, pc_ptr);
         particleData.getNamesUnalloc().remove(pc_name);
     }
 #endif
  
     if (input_bndry_planes) {
         // Read the "time.dat" file to know what data is available
         m_r2d->read_time_file();
  
         // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
         Real dt_dummy = 0.0;
         m_r2d->read_input_files(t_new[0],dt_dummy,m_bc_extdir_vals);
     }
  
     if (solverChoice.custom_rhotheta_forcing)
     {
         rhotheta_src.resize(max_level+1);
         for (int lev = 0; lev <= finest_level; lev++) {
             BoxList bl_src = vars_new[lev][Vars::cons].boxArray().boxList();
             for (auto& b : bl_src) {
                 if (!solverChoice.spatial_rhotheta_forcing)
                 {
                     // source is only defined in Z
                     b.setRange(0, 0, 1);
                     b.setRange(1, 0, 1);
                 }
             }
             BoxArray ba_src(std::move(bl_src));
             rhotheta_src[lev] = std::make_unique<MultiFab>(ba_src, vars_new[lev][Vars::cons].DistributionMap(), 1, 0);
             rhotheta_src[lev]->setVal(0.);
             prob->update_rhotheta_sources(t_new[0],
                                           rhotheta_src[lev].get(),
                                           geom[lev], z_phys_cc[lev]);
         }
     }
  
     if (solverChoice.have_geo_wind_profile)
     {
         h_u_geos.resize(max_level+1, Vector<Real>(0));
         d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
         h_v_geos.resize(max_level+1, Vector<Real>(0));
         d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
         for (int lev = 0; lev <= finest_level; lev++) {
             const int domlen = geom[lev].Domain().length(2);
             h_u_geos[lev].resize(domlen, 0.0_rt);
             d_u_geos[lev].resize(domlen, 0.0_rt);
             h_v_geos[lev].resize(domlen, 0.0_rt);
             d_v_geos[lev].resize(domlen, 0.0_rt);
             if (solverChoice.custom_geostrophic_profile) {
                 prob->update_geostrophic_profile(t_new[0],
                                                  h_u_geos[lev], d_u_geos[lev],
                                                  h_v_geos[lev], d_v_geos[lev],
                                                  geom[lev], z_phys_cc[lev]);
             } else {
                 if (SolverChoice::mesh_type == MeshType::VariableDz) {
                     amrex::Print() << "Note: 1-D geostrophic wind profile input is not defined for real terrain" << std::endl;
                 }
                 init_geo_wind_profile(solverChoice.abl_geo_wind_table,
                                       h_u_geos[lev], d_u_geos[lev],
                                       h_v_geos[lev], d_v_geos[lev],
                                       geom[lev],
                                       zlevels_stag[0]);
             }
         }
     }
  
     if (solverChoice.custom_moisture_forcing)
     {
         rhoqt_src.resize(max_level+1);
         for (int lev = 0; lev <= finest_level; lev++) {
             BoxList bl_src = vars_new[lev][Vars::cons].boxArray().boxList();
             for (auto& b : bl_src) {
                 if (!solverChoice.spatial_moisture_forcing)
                 {
                     // source is only defined in Z
                     b.setRange(0, 0, 1);
                     b.setRange(1, 0, 1);
                 }
             }
             BoxArray ba_src(std::move(bl_src));
             rhoqt_src[lev] = std::make_unique<MultiFab>(ba_src, vars_new[lev][Vars::cons].DistributionMap(), 1, 0);
             rhoqt_src[lev]->setVal(0.);
             prob->update_rhoqt_sources(t_new[0],
                                        rhoqt_src[lev].get(),
                                        geom[lev], z_phys_cc[lev]);
         }
     }
  
     if (solverChoice.custom_w_subsidence)
     {
         h_w_subsid.resize(max_level+1, Vector<Real>(0));
         d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
         for (int lev = 0; lev <= finest_level; lev++) {
             const int domlen = geom[lev].Domain().length(2) + 1; // lives on z-faces
             h_w_subsid[lev].resize(domlen, 0.0_rt);
             d_w_subsid[lev].resize(domlen, 0.0_rt);
             prob->update_w_subsidence(t_new[0],
                                       h_w_subsid[lev], d_w_subsid[lev], base_state[lev],
                                       geom[lev], z_phys_nd[lev]);
         }
     }
  
     if (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
         solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T)
     {
         initRayleigh();
         if (solverChoice.init_type == InitType::Input_Sounding)
         {
             // Overwrite ubar, vbar, and thetabar with input profiles;
             // wbar is assumed to be 0. Note: the tau coefficient set by
             // prob->erf_init_rayleigh() is still used
             bool restarting = (!restart_chkfile.empty());
             setRayleighRefFromSounding(restarting);
         }
     }
  
     // Read in sponge data from input file
     if(solverChoice.spongeChoice.sponge_type == "input_sponge")
     {
         initSponge();
         bool restarting = (!restart_chkfile.empty());
         setSpongeRefFromSounding(restarting);
     }
  
     if (solverChoice.pert_type == PerturbationType::Source ||
         solverChoice.pert_type == PerturbationType::Direct ||
         solverChoice.pert_type == PerturbationType::CPM) {
         if (is_it_time_for_action(istep[0], t_new[0], dt[0], pert_interval, -1.)) {
             turbPert.debug(t_new[0]);
         }
     }
  
     // We only write the file at level 0 for now
     if (output_bndry_planes)
     {
         // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
         m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
  
         Real time = 0.;
         if (time >= bndry_output_planes_start_time) {
             bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
             m_w2d->write_planes(0, time, vars_new, is_moist);
         }
     }
  
     // Fill boundary conditions in vars_new
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         auto& lev_new = vars_new[lev];
  
         // ***************************************************************************
         // Physical bc's at domain boundary
         // ***************************************************************************
         IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
         IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
  
         int ncomp_cons = lev_new[Vars::cons].nComp();
         bool do_fb     = true;
  
         (*physbcs_cons[lev])(lev_new[Vars::cons],lev_new[Vars::xvel],lev_new[Vars::yvel],0,ncomp_cons,
                              ngvect_cons,t_new[lev],BCVars::cons_bc,do_fb);
         (   *physbcs_u[lev])(lev_new[Vars::xvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                              ngvect_vels,t_new[lev],BCVars::xvel_bc,do_fb);
         (   *physbcs_v[lev])(lev_new[Vars::yvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                              ngvect_vels,t_new[lev],BCVars::yvel_bc,do_fb);
         (   *physbcs_w[lev])(lev_new[Vars::zvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
                              ngvect_vels,t_new[lev],BCVars::zvel_bc,do_fb);
     }
  
     //
     // If we are starting from scratch, we have the option to project the initial velocity field
     //    regardless of how we initialized.  Note that project_initial_velocity operates on vars_new.
     // pp_inc is used as scratch space here; we zero it out after the projection
     //
     if (restart_chkfile == "")
     {
         for (int lev = 0; lev <= finest_level; ++lev)
         {
             if (solverChoice.project_initial_velocity[lev] == 1) {
                 Real dummy_dt = 1.0;
                 if (verbose > 0) {
                     amrex::Print() << "Projecting initial velocity field at level " << lev << std::endl;
                 }
  
                 project_initial_velocity(lev, t_new[lev], dummy_dt);
  
                 pp_inc[lev].setVal(0.);
                 gradp[lev][GpVars::gpx].setVal(0.);
                 gradp[lev][GpVars::gpy].setVal(0.);
                 gradp[lev][GpVars::gpz].setVal(0.);
             } // project
         } // lev
     }
  
     // Copy from new into old just in case (after filling boundary conditions and possibly projecting)
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         int nc = vars_new[lev][Vars::cons].nComp();
  
         MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,nc,vars_new[lev][Vars::cons].nGrowVect());
         MultiFab::Copy(vars_old[lev][Vars::xvel],vars_new[lev][Vars::xvel],0,0, 1,vars_new[lev][Vars::xvel].nGrowVect());
         MultiFab::Copy(vars_old[lev][Vars::yvel],vars_new[lev][Vars::yvel],0,0, 1,vars_new[lev][Vars::yvel].nGrowVect());
         MultiFab::Copy(vars_old[lev][Vars::zvel],vars_new[lev][Vars::zvel],0,0, 1,vars_new[lev][Vars::zvel].nGrowVect());
     }
  
     // Compute the minimum dz in the domain at each level (to be used for setting the timestep)
     dz_min.resize(max_level+1);
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         dz_min[lev] = geom[lev].CellSize(2);
         if ( SolverChoice::mesh_type != MeshType::ConstantDz ) {
             dz_min[lev] *= (*detJ_cc[lev]).min(0);
         }
     }
  
  
     // We don't need to recompute dt[lev] on restart because we read it in from the checkpoint file.
     if (restart_chkfile.empty()) {
         ComputeDt();
     }
  
     // Check the viscous limit
     DiffChoice dc = solverChoice.diffChoice;
     if (dc.molec_diff_type == MolecDiffType::Constant ||
         dc.molec_diff_type == MolecDiffType::ConstantAlpha) {
         Real delta = std::min({geom[finest_level].CellSize(0),
                                geom[finest_level].CellSize(1),
                                dz_min[finest_level]});
         if (dc.dynamic_viscosity == 0) {
             Print() << "Note: Molecular diffusion specified but dynamic_viscosity has not been specified" << std::endl;
         } else {
             Real nu = dc.dynamic_viscosity / dc.rho0_trans;
             Real viscous_limit = 2.0 * delta*delta / nu;
             Print() << "smallest grid spacing at level " << finest_level << " = " << delta << std::endl;
             Print() << "dt at level " << finest_level << " = " << dt[finest_level] << std::endl;
             Print() << "Viscous CFL is " << dt[finest_level] / viscous_limit << std::endl;
             if (fixed_dt[finest_level] >= viscous_limit) {
                 Warning("Specified fixed_dt is above the viscous limit");
             } else if (dt[finest_level] >= viscous_limit) {
                 Warning("Adaptive dt based on convective CFL only is above the viscous limit");
             }
         }
     }
  
     // Fill ghost cells/faces
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         if (lev > 0 && cf_width >= 0) {
             Construct_ERFFillPatchers(lev);
         }
  
         auto& lev_new = vars_new[lev];
  
         //
         // Fill boundary conditions -- not sure why we need this here
         //
         bool fillset = false;
         if (lev == 0) {
             FillPatchCrseLevel(lev, t_new[lev],
                       {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]});
         } else {
             FillPatchFineLevel(lev, t_new[lev],
                       {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
                       {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                       base_state[lev], base_state[lev],
                       fillset);
         }
  
         //
         // We do this here to make sure level (lev-1) boundary conditions are filled
         // before we interpolate to level (lev) ghost cells
         //
         if (lev < finest_level) {
             auto& lev_old = vars_old[lev];
             MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
             MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
             MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
             MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
         }
  
         //
         // We fill the ghost cell values of the base state in case it wasn't done in the initialization
         //
         base_state[lev].FillBoundary(geom[lev].periodicity());
  
         // For moving terrain only
         if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
             MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
             base_state_new[lev].FillBoundary(geom[lev].periodicity());
         }
  
     }
  
     // Allow idealized cases over water, used to set lmask
     ParmParse pp("erf");
     int is_land;
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         if (pp.query("is_land", is_land, lev)) {
             if (is_land == 1) {
                 amrex::Print() << "Level " << lev << " is land" << std::endl;
             } else if (is_land == 0) {
                 amrex::Print() << "Level " << lev << " is water" << std::endl;
             } else {
                 Error("is_land should be 0 or 1");
             }
             lmask_lev[lev][0]->setVal(is_land);
             lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
         }
     }
  
     // If lev > 0, we need to fill bc's by interpolation from coarser grid
     for (int lev = 1; lev <= finest_level; ++lev)
     {
         Interp2DArrays(lev,grids[lev],dmap[lev]);
     } // lev
  
 #ifdef ERF_USE_WW3_COUPLING
     int my_lev = 0;
     amrex::Print() <<  " About to call send_to_ww3 from ERF.cpp" << std::endl;
     send_to_ww3(my_lev);
     amrex::Print() <<  " About to call read_waves from ERF.cpp"  << std::endl;
     read_waves(my_lev);
    // send_to_ww3(my_lev);
 #endif
  
     // Create wall distance field for RANS model
     for (int lev = 0; lev <= finest_level; lev++) {
         if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
             // Handle bottom boundary
             poisson_wall_dist(lev);
  
             // Correct the wall distance for immersed bodies
             if (solverChoice.advChoice.have_zero_flux_faces) {
                 thinbody_wall_dist(walldist[lev],
                                    solverChoice.advChoice.zero_xflux,
                                    solverChoice.advChoice.zero_yflux,
                                    solverChoice.advChoice.zero_zflux,
                                    geom[lev],
                                    z_phys_cc[lev]);
             }
         }
     }
  
     // Configure SurfaceLayer params if used
     // NOTE: we must set up the MOST routine after calling FillPatch
     //       in order to have lateral ghost cells filled (MOST + terrain interp).
     if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
     {
         bool has_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
                           (solverChoice.turbChoice[0].les_type  != LESType::None)          ||
                           (solverChoice.turbChoice[0].rans_type != RANSType::None)         ||
                           (solverChoice.turbChoice[0].pbl_type  != PBLType::None) );
         AMREX_ALWAYS_ASSERT(has_diff);
  
         bool rotate = solverChoice.use_rotate_surface_flux;
         if (rotate) {
             Print() << "Using surface layer model with stress rotations" << std::endl;
         }
  
         //
         // This constructor will make the SurfaceLayer object but not allocate the arrays at each level.
         //
         m_SurfaceLayer = std::make_unique<SurfaceLayer>(geom, rotate, pp_prefix, Qv_prim,
                                                         z_phys_nd,
                                                         solverChoice.mesh_type,
                                                         solverChoice.terrain_type,
                                                         solverChoice.turbChoice[finest_level],
 #ifdef ERF_USE_NETCDF
                                                         start_low_time, final_low_time, low_time_interval
 #else
                                                         0.0, 0.0
 #endif
                                                         );
         // This call will allocate the arrays at each level. If we regrid later, either changing
         // the number of levels or just the grids at each existing level, we will call an update routine
         // to redefine the internal arrays in m_SurfaceLayer.
         for (int lev = 0; lev <= finest_level; lev++)
         {
             Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
                                          &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
             m_SurfaceLayer->make_SurfaceLayer_at_level(lev,finest_level+1,
                                                        mfv_old, Theta_prim[lev], Qv_prim[lev],
                                                        Qr_prim[lev], z_phys_nd[lev],
                                                        Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
                                                        lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
                                                        sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
         }
  
         // If initializing from an input_sounding, make sure the surface layer
         // is using the same surface conditions
         if (solverChoice.init_type == InitType::Input_Sounding) {
             const Real theta0 = input_sounding_data.theta_ref_inp_sound;
             const Real qv0    = input_sounding_data.qv_ref_inp_sound;
             for (int lev = 0; lev <= finest_level; lev++) {
                 m_SurfaceLayer->set_t_surf(lev, theta0);
                 m_SurfaceLayer->set_q_surf(lev, qv0);
             }
         }
  
         if (restart_chkfile != "") {
             // Update surface fields if needed (and available)
             ReadCheckpointFileSurfaceLayer();
         }
  
         // We now configure ABLMost params here so that we can print the averages at t=0
         // Note we don't fill ghost cells here because this is just for diagnostics
         for (int lev = 0; lev <= finest_level; ++lev)
         {
             IntVect ng = Theta_prim[lev]->nGrowVect();
  
             MultiFab::Copy(  *Theta_prim[lev], vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, ng);
             MultiFab::Divide(*Theta_prim[lev], vars_new[lev][Vars::cons],      Rho_comp, 0, 1, ng);
  
             if (solverChoice.moisture_type != MoistureType::None) {
                 ng = Qv_prim[lev]->nGrowVect();
  
                 MultiFab::Copy(  *Qv_prim[lev], vars_new[lev][Vars::cons], RhoQ1_comp, 0, 1, ng);
                 MultiFab::Divide(*Qv_prim[lev], vars_new[lev][Vars::cons],   Rho_comp, 0, 1, ng);
  
                 int rhoqr_comp = solverChoice.moisture_indices.qr;
                 if (rhoqr_comp > -1) {
                     MultiFab::Copy(  *Qr_prim[lev], vars_new[lev][Vars::cons], rhoqr_comp, 0, 1, ng);
                     MultiFab::Divide(*Qr_prim[lev], vars_new[lev][Vars::cons],   Rho_comp, 0, 1, ng);
                 } else {
                     Qr_prim[lev]->setVal(0.0);
                 }
             }
             m_SurfaceLayer->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim, Qr_prim);
  
             if (restart_chkfile == "") {
                 // Only do this if starting from scratch; if restarting, then
                 // we don't want to call update_fluxes multiple times because
                 // it will change u* and theta* from their previous values
                 m_SurfaceLayer->update_pblh(lev, vars_new, z_phys_cc[lev].get(),
                                             solverChoice.moisture_indices);
 #ifdef ERF_USE_NETCDF
                 Real elapsed_time_since_start_low = t_new[lev] + (start_time - start_low_time);
 #else
                 Real elapsed_time_since_start_low = t_new[lev] + start_time;
 #endif
                 m_SurfaceLayer->update_fluxes(lev, elapsed_time_since_start_low,
                                               vars_new[lev][Vars::cons],
                                               z_phys_nd[lev],
                                               walldist[lev]);
  
                 // Initialize tke(x,y,z) as a function of u*(x,y)
                 if (solverChoice.turbChoice[lev].init_tke_from_ustar) {
                     Real qkefac = 1.0;
                     if (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25 ||
                         solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF)
                     {
                         // https://github.com/NCAR/MYNN-EDMF/blob/90f36c25259ec1960b24325f5b29ac7c5adeac73/module_bl_mynnedmf.F90#L1325-L1333
                         const Real B1 = solverChoice.turbChoice[lev].pbl_mynn.B1;
                         qkefac = 1.5 * std::pow(B1, 2.0/3.0);
                     }
                     m_SurfaceLayer->init_tke_from_ustar(lev, vars_new[lev][Vars::cons], z_phys_nd[lev], qkefac);
                 }
             }
         }
     } // end if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
  
     // Update micro vars and finish moisture model initializations before first plot file
     if (solverChoice.moisture_type != MoistureType::None) {
         for (int lev = 0; lev <= finest_level; ++lev) {
             micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons]);
             micro->FinishInit(lev, vars_new[lev][Vars::cons], z_phys_nd);
         }
     }
  
     // Fill time averaged velocities before first plot file
     if (solverChoice.time_avg_vel) {
         for (int lev = 0; lev <= finest_level; ++lev) {
             Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
                               vars_new[lev][Vars::xvel],
                               vars_new[lev][Vars::yvel],
                               vars_new[lev][Vars::zvel]);
         }
     }
  
     // check for additional plotting variables that are available after particle containers
     // are setup.
     const std::string& pv3d_1 = "plot_vars_1"  ; appendPlotVariables(pv3d_1,plot3d_var_names_1);
     const std::string& pv3d_2 = "plot_vars_2"  ; appendPlotVariables(pv3d_2,plot3d_var_names_2);
     const std::string& pv2d_1 = "plot2d_vars_1"; appendPlotVariables(pv2d_1,plot2d_var_names_1);
     const std::string& pv2d_2 = "plot2d_vars_2"; appendPlotVariables(pv2d_2,plot2d_var_names_2);
  
     if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > 0.) )
     {
         WriteCheckpointFile();
         last_check_file_step = 0;
         if (m_check_per > 0.) {last_check_file_time += m_check_per;}
     }
  
     if ( (restart_chkfile.empty()) ||
          (!restart_chkfile.empty() && plot_file_on_restart) )
     {
         if (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.)
         {
             Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
             if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 += m_plot3d_per_1;}
             last_plot3d_file_step_1 = istep[0];
         }
         if (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.)
         {
             Write3DPlotFile(2,plotfile3d_type_2,plot3d_var_names_2);
             if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 += m_plot3d_per_2;}
             last_plot3d_file_step_2 = istep[0];
         }
         if (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.)
         {
             Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
             if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 += m_plot2d_per_1;}
             last_plot2d_file_step_1 = istep[0];
         }
         if (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.)
         {
             Write2DPlotFile(2,plotfile2d_type_2,plot2d_var_names_2);
             if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 += m_plot2d_per_2;}
             last_plot2d_file_step_2 = istep[0];
         }
         for (int i = 0; i < m_subvol_int.size(); i++) {
             if (m_subvol_int[i] > 0 || m_subvol_per[i] > 0.) {
                 WriteSubvolume(i,subvol3d_var_names);
                 last_subvol_step[i] = istep[0];
                 if (m_subvol_per[i] > 0.) {last_subvol_time[i] += m_subvol_per[i];}
             }
         }
     }
  
     // Set these up here because we need to know which MPI rank "cell" is on...
     if (pp.contains("data_log"))
     {
         int num_datalogs = pp.countval("data_log");
         datalog.resize(num_datalogs);
         datalogname.resize(num_datalogs);
         pp.queryarr("data_log",datalogname,0,num_datalogs);
         for (int i = 0; i < num_datalogs; i++) {
             setRecordDataInfo(i,datalogname[i]);
         }
     }
  
     if (pp.contains("der_data_log"))
     {
         int num_der_datalogs = pp.countval("der_data_log");
         der_datalog.resize(num_der_datalogs);
         der_datalogname.resize(num_der_datalogs);
         pp.queryarr("der_data_log",der_datalogname,0,num_der_datalogs);
         for (int i = 0; i < num_der_datalogs; i++) {
             setRecordDerDataInfo(i,der_datalogname[i]);
         }
     }
  
     if (pp.contains("energy_data_log"))
     {
         int num_energy_datalogs = pp.countval("energy_data_log");
         tot_e_datalog.resize(num_energy_datalogs);
         tot_e_datalogname.resize(num_energy_datalogs);
         pp.queryarr("energy_data_log",tot_e_datalogname,0,num_energy_datalogs);
         for (int i = 0; i < num_energy_datalogs; i++) {
             setRecordEnergyDataInfo(i,tot_e_datalogname[i]);
         }
     }
  
     if (solverChoice.rad_type != RadiationType::None)
     {
         // Create data log for radiation model if requested
         rad[0]->setupDataLog();
     }
  
  
     if (restart_chkfile.empty() && profile_int > 0) {
         if (destag_profiles) {
             // all variables cell-centered
             write_1D_profiles(t_new[0]);
         } else {
             // some variables staggered
             write_1D_profiles_stag(t_new[0]);
         }
     }
  
     if (pp.contains("sample_point_log") && pp.contains("sample_point"))
     {
         int lev = 0;
  
         int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
         if (num_samplepts > 0) {
             Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
             samplepoint.resize(num_samplepts);
  
             pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
             for (int i = 0; i < num_samplepts; i++) {
                 IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                 samplepoint[i] = iv;
             }
         }
  
         int num_sampleptlogs = pp.countval("sample_point_log");
         AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
         if (num_sampleptlogs > 0) {
             sampleptlog.resize(num_sampleptlogs);
             sampleptlogname.resize(num_sampleptlogs);
             pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
  
             for (int i = 0; i < num_sampleptlogs; i++) {
                 setRecordSamplePointInfo(i,lev,samplepoint[i],sampleptlogname[i]);
             }
         }
  
     }
  
     if (pp.contains("sample_line_log") && pp.contains("sample_line"))
     {
         int lev = 0;
  
         int num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
         if (num_samplelines > 0) {
             Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
             sampleline.resize(num_samplelines);
  
             pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
             for (int i = 0; i < num_samplelines; i++) {
                 IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
                 sampleline[i] = iv;
             }
         }
  
         int num_samplelinelogs = pp.countval("sample_line_log");
         AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
         if (num_samplelinelogs > 0) {
             samplelinelog.resize(num_samplelinelogs);
             samplelinelogname.resize(num_samplelinelogs);
             pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
  
             for (int i = 0; i < num_samplelinelogs; i++) {
                 setRecordSampleLineInfo(i,lev,sampleline[i],samplelinelogname[i]);
             }
         }
  
     }
  
     if (is_it_time_for_action(istep[0], t_new[0], dt[0], sum_interval, sum_per)) {
         sum_integrated_quantities(t_new[0]);
         sum_derived_quantities(t_new[0]);
         sum_energy_quantities(t_new[0]);
     }
  
     // Create object to do line and plane sampling if needed
     bool do_line = false; bool do_plane = false;
     pp.query("do_line_sampling",do_line); pp.query("do_plane_sampling",do_plane);
     if (do_line) {
         if (line_sampling_interval < 0 && line_sampling_per < 0) {
             Abort("Need to specify line_sampling_interval or line_sampling_per");
         }
         line_sampler = std::make_unique<LineSampler>();
         line_sampler->write_coords(z_phys_cc);
     }
     if (do_plane) {
         if (plane_sampling_interval < 0 && plane_sampling_per < 0) {
             Abort("Need to specify plane_sampling_interval or plane_sampling_per");
         }
         plane_sampler = std::make_unique<PlaneSampler>();
     }
  
     if ( solverChoice.terrain_type == TerrainType::EB ||
          solverChoice.terrain_type == TerrainType::ImmersedForcing  ||
          solverChoice.buildings_type == BuildingsType::ImmersedForcing )
     {
         bool write_eb_surface = false;
         pp.query("write_eb_surface", write_eb_surface);
         if (write_eb_surface) {
             if (verbose > 0) {
                 amrex::Print() << "Writing the geometry to a vtp file.\n" << std::endl;
             }
             WriteEBSurface(grids[finest_level],dmap[finest_level],Geom(finest_level),&EBFactory(finest_level));
         }
     }
  
 }

Here is the call graph for this function:

◆ InitData_pre()

void ERF::InitData_pre ( )

 {
     // Initialize the start time for our CPU-time tracker
     startCPUTime = ParallelDescriptor::second();
  
     // Create the ReadBndryPlanes object so we can read boundary plane data
     // m_r2d is used by init_bcs so we must instantiate this class before
     if (input_bndry_planes) {
         Print() << "Defining r2d for the first time " << std::endl;
         m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
     }
  
     if (restart_chkfile.empty()) {
         // Start simulation from the beginning
         InitFromScratch(0.0);
     } else {
         // For initialization this is done in init_only; it is done here for restart
         init_bcs();
     }
  
     solverChoice.check_params(max_level,geom,phys_bc_type);
 }

◆ initHSE() [1/2]

void ERF::initHSE ( )

private

Initialize HSE.

 {
     for (int lev = 0; lev <= finest_level; lev++)
     {
         initHSE(lev);
     }
 }

◆ initHSE() [2/2]

void ERF::initHSE ( int lev )

private

Initialize density and pressure base state in hydrostatic equilibrium.

 {
     // This integrates up through column to update p_hse, pi_hse, th_hse;
     // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
  
     MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
     MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
     MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
     MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
     MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
  
     bool all_boxes_touch_bottom = true;
     Box domain(geom[lev].Domain());
  
     int icomp = 0; int ncomp = BaseState::num_comps;
  
     if (lev == 0) {
         BoxArray ba(base_state[lev].boxArray());
         for (int i = 0; i < ba.size(); i++) {
             if (ba[i].smallEnd(2) != domain.smallEnd(2)) {
                 all_boxes_touch_bottom = false;
             }
         }
     }
     else
     {
         //
         // We need to do this interp from coarse level in order to set the values of
         // the base state inside the domain but outside of the fine region
         //
         base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
         //
         // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
         //       have been pre-filled - this includes ghost cells both inside and outside
         //       the domain
         //
         InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                               IntVect(0,0,0), // do not fill ghost cells outside the domain
                               base_state[lev-1], icomp, icomp, ncomp,
                               geom[lev-1], geom[lev],
                               refRatio(lev-1), &cell_cons_interp,
                               domain_bcs_type, BCVars::cons_bc);
  
          // We need to do this here because the interpolation above may leave corners unfilled
          //    when the corners need to be filled by, for example, reflection of the fine ghost
          //    cell outside the fine region but inide the domain.
          (*physbcs_base[lev])(base_state[lev],icomp,ncomp,base_state[lev].nGrowVect());
     }
  
     if (all_boxes_touch_bottom || lev > 0) {
  
         // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
         if ( (solverChoice.init_type == InitType::MoistBaseState) ||
              (solverChoice.init_type == InitType::HindCast) )
         {
             prob->erf_init_dens_hse_moist(r_hse, z_phys_nd[lev], geom[lev]);
  
         }
         else if (solverChoice.init_type == InitType::ConstantDensity)
         {
             // In this case we set rho from user-specified values, then integrate
             //    to define p from HSE (even if gravity = 0), then compute theta from (p,rho)
             prob->erf_init_const_dens_hse(r_hse);
         }
         else if (solverChoice.init_type == InitType::Uniform)
         {
             // In this case we set both rho and theta from user-specified values
             AMREX_ALWAYS_ASSERT(!solverChoice.use_gravity || solverChoice.anelastic[lev]);
             prob->erf_init_const_dens_and_th_hse(r_hse,p_hse,pi_hse,th_hse,qv_hse,solverChoice.rdOcp);
         }
         else
         {
             // In this case we set rho from user-specified values, then integrate
             //    to define p from HSE (even if gravity = 0), then compute theta from (p,rho)
             prob->erf_init_dens_hse(r_hse, z_phys_nd[lev], z_phys_cc[lev], geom[lev]);
         }
  
         if (solverChoice.init_type != InitType::Uniform) {
             erf_enforce_hse(lev, r_hse, p_hse, pi_hse, th_hse, qv_hse, z_phys_cc[lev]);
         }
  
         //
         // Impose physical bc's on the base state
         //
         (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
  
     } else {
  
         BoxArray ba_new(domain);
  
         ChopGrids2D(ba_new, domain, ParallelDescriptor::NProcs());
  
         DistributionMapping dm_new(ba_new);
  
         MultiFab new_base_state(ba_new, dm_new, BaseState::num_comps, base_state[lev].nGrowVect());
         new_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
                                     base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
  
         MultiFab new_r_hse (new_base_state, make_alias, BaseState::r0_comp, 1);
         MultiFab new_p_hse (new_base_state, make_alias, BaseState::p0_comp, 1);
         MultiFab new_pi_hse(new_base_state, make_alias, BaseState::pi0_comp, 1);
         MultiFab new_th_hse(new_base_state, make_alias, BaseState::th0_comp, 1);
         MultiFab new_qv_hse(new_base_state, make_alias, BaseState::qv0_comp, 1);
  
         std::unique_ptr<MultiFab> new_z_phys_cc;
         std::unique_ptr<MultiFab> new_z_phys_nd;
         if (solverChoice.mesh_type != MeshType::ConstantDz) {
             new_z_phys_cc = std::make_unique<MultiFab>(ba_new,dm_new,1,1);
             new_z_phys_cc->ParallelCopy(*z_phys_cc[lev],0,0,1,1,1);
  
             BoxArray ba_new_nd(ba_new);
             ba_new_nd.surroundingNodes();
             new_z_phys_nd = std::make_unique<MultiFab>(ba_new_nd,dm_new,1,1);
             new_z_phys_nd->ParallelCopy(*z_phys_nd[lev],0,0,1,1,1);
         }
  
         // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
         if (solverChoice.init_type == InitType::MoistBaseState) {
             prob->erf_init_dens_hse_moist(new_r_hse, new_z_phys_nd, geom[lev]);
  
         } else if (solverChoice.init_type == InitType::ConstantDensity) {
  
             // In this case we set rho from user-specified values, then integrate
             //    to define p from HSE (even if gravity = 0), then compute theta from (p,rho)
             prob->erf_init_const_dens_hse(r_hse);
  
         } else if (solverChoice.init_type == InitType::Uniform) {
  
             // In this case we set both rho and theta from user-specified values
             AMREX_ALWAYS_ASSERT(!solverChoice.use_gravity || solverChoice.anelastic[lev]);
             prob->erf_init_const_dens_and_th_hse(r_hse,p_hse,pi_hse,th_hse,qv_hse,solverChoice.rdOcp);
  
         } else {
             prob->erf_init_dens_hse(new_r_hse, new_z_phys_nd, new_z_phys_cc, geom[lev]);
         }
  
         erf_enforce_hse(lev, new_r_hse, new_p_hse, new_pi_hse, new_th_hse, new_qv_hse, new_z_phys_cc);
  
         //
         // Impose physical bc's on the base state (we must make new, temporary bcs object because the z_phys_nd is different)
         //
         ERFPhysBCFunct_base* temp_physbcs_base =
             new ERFPhysBCFunct_base(lev, geom[lev], domain_bcs_type, domain_bcs_type_d, new_z_phys_nd,
                                     (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
         (*temp_physbcs_base)(new_base_state,0,new_base_state.nComp(),new_base_state.nGrowVect());
         delete temp_physbcs_base;
  
         // Now copy back into the original arrays
         base_state[lev].ParallelCopy(new_base_state,0,0,base_state[lev].nComp(),
                                      base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
     }
  
     //
     // Impose physical bc's on the base state -- the values outside the fine region
     //   but inside the domain have already been filled in the call above to InterpFromCoarseLevel
     //
     (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
 }

Here is the call graph for this function:

◆ initialize_integrator()

void ERF::initialize_integrator	(	int	lev,
		amrex::MultiFab &	cons_mf,
		amrex::MultiFab &	vel_mf
	)

private

 {
     const BoxArray& ba(cons_mf.boxArray());
     const DistributionMapping& dm(cons_mf.DistributionMap());
  
     int ncomp_cons = cons_mf.nComp();
  
     // Initialize the integrator memory
     Vector<MultiFab> int_state; // integration state data structure example
     int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons));         // cons
     int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
     int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
     int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
  
     mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
     mri_integrator_mem[lev]->setNoSubstepping((solverChoice.substepping_type[lev] == SubsteppingType::None));
     mri_integrator_mem[lev]->setAnelastic(solverChoice.anelastic[lev]);
     mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
     mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
 }

◆ InitializeFromFile()

void ERF::InitializeFromFile ( )

private

◆ InitializeLevelFromData()

void ERF::InitializeLevelFromData	(	int	lev,
		const amrex::MultiFab &	initial_data
	)

private

◆ initializeMicrophysics()

void ERF::initializeMicrophysics ( const int & a_nlevsmax )

private

Parameters

a_nlevsmax number of AMR levels

 {
     if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
  
         micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
  
     } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
 #ifdef ERF_USE_PARTICLES
         micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
         /* Lagrangian microphysics models will have a particle container; it needs to be added
            to ERF::particleData */
         const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
         /* The particle container has not yet been constructed and initialized, so just add
            its name here for now (so that functions to set plotting variables can see it). */
         particleData.addName( pc_name );
  
 #else
         Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
 #endif
     }
  
     qmoist.resize(a_nlevsmax);
     return;
 }

Here is the call graph for this function:

◆ initRayleigh()

void ERF::initRayleigh ( )

private

Initialize Rayleigh damping profiles.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

 {
     const int khi = geom[0].Domain().bigEnd(2);
     solverChoice.dampingChoice.rayleigh_ztop = (solverChoice.terrain_type == TerrainType::None) ?  geom[0].ProbHi(2) : zlevels_stag[0][khi+1];
  
     h_rayleigh_ptrs.resize(max_level+1);
     d_rayleigh_ptrs.resize(max_level+1);
  
     h_sinesq_ptrs.resize(max_level+1);
     d_sinesq_ptrs.resize(max_level+1);
  
     h_sinesq_stag_ptrs.resize(max_level+1);
     d_sinesq_stag_ptrs.resize(max_level+1);
  
     for (int lev = 0; lev <= finest_level; lev++)
     {
         // These have 4 components: ubar, vbar, wbar, thetabar
         h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
         d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
  
         const int zlen_rayleigh = geom[lev].Domain().length(2);
  
         // Allocate space for these 1D vectors
         for (int n = 0; n < Rayleigh::nvars; n++) {
             h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
             d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
         }
  
         h_sinesq_ptrs[lev].resize(zlen_rayleigh);
         d_sinesq_ptrs[lev].resize(zlen_rayleigh);
  
         h_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
         d_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
  
         Real ztop     = solverChoice.dampingChoice.rayleigh_ztop;
         Real zdamp    = solverChoice.dampingChoice.rayleigh_zdamp;
  
         for (int k = 0; k < zlen_rayleigh; k++) {
             Real z = 0.5 * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
             if (z > (ztop - zdamp)) {
                 Real zfrac = 1.0 - (ztop - z) / zdamp;
                 Real s = std::sin(PIoTwo*zfrac);
                 h_sinesq_ptrs[lev][k] = s*s;
             } else {
                 h_sinesq_ptrs[lev][k] = 0.0;
             }
         }
  
         for (int k = 0; k < zlen_rayleigh+1; k++) {
             Real z = zlevels_stag[lev][k];
             if (z > (ztop - zdamp)) {
                 Real zfrac = 1.0 - (ztop - z) / zdamp;
                 Real s = std::sin(PIoTwo*zfrac);
                 h_sinesq_stag_ptrs[lev][k] = s*s;
             } else {
                 h_sinesq_stag_ptrs[lev][k] = 0.0;
             }
         }
  
         // Init the host vectors for the reference states
         prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_nd[lev], solverChoice.dampingChoice.rayleigh_zdamp);
  
         // Copy from host vectors to device vectors
         for (int n = 0; n < Rayleigh::nvars; n++) {
             Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
                       d_rayleigh_ptrs[lev][n].begin());
         }
         Gpu::copy(Gpu::hostToDevice, h_sinesq_ptrs[lev].begin(), h_sinesq_ptrs[lev].end(), d_sinesq_ptrs[lev].begin());
         Gpu::copy(Gpu::hostToDevice, h_sinesq_stag_ptrs[lev].begin(), h_sinesq_stag_ptrs[lev].end(), d_sinesq_stag_ptrs[lev].begin());
     }
 }

◆ initSponge()

void ERF::initSponge ( )

private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

 {
     h_sponge_ptrs.resize(max_level+1);
     d_sponge_ptrs.resize(max_level+1);
  
     for (int lev = 0; lev <= finest_level; lev++)
     {
         // These have 2 components: ubar, vbar
         h_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
         d_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
  
         const int zlen_sponge = geom[lev].Domain().length(2);
  
         // Allocate space for these 1D vectors
         for (int n = 0; n < Sponge::nvars_sponge; n++) {
             h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
             d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
         }
  
     }
 }

◆ input_sponge()

void ERF::input_sponge ( int lev )

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters

lev	Integer specifying the current level

 {
     // We only want to read the file once
     if (lev == 0) {
         if (input_sponge_data.input_sponge_file.empty())
             Error("input_sounding file name must be provided via input");
  
         // this will interpolate the input profiles to the nominal height levels
         // (ranging from 0 to the domain top)
         input_sponge_data.read_from_file(geom[lev], zlevels_stag[lev]);
     }
 }

◆ Interp2DArrays()

void ERF::Interp2DArrays	(	int	lev,
		const amrex::BoxArray &	my_ba2d,
		const amrex::DistributionMapping &	my_dm
	)

 {
     if (lon_m[lev-1] && !lon_m[lev]) {
         auto ngv = lon_m[lev-1]->nGrowVect(); ngv[2] = 0;
         lon_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
         InterpFromCoarseLevel(*lon_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                               *lon_m[lev-1], 0, 0, 1,
                               geom[lev-1], geom[lev],
                               refRatio(lev-1), &cell_cons_interp,
                               domain_bcs_type, BCVars::cons_bc);
     }
     if (lat_m[lev-1] && !lat_m[lev]) {
         auto ngv = lat_m[lev-1]->nGrowVect(); ngv[2] = 0;
         lat_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
         InterpFromCoarseLevel(*lat_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                               *lat_m[lev-1], 0, 0, 1,
                               geom[lev-1], geom[lev],
                               refRatio(lev-1), &cell_cons_interp,
                               domain_bcs_type, BCVars::cons_bc);
     }
     if (sinPhi_m[lev-1] && !sinPhi_m[lev]) {
         auto ngv = sinPhi_m[lev-1]->nGrowVect(); ngv[2] = 0;
         sinPhi_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
         InterpFromCoarseLevel(*sinPhi_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                               *sinPhi_m[lev-1], 0, 0, 1,
                               geom[lev-1], geom[lev],
                               refRatio(lev-1), &cell_cons_interp,
                               domain_bcs_type, BCVars::cons_bc);
     }
     if (cosPhi_m[lev-1] && !cosPhi_m[lev]) {
         auto ngv = cosPhi_m[lev-1]->nGrowVect(); ngv[2] = 0;
         cosPhi_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
         InterpFromCoarseLevel(*cosPhi_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                               *cosPhi_m[lev-1], 0, 0, 1,
                               geom[lev-1], geom[lev],
                               refRatio(lev-1), &cell_cons_interp,
                               domain_bcs_type, BCVars::cons_bc);
     }
     if (sst_lev[lev-1][0] && !sst_lev[lev][0]) {
         int ntimes = sst_lev[lev-1].size();
         sst_lev[lev].resize(ntimes);
         auto ngv = sst_lev[lev-1][0]->nGrowVect(); ngv[2] = 0;
         for (int n = 0; n < ntimes; n++) {
             sst_lev[lev][n] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
             InterpFromCoarseLevel(*sst_lev[lev][n], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                                   *sst_lev[lev-1][n], 0, 0, 1,
                                   geom[lev-1], geom[lev],
                                   refRatio(lev-1), &cell_cons_interp,
                                   domain_bcs_type, BCVars::cons_bc);
         }
     }
     if (tsk_lev[lev-1][0] && !tsk_lev[lev][0]) {
         int ntimes = tsk_lev[lev-1].size();
         tsk_lev[lev].resize(ntimes);
         auto ngv = tsk_lev[lev-1][0]->nGrowVect(); ngv[2] = 0;
         for (int n = 0; n < ntimes; n++) {
             tsk_lev[lev][n] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
             InterpFromCoarseLevel(*tsk_lev[lev][n], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
                                   *tsk_lev[lev-1][n], 0, 0, 1,
                                   geom[lev-1], geom[lev],
                                   refRatio(lev-1), &cell_cons_interp,
                                   domain_bcs_type, BCVars::cons_bc);
         }
     }
  
     Real time_for_fp = 0.; // This is not actually used
     Vector<Real> ftime    = {time_for_fp, time_for_fp};
     Vector<Real> ctime    = {time_for_fp, time_for_fp};
     if (lat_m[lev]) {
         // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
         Vector<MultiFab*> fmf = {lat_m[lev  ].get(), lat_m[lev  ].get()};
         Vector<MultiFab*> cmf = {lat_m[lev-1].get(), lat_m[lev-1].get()};
         IntVect ngv = lat_m[lev]->nGrowVect(); ngv[2] = 0;
         Interpolater* mapper = &cell_cons_interp;
         FillPatchTwoLevels(*lat_m[lev].get(), ngv, IntVect(0,0,0),
                            time_for_fp, cmf, ctime, fmf, ftime,
                            0, 0, 1, geom[lev-1], geom[lev],
                            refRatio(lev-1), mapper, domain_bcs_type,
                            BCVars::cons_bc);
     }
     if (lon_m[lev]) {
         // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
         Vector<MultiFab*> fmf = {lon_m[lev  ].get(), lon_m[lev  ].get()};
         Vector<MultiFab*> cmf = {lon_m[lev-1].get(), lon_m[lev-1].get()};
         IntVect ngv = lon_m[lev]->nGrowVect(); ngv[2] = 0;
         Interpolater* mapper = &cell_cons_interp;
         FillPatchTwoLevels(*lon_m[lev].get(), ngv, IntVect(0,0,0),
                            time_for_fp, cmf, ctime, fmf, ftime,
                            0, 0, 1, geom[lev-1], geom[lev],
                            refRatio(lev-1), mapper, domain_bcs_type,
                            BCVars::cons_bc);
     } // lon_m
     if (sinPhi_m[lev]) {
         // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
         Vector<MultiFab*> fmf = {sinPhi_m[lev  ].get(), sinPhi_m[lev  ].get()};
         Vector<MultiFab*> cmf = {sinPhi_m[lev-1].get(), sinPhi_m[lev-1].get()};
         IntVect ngv = sinPhi_m[lev]->nGrowVect(); ngv[2] = 0;
         Interpolater* mapper = &cell_cons_interp;
         FillPatchTwoLevels(*sinPhi_m[lev].get(), ngv, IntVect(0,0,0),
                            time_for_fp, cmf, ctime, fmf, ftime,
                            0, 0, 1, geom[lev-1], geom[lev],
                            refRatio(lev-1), mapper, domain_bcs_type,
                            BCVars::cons_bc);
     } // sinPhi
     if (cosPhi_m[lev]) {
         // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
         Vector<MultiFab*> fmf = {cosPhi_m[lev  ].get(), cosPhi_m[lev  ].get()};
         Vector<MultiFab*> cmf = {cosPhi_m[lev-1].get(), cosPhi_m[lev-1].get()};
         IntVect ngv = cosPhi_m[lev]->nGrowVect(); ngv[2] = 0;
         Interpolater* mapper = &cell_cons_interp;
         FillPatchTwoLevels(*cosPhi_m[lev].get(), ngv, IntVect(0,0,0),
                            time_for_fp, cmf, ctime, fmf, ftime,
                            0, 0, 1, geom[lev-1], geom[lev],
                            refRatio(lev-1), mapper, domain_bcs_type,
                            BCVars::cons_bc);
     } // cosPhi
     if (sst_lev[lev][0]) {
         // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
     int ntimes = sst_lev[lev].size();
     for (int n = 0; n < ntimes; n++) {
             Vector<MultiFab*> fmf = {sst_lev[lev  ][n].get(), sst_lev[lev  ][n].get()};
             Vector<MultiFab*> cmf = {sst_lev[lev-1][n].get(), sst_lev[lev-1][n].get()};
             IntVect ngv = sst_lev[lev][n]->nGrowVect(); ngv[2] = 0;
             Interpolater* mapper = &cell_cons_interp;
             FillPatchTwoLevels(*sst_lev[lev][n].get(), ngv, IntVect(0,0,0),
                                time_for_fp, cmf, ctime, fmf, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                refRatio(lev-1), mapper, domain_bcs_type,
                                BCVars::cons_bc);
         } // ntimes
     } // sst_lev
     if (tsk_lev[lev][0]) {
         // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
     int ntimes = tsk_lev[lev].size();
     for (int n = 0; n < ntimes; n++) {
             Vector<MultiFab*> fmf = {tsk_lev[lev  ][n].get(), tsk_lev[lev  ][n].get()};
             Vector<MultiFab*> cmf = {tsk_lev[lev-1][n].get(), tsk_lev[lev-1][n].get()};
             IntVect ngv = tsk_lev[lev][n]->nGrowVect(); ngv[2] = 0;
             Interpolater* mapper = &cell_cons_interp;
             FillPatchTwoLevels(*tsk_lev[lev][n].get(), ngv, IntVect(0,0,0),
                                time_for_fp, cmf, ctime, fmf, ftime,
                                0, 0, 1, geom[lev-1], geom[lev],
                                refRatio(lev-1), mapper, domain_bcs_type,
                                BCVars::cons_bc);
         } // ntimes
     } // tsk_lev
 }

Here is the call graph for this function:

◆ is_it_time_for_action()

bool ERF::is_it_time_for_action	(	int	nstep,
		amrex::Real	time,
		amrex::Real	dt,
		int	action_interval,
		amrex::Real	action_per
	)

static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters

nstep	Timestep number
time	Current time
dtlev	Timestep for the current level
action_interval	Interval in number of timesteps for taking action
action_per	Interval in simulation time for taking action

 {
   bool int_test = (action_interval > 0 && nstep % action_interval == 0);
  
   bool per_test = false;
   if (action_per > 0.0) {
     const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
     const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
  
     if (num_per_old != num_per_new) {
       per_test = true;
     }
   }
  
   return int_test || per_test;
 }

◆ make_eb_box()

void ERF::make_eb_box ( )

◆ make_eb_regular()

void ERF::make_eb_regular ( )

◆ make_physbcs()

void ERF::make_physbcs ( int lev )

private

 {
     if (SolverChoice::mesh_type == MeshType::VariableDz) {
         AMREX_ALWAYS_ASSERT(z_phys_nd[lev] != nullptr);
     }
  
     physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                                m_bc_extdir_vals, m_bc_neumann_vals,
                                                                z_phys_nd[lev], solverChoice.use_real_bcs, th_bc_data[lev].data());
     physbcs_u[lev]    = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                             m_bc_extdir_vals, m_bc_neumann_vals,
                                                             z_phys_nd[lev], solverChoice.use_real_bcs, xvel_bc_data[lev].data());
     physbcs_v[lev]    = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                             m_bc_extdir_vals, m_bc_neumann_vals,
                                                             z_phys_nd[lev], solverChoice.use_real_bcs, yvel_bc_data[lev].data());
     physbcs_w[lev]    = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                             m_bc_extdir_vals, m_bc_neumann_vals,
                                                             solverChoice.terrain_type, mapfac[lev], z_phys_nd[lev],
                                                             solverChoice.use_real_bcs, zvel_bc_data[lev].data());
     physbcs_base[lev] = std::make_unique<ERFPhysBCFunct_base> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d, z_phys_nd[lev],
                                                                (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
 }

Here is the call graph for this function:

◆ make_subdomains()

void ERF::make_subdomains	(	const amrex::BoxList &	ba,
		amrex::Vector< amrex::BoxArray > &	bins
	)

 {
     Vector<BoxList> bins_bl;
  
     // Clear out any old bins
     bins.clear();
  
     // Iterate over boxes
     for (auto bx : bl)
     {
         bool added = false;
  
         // Try to add box to existing bin
         for (int j = 0; j < bins_bl.size(); ++j) {
             BoxList& bin = bins_bl[j];
             bool touches = false;
  
             for (auto& b : bin)
             {
                 Box gbx(bx); gbx.grow(1);
                 if (gbx.intersects(b)) {
                     touches = true;
                     break;
                 }
             }
  
             if (touches) {
                 bin.push_back(bx);
                 added = true;
                 break;
             }
         }
  
         // If box couldn't be added to existing bin, create new bin
         if (!added) {
             BoxList new_bin;
             new_bin.push_back(bx);
             bins_bl.push_back(new_bin);
         }
     }
  
     // Convert the BoxLists to BoxArrays
     for (int i = 0; i < bins_bl.size(); ++i) {
         bins.push_back(BoxArray(bins_bl[i]));
     }
 }

◆ MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage	(	amrex::Vector< amrex::Real > &	h_havg,
		amrex::MultiFab &	S,
		int	n
	)

 {
     // Get the number of cells in z at level 0
     int dir_z = AMREX_SPACEDIM-1;
     auto domain = geom[0].Domain();
     int size_z = domain.length(dir_z);
     int start_z = domain.smallEnd()[dir_z];
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
  
     // resize the level 0 horizontal average vectors
     h_havg.resize(size_z, 0.0_rt);
  
     // Get the cell centered data and construct sums
 #ifdef _OPENMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(S); mfi.isValid(); ++mfi) {
         const Box& box = mfi.validbox();
         const IntVect& se = box.smallEnd();
         const IntVect& be = box.bigEnd();
  
         auto      fab_arr = S[mfi].array();
  
         FArrayBox fab_reduce(box, 1, The_Async_Arena());
         auto arr_reduce   = fab_reduce.array();
  
         ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
             arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
         });
  
         for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
             Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
             h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
         }
     }
  
     // combine sums from different MPI ranks
     ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), h_havg.size());
  
     // divide by the total number of cells we are averaging over
     for (int k = 0; k < size_z; ++k) {
         h_havg[k]     /= area_z;
     }
 }

Here is the call graph for this function:

◆ MakeEBGeometry()

void ERF::MakeEBGeometry ( )

◆ MakeFilename_EyeTracker_latlon()

std::string ERF::MakeFilename_EyeTracker_latlon ( int nstep )

                                              {
     // Ensure output directory exists
     const std::string dir = "Output_StormTracker/latlon";
     if (!fs::exists(dir)) {
         fs::create_directories(dir);
     }
  
     // Construct filename with zero-padded step
     std::ostringstream oss;
     oss << dir << "/storm_track_latlon" << std::setw(7) << std::setfill('0') << nstep << ".txt";
     return oss.str();
 }

◆ MakeFilename_EyeTracker_maxvel()

std::string ERF::MakeFilename_EyeTracker_maxvel ( int nstep )

                                              {
     // Ensure output directory exists
     const std::string dir = "Output_StormTracker/maxvel";
     if (!fs::exists(dir)) {
         fs::create_directories(dir);
     }
  
     // Construct filename with zero-padded step
     std::ostringstream oss;
     oss << dir << "/storm_maxvel_" << std::setw(7) << std::setfill('0') << nstep << ".txt";
     return oss.str();
 }

◆ MakeFilename_EyeTracker_minpressure()

std::string ERF::MakeFilename_EyeTracker_minpressure ( int nstep )

                                                   {
     // Ensure output directory exists
     const std::string dir = "Output_StormTracker/minpressure";
     if (!fs::exists(dir)) {
         fs::create_directories(dir);
     }
  
     // Construct filename with zero-padded step
     std::ostringstream oss;
     oss << dir << "/storm_minpressure_" << std::setw(7) << std::setfill('0') << nstep << ".txt";
     return oss.str();
 }

◆ MakeHorizontalAverages()

void ERF::MakeHorizontalAverages ( )

 {
     int lev = 0;
  
     // First, average down all levels (if doing two-way coupling)
     if (solverChoice.coupling_type == CouplingType::TwoWay) {
         AverageDown();
     }
  
     MultiFab mf(grids[lev], dmap[lev], 5, 0);
  
     int zdir = 2;
     auto domain = geom[0].Domain();
  
     bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
     bool is_anelastic = (solverChoice.anelastic[lev] == 1);
  
     for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.validbox();
         auto  fab_arr = mf.array(mfi);
         auto const  hse_arr = base_state[lev].const_array(mfi);
         auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
         ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
             Real dens = cons_arr(i, j, k, Rho_comp);
             fab_arr(i, j, k, 0) = dens;
             fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
             if (!use_moisture) {
                 if (is_anelastic) {
                     fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
                 } else {
                     fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp));
                 }
             }
         });
     }
  
     if (use_moisture)
     {
         for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
             const Box& bx = mfi.validbox();
             auto  fab_arr = mf.array(mfi);
             auto const  hse_arr = base_state[lev].const_array(mfi);
             auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
             int ncomp = vars_new[lev][Vars::cons].nComp();
  
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                 Real dens = cons_arr(i, j, k, Rho_comp);
                 if (is_anelastic) {
                     fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
                 } else {
                     Real qv = cons_arr(i, j, k, RhoQ1_comp) / dens;
                     fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
                 }
                 fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : 0.0);
                 fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : 0.0);
             });
         }
  
         Gpu::HostVector<Real> h_avg_qv          = sumToLine(mf,3,1,domain,zdir);
         Gpu::HostVector<Real> h_avg_qc          = sumToLine(mf,4,1,domain,zdir);
     }
  
     // Sum in the horizontal plane
     Gpu::HostVector<Real> h_avg_density     = sumToLine(mf,0,1,domain,zdir);
     Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
     Gpu::HostVector<Real> h_avg_pressure    = sumToLine(mf,2,1,domain,zdir);
  
     // Divide by the total number of cells we are averaging over
     int size_z = domain.length(zdir);
     Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
     int klen = static_cast<int>(h_avg_density.size());
  
     for (int k = 0; k < klen; ++k) {
         h_havg_density[k]     /= area_z;
         h_havg_temperature[k] /= area_z;
         h_havg_pressure[k]    /= area_z;
         if (solverChoice.moisture_type != MoistureType::None)
         {
             h_havg_qc[k]          /= area_z;
             h_havg_qv[k]          /= area_z;
         }
     } // k
  
     // resize device vectors
     d_havg_density.resize(size_z, 0.0_rt);
     d_havg_temperature.resize(size_z, 0.0_rt);
     d_havg_pressure.resize(size_z, 0.0_rt);
  
     // copy host vectors to device vectors
     Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
     Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
     Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
  
     if (solverChoice.moisture_type != MoistureType::None)
     {
         d_havg_qv.resize(size_z, 0.0_rt);
         d_havg_qc.resize(size_z, 0.0_rt);
         Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
         Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
     }
 }

Here is the call graph for this function:

◆ MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse	(	int	lev,
		amrex::Real	time,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

override

 {
     //
     // Note that "time" here is elapsed time
     //
     AMREX_ALWAYS_ASSERT(lev > 0);
  
     if (verbose) {
         amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
     }
  
     //
     // Grow the subdomains vector and build the subdomains vector at this level
     //
     subdomains.resize(lev+1);
     //
     // Create subdomains at each level within the domain such that
     // 1) all boxes in a given subdomain are "connected"
     // 2) no boxes in a subdomain touch any boxes in any other subdomain
     //
     if ( (solverChoice.anelastic[lev] == 0) && (solverChoice.project_initial_velocity[lev] == 0) ) {
         BoxArray dom(geom[lev].Domain());
         subdomains[lev].push_back(dom);
     } else {
         make_subdomains(ba.simplified_list(), subdomains[lev]);
     }
  
     if (lev == 0) init_bcs();
  
     //********************************************************************************************
     // This allocates all kinds of things, including but not limited to: solution arrays,
     //      terrain arrays, ba2d, metric terms and base state.
     // *******************************************************************************************
     init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev], base_state[lev], z_phys_nd[lev]);
  
     //
     // Note that t_new = time here is elapsed time
     //
     t_new[lev] = time;
     t_old[lev] = time - 1.e200;
  
     // ********************************************************************************************
     // Build the data structures for metric quantities used with terrain-fitted coordinates
     // ********************************************************************************************
     if ( solverChoice.terrain_type == TerrainType::EB ||
          solverChoice.terrain_type == TerrainType::ImmersedForcing ||
          solverChoice.buildings_type == BuildingsType::ImmersedForcing)
     {
         const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
         const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
         if (solverChoice.terrain_type == TerrainType::EB) {
             eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
         } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                    solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
             eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
         }
     }
     init_zphys(lev, time);
     update_terrain_arrays(lev);
  
     //
     // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
     //     *and* if there is two-way coupling
     //
     if ( (SolverChoice::mesh_type != MeshType::ConstantDz) && (solverChoice.coupling_type == CouplingType::TwoWay) ) {
         for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
             average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
             average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
         }
     }
  
     // ********************************************************************************************
     // Build the data structures for canopy model (depends upon z_phys)
     // ********************************************************************************************
     if (solverChoice.do_forest_drag) {
         m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
     }
  
     //********************************************************************************************
     // Radiation
     // *******************************************************************************************
     if (solverChoice.rad_type != RadiationType::None)
     {
         rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
     }
  
     // *****************************************************************************************************
     // Initialize the boundary conditions (after initializing the terrain but before calling
     //     initHSE or FillCoarsePatch)
     // *****************************************************************************************************
     make_physbcs(lev);
  
     // ********************************************************************************************
     // Update the base state at this level by interpolation from coarser level
     // ********************************************************************************************
     InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
                           IntVect(0,0,0), // do not fill ghost cells outside the domain
                           base_state[lev-1], 0, 0, base_state[lev].nComp(),
                           geom[lev-1], geom[lev],
                           refRatio(lev-1), &cell_cons_interp,
                           domain_bcs_type, BCVars::cons_bc);
  
     // Impose bc's outside the domain
     (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
  
     //********************************************************************************************
     // Microphysics
     // *******************************************************************************************
     int q_size  = micro->Get_Qmoist_Size(lev);
     qmoist[lev].resize(q_size);
     micro->Define(lev, solverChoice);
     if (solverChoice.moisture_type != MoistureType::None)
     {
         micro->Init(lev, vars_new[lev][Vars::cons],
                     grids[lev], Geom(lev), 0.0,
                     z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
     }
     for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
         qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
     }
  
     // ********************************************************************************************
     // Build the data structures for calculating diffusive/turbulent terms
     // ********************************************************************************************
     update_diffusive_arrays(lev, ba, dm);
  
     // ********************************************************************************************
     // Build the data structures for holding sea surface temps and skin temps
     // ********************************************************************************************
     sst_lev[lev].resize(1);     sst_lev[lev][0] = nullptr;
     tsk_lev[lev].resize(1);     tsk_lev[lev][0] = nullptr;
  
     // ********************************************************************************************
     // Fill data at the new level by interpolation from the coarser level
     // Note that internal to FillCoarsePatch we will convert velocity to momentum,
     //      then interpolate momentum, then convert momentum back to velocity
     // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
     // ********************************************************************************************
  
 #ifdef ERF_USE_NETCDF
     if ( ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) ) &&
          !nc_init_file[lev].empty() )
     {
         // Just making sure that ghost cells aren't uninitialized...
         vars_new[lev][Vars::cons].setVal(0.0); vars_old[lev][Vars::cons].setVal(0.0);
         vars_new[lev][Vars::xvel].setVal(0.0); vars_old[lev][Vars::xvel].setVal(0.0);
         vars_new[lev][Vars::yvel].setVal(0.0); vars_old[lev][Vars::yvel].setVal(0.0);
         vars_new[lev][Vars::zvel].setVal(0.0); vars_old[lev][Vars::zvel].setVal(0.0);
  
         AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
         if (solverChoice.init_type == InitType::Metgrid) {
             init_from_metgrid(lev);
         } else if (solverChoice.init_type == InitType::WRFInput) {
             init_from_wrfinput(lev, *mf_C1H, *mf_C2H, *mf_MUB, *mf_PSFC[lev]);
         }
         init_zphys(lev, time);
         update_terrain_arrays(lev);
         make_physbcs(lev);
  
         dz_min[lev] = (*detJ_cc[lev]).min(0) * geom[lev].CellSize(2);
  
     } else {
 #endif
     //
     // Interpolate the solution data
     //
     FillCoarsePatch(lev, time);
  
     //
     // Interpolate the 2D arrays at the lower boundary
     // Note that ba2d is constructed already in init_stuff, but we have not yet defined dmap[lev]
     // so we must explicitly pass dm.
     Interp2DArrays(lev,ba2d[lev],dm);
 #ifdef ERF_USE_NETCDF
     }
 #endif
  
     // ********************************************************************************************
     // Initialize the integrator class
     // ********************************************************************************************
     dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
     initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
  
     // ********************************************************************************************
     // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
     // ********************************************************************************************
     if (lev > 0 && cf_width >= 0) {
         Construct_ERFFillPatchers(lev);
            Define_ERFFillPatchers(lev);
     }
  
     //********************************************************************************************
     // Land Surface Model
     // *******************************************************************************************
     int lsm_data_size  = lsm.Get_Data_Size();
     int lsm_flux_size  = lsm.Get_Flux_Size();
     lsm_data[lev].resize(lsm_data_size);
     lsm_data_name.resize(lsm_data_size);
     lsm_flux[lev].resize(lsm_flux_size);
     lsm_flux_name.resize(lsm_flux_size);
     lsm.Define(lev, solverChoice);
     if (solverChoice.lsm_type != LandSurfaceType::None)
     {
         lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
     }
     for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
         lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
         lsm_data_name[mvar] = lsm.Get_DataName(mvar);
     }
     for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
         lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
         lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
     }
  
     // ********************************************************************************************
     // Create the SurfaceLayer arrays at this (new) level
     // ********************************************************************************************
     if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
         Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
                                      &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
         m_SurfaceLayer->make_SurfaceLayer_at_level(lev,lev+1,
                                                    mfv_old, Theta_prim[lev], Qv_prim[lev],
                                                    Qr_prim[lev], z_phys_nd[lev],
                                                    Hwave[lev].get(), Lwave[lev].get(), eddyDiffs_lev[lev].get(),
                                                    lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
                                                    sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
     }
  
 #ifdef ERF_USE_PARTICLES
     // particleData.Redistribute();
 #endif
 }

Here is the call graph for this function:

◆ MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch	(	int	lev,
		amrex::Real	time,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

override

 {
     //
     // Note that "time" here is elapsed time
     //
     BoxArray ba;
     DistributionMapping dm;
     Box domain(Geom(0).Domain());
     if (lev == 0 && restart_chkfile.empty() &&
         (max_grid_size[0][0] >= domain.length(0)) &&
         (max_grid_size[0][1] >= domain.length(1)) &&
         ba_in.size() != ParallelDescriptor::NProcs())
     {
         // We only decompose in z if max_grid_size_z indicates we should
         bool decompose_in_z = (max_grid_size[0][2] < domain.length(2));
  
         ba = ERFPostProcessBaseGrids(Geom(0).Domain(),decompose_in_z);
         dm = DistributionMapping(ba);
     } else {
         ba = ba_in;
         dm = dm_in;
     }
  
     // ********************************************************************************************
     // Define grids[lev] to be ba
     // ********************************************************************************************
     SetBoxArray(lev, ba);
  
     // ********************************************************************************************
     // Define dmap[lev] to be dm
     // ********************************************************************************************
     SetDistributionMap(lev, dm);
  
     if (verbose) {
         amrex::Print() <<            "BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
         // amrex::Print() <<" SIMPLIFIED BA FROM SCRATCH AT LEVEL " << lev << " " << ba.simplified_list() << std::endl;
     }
  
     subdomains.resize(lev+1);
     if ( (lev == 0) || (
          (solverChoice.anelastic[lev] == 0) && (solverChoice.project_initial_velocity[lev] == 0) &&
          (solverChoice.init_type != InitType::WRFInput) && (solverChoice.init_type != InitType::Metgrid) ) ) {
         BoxArray dom(geom[lev].Domain());
         subdomains[lev].push_back(dom);
     } else {
         //
         // Create subdomains at each level within the domain such that
         // 1) all boxes in a given subdomain are "connected"
         // 2) no boxes in a subdomain touch any boxes in any other subdomain
         //
         make_subdomains(ba.simplified_list(), subdomains[lev]);
     }
  
     if (lev == 0) init_bcs();
  
     if ( solverChoice.terrain_type == TerrainType::EB ||
          solverChoice.terrain_type == TerrainType::ImmersedForcing ||
          solverChoice.buildings_type == BuildingsType::ImmersedForcing)
     {
         const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
         const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
         if (solverChoice.terrain_type == TerrainType::EB) {
             eb[lev]->make_all_factories(lev, geom[lev], grids[lev], dmap[lev], eb_level);
         } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                    solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
             eb[lev]->make_cc_factory(lev, geom[lev], grids[lev], dmap[lev], eb_level);
         }
     }
  
     auto& lev_new = vars_new[lev];
     auto& lev_old = vars_old[lev];
  
     //********************************************************************************************
     // This allocates all kinds of things, including but not limited to: solution arrays,
     //      terrain arrays, metric terms and base state.
     // *******************************************************************************************
     init_stuff(lev, ba, dm, lev_new, lev_old, base_state[lev], z_phys_nd[lev]);
  
     //********************************************************************************************
     // Land Surface Model
     // *******************************************************************************************
     int lsm_data_size  = lsm.Get_Data_Size();
     int lsm_flux_size  = lsm.Get_Flux_Size();
     lsm_data[lev].resize(lsm_data_size);
     lsm_data_name.resize(lsm_data_size);
     lsm_flux[lev].resize(lsm_flux_size);
     lsm_flux_name.resize(lsm_flux_size);
     lsm.Define(lev, solverChoice);
     if (solverChoice.lsm_type != LandSurfaceType::None)
     {
         lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
     }
     for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
         lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
         lsm_data_name[mvar] = lsm.Get_DataName(mvar);
     }
     for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
         lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
         lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
     }
  
  
  
     // ********************************************************************************************
     // Build the data structures for calculating diffusive/turbulent terms
     // ********************************************************************************************
     update_diffusive_arrays(lev, ba, dm);
  
     // ********************************************************************************************
     // Build the data structures for holding sea surface temps and skin temps
     // ********************************************************************************************
     sst_lev[lev].resize(1);     sst_lev[lev][0] = nullptr;
     tsk_lev[lev].resize(1);     tsk_lev[lev][0] = nullptr;
  
     // ********************************************************************************************
     // Thin immersed body
     // *******************************************************************************************
     init_thin_body(lev, ba, dm);
  
     // ********************************************************************************************
     // Initialize the integrator class
     // ********************************************************************************************
     initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
  
     // ********************************************************************************************
     // Initialize the data itself
     // If (init_type == InitType::WRFInput) then we are initializing terrain and the initial data in
     //                                      the same call so we must call init_only before update_terrain_arrays
     // If (init_type != InitType::WRFInput) then we want to initialize the terrain before the initial data
     //                                      since we may need to use the grid information before constructing
     //                                      initial idealized data
     // ********************************************************************************************
     if (restart_chkfile.empty()) {
         if ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) )
         {
             AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
             //
             // Note that "time" here is elapsed time, and start_time is the start_time from wrfinput/metgrid files
             //
             init_only(lev, time);
             init_zphys(lev, time);
             update_terrain_arrays(lev);
             make_physbcs(lev);
         } else {
             //
             // Note that "time" here is elapsed time, and start_time = 0 when not using wrfinput/metgrid
             //
             init_zphys(lev, time);
             update_terrain_arrays(lev);
             // Note that for init_type != InitType::WRFInput and != InitType::Metgrid,
             // make_physbcs is called inside init_only
             init_only(lev, time);
         }
     } else {
         // if restarting and nudging from input sounding, load the input sounding files
         if (lev == 0 && solverChoice.init_type == InitType::Input_Sounding && solverChoice.nudging_from_input_sounding)
         {
             if (input_sounding_data.input_sounding_file.empty()) {
                 Error("input_sounding file name must be provided via input");
             }
  
             input_sounding_data.resize_arrays();
  
             // this will interpolate the input profiles to the nominal height levels
             // (ranging from 0 to the domain top)
             for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
                 input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n);
             }
  
             // this will calculate the hydrostatically balanced density and pressure
             // profiles following WRF ideal.exe
             if (solverChoice.sounding_type == SoundingType::Ideal) {
                 input_sounding_data.calc_rho_p(0);
             } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
                        solverChoice.sounding_type == SoundingType::DryIsentropic) {
                 input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
                 input_sounding_data.calc_rho_p_isentropic(0);
             }
         }
  
         // We re-create terrain_blanking on restart rather than storing it in the checkpoint
         if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
             solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
             int ngrow = ComputeGhostCells(solverChoice) + 2;
             terrain_blanking[lev]->setVal(1.0);
             MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
             terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
         }
     }
  
      // Read in tables needed for windfarm simulations
     // fill in Nturb multifab - number of turbines in each mesh cell
     // write out the vtk files for wind turbine location and/or
     // actuator disks
     #ifdef ERF_USE_WINDFARM
         init_windfarm(lev);
     #endif
  
     // ********************************************************************************************
     // Build the data structures for canopy model (depends upon z_phys)
     // ********************************************************************************************
     if (restart_chkfile.empty()) {
         if (solverChoice.do_forest_drag) {
             m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
         }
     }
  
     //********************************************************************************************
     // Microphysics
     // *******************************************************************************************
     int q_size  = micro->Get_Qmoist_Size(lev);
     qmoist[lev].resize(q_size);
     micro->Define(lev, solverChoice);
     if (solverChoice.moisture_type != MoistureType::None)
     {
         micro->Init(lev, vars_new[lev][Vars::cons],
                     grids[lev], Geom(lev), 0.0,
                     z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
     }
     for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
         qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
     }
  
     //********************************************************************************************
     // Radiation
     // *******************************************************************************************
     if (solverChoice.rad_type != RadiationType::None)
     {
         rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
     }
  
     // ********************************************************************************************
     // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
     // ********************************************************************************************
     if (lev > 0 && cf_width >= 0) {
         Construct_ERFFillPatchers(lev);
            Define_ERFFillPatchers(lev);
     }
  
 #ifdef ERF_USE_PARTICLES
     if (restart_chkfile.empty()) {
         if (lev == 0) {
             initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,time);
         } else {
             particleData.Redistribute();
         }
     }
 #endif
 }

Here is the call graph for this function:

◆ MakeVTKFilename()

std::string ERF::MakeVTKFilename ( int nstep )

                               {
     // Ensure output directory exists
     const std::string dir = "Output_StormTracker";
     if (!fs::exists(dir)) {
         fs::create_directory(dir);
     }
  
     std::ostringstream oss;
     oss << dir << "/storm_track_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
     return oss.str();
 }

◆ MakeVTKFilename_EyeTracker_xy()

std::string ERF::MakeVTKFilename_EyeTracker_xy ( int nstep )

                                             {
     // Ensure output directory exists
     const std::string dir = "Output_StormTracker/xy";
     if (!fs::exists(dir)) {
         fs::create_directories(dir);
     }
  
     // Construct filename with zero-padded step
     std::ostringstream oss;
     oss << dir << "/storm_track_xy_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
     return oss.str();
 }

◆ MakeVTKFilename_TrackerCircle()

std::string ERF::MakeVTKFilename_TrackerCircle ( int nstep )

                                             {
     // Ensure output directory exists
     const std::string dir = "Output_StormTracker/tracker_circle";
     if (!fs::exists(dir)) {
         fs::create_directories(dir);
     }
  
     // Construct filename with zero-padded step
     std::ostringstream oss;
     oss << dir << "/storm_tracker_circle_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
     return oss.str();
 }

◆ nghost_eb_basic()

static int ERF::nghost_eb_basic ( )

inlinestaticprivate

1639 { return 5; }

◆ nghost_eb_full()

static int ERF::nghost_eb_full ( )

inlinestaticprivate

1646 { return 4; }

◆ nghost_eb_volume()

static int ERF::nghost_eb_volume ( )

inlinestaticprivate

1643 { return 5; }

◆ NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )

inlineprivatenoexcept

     {
         return datalog.size();
     }

◆ NumDerDataLogs()

AMREX_FORCE_INLINE int ERF::NumDerDataLogs ( )

inlineprivatenoexcept

     {
         return der_datalog.size();
     }

◆ NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )

inlineprivatenoexcept

     {
         return samplelinelog.size();
     }

◆ NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )

inlineprivatenoexcept

     {
         return sampleline.size();
     }

◆ NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )

inlineprivatenoexcept

     {
         return sampleptlog.size();
     }

◆ NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )

inlineprivatenoexcept

     {
         return samplepoint.size();
     }

◆ operator=() [1/2]

ERF& ERF::operator= ( const ERF & other )

delete

◆ operator=() [2/2]

ERF& ERF::operator= ( ERF && other )

deletenoexcept

◆ ParameterSanityChecks()

void ERF::ParameterSanityChecks ( )

private

 {
     AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt[0] > 0.);
  
     // We don't allow use_real_bcs to be true if init_type is not either InitType::WRFInput or InitType::Metgrid
     AMREX_ALWAYS_ASSERT( !solverChoice.use_real_bcs ||
                         ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) );
  
     AMREX_ALWAYS_ASSERT(real_width >= 0);
  
     if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
         Abort("You must set cf_width >= cf_set_width >= 0");
     }
     if (max_level > 0 && cf_set_width > 0) {
         for (int lev = 1; lev <= max_level; lev++) {
             if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
                 cf_set_width%ref_ratio[lev-1][1] != 0 ||
                 cf_set_width%ref_ratio[lev-1][2] != 0 ) {
                 Abort("You must set cf_width to be a multiple of ref_ratio");
             }
         }
     }
  
     // If fixed_mri_dt_ratio is set, it must be even
     if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
     {
         Abort("If you specify fixed_mri_dt_ratio, it must be even");
     }
  
     for (int lev = 0; lev <= max_level; lev++)
     {
         // We ignore fixed_fast_dt if not substepping
         if (solverChoice.substepping_type[lev] == SubsteppingType::None) {
             fixed_fast_dt[lev] = -1.0;
         }
  
         // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
         if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio <= 0)
         {
             Real eps = 1.e-12;
             int ratio = static_cast<int>( ( (1.0+eps) * fixed_dt[lev] ) / fixed_fast_dt[lev] );
             if (fixed_dt[lev] / fixed_fast_dt[lev] != ratio)
             {
                 Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
             }
         }
  
         // If all three are specified, they must be consistent
         if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. &&  fixed_mri_dt_ratio > 0)
         {
             if (fixed_dt[lev] / fixed_fast_dt[lev] != fixed_mri_dt_ratio)
             {
                 Abort("Dt is over-specfied");
             }
         }
     } // lev
  
     if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
         Abort("For two-way coupling you must set cf_width = 0");
     }
 }

Here is the call graph for this function:

◆ PlotFileName()

std::string ERF::PlotFileName ( int lev ) const

private

◆ PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string > plot_var_names )

staticprivate

 {
     Vector<std::string> names;
  
     names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
  
     return names;
  
 }

◆ poisson_wall_dist()

void ERF::poisson_wall_dist ( int lev )

Calculate wall distances using the Poisson equation

The zlo boundary is assumed to correspond to the land surface. If there are no boundary walls, then the other use case is to calculate wall distances for immersed boundaries (embedded or thin body).

See Tucker, P. G. (2003). Differential equation-based wall distance computation for DES and RANS. Journal of Computational Physics, 190(1), 229–248. https://doi.org/10.1016/S0021-9991(03)00272-9

 {
     BL_PROFILE("ERF::poisson_wall_dist()");
  
     bool havewall{false};
     Orientation zlo(Direction::z, Orientation::low);
     if ( ( phys_bc_type[zlo] == ERF_BC::surface_layer                      ) ||
          ( phys_bc_type[zlo] == ERF_BC::no_slip_wall                       ) )/*||
          ((phys_bc_type[zlo] == ERF_BC::slip_wall) && (dom_hi.z > dom_lo.z)) )*/
     {
         havewall = true;
     }
  
     auto const& geomdata = geom[lev];
     auto const& dxinv    = geomdata.InvCellSizeArray();
  
     auto const& zphys_arr = z_phys_nd[lev]->const_arrays();
  
     if (havewall) {
 #if 1
         // Bypass wall dist calc in the trivial cases
  
         if (solverChoice.mesh_type == MeshType::ConstantDz) {
             Print() << "Directly calculating direct wall distance for constant dz" << std::endl;
             const Real* prob_lo = geomdata.ProbLo();
             const Real* dx = geomdata.CellSize();
             for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.validbox();
                 auto dist_arr = walldist[lev]->array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                     dist_arr(i, j, k) = prob_lo[2] + (k + 0.5) * dx[2];
                 });
             }
             return;
         }
  
         if (solverChoice.mesh_type == MeshType::StretchedDz) {
             Print() << "Directly calculating direct wall distance for stretched dz" << std::endl;
             for (MFIter mfi(*walldist[lev],TileNoZ()); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.validbox();
                 auto dist_arr = walldist[lev]->array(mfi);
                 const auto zcc_arr = z_phys_cc[lev]->const_array(mfi);
                 const auto znd_arr = z_phys_nd[lev]->const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                     dist_arr(i, j, k) = zcc_arr(i, j, k) - znd_arr(i, j, 0);
                 });
             }
             return;
         }
 #endif
     }
     else
     {
         Error("No solid boundaries in the computational domain");
     }
  
     Print() << "Calculating Poisson wall distance for general terrain" << std::endl;
  
     // Make sure the solver only sees the levels over which we are solving
     Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
     Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(walldist[lev]->boxArray());
     Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(walldist[lev]->DistributionMap());
  
     Vector<MultiFab> rhs;
     Vector<MultiFab> phi;
  
     if (solverChoice.terrain_type == TerrainType::EB) {
         amrex::Error("Wall dist calc not implemented for EB");
     } else {
         rhs.resize(1);   rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
         phi.resize(1);   phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
     }
  
     rhs[0].setVal(-1.0);
  
     auto const dom_lo = lbound(geom[lev].Domain());
     auto const dom_hi = ubound(geom[lev].Domain());
  
     // ****************************************************************************
     // Initialize phi
     // (It is essential that we do this in order to fill the corners; this is
     // used if we include blanking.)
     // ****************************************************************************
     phi[0].setVal(0.0);
  
     // ****************************************************************************
     // Interior boundaries are marked with phi=0
     // ****************************************************************************
 #if 0
     // Define an overset mask (0 or 1) to set dirichlet nodes on walls
     // 1 means the node is an unknown. 0 means it's known.
     iMultiFab mask(ba_tmp[0], dm_tmp[0], 1, 0);
     Vector<const iMultiFab*> overset_mask = {&mask};
  
     mask.setVal(1);
     if (solverChoice.advChoice.have_zero_flux_faces) {
         Warning("Poisson distance is inaccurate for bodies in open domains that are small compared to the domain size, skipping");
         return;
  
         Gpu::DeviceVector<IntVect> xfacelist, yfacelist, zfacelist;
  
         xfacelist.resize(solverChoice.advChoice.zero_xflux.size());
         yfacelist.resize(solverChoice.advChoice.zero_yflux.size());
         zfacelist.resize(solverChoice.advChoice.zero_zflux.size());
  
         if (xfacelist.size() > 0) {
             Gpu::copy(amrex::Gpu::hostToDevice,
                       solverChoice.advChoice.zero_xflux.begin(),
                       solverChoice.advChoice.zero_xflux.end(),
                       xfacelist.begin());
             Print() << "  masking interior xfaces" << std::endl;
         }
         if (yfacelist.size() > 0) {
             Gpu::copy(amrex::Gpu::hostToDevice,
                       solverChoice.advChoice.zero_yflux.begin(),
                       solverChoice.advChoice.zero_yflux.end(),
                       yfacelist.begin());
             Print() << "  masking interior yfaces" << std::endl;
         }
         if (zfacelist.size() > 0) {
             Gpu::copy(amrex::Gpu::hostToDevice,
                       solverChoice.advChoice.zero_zflux.begin(),
                       solverChoice.advChoice.zero_zflux.end(),
                       zfacelist.begin());
             Print() << "  masking interior zfaces" << std::endl;
         }
  
         for (MFIter mfi(phi[0]); mfi.isValid(); ++mfi) {
             const Box& bx = mfi.validbox();
  
             auto phi_arr  = phi[0].array(mfi);
             auto mask_arr = mask.array(mfi);
  
             if (xfacelist.size() > 0) {
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                     for (int iface=0; iface < xfacelist.size(); ++iface) {
                         if ((i == xfacelist[iface][0]) &&
                             (j == xfacelist[iface][1]) &&
                             (k == xfacelist[iface][2]))
                         {
                             mask_arr(i, j  , k  ) = 0;
                             mask_arr(i, j  , k+1) = 0;
                             mask_arr(i, j+1, k  ) = 0;
                             mask_arr(i, j+1, k+1) = 0;
                         }
                     }
                 });
             }
  
             if (yfacelist.size() > 0) {
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                     for (int iface=0; iface < yfacelist.size(); ++iface) {
                         if ((i == yfacelist[iface][0]) &&
                             (j == yfacelist[iface][1]) &&
                             (k == yfacelist[iface][2]))
                         {
                             mask_arr(i  , j, k  ) = 0;
                             mask_arr(i  , j, k+1) = 0;
                             mask_arr(i+1, j, k  ) = 0;
                             mask_arr(i+1, j, k+1) = 0;
                         }
                     }
                 });
             }
  
             if (zfacelist.size() > 0) {
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
                     for (int iface=0; iface < zfacelist.size(); ++iface) {
                         if ((i == xfacelist[iface][0]) &&
                             (j == xfacelist[iface][1]) &&
                             (k == xfacelist[iface][2]))
                         {
                             mask_arr(i  , j  , k) = 0;
                             mask_arr(i  , j+1, k) = 0;
                             mask_arr(i+1, j  , k) = 0;
                             mask_arr(i+1, j+1, k) = 0;
                         }
                     }
                 });
             }
         }
     }
 #endif
  
     // ****************************************************************************
     // Setup BCs, with solid domain boundaries being dirichlet
     // ****************************************************************************
     amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bc3d_lo, bc3d_hi;
     for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
         if (geom[0].isPeriodic(dir)) {
             bc3d_lo[dir] = LinOpBCType::Periodic;
             bc3d_hi[dir] = LinOpBCType::Periodic;
         } else {
             bc3d_lo[dir] = LinOpBCType::Neumann;
             bc3d_hi[dir] = LinOpBCType::Neumann;
         }
     }
     if (havewall) {
         Print() << "  Poisson zlo BC is dirichlet" << std::endl;
         bc3d_lo[2] = LinOpBCType::Dirichlet;
     }
     Print() << "  bc lo : " << bc3d_lo << std::endl;
     Print() << "  bc hi : " << bc3d_hi << std::endl;
  
     if (!solverChoice.advChoice.have_zero_flux_faces && !havewall) {
         Error("No solid boundaries in the computational domain");
     }
  
     LPInfo info; // defaults
  
 /* Nodal solver cannot have hidden dimensions */
 #if 0
     // Allow a hidden direction if the domain is one cell wide
     if (dom_lo.x == dom_hi.x) {
         info.setHiddenDirection(0);
         Print() << "  domain is 2D in yz" << std::endl;
     } else if (dom_lo.y == dom_hi.y) {
         info.setHiddenDirection(1);
         Print() << "  domain is 2D in xz" << std::endl;
     } else if (dom_lo.z == dom_hi.z) {
         info.setHiddenDirection(2);
         Print() << "  domain is 2D in xy" << std::endl;
     }
 #endif
  
 #if 0
     Vector<EBFArrayBoxFactory const*> factory_vec;
     factory_vec.push_back(static_cast<FabFactory<FArrayBox> const*>(&EBFactory(lev));
 #endif
  
     // ****************************************************************************
     // Setup Poisson problem
     // (A \alpha - B \nabla \cdot \beta \nabla ) \phi = f
     //
     // In physical space:
     //   \nabla \cdot \nabla \phi = -1
     //
     // In computational space:
     //   grad(phi) = T^T \nabla \phi
     // and
     //   \nabla \cdot (h_zeta T (T^T \nabla \phi)) = -h_zeta
     // where T = inv(J), T^T is the transpose of inv(J)
     // ****************************************************************************
     constexpr Real constA = 0.0;
     constexpr Real constB = -1.0;
  
     MLABecLaplacian mlabec(geom_tmp, ba_tmp, dm_tmp, info);
  
     mlabec.setScalars(constA, constB);
     mlabec.setACoeffs(0, 0.0);
 #if 1
     // Set beta coefficients at faces
     Array<MultiFab, AMREX_SPACEDIM> beta;
  
     for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
         BoxArray ba_face = ba_tmp[0];
         ba_face.surroundingNodes(idim);  // Convert to face-centered in direction idim
         beta[idim].define(ba_face, dm_tmp[0], 1, 0);
     }
  
     auto beta0_arr = beta[0].arrays();
     auto beta1_arr = beta[1].arrays();
     auto beta2_arr = beta[2].arrays();
  
     // Note: This ignores the off-diagonal components of (h_zeta T T^T), which
     //       is equivalent to assuming that h_xi and h_eta are small.
  
     ParallelFor(beta[0], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
         beta0_arr[b](i, j, k) = Compute_h_zeta_AtIface(i, j, k, dxinv, zphys_arr[b]);;
     });
     ParallelFor(beta[1], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
         beta1_arr[b](i, j, k) = Compute_h_zeta_AtJface(i, j, k, dxinv, zphys_arr[b]);;
     });
     ParallelFor(beta[2], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
         Real inv_h_zeta = 1.0 / Compute_h_zeta_AtKface(i, j, k, dxinv, zphys_arr[b]);
         Real h_xi = Compute_h_xi_AtKface(i, j, k, dxinv, zphys_arr[b]);
         Real h_eta = Compute_h_eta_AtKface(i, j, k, dxinv, zphys_arr[b]);
         beta2_arr[b](i, j, k) = inv_h_zeta * (1 + h_xi*h_xi + h_eta*h_eta);
     });
  
     mlabec.setBCoeffs(0, GetArrOfConstPtrs(beta));
  
     // Set RHS := -h_zeta
     auto rhs_arr = rhs[0].arrays();
     ParallelFor(rhs[0], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
         rhs_arr[b](i, j, k) = -Compute_h_zeta_AtCellCenter(i, j, k, dxinv, zphys_arr[b]);
     });
 #else
     mlabec.setBCoeffs(0, 1.0);
 #endif
  
     mlabec.setDomainBC(bc3d_lo, bc3d_hi);
  
     if (lev > 0) {
         mlabec.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
     }
  
     // If we have inhomogeneous BCs -- do this after setCoarseFineBC
     mlabec.setLevelBC(0, nullptr);
  
     // ****************************************************************************
     // Solve Poisson problem with MLMG
     // ****************************************************************************
     const Real reltol = solverChoice.poisson_reltol;
     const Real abstol = solverChoice.poisson_abstol;
     const int n_corr = solverChoice.ncorr;
     constexpr int max_iter = 100;
  
     MLMG mlmg(mlabec);
     mlmg.setMaxIter(max_iter);
     mlmg.setVerbose(mg_verbose);
     mlmg.setBottomVerbose(0);
  
     for (int icorr=0; icorr <= n_corr; ++icorr) {
         Print()<< "Solving wall distance poisson, icorr=" << icorr << std::endl;
  
         mlmg.solve(GetVecOfPtrs(phi),
                    GetVecOfConstPtrs(rhs),
                    reltol, abstol);
  
         // ****************************************************************************
         // Apply BCs: dirichlet (odd) on zlo, neumann (even) / periodic elsewhere
         // ****************************************************************************
  
         // Overwrite with periodic fill outside domain and fine-fine fill inside
         phi[0].FillBoundary(geom[lev].periodicity());
  
         if (!geom[lev].isPeriodic(0)) {
             for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
             {
                 Box bx = mfi.tilebox();
                 const Array4<Real>& phi_arr = phi[0].array(mfi);
                 if (bx.smallEnd(0) <= dom_lo.x) {
                     ParallelFor(makeSlab(bx,0,dom_lo.x),
                     [=] AMREX_GPU_DEVICE (int i, int j, int k)
                     {
                         phi_arr(i-1,j,k) =  phi_arr(i,j,k); // even BC
                     });
                 } // lo x
                 if (bx.bigEnd(0) >= dom_hi.x) {
                     ParallelFor(makeSlab(bx,0,dom_hi.x),
                     [=] AMREX_GPU_DEVICE (int i, int j, int k)
                     {
                         phi_arr(i+1,j,k) =  phi_arr(i,j,k); // even BC
                     });
                 } // hi x
             } // mfi
         } // not periodic in x
  
         if (!geom[lev].isPeriodic(1)) {
             for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
             {
                 Box bx = mfi.tilebox();
                 Box bx2(bx); bx2.grow(0,1);
                 const Array4<Real>& phi_arr = phi[0].array(mfi);
                 if (bx.smallEnd(1) <= dom_lo.y) {
                     ParallelFor(makeSlab(bx2,1,dom_lo.y),
                     [=] AMREX_GPU_DEVICE (int i, int j, int k)
                     {
                         phi_arr(i,j-1,k) =  phi_arr(i,j,k); // even BC
                     });
                 } // lo y
                 if (bx.bigEnd(1) >= dom_hi.y) {
                     ParallelFor(makeSlab(bx2,1,dom_hi.y),
                     [=] AMREX_GPU_DEVICE (int i, int j, int k)
                     {
                         phi_arr(i,j+1,k) =  phi_arr(i,j,k); // even BC
                     });
                 } // hi y
  
             } // mfi
         } // not periodic in y
  
         for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
         {
             Box bx = mfi.tilebox();
             Box bx3(bx); bx3.grow(0,1); bx3.grow(1,1);
             const Array4<Real>& phi_arr = phi[0].array(mfi);
             if (bx.smallEnd(2) <= dom_lo.z) {
                 ParallelFor(makeSlab(bx3,2,dom_lo.z),
                 [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     phi_arr(i,j,k-1) = -phi_arr(i,j,k); // ODD BC
                 });
             } // lo z
             if (bx.bigEnd(2) >= dom_hi.z) {
                 ParallelFor(makeSlab(bx3,2,dom_hi.z),
                 [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     phi_arr(i,j,k+1) =  phi_arr(i,j,k); // even BC
                 });
             } // hi z
         } // mfi
  
         // ****************************************************************************
         // Compute grad(phi) to get distances
         // ****************************************************************************
         auto const& phi_arr = phi[0].const_arrays();
         //auto rhs_arr = rhs[0].arrays();
         auto dist_arr = walldist[lev]->arrays();
  
         ParallelFor(*walldist[lev], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
             Real dpdx{0}, dpdy{0}, dpdz{0};
  
             dpdx = terrpoisson_flux_x(i, j, k, phi_arr[b], zphys_arr[b], dxinv[0]);
             dpdy = terrpoisson_flux_y(i, j, k, phi_arr[b], zphys_arr[b], dxinv[1]);
             if (k == dom_lo.z) {
                 dpdz = terrpoisson_flux_zlo_dir(i, j, k, phi_arr[b], zphys_arr[b], dxinv[0], dxinv[1]);
             } else {
                 // This returns 0 at the wall, hence the need for the separate calc above
                 dpdz = terrpoisson_flux_z(i, j, k, phi_arr[b], zphys_arr[b], dxinv[0], dxinv[1]);
             }
  
             Real magsqr_dphi = dpdx*dpdx + dpdy*dpdy + dpdz*dpdz;
             Real mag_dphi = std::sqrt(magsqr_dphi);
 #if 1
             // Tucker 2003 Eqn 2
             dist_arr[b](i, j, k) = -mag_dphi + std::sqrt(magsqr_dphi + 2*phi_arr[b](i, j, k));
 #else
             // DEBUG: output phi instead
             if (i==0 && j==0) AllPrint() << "walldist"<<IntVect(i,j,k) << " = " << dist_arr[b](i,j,k) << std::endl;
             dist_arr[b](i, j, k) = phi_arr[b](i, j, k);
 #endif
             // Update RHS source term to explicitly include cross-terms
             if (n_corr > 0) {
                 // d/dxi ( h_xi * dphi/dzeta )
                 Real phi_zeta_xlo = 0.25 * dxinv[2] * ( phi_arr[b](i  , j, k+1) - phi_arr[b](i  , j, k-1)
                                                       + phi_arr[b](i-1, j, k+1) - phi_arr[b](i-1, j, k-1) );
                 Real phi_zeta_xhi = 0.25 * dxinv[2] * ( phi_arr[b](i  , j, k+1) - phi_arr[b](i  , j, k-1)
                                                       + phi_arr[b](i+1, j, k+1) - phi_arr[b](i+1, j, k-1) );
                 Real h_xi_xlo = Compute_h_xi_AtIface(i  , j, k, dxinv, zphys_arr[b]);
                 Real h_xi_xhi = Compute_h_xi_AtIface(i+1, j, k, dxinv, zphys_arr[b]);
  
                 // d/deta ( h_eta * dphi/dzeta )
                 Real phi_zeta_ylo = 0.25 * dxinv[2] * ( phi_arr[b](i, j  , k+1) - phi_arr[b](i, j  , k-1)
                                                       + phi_arr[b](i, j-1, k+1) - phi_arr[b](i, j-1, k-1) );
                 Real phi_zeta_yhi = 0.25 * dxinv[2] * ( phi_arr[b](i, j  , k+1) - phi_arr[b](i, j  , k-1)
                                                       + phi_arr[b](i, j+1, k+1) - phi_arr[b](i, j+1, k-1) );
                 Real h_eta_ylo = Compute_h_eta_AtJface(i, j  , k, dxinv, zphys_arr[b]);
                 Real h_eta_yhi = Compute_h_eta_AtJface(i, j+1, k, dxinv, zphys_arr[b]);
  
                 // d/dzeta ( h_xi * dphi/dxi )
                 Real phi_xi_zlo = 0.25 * dxinv[0] * ( phi_arr[b](i+1, j, k  ) - phi_arr[b](i-1, j, k  )
                                                     + phi_arr[b](i+1, j, k-1) - phi_arr[b](i-1, j, k-1) );
                 Real phi_xi_zhi = 0.25 * dxinv[0] * ( phi_arr[b](i+1, j, k  ) - phi_arr[b](i-1, j, k  )
                                                     + phi_arr[b](i+1, j, k+1) - phi_arr[b](i-1, j, k+1) );
                 Real h_xi_zlo = Compute_h_xi_AtKface(i, j, k  , dxinv, zphys_arr[b]);
                 Real h_xi_zhi = Compute_h_xi_AtKface(i, j, k+1, dxinv, zphys_arr[b]);
  
                 // d/dzeta ( h_eta * dphi/deta )
                 Real phi_eta_zlo = 0.25 * dxinv[1] * ( phi_arr[b](i, j+1, k  ) - phi_arr[b](i, j-1, k  )
                                                      + phi_arr[b](i, j+1, k-1) - phi_arr[b](i, j-1, k-1) );
                 Real phi_eta_zhi = 0.25 * dxinv[1] * ( phi_arr[b](i, j+1, k  ) - phi_arr[b](i, j-1, k  )
                                                      + phi_arr[b](i, j+1, k+1) - phi_arr[b](i, j-1, k+1) );
                 Real h_eta_zlo = Compute_h_eta_AtKface(i, j, k  , dxinv, zphys_arr[b]);
                 Real h_eta_zhi = Compute_h_eta_AtKface(i, j, k+1, dxinv, zphys_arr[b]);
  
                 Real detJ = Compute_h_zeta_AtCellCenter(i, j, k, dxinv, zphys_arr[b]);
  
                 rhs_arr[b](i, j, k) = -detJ
                                     + dxinv[0] * ( h_xi_xhi * phi_zeta_xhi - h_xi_xlo * phi_zeta_xlo)
                                     + dxinv[1] * ( h_eta_yhi * phi_zeta_yhi - h_eta_ylo * phi_zeta_ylo)
                                     + dxinv[2] * ( h_xi_zhi * phi_xi_zhi - h_xi_zlo * phi_xi_zlo
                                                  + h_eta_zhi * phi_eta_zhi - h_eta_zlo * phi_eta_zlo);
             }
         });
     } // corrector loop
 }

Here is the call graph for this function:

◆ post_timestep()

void ERF::post_timestep	(	int	nstep,
		amrex::Real	time,
		amrex::Real	dt_lev
	)

 {
     BL_PROFILE("ERF::post_timestep()");
  
 #ifdef ERF_USE_PARTICLES
     particleData.Redistribute();
 #endif
  
     if (solverChoice.coupling_type == CouplingType::TwoWay)
     {
         int ncomp = vars_new[0][Vars::cons].nComp();
         for (int lev = finest_level-1; lev >= 0; lev--)
         {
             // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
             // m is the map scale factor at cell centers
             // Here we pre-divide (rho S) by m^2 before refluxing
             for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.tilebox();
                 const Array4<      Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
                 const Array4<const Real>  mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                 const Array4<const Real>  mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                 if (SolverChoice::mesh_type == MeshType::ConstantDz) {
                     ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                     {
                         cons_arr(i,j,k,n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                     });
                 } else {
                     const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                     ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                     {
                         cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                     });
                 }
             } // mfi
  
             // This call refluxes all "slow" cell-centered variables
             // (i.e. not density or (rho theta) or velocities) from the lev/lev+1 interface onto lev
             getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 2, 2, ncomp-2);
  
             // Here we multiply (rho S) by m^2 after refluxing
             for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.tilebox();
                 const Array4<      Real>   cons_arr = vars_new[lev][Vars::cons].array(mfi);
                 const Array4<const Real>  mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                 const Array4<const Real>  mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                 if (SolverChoice::mesh_type == MeshType::ConstantDz) {
                     ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                     {
                         cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                     });
                 } else {
                     const Array4<const Real>   detJ_arr = detJ_cc[lev]->const_array(mfi);
                     ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
                     {
                         cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
                     });
                 }
             } // mfi
  
             // We need to do this before anything else because refluxing changes the
             // values of coarse cells underneath fine grids with the assumption they'll
             // be over-written by averaging down
             int src_comp;
             if (solverChoice.anelastic[lev]) {
                 src_comp = 1;
             } else {
                 src_comp = 0;
             }
             int num_comp = ncomp - src_comp;
             AverageDownTo(lev,src_comp,num_comp);
         }
     }
  
     if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
         sum_integrated_quantities(time);
         sum_derived_quantities(time);
         sum_energy_quantities(time);
     }
  
     if (solverChoice.pert_type == PerturbationType::Source ||
         solverChoice.pert_type == PerturbationType::Direct ||
         solverChoice.pert_type == PerturbationType::CPM) {
         if (is_it_time_for_action(nstep, time, dt_lev0, pert_interval, -1.)) {
             turbPert.debug(time);
         }
     }
  
     if (profile_int > 0 && (nstep+1) % profile_int == 0) {
         if (destag_profiles) {
             // all variables cell-centered
             write_1D_profiles(time);
         } else {
             // some variables staggered
             write_1D_profiles_stag(time);
         }
     }
  
     if (solverChoice.rad_type != RadiationType::None)
     {
         if ( rad_datalog_int > 0 &&
              (((nstep+1) % rad_datalog_int == 0) || (nstep==0)) ) {
             if (rad[0]->hasDatalog()) {
                 rad[0]->WriteDataLog(time+start_time);
             }
         }
     }
  
     if (output_1d_column) {
 #ifdef ERF_USE_NETCDF
       if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
       {
          int lev_column = 0;
          for (int lev = finest_level; lev >= 0; lev--)
          {
             Real dx_lev = geom[lev].CellSize(0);
             Real dy_lev = geom[lev].CellSize(1);
             int i_lev = static_cast<int>(std::floor(column_loc_x / dx_lev));
             int j_lev = static_cast<int>(std::floor(column_loc_y / dy_lev));
             if (grids[lev].contains(IntVect(i_lev,j_lev,0))) lev_column = lev;
          }
          writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
       }
 #else
       Abort("To output 1D column files ERF must be compiled with NetCDF");
 #endif
     }
  
     if (output_bndry_planes)
     {
       if (is_it_time_for_action(istep[0], time, dt_lev0, bndry_output_planes_interval, bndry_output_planes_per) &&
           time >= bndry_output_planes_start_time)
       {
          bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
          m_w2d->write_planes(istep[0], time, vars_new, is_moist);
       }
     }
  
     // Write plane/line sampler data
     if (line_sampler && is_it_time_for_action(nstep+1, time, dt_lev0, line_sampling_interval, line_sampling_per)) {
         line_sampler->get_sample_data(geom, vars_new);
         line_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
     }
     if (plane_sampler && is_it_time_for_action(nstep+1, time, dt_lev0, plane_sampling_interval, plane_sampling_per)) {
         plane_sampler->get_sample_data(geom, vars_new);
         plane_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
     }
  
     // Moving terrain
     if ( solverChoice.terrain_type == TerrainType::MovingFittedMesh )
     {
       for (int lev = finest_level; lev >= 0; lev--)
       {
         // Copy z_phs_nd and detJ_cc at end of timestep
         MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
         MultiFab::Copy(  *detJ_cc[lev],   *detJ_cc_new[lev], 0, 0, 1,   detJ_cc[lev]->nGrowVect());
         MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
  
         make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
       }
     }
  
     if ( solverChoice.io_hurricane_eye_tracker and (nstep == 0 or (nstep+1)%m_plot3d_int_1 == 0) )
     {
         int levc=finest_level;
  
         HurricaneEyeTracker(geom[levc],
                             vars_new[levc],
                             solverChoice.moisture_type,
                             solverChoice.hindcast_lateral_forcing? &forecast_state_interp[levc] : nullptr,
                             solverChoice.hurricane_eye_latitude,
                             solverChoice.hurricane_eye_longitude,
                             hurricane_eye_track_xy,
                             hurricane_eye_track_latlon,
                             hurricane_tracker_circle);
  
         MultiFab& U_new = vars_new[levc][Vars::xvel];
         MultiFab& V_new = vars_new[levc][Vars::yvel];
         MultiFab& W_new = vars_new[levc][Vars::zvel];
  
         MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
         average_face_to_cellcenter(mf_cc_vel,0,{AMREX_D_DECL(&U_new,&V_new,&W_new)},0);
  
         HurricaneMaxVelTracker(geom[levc],
                                mf_cc_vel,
                                t_new[0],
                                hurricane_eye_track_xy,
                                hurricane_maxvel_vs_time);
  
         HurricaneMinPressureTracker(solverChoice.moisture_type,
                                     geom[levc],
                                     vars_new[levc][Vars::cons],
                                     t_new[0],
                                     hurricane_eye_track_xy,
                                     hurricane_minpressure_vs_time);
  
         std::string filename_tracker = MakeVTKFilename_TrackerCircle(nstep);
         std::string filename_xy      = MakeVTKFilename_EyeTracker_xy(nstep);
         std::string filename_latlon  = MakeFilename_EyeTracker_latlon(nstep);
         std::string filename_maxvel  = MakeFilename_EyeTracker_maxvel(nstep);
         std::string filename_minpressure  = MakeFilename_EyeTracker_minpressure(nstep);
  
         if (ParallelDescriptor::IOProcessor()) {
             WriteVTKPolyline(filename_tracker, hurricane_tracker_circle);
             WriteVTKPolyline(filename_xy, hurricane_eye_track_xy);
             WriteLinePlot(filename_latlon, hurricane_eye_track_latlon);
             WriteLinePlot(filename_maxvel, hurricane_maxvel_vs_time);
             WriteLinePlot(filename_minpressure, hurricane_minpressure_vs_time);
         }
     }
 } // post_timestep

Here is the call graph for this function:

◆ post_update()

void ERF::post_update	(	amrex::MultiFab &	state_mf,
		amrex::Real	time,
		const amrex::Geometry &	geom
	)

private

◆ print_banner()

void ERF::print_banner	(	MPI_Comm	comm,
		std::ostream &	out
	)

static

                      : " << msg << std::endl;
 }
  
 void ERF::print_banner (MPI_Comm comm, std::ostream& out)
 {
 #ifdef AMREX_USE_MPI
     int irank = 0;
     int num_ranks = 1;
     MPI_Comm_size(comm, &num_ranks);
     MPI_Comm_rank(comm, &irank);
  
     // Only root process does the printing
     if (irank != 0) return;
 #else
     amrex::ignore_unused(comm);
 #endif
  
     auto etime = std::chrono::system_clock::now();
     auto etimet = std::chrono::system_clock::to_time_t(etime);
 #ifndef _WIN32
     char time_buf[64];
     ctime_r(&etimet, time_buf);
     const std::string tstamp(time_buf);
 #else
     char* time_buf = new char[64];
     ctime_s(time_buf, 64, &etimet);
     const std::string tstamp(time_buf);
 #endif
  
     const char* githash1 = amrex::buildInfoGetGitHash(1);
     const char* githash2 = amrex::buildInfoGetGitHash(2);
  
     // clang-format off
     out << dbl_line
         << "                ERF (https://github.com/erf-model/ERF)"
         << std::endl << std::endl
         << "  ERF Git SHA      :: " << githash1 << std::endl
         << "  AMReX Git SHA    :: " << githash2 << std::endl
         << "  AMReX version    :: " << amrex::Version() << std::endl << std::endl
         << "  Exec. time       :: " << tstamp
         << "  Build time       :: " << amrex::buildInfoGetBuildDate() << std::endl
         << "  C++ compiler     :: " << amrex::buildInfoGetComp()
         << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
         << "  MPI              :: "
 #ifdef AMREX_USE_MPI
         << "ON    (Num. ranks = " << num_ranks << ")" << std::endl
 #else
         << "OFF " << std::endl
 #endif
         << "  GPU              :: "
 #ifdef AMREX_USE_GPU
         << "ON    "
 #if defined(AMREX_USE_CUDA)
         << "(Backend: CUDA)"
 #elif defined(AMREX_USE_HIP)
         << "(Backend: HIP)"
 #elif defined(AMREX_USE_SYCL)
         << "(Backend: SYCL)"
 #endif
         << std::endl
 #else
         << "OFF" << std::endl
 #endif
         << "  OpenMP           :: "
 #ifdef AMREX_USE_OMP
         << "ON    (Num. threads = " << omp_get_max_threads() << ")" << std::endl
 #else
         << "OFF" << std::endl
 #endif
         << std::endl;
  

Referenced by main().

Here is the caller graph for this function:

◆ print_error()

void ERF::print_error	(	MPI_Comm	comm,
		const std::string &	msg
	)

static

         :
     input_file   : Input file with simulation settings
  
 Optional:
     param=value  : Overrides for parameters during runtime
 )doc" << std::endl;
 }
  
 void ERF::print_error (MPI_Comm comm, const std::string& msg)
 {
 #ifdef AMREX_USE_MPI
     int irank = 0;
     int num_ranks = 1;
     MPI_Comm_size(comm, &num_ranks);
     MPI_Comm_rank(comm, &irank);
  

Referenced by main().

Here is the caller graph for this function:

◆ print_summary()

static void ERF::print_summary ( std::ostream & )

static

◆ print_tpls()

void ERF::print_tpls ( std::ostream & out )

static

                       ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
         << dash_line << std::endl;
     // clang-format on
 }
  
 void ERF::print_tpls (std::ostream& out)
 {
     amrex::Vector<std::string> tpls;
  
 #ifdef ERF_USE_NETCDF
     tpls.push_back(std::string("NetCDF    ") + NC_VERSION);
 #endif
 #ifdef AMREX_USE_SUNDIALS
     tpls.push_back(std::string("SUNDIALS     ") + SUNDIALS_VERSION);
 #endif
  
     if (!tpls.empty()) {
         out << "  Enabled third-party libraries: ";
         for (const auto& val : tpls) {

◆ print_usage()

void ERF::print_usage	(	MPI_Comm	comm,
		std::ostream &	out
	)

static

 {
 #ifdef AMREX_USE_MPI
     int irank = 0;
     int num_ranks = 1;
     MPI_Comm_size(comm, &num_ranks);
     MPI_Comm_rank(comm, &irank);
  
     // Only root process does the printing
     if (irank != 0) return;
 #else
     amrex::ignore_unused(comm);
 #endif
  
     out << R"doc(Usage:
     ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

◆ project_initial_velocity()

void ERF::project_initial_velocity	(	int	lev,
		amrex::Real	time,
		amrex::Real	dt
	)

Project the single-level velocity field to enforce the anelastic constraint Note that the level may or may not be level 0.

 {
     BL_PROFILE("ERF::project_initial_velocity()");
     // Impose FillBoundary on density since we use it in the conversion of velocity to momentum
     vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
  
     const MultiFab* c_vfrac = nullptr;
     if (solverChoice.terrain_type == TerrainType::EB) {
         c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
     }
  
     VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
                        vars_new[lev][Vars::yvel], IntVect{0},
                        vars_new[lev][Vars::zvel], IntVect{0},
                        vars_new[lev][Vars::cons],
                        rU_new[lev], rV_new[lev], rW_new[lev],
                        Geom(lev).Domain(), domain_bcs_type, c_vfrac);
  
     Vector<MultiFab> tmp_mom;
  
     tmp_mom.push_back(MultiFab(vars_new[lev][Vars::cons],make_alias,0,1));
     tmp_mom.push_back(MultiFab(rU_new[lev],make_alias,0,1));
     tmp_mom.push_back(MultiFab(rV_new[lev],make_alias,0,1));
     tmp_mom.push_back(MultiFab(rW_new[lev],make_alias,0,1));
  
     // If at lev > 0 we must first fill the velocities at the c/f interface -- this must
     //    be done *after* the projection at lev-1
     if (lev > 0) {
         int levc = lev-1;
  
         const MultiFab* c_vfrac_crse = nullptr;
         if (solverChoice.terrain_type == TerrainType::EB) {
             c_vfrac_crse = &((get_eb(levc).get_const_factory())->getVolFrac());
         }
  
         MultiFab& S_new_crse = vars_new[levc][Vars::cons];
         MultiFab& U_new_crse = vars_new[levc][Vars::xvel];
         MultiFab& V_new_crse = vars_new[levc][Vars::yvel];
         MultiFab& W_new_crse = vars_new[levc][Vars::zvel];
  
         VelocityToMomentum(U_new_crse, IntVect{0}, V_new_crse, IntVect{0}, W_new_crse, IntVect{0}, S_new_crse,
                            rU_new[levc], rV_new[levc], rW_new[levc],
                            Geom(levc).Domain(), domain_bcs_type, c_vfrac_crse);
  
         rU_new[levc].FillBoundary(geom[levc].periodicity());
         FPr_u[levc].RegisterCoarseData({&rU_new[levc], &rU_new[levc]}, {time, time+l_dt});
  
         rV_new[levc].FillBoundary(geom[levc].periodicity());
         FPr_v[levc].RegisterCoarseData({&rV_new[levc], &rV_new[levc]}, {time, time+l_dt});
  
         rW_new[levc].FillBoundary(geom[levc].periodicity());
         FPr_w[levc].RegisterCoarseData({&rW_new[levc], &rW_new[levc]}, {time, time+l_dt});
     }
  
     Real l_time = 0.0;
     project_momenta(lev, l_time, l_dt, tmp_mom);
  
     MomentumToVelocity(vars_new[lev][Vars::xvel],
                        vars_new[lev][Vars::yvel],
                        vars_new[lev][Vars::zvel],
                        vars_new[lev][Vars::cons],
                        rU_new[lev], rV_new[lev], rW_new[lev],
                        Geom(lev).Domain(), domain_bcs_type, c_vfrac);
  }

Here is the call graph for this function:

◆ project_momenta()

void ERF::project_momenta	(	int	lev,
		amrex::Real	l_time,
		amrex::Real	l_dt,
		amrex::Vector< amrex::MultiFab > &	vars
	)

Project the single-level momenta to enforce the anelastic constraint Note that the level may or may not be level 0.

 {
     BL_PROFILE("ERF::project_momenta()");
     //
     // If at lev > 0 we must first fill the momenta at the c/f interface with interpolated coarse values
     //
     if (lev > 0) {
         PhysBCFunctNoOp null_bc;
         FPr_u[lev-1].FillSet(mom_mf[IntVars::xmom], l_time, null_bc, domain_bcs_type);
         FPr_v[lev-1].FillSet(mom_mf[IntVars::ymom], l_time, null_bc, domain_bcs_type);
         FPr_w[lev-1].FillSet(mom_mf[IntVars::zmom], l_time, null_bc, domain_bcs_type);
     }
  
     // Make sure the solver only sees the levels over which we are solving
     Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(mom_mf[Vars::cons].boxArray());
     Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap());
     Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
  
     Box domain = geom[lev].Domain();
  
     MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
  
     Vector<MultiFab> rhs;
     Vector<MultiFab> phi;
  
     if (solverChoice.terrain_type == TerrainType::EB)
     {
         rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
         phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1, MFInfo(), EBFactory(lev));
     } else {
         rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
         phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
     }
  
     MultiFab rhs_lev(rhs[0], make_alias, 0, 1);
     MultiFab phi_lev(phi[0], make_alias, 0, 1);
  
     auto dx    = geom[lev].CellSizeArray();
     auto dxInv = geom[lev].InvCellSizeArray();
  
     // Inflow on an x-face -- note only the normal velocity is used in the projection
     if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow") {
         (*physbcs_u[lev])(vars_new[lev][Vars::xvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                         IntVect{1,0,0},t_new[lev],BCVars::xvel_bc,false);
     }
  
     // Inflow on a  y-face -- note only the normal velocity is used in the projection
     if (domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
         (*physbcs_v[lev])(vars_new[lev][Vars::yvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
                           IntVect{0,1,0},t_new[lev],BCVars::yvel_bc,false);
     }
  
     if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow" ||
         domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
  
         const MultiFab* c_vfrac = nullptr;
         if (solverChoice.terrain_type == TerrainType::EB) {
             c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
         }
  
         VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
                             vars_new[lev][Vars::yvel], IntVect{0},
                             vars_new[lev][Vars::zvel], IntVect{0},
                             vars_new[lev][Vars::cons],
                             mom_mf[IntVars::xmom],
                             mom_mf[IntVars::ymom],
                             mom_mf[IntVars::zmom],
                             Geom(lev).Domain(),
                             domain_bcs_type, c_vfrac);
     }
  
     // If !fixed_density, we must convert (rho u) which came in
     // to (rho0 u) which is what we will project
     if (!solverChoice.fixed_density[lev]) {
         ConvertForProjection(mom_mf[Vars::cons], r_hse,
                              mom_mf[IntVars::xmom],
                              mom_mf[IntVars::ymom],
                              mom_mf[IntVars::zmom],
                              Geom(lev).Domain(),
                              domain_bcs_type);
     }
  
     //
     // ****************************************************************************
     // Now convert the rho0w MultiFab to hold Omega rather than rhow
     // ****************************************************************************
     //
     if (solverChoice.mesh_type == MeshType::VariableDz)
     {
         for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
             const Array4<Real const>& rho0u_arr = mom_mf[IntVars::xmom].const_array(mfi);
             const Array4<Real const>& rho0v_arr = mom_mf[IntVars::ymom].const_array(mfi);
             const Array4<Real      >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
  
             const Array4<Real const>&     z_nd = z_phys_nd[lev]->const_array(mfi);
             const Array4<Real const>&     mf_u =  mapfac[lev][MapFacType::u_x]->const_array(mfi);
             const Array4<Real const>&     mf_v =  mapfac[lev][MapFacType::v_y]->const_array(mfi);
  
             //
             // Define Omega from (rho0 W) but store it in the same array
             //
             Box tbz = mfi.nodaltilebox(2);
             ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                 if (k == 0) {
                     rho0w_arr(i,j,k) = Real(0.0);
                 } else {
                     Real rho0w = rho0w_arr(i,j,k);
                     rho0w_arr(i,j,k) = OmegaFromW(i,j,k,rho0w,
                                                   rho0u_arr,rho0v_arr,
                                                   mf_u,mf_v,z_nd,dxInv);
                 }
             });
         } // mfi
     }
  
     // ****************************************************************************
     // Allocate fluxes
     // ****************************************************************************
     Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
     fluxes.resize(1);
     for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
         if (solverChoice.terrain_type == TerrainType::EB) {
             fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
         } else {
             fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
         }
     }
  
     // ****************************************************************************
     // Initialize phi to 0
     // (It is essential that we do this in order to fill the corners; these are never
     //  used but the Saxpy requires the values to be initialized.)
     // ****************************************************************************
     phi_lev.setVal(0.0);
  
     // ****************************************************************************
     // Break into subdomains
     // ****************************************************************************
  
     std::map<int,int> index_map;
  
     BoxArray ba(grids[lev]);
  
     Vector<MultiFab> rhs_sub; rhs_sub.resize(1);
     Vector<MultiFab> phi_sub; phi_sub.resize(1);
     Vector<Array<MultiFab,AMREX_SPACEDIM>> fluxes_sub; fluxes_sub.resize(1);
  
     MultiFab ax_sub, ay_sub, az_sub, dJ_sub, znd_sub;
     MultiFab mfmx_sub, mfmy_sub;
  
     Array<MultiFab,AMREX_SPACEDIM> rho0_u_sub;
     Array<MultiFab const*, AMREX_SPACEDIM> rho0_u_const;
  
     // If we are going to solve with MLMG then we do not need to break this into subdomains
     bool will_solve_with_mlmg = false;
     if (solverChoice.mesh_type == MeshType::ConstantDz) {
         will_solve_with_mlmg = true;
 #ifdef ERF_USE_FFT
         if (use_fft) {
             bool all_boxes_ok = true;
             for (int isub = 0; isub < subdomains[lev].size(); ++isub) {
                 Box my_region(subdomains[lev][isub].minimalBox());
                 bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
                 if (!boxes_make_rectangle) {
                     all_boxes_ok = false;
                 }
             } // isub
             if (all_boxes_ok) {
                 will_solve_with_mlmg = false;
             }
         } // use_fft
 #else
         if (use_fft) {
             amrex::Warning("You set use_fft=true but didn't build with USE_FFT = TRUE; defaulting to MLMG");
         }
 #endif
     } // No terrain or grid stretching
  
     for (int isub = 0; isub < subdomains[lev].size(); ++isub)
     {
         BoxList bl_sub;
         Vector<int> dm_sub;
  
         for (int j = 0; j < ba.size(); j++)
         {
             if (subdomains[lev][isub].intersects(ba[j]))
             {
                 //
                 // Note that bl_sub.size() is effectively a counter which is
                 // incremented above
                 //
                 // if (ParallelDescriptor::MyProc() == j) {
                 // }
                 index_map[bl_sub.size()] = j;
  
                 bl_sub.push_back(grids[lev][j]);
                 dm_sub.push_back(dmap[lev][j]);
             } // intersects
         } // loop over ba (j)
  
         BoxArray ba_sub(bl_sub);
  
         BoxList bl2d_sub = ba_sub.boxList();
         for (auto& b : bl2d_sub) {
             b.setRange(2,0);
         }
         BoxArray ba2d_sub(std::move(bl2d_sub));
  
         // Define MultiFabs that hold only the data in this particular subdomain
         if (solverChoice.terrain_type == TerrainType::EB) {
             if (ba_sub != ba) {
                 amrex::Print() << "EB Solves with multiple regions is not yet supported" << std::endl;
             }
             rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs_lev.nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
             phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi_lev.nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
  
             mfmx_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_x]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
             mfmy_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_y]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
               dJ_sub.define(ba_sub, DistributionMapping(dm_sub), 1, detJ_cc[lev]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
  
             for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
                 fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
                                                    IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false), EBFactory(lev));
             }
             rho0_u_sub[0].define(convert(ba_sub, IntVect::TheDimensionVector(0)), DistributionMapping(dm_sub), 1,
                                          mom_mf[IntVars::xmom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
             rho0_u_sub[1].define(convert(ba_sub, IntVect::TheDimensionVector(1)), DistributionMapping(dm_sub), 1,
                                          mom_mf[IntVars::ymom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
             rho0_u_sub[2].define(convert(ba_sub, IntVect::TheDimensionVector(2)), DistributionMapping(dm_sub), 1,
                                          mom_mf[IntVars::zmom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
         } else {
             rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
             phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
  
             mfmx_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_x]->nGrowVect(), MFInfo{}.SetAlloc(false));
             mfmy_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_y]->nGrowVect(), MFInfo{}.SetAlloc(false));
               dJ_sub.define(ba_sub, DistributionMapping(dm_sub), 1, detJ_cc[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
  
             for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
                 fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
                                                    IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false));
             }
             rho0_u_sub[0].define(convert(ba_sub, IntVect::TheDimensionVector(0)), DistributionMapping(dm_sub), 1,
                                          mom_mf[IntVars::xmom].nGrowVect(), MFInfo{}.SetAlloc(false));
             rho0_u_sub[1].define(convert(ba_sub, IntVect::TheDimensionVector(1)), DistributionMapping(dm_sub), 1,
                                          mom_mf[IntVars::ymom].nGrowVect(), MFInfo{}.SetAlloc(false));
             rho0_u_sub[2].define(convert(ba_sub, IntVect::TheDimensionVector(2)), DistributionMapping(dm_sub), 1,
                                          mom_mf[IntVars::zmom].nGrowVect(), MFInfo{}.SetAlloc(false));
         }
  
         // Link the new MultiFabs to the FABs in the original MultiFabs (no copy required)
         for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
         {
             int orig_index = index_map[mfi.index()];
             rhs_sub[0].setFab(mfi, FArrayBox(rhs_lev[orig_index], amrex::make_alias, 0, 1));
             phi_sub[0].setFab(mfi, FArrayBox(phi_lev[orig_index], amrex::make_alias, 0, 1));
  
             mfmx_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::m_x])[orig_index], amrex::make_alias, 0, 1));
             mfmy_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::m_y])[orig_index], amrex::make_alias, 0, 1));
  
             fluxes_sub[0][0].setFab(mfi,FArrayBox(fluxes[0][0][orig_index], amrex::make_alias, 0, 1));
             fluxes_sub[0][1].setFab(mfi,FArrayBox(fluxes[0][1][orig_index], amrex::make_alias, 0, 1));
             fluxes_sub[0][2].setFab(mfi,FArrayBox(fluxes[0][2][orig_index], amrex::make_alias, 0, 1));
  
             rho0_u_sub[0].setFab(mfi,FArrayBox(mom_mf[IntVars::xmom][orig_index], amrex::make_alias, 0, 1));
             rho0_u_sub[1].setFab(mfi,FArrayBox(mom_mf[IntVars::ymom][orig_index], amrex::make_alias, 0, 1));
             rho0_u_sub[2].setFab(mfi,FArrayBox(mom_mf[IntVars::zmom][orig_index], amrex::make_alias, 0, 1));
         }
  
         rho0_u_const[0] = &rho0_u_sub[0];
         rho0_u_const[1] = &rho0_u_sub[1];
         rho0_u_const[2] = &rho0_u_sub[2];
  
         if (solverChoice.mesh_type != MeshType::ConstantDz) {
             ax_sub.define(convert(ba_sub,IntVect(1,0,0)), DistributionMapping(dm_sub), 1,
                           ax[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
             ay_sub.define(convert(ba_sub,IntVect(0,1,0)), DistributionMapping(dm_sub), 1,
                           ay[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
             az_sub.define(convert(ba_sub,IntVect(0,0,1)), DistributionMapping(dm_sub), 1,
                           az[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
             znd_sub.define(convert(ba_sub,IntVect(1,1,1)), DistributionMapping(dm_sub), 1,
                            z_phys_nd[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
  
             for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
                 int orig_index = index_map[mfi.index()];
                 ax_sub.setFab(mfi, FArrayBox((*ax[lev])[orig_index], amrex::make_alias, 0, 1));
                 ay_sub.setFab(mfi, FArrayBox((*ay[lev])[orig_index], amrex::make_alias, 0, 1));
                 az_sub.setFab(mfi, FArrayBox((*az[lev])[orig_index], amrex::make_alias, 0, 1));
                 znd_sub.setFab(mfi, FArrayBox((*z_phys_nd[lev])[orig_index], amrex::make_alias, 0, 1));
                 dJ_sub.setFab(mfi, FArrayBox((*detJ_cc[lev])[orig_index], amrex::make_alias, 0, 1));
             }
         }
  
         if (solverChoice.terrain_type == TerrainType::EB) {
             for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
                 int orig_index = index_map[mfi.index()];
                 dJ_sub.setFab(mfi, FArrayBox((*detJ_cc[lev])[orig_index], amrex::make_alias, 0, 1));
             }
         }
  
         // ****************************************************************************
         // Compute divergence which will form RHS
         // Note that we replace "rho0w" with the contravariant momentum, Omega
         // ****************************************************************************
  
         compute_divergence(lev, rhs_sub[0], rho0_u_const, geom_tmp[0]);
  
         Real rhsnorm;
  
         // Max norm over the entire MultiFab
         rhsnorm = rhs_sub[0].norm0();
  
         if (mg_verbose > 0) {
             bool local = false;
             Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
             Print() << "Max/L2 norm of divergence before solve in subdomain " << isub << " at level " << lev << " : " << rhsnorm << " " <<
                         rhs_sub[0].norm2() << " and volume-weighted sum " << sum << std::endl;
         }
  
         if (lev == 0 && solverChoice.use_real_bcs)
         {
             // We always use VariableDz if use_real_bcs is true
             AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::VariableDz);
  
             // Note that we always impose the projections one level at a time so this will always be a vector of length 1
             Array<MultiFab*, AMREX_SPACEDIM> rho0_u_vec =
                {&mom_mf[IntVars::xmom], &mom_mf[IntVars::ymom], &mom_mf[IntVars::zmom]};
             Array<MultiFab*, AMREX_SPACEDIM> area_vec = {ax[lev].get(), ay[lev].get(), az[lev].get()};
             //
             // Modify ax,ay,ax to include the map factors as used in the divergence calculation
             // We do this here so that it is seen in the call to enforceInOutSolvability
             //
             for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
             {
                 Box xbx = mfi.nodaltilebox(0);
                 Box ybx = mfi.nodaltilebox(1);
                 Box zbx = mfi.nodaltilebox(2);
                 const Array4<Real      >& ax_ar = ax[lev]->array(mfi);
                 const Array4<Real      >& ay_ar = ay[lev]->array(mfi);
                 const Array4<Real      >& az_ar = az[lev]->array(mfi);
                 const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                 const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                 const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                 const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                 ParallelFor(xbx,ybx,zbx,
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     ax_ar(i,j,k) /= mf_uy(i,j,0);
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     ay_ar(i,j,k) /= mf_vx(i,j,0);
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
                 });
             } // mfi
  
             if (mg_verbose > 0) {
                 Print() << "Calling enforceInOutSolvability" << std::endl;
             }
             enforceInOutSolvability(lev, rho0_u_vec, area_vec, geom[lev]);
  
             //
             // Return ax,ay,ax to their original definition
             //
             for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
             {
                 Box xbx = mfi.nodaltilebox(0);
                 Box ybx = mfi.nodaltilebox(1);
                 Box zbx = mfi.nodaltilebox(2);
                 const Array4<Real      >& ax_ar = ax[lev]->array(mfi);
                 const Array4<Real      >& ay_ar = ay[lev]->array(mfi);
                 const Array4<Real      >& az_ar = az[lev]->array(mfi);
                 const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                 const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                 const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                 const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                 ParallelFor(xbx,ybx,zbx,
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     ax_ar(i,j,k) *= mf_uy(i,j,0);
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     ay_ar(i,j,k) *= mf_vx(i,j,0);
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
                 });
             } // mfi
  
             compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
  
             // Re-define max norm over the entire MultiFab
             rhsnorm = rhs_lev.norm0();
  
             if (mg_verbose > 0)
             {
                 bool local = false;
                 Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
                 Print() << "Max/L2 norm of divergence before solve at level " << lev << " : " << rhsnorm << " " <<
                             rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
             }
         } // lev 0 && use_real_bcs
  
         // *******************************************************************************************
         // Enforce solvability if the problem is singular (i.e all sides Neumann or periodic)
         // Note that solves at lev > 0 are always singular because we impose Neumann bc's on all sides
         // *******************************************************************************************
         bool is_singular = true;
         if (lev == 0) {
             if ( (domain_bc_type[0] == "Outflow" || domain_bc_type[0] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
             if ( (domain_bc_type[1] == "Outflow" || domain_bc_type[1] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
             if ( (domain_bc_type[3] == "Outflow" || domain_bc_type[3] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
             if ( (domain_bc_type[4] == "Outflow" || domain_bc_type[4] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
             if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open")                               ) is_singular = false;
         } else {
             Box my_region(subdomains[lev][isub].minimalBox());
             if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open") && (my_region.bigEnd(2) == domain.bigEnd(2)) ) is_singular = false;
         }
  
         if (is_singular)
         {
             bool local = false;
             Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
  
             Real vol;
             if (solverChoice.mesh_type == MeshType::ConstantDz) {
                 vol = rhs_sub[0].boxArray().numPts();
             } else {
                 vol = dJ_sub.sum();
             }
  
             sum /= (vol * dx[0] * dx[1] * dx[2]);
  
             for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
             {
                 rhs_sub[0][mfi.index()].template minus<RunOn::Device>(sum);
             }
             if (mg_verbose > 0) {
                 amrex::Print() << " Subtracting " << sum << " from rhs in subdomain " << isub << std::endl;
  
                 sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
                 Print() << "Sum after subtraction " << sum << " in subdomain " << isub << std::endl;
             }
  
         } // if is_singular
  
         rhsnorm = rhs_sub[0].norm0();
  
         // ****************************************************************************
         // No need to build the solver if RHS == 0
         // ****************************************************************************
         if (rhsnorm <= solverChoice.poisson_abstol) return;
  
         Real start_step = static_cast<Real>(ParallelDescriptor::second());
  
         if (mg_verbose > 0) {
             amrex::Print() << " Solving in subdomain " << isub << " of " << subdomains[lev].size() << " bins at level " << lev << std::endl;
         }
  
         if (solverChoice.mesh_type == MeshType::VariableDz) {
             //
             // Modify ax,ay,ax to include the map factors as used in the divergence calculation
             // We do this here to set the coefficients used in the stencil -- the extra factor
             // of the mapfac comes from the gradient
             //
             for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
             {
                 Box xbx = mfi.nodaltilebox(0);
                 Box ybx = mfi.nodaltilebox(1);
                 Box zbx = mfi.nodaltilebox(2);
                 const Array4<Real      >& ax_ar = ax_sub.array(mfi);
                 const Array4<Real      >& ay_ar = ay_sub.array(mfi);
                 const Array4<Real      >& az_ar = az_sub.array(mfi);
                 const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
                 const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                 const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                 const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
                 const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                 const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                 ParallelFor(xbx,ybx,zbx,
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     ax_ar(i,j,k) *= (mf_ux(i,j,0) / mf_uy(i,j,0));
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     ay_ar(i,j,k) *= (mf_vy(i,j,0) / mf_vx(i,j,0));
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
                 });
             } // mfi
         }
  
         if (solverChoice.terrain_type != TerrainType::EB) {
  
 #ifdef ERF_USE_FFT
         Box my_region(subdomains[lev][isub].minimalBox());
 #endif
  
         // ****************************************************************************
         // No terrain or grid stretching
         // ****************************************************************************
         if (solverChoice.mesh_type == MeshType::ConstantDz) {
             if (will_solve_with_mlmg) {
                 solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub, geom[lev], ref_ratio, domain_bc_type,
                                 mg_verbose, solverChoice.poisson_reltol, solverChoice.poisson_abstol);
             } else {
 #ifdef ERF_USE_FFT
                 solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
 #endif
             }
         } // No terrain or grid stretching
         // ****************************************************************************
         // Grid stretching (flat terrain)
         // ****************************************************************************
         else if (solverChoice.mesh_type == MeshType::StretchedDz) {
 #ifndef ERF_USE_FFT
             amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT solver");
 #else
             bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
             if (!boxes_make_rectangle) {
                 amrex::Abort("FFT won't work unless the union of boxes is rectangular");
             } else {
                 if (!use_fft) {
                     amrex::Warning("Using FFT even though you didn't set use_fft to true; it's the best choice");
                 }
                 solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
             }
 #endif
         } // grid stretching
  
         // ****************************************************************************
         // General terrain
         // ****************************************************************************
         else if (solverChoice.mesh_type == MeshType::VariableDz) {
 #ifdef ERF_USE_FFT
             bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
             if (!boxes_make_rectangle) {
                 amrex::Abort("FFT preconditioner for GMRES won't work unless the union of boxes is rectangular");
             } else {
                 solve_with_gmres(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0], ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
             }
 #else
             amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT preconditioner for GMRES");
 #endif
  
             //
             // Restore ax,ay,ax to their original definitions
             //
             for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
             {
                 Box xbx = mfi.nodaltilebox(0);
                 Box ybx = mfi.nodaltilebox(1);
                 Box zbx = mfi.nodaltilebox(2);
                 const Array4<Real      >& ax_ar = ax_sub.array(mfi);
                 const Array4<Real      >& ay_ar = ay_sub.array(mfi);
                 const Array4<Real      >& az_ar = az_sub.array(mfi);
                 const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
                 const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
                 const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
                 const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
                 const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                 const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                 ParallelFor(xbx,ybx,zbx,
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     ax_ar(i,j,k) *= (mf_uy(i,j,0) / mf_ux(i,j,0));
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     ay_ar(i,j,k) *= (mf_vx(i,j,0) / mf_vy(i,j,0));
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
                 });
             } // mfi
  
         } // MeshType::VariableDz
  
         // ****************************************************************************
         // Print time in solve
         // ****************************************************************************
         Real end_step = static_cast<Real>(ParallelDescriptor::second());
         if (mg_verbose > 0) {
             amrex::Print() << "Time in solve " << end_step - start_step << std::endl;
         }
  
         } // not EB
     } // loop over subdomains (i)
  
     // ****************************************************************************
     // When using multigrid we can solve for all of the level at once, even if there
     //      are disjoint regions
     // ****************************************************************************
     if (solverChoice.terrain_type == TerrainType::EB) {
         Real start_step_eb = static_cast<Real>(ParallelDescriptor::second());
         solve_with_EB_mlmg(lev, rhs_sub, phi_sub, fluxes_sub,
                            *(get_eb(lev).get_const_factory()),
                            *(get_eb(lev).get_u_const_factory()),
                            *(get_eb(lev).get_v_const_factory()),
                            *(get_eb(lev).get_w_const_factory()),
                            geom[lev], ref_ratio, domain_bc_type,
                            mg_verbose, solverChoice.poisson_reltol, solverChoice.poisson_abstol);
         Real end_step_eb = static_cast<Real>(ParallelDescriptor::second());
         if (mg_verbose > 0) {
             amrex::Print() << "Time in solve " << end_step_eb - start_step_eb << std::endl;
         }
     }
  
     // ****************************************************************************
     // Subtract dt grad(phi) from the momenta (rho0u, rho0v, Omega)
     // ****************************************************************************
     MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0);
     MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0);
     MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0);
  
     // ****************************************************************************
     // Define gradp from fluxes -- note that fluxes is dt * change in Gp
     //   (weighted by map factor!)
     // ****************************************************************************
     MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
     MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
     MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
  
     gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
     gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
     gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
  
     //
     // This call is only to verify the divergence after the solve
     // It is important we do this before computing the rho0w_arr from Omega back to rho0w
     //
     // ****************************************************************************
     // THIS IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE
     // ****************************************************************************
     //
     if (mg_verbose > 0)
     {
         rho0_u_const[0] = &mom_mf[IntVars::xmom];
         rho0_u_const[1] = &mom_mf[IntVars::ymom];
         rho0_u_const[2] = &mom_mf[IntVars::zmom];
  
         compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
  
         bool local = false;
         Real sum = volWgtSumMF(lev,rhs_lev,0,*detJ_cc[lev],*mapfac[lev][MapFacType::m_x],*mapfac[lev][MapFacType::m_y],false,local);
  
         if (mg_verbose > 0) {
             Print() << "Max/L2 norm of divergence after  solve at level " << lev << " : " << rhs_lev.norm0() << " " <<
                         rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
         }
  
 #if 0
          // FOR DEBUGGING ONLY
          for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
          {
             const Array4<Real const>& rhs_arr = rhs_lev.const_array(mfi);
             Box bx = mfi.validbox();
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                 if (std::abs(rhs_arr(i,j,k)) > 1.e-10) {
                     amrex::AllPrint() << "RHS after solve at " <<
                                           IntVect(i,j,k) << " " << rhs_arr(i,j,k) << std::endl;
                 }
             });
          } // mfi
 #endif
  
     } // mg_verbose
  
     //
     // ****************************************************************************
     // Now convert the rho0w MultiFab back to holding (rho0w) rather than Omega
     // ****************************************************************************
     //
     if (solverChoice.mesh_type == MeshType::VariableDz)
     {
         for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi)
         {
              Box tbz = mfi.nodaltilebox(2);
              const Array4<Real      >& rho0u_arr = mom_mf[IntVars::xmom].array(mfi);
              const Array4<Real      >& rho0v_arr = mom_mf[IntVars::ymom].array(mfi);
              const Array4<Real      >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
              const Array4<Real const>&      z_nd = z_phys_nd[lev]->const_array(mfi);
              const Array4<Real const>&      mf_u =  mapfac[lev][MapFacType::u_x]->const_array(mfi);
              const Array4<Real const>&      mf_v =  mapfac[lev][MapFacType::v_y]->const_array(mfi);
              ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
                  Real omega = rho0w_arr(i,j,k);
                  rho0w_arr(i,j,k) = WFromOmega(i,j,k,omega,
                                                rho0u_arr,rho0v_arr,
                                                mf_u,mf_v,z_nd,dxInv);
              });
         } // mfi
     }
  
     // If !fixed_density, we must convert (rho0 u) back
     // to (rho0 u) which is what we will pass back out
     if (!solverChoice.fixed_density[lev]) {
         ConvertForProjection(r_hse, mom_mf[Vars::cons],
                              mom_mf[IntVars::xmom],
                              mom_mf[IntVars::ymom],
                              mom_mf[IntVars::zmom],
                              Geom(lev).Domain(),
                              domain_bcs_type);
     }
  
     // ****************************************************************************
     // Update pressure variable with phi -- note that phi is dt * change in pressure
     // ****************************************************************************
     MultiFab::Saxpy(pp_inc[lev], 1.0/l_dt, phi_lev,0,0,1,1);
 }

Here is the call graph for this function:

◆ project_velocity_tb()

void ERF::project_velocity_tb	(	int	lev,
		amrex::Real	dt,
		amrex::Vector< amrex::MultiFab > &	vars
	)

Project the single-level velocity field to enforce incompressibility with a thin body

 {
     BL_PROFILE("ERF::project_velocity_tb()");
     AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
  
     // Make sure the solver only sees the levels over which we are solving
     Vector<BoxArray>            ba_tmp;   ba_tmp.push_back(vmf[Vars::cons].boxArray());
     Vector<DistributionMapping> dm_tmp;   dm_tmp.push_back(vmf[Vars::cons].DistributionMap());
     Vector<Geometry>          geom_tmp; geom_tmp.push_back(geom[lev]);
  
     // Use the default settings
     LPInfo info;
     std::unique_ptr<MLPoisson> p_mlpoisson;
 #if 0
     if (overset_imask[0]) {
         // Add overset mask around thin body
         p_mlpoisson = std::make_unique<MLPoisson>(geom, grids, dmap, GetVecOfConstPtrs(overset_imask), info);
     }
     else
 #endif
     {
         // Use the default settings
         p_mlpoisson = std::make_unique<MLPoisson>(geom_tmp, ba_tmp, dm_tmp, info);
     }
  
     auto bclo = get_lo_projection_bc(geom[lev],domain_bc_type);
     auto bchi = get_hi_projection_bc(geom[lev],domain_bc_type);
  
     bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true;
     p_mlpoisson->setDomainBC(bclo, bchi);
  
     if (lev > 0) {
         p_mlpoisson->setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
     }
  
     p_mlpoisson->setLevelBC(0, nullptr);
  
     Vector<MultiFab> rhs;
     Vector<MultiFab> phi;
     Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
     Vector<Array<MultiFab,AMREX_SPACEDIM> > deltaf; // f^* - f^{n-1}
     Vector<Array<MultiFab,AMREX_SPACEDIM> > u_plus_dtdf; // u + dt*deltaf
  
     // Used to pass array of const MFs to ComputeDivergence
     Array<MultiFab const*, AMREX_SPACEDIM> u;
  
     rhs.resize(1);
     phi.resize(1);
     fluxes.resize(1);
     deltaf.resize(1);
     u_plus_dtdf.resize(1);
  
     rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
     phi[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
     rhs[0].setVal(0.0);
     phi[0].setVal(0.0);
  
     for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
              fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
         u_plus_dtdf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
  
              deltaf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
              deltaf[0][idim].setVal(0.0); // start with f^* == f^{n-1}
     }
  
 #if 0
     // DEBUG
     u[0] = &(vmf[Vars::xvel]);
     u[1] = &(vmf[Vars::yvel]);
     u[2] = &(vmf[Vars::zvel]);
     computeDivergence(rhs[0], u, geom[0]);
     Print() << "Max norm of divergence before solve at level 0 : " << rhs[0].norm0() << std::endl;
 #endif
  
     for (int itp = 0; itp < solverChoice.ncorr; ++itp)
     {
         // Calculate u + dt*deltaf
         for (int idim = 0; idim < 3; ++idim) {
             MultiFab::Copy(u_plus_dtdf[0][idim], deltaf[0][idim], 0, 0, 1, 0);
             u_plus_dtdf[0][0].mult(-l_dt,0,1,0);
         }
         MultiFab::Add(u_plus_dtdf[0][0], vmf[Vars::xvel], 0, 0, 1, 0);
         MultiFab::Add(u_plus_dtdf[0][1], vmf[Vars::yvel], 0, 0, 1, 0);
         MultiFab::Add(u_plus_dtdf[0][2], vmf[Vars::zvel], 0, 0, 1, 0);
  
         u[0] = &(u_plus_dtdf[0][0]);
         u[1] = &(u_plus_dtdf[0][1]);
         u[2] = &(u_plus_dtdf[0][2]);
         computeDivergence(rhs[0], u, geom_tmp[0]);
  
 #if 0
         // DEBUG
         if (itp==0) {
             for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real const>& divU = rhs[0].const_array(mfi);
                 const Array4<Real const>& uarr = vmf[Vars::xvel].const_array(mfi);
                 const Array4<Real const>& varr = vmf[Vars::yvel].const_array(mfi);
                 const Array4<Real const>& warr = vmf[Vars::zvel].const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                 {
                     if ((i>=120) && (i<=139) && (j==0) && ((k>=127)&&(k<=128))) {
                         amrex::AllPrint() << "before project div"<<IntVect(i,j,k)<<" = "<< divU(i,j,k)
                             << " u: " << uarr(i,j,k) << " " << uarr(i+1,j,k)
                             << " v: " << varr(i,j,k) << " " << varr(i,j+1,k)
                             << " w: " << warr(i,j,k) << " " << warr(i,j,k+1)
                             << std::endl;
                     }
                 });
             }
         }
 #endif
  
         // If all Neumann BCs, adjust RHS to make sure we can converge
         if (need_adjust_rhs) {
             bool local = false;
             Real offset = volWgtSumMF(lev,rhs[0],0,*detJ_cc[lev],*mapfac[lev][MapFacType::m_x],*mapfac[lev][MapFacType::m_y],false,local);
             // amrex::Print() << "Poisson solvability offset = " << offset << std::endl;
             rhs[0].plus(-offset, 0, 1);
         }
  
         // Initialize phi to 0
         phi[0].setVal(0.0);
  
         MLMG mlmg(*p_mlpoisson);
         int max_iter = 100;
         mlmg.setMaxIter(max_iter);
  
         mlmg.setVerbose(mg_verbose);
         //mlmg.setBottomVerbose(mg_verbose);
  
         // solve for dt*p
         mlmg.solve(GetVecOfPtrs(phi),
                    GetVecOfConstPtrs(rhs),
                    solverChoice.poisson_reltol,
                    solverChoice.poisson_abstol);
  
         mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
  
         // Calculate new intermediate body force with updated gradp
         if (thin_xforce[lev]) {
             MultiFab::Copy(   deltaf[0][0], fluxes[0][0], 0, 0, 1, 0);
             ApplyInvertedMask(deltaf[0][0], *xflux_imask[0]);
         }
         if (thin_yforce[lev]) {
             MultiFab::Copy(   deltaf[0][1], fluxes[0][1], 0, 0, 1, 0);
             ApplyInvertedMask(deltaf[0][1], *yflux_imask[0]);
         }
         if (thin_zforce[lev]) {
             MultiFab::Copy(   deltaf[0][2], fluxes[0][2], 0, 0, 1, 0);
             ApplyInvertedMask(deltaf[0][2], *zflux_imask[0]);
         }
  
         // DEBUG
         //        for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
         //        {
         //            const Box& bx = mfi.tilebox();
         //            const Array4<Real const>& dfz_arr = deltaf[0][2].const_array(mfi);
         //            ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         //            {
         //                if ((i>=120) && (i<=139) && (j==0) && (k==128)) {
         //                    amrex::AllPrint()
         //                        << " piter" << itp
         //                        << " dfz"<<IntVect(i,j,k)<<" = "<< dfz_arr(i,j,k)
         //                        << std::endl;
         //                }
         //            });
         //        }
  
         // Update pressure variable with phi -- note that phi is change in pressure, not the full pressure
         MultiFab::Saxpy(pp_inc[lev], 1.0, phi[0],0,0,1,0);
  
         // Subtract grad(phi) from the velocity components
         Real beta = 1.0;
         MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
         MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
         MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
         if (thin_xforce[lev]) {
             ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
         }
         if (thin_yforce[lev]) {
             ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
         }
         if (thin_zforce[lev]) {
             ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
         }
     } // itp: pressure-force iterations
  
     // ****************************************************************************
     // Define gradp from fluxes -- note that fluxes is dt * change in Gp
     // ****************************************************************************
     MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
     MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
     MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
  
     gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
     gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
     gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
  
     // Subtract grad(phi) from the velocity components
 //    Real beta = 1.0;
 //    for (int ilev = lev_min; ilev <= lev_max; ++ilev) {
 //        MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
 //        MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
 //        MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
 //        if (thin_xforce[lev]) {
 //            ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
 //        }
 //        if (thin_yforce[lev]) {
 //            ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
 //        }
 //        if (thin_zforce[lev]) {
 //            ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
 //        }
 //    }
  
 #if 0
     // Confirm that the velocity is now divergence free
     u[0] = &(vmf[Vars::xvel]);
     u[1] = &(vmf[Vars::yvel]);
     u[2] = &(vmf[Vars::zvel]);
     computeDivergence(rhs[0], u, geom_tmp[0]);
     Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
  
 #endif
 }

Here is the call graph for this function:

◆ ReadCheckpointFile()

void ERF::ReadCheckpointFile ( )

ERF function for reading data from a checkpoint file during restart.

 {
     Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
  
     // Header
     std::string File(restart_chkfile + "/Header");
  
     VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
  
     Vector<char> fileCharPtr;
     ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
     std::string fileCharPtrString(fileCharPtr.dataPtr());
     std::istringstream is(fileCharPtrString, std::istringstream::in);
  
     std::string line, word;
  
     int chk_ncomp_cons, chk_ncomp;
  
     // read in title line
     std::getline(is, line);
  
     // read in finest_level
     is >> finest_level;
     GotoNextLine(is);
  
     // read the number of components
     // for each variable we store
  
     // conservative, cell-centered vars
     is >> chk_ncomp_cons;
     GotoNextLine(is);
  
     // x-velocity on faces
     is >> chk_ncomp;
     GotoNextLine(is);
     AMREX_ASSERT(chk_ncomp == 1);
  
     // y-velocity on faces
     is >> chk_ncomp;
     GotoNextLine(is);
     AMREX_ASSERT(chk_ncomp == 1);
  
     // z-velocity on faces
     is >> chk_ncomp;
     GotoNextLine(is);
     AMREX_ASSERT(chk_ncomp == 1);
  
     // read in array of istep
     std::getline(is, line);
     {
         std::istringstream lis(line);
         int i = 0;
         while (lis >> word) {
             istep[i++] = std::stoi(word);
         }
     }
  
     // read in array of dt
     std::getline(is, line);
     {
         std::istringstream lis(line);
         int i = 0;
         while (lis >> word) {
             dt[i++] = std::stod(word);
         }
     }
  
     // read in array of t_new
     std::getline(is, line);
     {
         std::istringstream lis(line);
         int i = 0;
         while (lis >> word) {
             t_new[i++] = std::stod(word);
         }
     }
  
     for (int lev = 0; lev <= finest_level; ++lev) {
         // read in level 'lev' BoxArray from Header
         BoxArray ba;
         ba.readFrom(is);
         GotoNextLine(is);
  
         // create a distribution mapping
         DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
  
         MakeNewLevelFromScratch (lev, t_new[lev], ba, dm);
     }
  
     // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
     // NOTE: Data is written over ncomp, so check that we match the header file
     int ncomp_cons = vars_new[0][Vars::cons].nComp();
  
     // NOTE: QKE was removed so this is for backward compatibility
     AMREX_ASSERT((chk_ncomp_cons==ncomp_cons) || ((chk_ncomp_cons-1)==ncomp_cons));
     //
     // See if we have a written separate file that tells how many components and how many ghost cells
     // we have of the base state
     //
     // If we can't find the file, then set the number of components to the original number = 3
     //
     int ncomp_base_to_read = 3;
     IntVect ng_base = IntVect{1};
     {
          std::string BaseStateFile(restart_chkfile + "/num_base_state_comps");
  
          if (amrex::FileExists(BaseStateFile))
          {
              Vector<char> BaseStatefileCharPtr;
              ParallelDescriptor::ReadAndBcastFile(BaseStateFile, BaseStatefileCharPtr);
              std::string BaseStatefileCharPtrString(BaseStatefileCharPtr.dataPtr());
  
              // We set this to the default value of 3 but allow it be larger if th0 and qv0 were written
              std::istringstream isb(BaseStatefileCharPtrString, std::istringstream::in);
              isb >> ncomp_base_to_read;
              isb >> ng_base;
          }
     }
  
     // read in the MultiFab data
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         // NOTE: For backward compatibility (chk file has QKE)
         if ((chk_ncomp_cons-1)==ncomp_cons) {
             MultiFab cons(grids[lev],dmap[lev],chk_ncomp_cons,0);
             VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
  
             // Copy up to RhoKE_comp
             MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,(RhoKE_comp+1),0);
  
             // Only if we have a PBL model do we need to copy QKE is src to KE in dst
             if ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
                  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ) {
                 MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+1),RhoKE_comp,1,0);
                 vars_new[lev][Vars::cons].mult(0.5,RhoKE_comp,1,0);
             }
  
             // Copy other components
             int ncomp_remainder = ncomp_cons - (RhoKE_comp + 1);
             MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+2),(RhoKE_comp+1),ncomp_remainder,0);
  
             vars_new[lev][Vars::cons].setBndry(1.0e34);
         } else {
             MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
             VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
             MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
             vars_new[lev][Vars::cons].setBndry(1.0e34);
         }
  
         MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
         VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
         MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
         vars_new[lev][Vars::xvel].setBndry(1.0e34);
  
         MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
         VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
         MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
         vars_new[lev][Vars::yvel].setBndry(1.0e34);
  
         MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
         VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
         MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
         vars_new[lev][Vars::zvel].setBndry(1.0e34);
  
         if (solverChoice.anelastic[lev] == 1) {
             MultiFab ppinc(grids[lev],dmap[lev],1,0);
             VisMF::Read(ppinc, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PP_Inc"));
             MultiFab::Copy(pp_inc[lev],ppinc,0,0,1,0);
             pp_inc[lev].FillBoundary(geom[lev].periodicity());
  
             MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
             VisMF::Read(gpx, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpx"));
             MultiFab::Copy(gradp[lev][GpVars::gpx],gpx,0,0,1,0);
             gradp[lev][GpVars::gpx].FillBoundary(geom[lev].periodicity());
  
             MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
             VisMF::Read(gpy, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpy"));
             MultiFab::Copy(gradp[lev][GpVars::gpy],gpy,0,0,1,0);
             gradp[lev][GpVars::gpy].FillBoundary(geom[lev].periodicity());
  
             MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
             VisMF::Read(gpz, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpz"));
             MultiFab::Copy(gradp[lev][GpVars::gpz],gpz,0,0,1,0);
             gradp[lev][GpVars::gpz].FillBoundary(geom[lev].periodicity());
         }
  
         // Note that we read the ghost cells of the base state (unlike above)
  
         // The original base state only had 3 components and 1 ghost cell -- we read this
         // here to be consistent with the old style
         MultiFab base(grids[lev],dmap[lev],ncomp_base_to_read,ng_base);
         VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
  
         MultiFab::Copy(base_state[lev],base,0,0,ncomp_base_to_read,ng_base);
  
         // Create theta0 from p0, rh0
         if (ncomp_base_to_read < 4) {
             for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 // We only compute theta_0 on valid cells since we will impose domain BC's after restart
                 const Box& bx = mfi.tilebox();
                 Array4<Real> const& fab = base_state[lev].array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     fab(i,j,k,BaseState::th0_comp) = getRhoThetagivenP(fab(i,j,k,BaseState::p0_comp))
                                                      / fab(i,j,k,BaseState::r0_comp);
                 });
             }
         }
         // Default theta0 to 0
         if (ncomp_base_to_read < 5) {
             for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 // We only compute theta_0 on valid cells since we will impose domain BC's after restart
                 const Box& bx = mfi.tilebox();
                 Array4<Real> const& fab = base_state[lev].array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     fab(i,j,k,BaseState::qv0_comp) = 0.0;
                 });
             }
         }
         base_state[lev].FillBoundary(geom[lev].periodicity());
  
         if (SolverChoice::mesh_type != MeshType::ConstantDz)  {
            // Note that we also read the ghost cells of z_phys_nd
            IntVect ng = z_phys_nd[lev]->nGrowVect();
            MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
            VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
            MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
            update_terrain_arrays(lev);
  
            // Compute the min dz and pass to the micro model
            Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
            micro->Set_dzmin(lev, dzmin);
  
            if (SolverChoice::mesh_type == MeshType::VariableDz) {
                MultiFab z_slab(convert(ba2d[lev],IntVect(1,1,1)),dmap[lev],1,0);
                int klo = geom[lev].Domain().smallEnd(2);
                for (MFIter mfi(z_slab); mfi.isValid(); ++mfi) {
                    Box nbx = mfi.tilebox();
                    Array4<Real const> const& z_arr      = z_phys_nd[lev]->const_array(mfi);
                    Array4<Real      > const& z_slab_arr = z_slab.array(mfi);
                    ParallelFor(nbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                    {
                        z_slab_arr(i,j,k) = z_arr(i,j,klo);
                    });
                }
                Real z_min = z_slab.min(0);
                Real z_max = z_slab.max(0);
  
                auto dz = geom[lev].CellSize()[2];
                if (z_max - z_min < 1.e-8 * dz) {
                    SolverChoice::set_mesh_type(MeshType::StretchedDz);
                    if (verbose > 0) {
                        amrex::Print() << "Resetting mesh type to StretchedDz since terrain is flat" << std::endl;
                    }
                }
            }
         }
  
         // Read in the moisture model restart variables
         std::vector<int> qmoist_indices;
         std::vector<std::string> qmoist_names;
         micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
         int qmoist_nvar = qmoist_indices.size();
         for (int var = 0; var < qmoist_nvar; var++) {
             const int ncomp  = 1;
             IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
             MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
             VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", qmoist_names[var]));
             MultiFab::Copy(*(qmoist[lev][qmoist_indices[var]]),moist_vars,0,0,ncomp,ng_moist);
         }
  
 #if defined(ERF_USE_WINDFARM)
         if(solverChoice.windfarm_type == WindFarmType::Fitch or
            solverChoice.windfarm_type == WindFarmType::EWP or
            solverChoice.windfarm_type == WindFarmType::SimpleAD){
             IntVect ng = Nturb[lev].nGrowVect();
             MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
             VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
             MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
         }
 #endif
  
         // Read the LSM data
         if (solverChoice.lsm_type != LandSurfaceType::None) {
             for (int ivar(0); ivar<lsm_data[lev].size(); ++ivar) {
                 BoxArray ba = lsm_data[lev][ivar]->boxArray();
                 DistributionMapping dm = lsm_data[lev][ivar]->DistributionMap();
                 IntVect ng = lsm_data[lev][ivar]->nGrowVect();
                 int nvar   = lsm_data[lev][ivar]->nComp();
                 MultiFab lsm_vars(ba,dm,nvar,ng);
                 VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmData" + std::to_string(ivar)));
                 MultiFab::Copy(*(lsm_data[lev][ivar]),lsm_vars,0,0,nvar,ng);
             }
             for (int iflux(0); iflux<lsm_flux[lev].size(); ++iflux) {
                 BoxArray ba = lsm_flux[lev][iflux]->boxArray();
                 DistributionMapping dm = lsm_flux[lev][iflux]->DistributionMap();
                 IntVect ng = lsm_flux[lev][iflux]->nGrowVect();
                 int nvar   = lsm_flux[lev][iflux]->nComp();
                 MultiFab lsm_vars(ba,dm,nvar,ng);
                 VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmFlux" + std::to_string(iflux)));
                 MultiFab::Copy(*(lsm_flux[lev][iflux]),lsm_vars,0,0,nvar,ng);
             }
         }
  
  
         IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
         MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
  
         std::string MapFacMFileName(restart_chkfile + "/Level_0/MapFactor_mx_H");
         if (amrex::FileExists(MapFacMFileName)) {
             VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_mx"));
         } else {
             VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
         }
         MultiFab::Copy(*mapfac[lev][MapFacType::m_x],mf_m,0,0,1,ng);
  
 #if 0
         if (MapFacType::m_x != MapFacType::m_y) {
             VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_my"));
             MultiFab::Copy(*mapfac[lev][MapFacType::m_y],mf_m,0,0,1,ng);
         }
 #endif
  
         ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
         MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
  
         std::string MapFacUFileName(restart_chkfile + "/Level_0/MapFactor_ux_H");
         if (amrex::FileExists(MapFacUFileName)) {
             VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_ux"));
         } else {
             VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
         }
         MultiFab::Copy(*mapfac[lev][MapFacType::u_x],mf_u,0,0,1,ng);
  
 #if 0
         if (MapFacType::u_x != MapFacType::u_y) {
             VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_uy"));
             MultiFab::Copy(*mapfac[lev][MapFacType::u_y],mf_u,0,0,1,ng);
         }
 #endif
  
         ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
         MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
  
         std::string MapFacVFileName(restart_chkfile + "/Level_0/MapFactor_vx_H");
         if (amrex::FileExists(MapFacVFileName)) {
             VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vx"));
         } else {
             VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
         }
         MultiFab::Copy(*mapfac[lev][MapFacType::v_x],mf_v,0,0,1,ng);
  
 #if 0
         if (MapFacType::v_x != MapFacType::v_y) {
             VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vy"));
             MultiFab::Copy(*mapfac[lev][MapFacType::v_y],mf_v,0,0,1,ng);
         }
 #endif
  
  
         // NOTE: We read MOST data in ReadCheckpointFileMOST (see below)!
  
         // See if we wrote out SST data
         std::string FirstSSTFileName(restart_chkfile + "/Level_0/SST_0_H");
         if (amrex::FileExists(FirstSSTFileName))
         {
             amrex::Print() << "Reading SST data" << std::endl;
             int ntimes = 1;
             ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
             MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
             sst_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
             for (int nt(0); nt<ntimes; ++nt) {
                 VisMF::Read(sst_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
                                                              "SST_" + std::to_string(nt)));
                 MultiFab::Copy(*sst_lev[lev][nt],sst_at_t,0,0,1,ng);
             }
         }
  
         // See if we wrote out TSK data
         std::string FirstTSKFileName(restart_chkfile + "/Level_0/TSK_0_H");
         if (amrex::FileExists(FirstTSKFileName))
         {
             amrex::Print() << "Reading TSK data" << std::endl;
             int ntimes = 1;
             ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
             MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
             tsk_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
             for (int nt(0); nt<ntimes; ++nt) {
                 VisMF::Read(tsk_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
                                                              "TSK_" + std::to_string(nt)));
                 MultiFab::Copy(*tsk_lev[lev][nt],tsk_at_t,0,0,1,ng);
             }
         }
  
         std::string LMaskFileName(restart_chkfile + "/Level_0/LMASK_0_H");
         if (amrex::FileExists(LMaskFileName))
         {
             amrex::Print() << "Reading LMASK data" << std::endl;
             int ntimes = 1;
             ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
             MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
             for (int nt(0); nt<ntimes; ++nt) {
                 VisMF::Read(lmask_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
                                                               "LMASK_" + std::to_string(nt)));
                 for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
                     const Box& bx = mfi.growntilebox();
                     Array4<int>  const& dst_arr = lmask_lev[lev][nt]->array(mfi);
                     Array4<Real> const& src_arr = lmask_at_t.array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                     {
                         dst_arr(i,j,k) = int(src_arr(i,j,k));
                     });
                 }
             }
         } else {
             // Allow idealized cases over water, used to set lmask
             ParmParse pp("erf");
             int is_land;
             if (pp.query("is_land", is_land, lev)) {
                 if (is_land == 1) {
                     amrex::Print() << "Level " << lev << " is land" << std::endl;
                 } else if (is_land == 0) {
                     amrex::Print() << "Level " << lev << " is water" << std::endl;
                 } else {
                     Error("is_land should be 0 or 1");
                 }
                 lmask_lev[lev][0]->setVal(is_land);
             } else {
                 // Default to land everywhere if not specified
                 lmask_lev[lev][0]->setVal(1);
             }
             lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
         }
  
         IntVect ngv = ng; ngv[2] = 0;
  
         // Read lat/lon if it exists
         if (solverChoice.has_lat_lon) {
             amrex::Print() << "Reading Lat/Lon variables" << std::endl;
             MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
             MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
             VisMF::Read(lat, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LAT"));
             VisMF::Read(lon, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LON"));
             lat_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
             lon_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
             MultiFab::Copy(*lat_m[lev],lat,0,0,1,ngv);
             MultiFab::Copy(*lon_m[lev],lon,0,0,1,ngv);
         }
  
 #ifdef ERF_USE_NETCDF
         // Read sinPhi and cosPhi if it exists
         if (solverChoice.variable_coriolis) {
             amrex::Print() << "Reading Coriolis factors" << std::endl;
             MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
             MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
             VisMF::Read(sphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SinPhi"));
             VisMF::Read(cphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "CosPhi"));
             sinPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
             cosPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
             MultiFab::Copy(*sinPhi_m[lev],sphi,0,0,1,ngv);
             MultiFab::Copy(*cosPhi_m[lev],cphi,0,0,1,ngv);
         }
  
         if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
             if (lev == 0) {
                 MultiFab tmp1d(ba1d[0],dmap[0],1,0);
  
                 VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C1H"));
                 MultiFab::Copy(*mf_C1H,tmp1d,0,0,1,0);
  
                 VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C2H"));
                 MultiFab::Copy(*mf_C2H,tmp1d,0,0,1,0);
  
                 MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
  
                 VisMF::Read(tmp2d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MUB"));
                 MultiFab::Copy(*mf_MUB,tmp2d,0,0,1,mf_MUB->nGrowVect());
             }
         }
 #endif
     } // for lev
  
 #ifdef ERF_USE_PARTICLES
     restartTracers((ParGDBBase*)GetParGDB(),restart_chkfile);
     if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
         dynamic_cast<LagrangianMicrophysics&>(*micro).restartParticles((ParGDBBase*)GetParGDB(),restart_chkfile);
     }
 #endif
  
 #if 0
 #ifdef ERF_USE_NETCDF
     // Read bdy_data files
     if ( ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
          solverChoice.use_real_bcs )
     {
         int ioproc = ParallelDescriptor::IOProcessorNumber();  // I/O rank
         int num_time;
         int num_var;
         Vector<Box> bx_v;
         if (ParallelDescriptor::IOProcessor()) {
             // Open header file and read from it
             std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
             bdy_h_file >> num_time;
             bdy_h_file >> num_var;
             bdy_h_file >> start_bdy_time;
             bdy_h_file >> bdy_time_interval;
             bdy_h_file >> real_width;
             bx_v.resize(4*num_var);
             for (int ivar(0); ivar<num_var; ++ivar) {
                 bdy_h_file >> bx_v[4*ivar  ];
                 bdy_h_file >> bx_v[4*ivar+1];
                 bdy_h_file >> bx_v[4*ivar+2];
                 bdy_h_file >> bx_v[4*ivar+3];
             }
  
             // IO size the FABs
             bdy_data_xlo.resize(num_time);
             bdy_data_xhi.resize(num_time);
             bdy_data_ylo.resize(num_time);
             bdy_data_yhi.resize(num_time);
             for (int itime(0); itime<num_time; ++itime) {
                 bdy_data_xlo[itime].resize(num_var);
                 bdy_data_xhi[itime].resize(num_var);
                 bdy_data_ylo[itime].resize(num_var);
                 bdy_data_yhi[itime].resize(num_var);
                 for (int ivar(0); ivar<num_var; ++ivar) {
                     bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
                     bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
                     bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
                     bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
                 }
             }
  
             // Open data file and read from it
             std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
             for (int itime(0); itime<num_time; ++itime) {
                 for (int ivar(0); ivar<num_var; ++ivar) {
                     bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
                     bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
                     bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
                     bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
                 }
             }
         } // IO
  
         // Broadcast the data
         ParallelDescriptor::Barrier();
         ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
         ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
         ParallelDescriptor::Bcast(&real_width,1,ioproc);
         ParallelDescriptor::Bcast(&num_time,1,ioproc);
         ParallelDescriptor::Bcast(&num_var,1,ioproc);
  
         // Everyone size their boxes
         bx_v.resize(4*num_var);
  
         ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
  
         // Everyone but IO size their FABs
         if (!ParallelDescriptor::IOProcessor()) {
           bdy_data_xlo.resize(num_time);
           bdy_data_xhi.resize(num_time);
           bdy_data_ylo.resize(num_time);
           bdy_data_yhi.resize(num_time);
           for (int itime(0); itime<num_time; ++itime) {
             bdy_data_xlo[itime].resize(num_var);
             bdy_data_xhi[itime].resize(num_var);
             bdy_data_ylo[itime].resize(num_var);
             bdy_data_yhi[itime].resize(num_var);
             for (int ivar(0); ivar<num_var; ++ivar) {
               bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
               bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
               bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
               bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
             }
           }
         }
  
         for (int itime(0); itime<num_time; ++itime) {
             for (int ivar(0); ivar<num_var; ++ivar) {
                 ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
                 ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
                 ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
                 ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
             }
         }
     } // init_type == WRFInput or Metgrid
 #endif
 #endif
 }

Here is the call graph for this function:

◆ ReadCheckpointFileSurfaceLayer()

void ERF::ReadCheckpointFileSurfaceLayer ( )

ERF function for reading additional data for MOST from a checkpoint file during restart.

This is called after the ABLMost object is instantiated.

 {
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         amrex::Print() << "Reading MOST variables" << std::endl;
  
         IntVect ng(1,1,0);
         MultiFab  m_var(ba2d[lev],dmap[lev],1,ng);
         MultiFab* dst = nullptr;
  
         // U*
         std::string UstarFileName(restart_chkfile + "/Level_0/Ustar_H");
         if (amrex::FileExists(UstarFileName)) {
             dst = m_SurfaceLayer->get_u_star(lev);
             VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Ustar"));
             MultiFab::Copy(*dst,m_var,0,0,1,ng);
         }
  
         // W*
         std::string WstarFileName(restart_chkfile + "/Level_0/Wstar_H");
         if (amrex::FileExists(WstarFileName)) {
             dst = m_SurfaceLayer->get_w_star(lev);
             VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Wstar"));
             MultiFab::Copy(*dst,m_var,0,0,1,ng);
         }
  
         // T*
         std::string TstarFileName(restart_chkfile + "/Level_0/Tstar_H");
         if (amrex::FileExists(TstarFileName)) {
             dst = m_SurfaceLayer->get_t_star(lev);
             VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Tstar"));
             MultiFab::Copy(*dst,m_var,0,0,1,ng);
         }
  
         // Q*
         std::string QstarFileName(restart_chkfile + "/Level_0/Qstar_H");
         if (amrex::FileExists(QstarFileName)) {
             dst = m_SurfaceLayer->get_q_star(lev);
             VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qstar"));
             MultiFab::Copy(*dst,m_var,0,0,1,ng);
         }
  
         // Olen
         std::string OlenFileName(restart_chkfile + "/Level_0/Olen_H");
         if (amrex::FileExists(OlenFileName)) {
             dst = m_SurfaceLayer->get_olen(lev);
             VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Olen"));
             MultiFab::Copy(*dst,m_var,0,0,1,ng);
         }
  
         // Qsurf
         std::string QsurfFileName(restart_chkfile + "/Level_0/Qsurf_H");
         if (amrex::FileExists(QsurfFileName)) {
             dst = m_SurfaceLayer->get_q_surf(lev);
             VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qsurf"));
             MultiFab::Copy(*dst,m_var,0,0,1,ng);
         }
  
         // PBLH
         std::string PBLHFileName(restart_chkfile + "/Level_0/PBLH_H");
         if (amrex::FileExists(PBLHFileName)) {
             dst = m_SurfaceLayer->get_pblh(lev);
             VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PBLH"));
             MultiFab::Copy(*dst,m_var,0,0,1,ng);
         }
  
         // Z0
         std::string Z0FileName(restart_chkfile + "/Level_0/Z0_H");
         if (amrex::FileExists(Z0FileName)) {
             dst = m_SurfaceLayer->get_z0(lev);
             VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z0"));
             MultiFab::Copy(*dst,m_var,0,0,1,ng);
         }
     }
 }

◆ ReadParameters()

void ERF::ReadParameters ( )

private

 {
     std::string prob_name = "Unknown";
     ParmParse pp_pn("erf"); pp_pn.queryAdd("prob_name", prob_name);
     Print() << "Problem name (from inputs file) is " << prob_name << std::endl;
  
     ParmParse pp(pp_prefix);
     ParmParse pp_amr("amr");
     {
         pp.query("regrid_level_0_on_restart", regrid_level_0_on_restart);
         pp.query("regrid_int", regrid_int);
         pp.query("check_file", check_file);
  
         // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
         //    for those or "erf.restart" / "erf.m_check_int" with the former taking
         //    precedence if both are specified
         pp.query("check_int", m_check_int);
         pp.query("check_per", m_check_per);
         pp_amr.query("check_int", m_check_int);
         pp_amr.query("check_per", m_check_per);
  
         pp.query("restart", restart_chkfile);
         pp_amr.query("restart", restart_chkfile);
  
         // Verbosity
         pp.query("v", verbose);
         pp.query("mg_v", mg_verbose);
         pp.query("use_fft", use_fft);
 #ifndef ERF_USE_FFT
         if (use_fft) {
             Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
         }
 #endif
  
         // Check for NaNs?
         pp.query("check_for_nans", check_for_nans);
  
         // Frequency of diagnostic output
         pp.query("sum_interval", sum_interval);
         pp.query("sum_period"  , sum_per);
  
         pp.query("pert_interval", pert_interval);
  
         // Time step controls
         pp.query("cfl", cfl);
         pp.query("substepping_cfl", sub_cfl);
         pp.query("init_shrink", init_shrink);
         pp.query("change_max", change_max);
         pp.query("dt_max_initial", dt_max_initial);
         pp.query("dt_max", dt_max);
  
         fixed_dt.resize(max_level+1,-1.);
         fixed_fast_dt.resize(max_level+1,-1.);
  
         pp.query("fixed_dt", fixed_dt[0]);
         pp.query("fixed_fast_dt", fixed_fast_dt[0]);
  
         int nlevs_max = max_level + 1;
         istep.resize(nlevs_max, 0);
         nsubsteps.resize(nlevs_max, 1);
         // This is the default
         for (int lev = 1; lev <= max_level; ++lev) {
             nsubsteps[lev] = MaxRefRatio(lev-1);
         }
  
         if (max_level > 0) {
             ParmParse pp_erf("erf");
             int count = pp_erf.countval("dt_ref_ratio");
             if (count > 0) {
                 Vector<int> nsub;
                 nsub.resize(nlevs_max, 0);
                 if (count == 1) {
                     pp_erf.queryarr("dt_ref_ratio", nsub, 0, 1);
                     for (int lev = 1; lev <= max_level; ++lev) {
                         nsubsteps[lev] = nsub[0];
                     }
                 } else {
                     pp_erf.queryarr("dt_ref_ratio", nsub, 0, max_level);
                     for (int lev = 1; lev <= max_level; ++lev) {
                         nsubsteps[lev] = nsub[lev-1];
                     }
                 }
             }
         }
  
         // Make sure we do this after we have defined nsubsteps above
         for (int lev = 1; lev <= max_level; lev++)
         {
             fixed_dt[lev]      = fixed_dt[lev-1]      / static_cast<Real>(nsubsteps[lev]);
             fixed_fast_dt[lev] = fixed_fast_dt[lev-1] / static_cast<Real>(nsubsteps[lev]);
         }
  
         pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
  
         // We use this to keep track of how many boxes we read in from WRF initialization
         num_files_at_level.resize(max_level+1,0);
  
         // We use this to keep track of how many boxes are specified thru the refinement indicators
         num_boxes_at_level.resize(max_level+1,0);
             boxes_at_level.resize(max_level+1);
  
         // We always have exactly one file at level 0
         num_boxes_at_level[0] = 1;
         boxes_at_level[0].resize(1);
         boxes_at_level[0][0] = geom[0].Domain();
  
 #ifdef ERF_USE_NETCDF
         nc_init_file.resize(max_level+1);
         have_read_nc_init_file.resize(max_level+1);
  
         // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
         //        but we always have exactly one file at level 0
         for (int lev = 0; lev <= max_level; lev++) {
             const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
             if (pp.contains(nc_file_names.c_str())) {
                 int num_files = pp.countval(nc_file_names.c_str());
                 num_files_at_level[lev] = num_files;
                 nc_init_file[lev].resize(num_files);
                 have_read_nc_init_file[lev].resize(num_files);
                 pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
                 for (int j = 0; j < num_files; j++) {
                     Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
                     have_read_nc_init_file[lev][j] = 0;
                 } // j
             } // if pp.contains
         } // lev
  
         // NetCDF wrfbdy lateral boundary file
         if (pp.query("nc_bdy_file", nc_bdy_file)) {
             Print() << "Reading NC bdy file name " << nc_bdy_file << std::endl;
         }
  
         // NetCDF wrflow lateral boundary file
         if (pp.query("nc_low_file", nc_low_file)) {
             Print() << "Reading NC low file name " << nc_low_file << std::endl;
         }
  
 #endif
  
         // Options for vertical interpolation of met_em*.nc data.
         pp.query("metgrid_debug_quiescent",  metgrid_debug_quiescent);
         pp.query("metgrid_debug_isothermal", metgrid_debug_isothermal);
         pp.query("metgrid_debug_dry",        metgrid_debug_dry);
         pp.query("metgrid_debug_psfc",       metgrid_debug_psfc);
         pp.query("metgrid_debug_msf",        metgrid_debug_msf);
         pp.query("metgrid_interp_theta",     metgrid_interp_theta);
         pp.query("metgrid_basic_linear",     metgrid_basic_linear);
         pp.query("metgrid_use_below_sfc",    metgrid_use_below_sfc);
         pp.query("metgrid_use_sfc",          metgrid_use_sfc);
         pp.query("metgrid_retain_sfc",       metgrid_retain_sfc);
         pp.query("metgrid_proximity",        metgrid_proximity);
         pp.query("metgrid_order",            metgrid_order);
         pp.query("metgrid_force_sfc_k",      metgrid_force_sfc_k);
  
         // Set default to FullState for now ... later we will try Perturbation
         interpolation_type = StateInterpType::FullState;
         pp.query_enum_case_insensitive("interpolation_type"  ,interpolation_type);
  
         PlotFileType plotfile3d_type_temp = PlotFileType::None;
         pp.query_enum_case_insensitive("plotfile_type"  ,plotfile3d_type_temp);
         pp.query_enum_case_insensitive("plotfile_type_1",plotfile3d_type_1);
         pp.query_enum_case_insensitive("plotfile_type_2",plotfile3d_type_2);
  
         PlotFileType plotfile2d_type_temp = PlotFileType::None;
         pp.query_enum_case_insensitive("plotfile2d_type"  ,plotfile2d_type_temp);
         pp.query_enum_case_insensitive("plotfile2d_type_1",plotfile2d_type_1);
         pp.query_enum_case_insensitive("plotfile2d_type_2",plotfile2d_type_2);
         //
         // This option is for backward consistency -- if only plotfile_type is set,
         //     then it will be used for both 1 and 2 if and only if they are not set
         //
         // Default is native amrex if no type is specified
         //
         if (plotfile3d_type_temp == PlotFileType::None) {
             if (plotfile3d_type_1 == PlotFileType::None) {
                 plotfile3d_type_1  = PlotFileType::Amrex;
             }
             if (plotfile3d_type_2 == PlotFileType::None) {
                 plotfile3d_type_2  = PlotFileType::Amrex;
             }
         } else {
             if (plotfile3d_type_1 == PlotFileType::None) {
                 plotfile3d_type_1  = plotfile3d_type_temp;
             } else {
                 Abort("You must set either plotfile_type or plotfile_type_1, not both");
             }
             if (plotfile3d_type_2 == PlotFileType::None) {
                 plotfile3d_type_2  = plotfile3d_type_temp;
             } else {
                 Abort("You must set either plotfile_type or plotfile_type_2, not both");
             }
         }
         if (plotfile2d_type_temp == PlotFileType::None) {
             if (plotfile2d_type_1 == PlotFileType::None) {
                 plotfile2d_type_1  = PlotFileType::Amrex;
             }
             if (plotfile2d_type_2 == PlotFileType::None) {
                 plotfile2d_type_2  = PlotFileType::Amrex;
             }
         } else {
             if (plotfile2d_type_1 == PlotFileType::None) {
                 plotfile2d_type_1  = plotfile2d_type_temp;
             } else {
                 Abort("You must set either plotfile2d_type or plotfile2d_type_1, not both");
             }
             if (plotfile2d_type_2 == PlotFileType::None) {
                 plotfile2d_type_2  = plotfile2d_type_temp;
             } else {
                 Abort("You must set either plotfile2d_type or plotfile2d_type_2, not both");
             }
         }
 #ifndef ERF_USE_NETCDF
         if (plotfile3d_type_1 == PlotFileType::Netcdf ||
             plotfile3d_type_2 == PlotFileType::Netcdf ||
             plotfile2d_type_1 == PlotFileType::Netcdf ||
             plotfile2d_type_2 == PlotFileType::Netcdf) {
             Abort("Plotfile type = Netcdf is not allowed without USE_NETCDF = TRUE");
         }
 #endif
  
         pp.query("plot_file_1"  ,   plot3d_file_1);
         pp.query("plot_file_2"  ,   plot3d_file_2);
         pp.query("plot2d_file_1",   plot2d_file_1);
         pp.query("plot2d_file_2",   plot2d_file_2);
  
         pp.query("plot_int_1"   , m_plot3d_int_1);
         pp.query("plot_int_2"   , m_plot3d_int_2);
         pp.query("plot_per_1"   , m_plot3d_per_1);
         pp.query("plot_per_2"   , m_plot3d_per_2);
  
         pp.query("plot2d_int_1" , m_plot2d_int_1);
         pp.query("plot2d_int_2" , m_plot2d_int_2);
         pp.query("plot2d_per_1",  m_plot2d_per_1);
         pp.query("plot2d_per_2",  m_plot2d_per_2);
  
         pp.query("subvol_file",   subvol_file);
  
         // Should we use format like plt1970-01-01_00:00:00.000000 (if true) or plt00001 (if false)
         pp.query("use_real_time_in_pltname", use_real_time_in_pltname);
  
         // If use_real_time_in_pltname is false, how many digits should we use for the timestep?
         pp.query("file_name_digits", file_name_digits);
  
         // Default if subvol_int not specified
         m_subvol_int.resize(1); m_subvol_int[0] = -1;
         m_subvol_per.resize(1); m_subvol_per[0] = -1.0;
         last_subvol_step.resize(1);
         last_subvol_time.resize(1);
  
         int nsi = pp.countval("subvol_int");
         int nsr = pp.countval("subvol_per");
  
         // We must specify only subvol_int OR subvol_per
         AMREX_ALWAYS_ASSERT (!(nsi > 0 && nsr > 0));
  
         int nsub = -1;
         if (nsi > 0 || nsr > 0) {
             ParmParse pp_sv("erf.subvol");
             int n1 = pp_sv.countval("origin"); int n2 = pp_sv.countval("nxnynz"); int n3 = pp_sv.countval("dxdydz");
             if (n1 != n2 || n1 != n3 || n2 != n3) {
                 Abort("WriteSubvolume: must have same number of entries in origin, nxnynz, and dxdydz.");
             }
             if ( n1%AMREX_SPACEDIM != 0) {
                 Abort("WriteSubvolume: origin, nxnynz, and dxdydz must have multiples of AMReX_SPACEDIM");
             }
             nsub = n1/AMREX_SPACEDIM;
             m_subvol_int.resize(nsub);
             last_subvol_step.resize(nsub);
             last_subvol_time.resize(nsub);
             m_subvol_int.resize(nsub);
             m_subvol_per.resize(nsub);
         }
  
         if (nsi > 0) {
             for (int i = 1; i < nsub; i++) m_subvol_per[i] = -1.0;
             if ( nsi == 1) {
                 m_subvol_int[0] = -1;
                 pp.get("subvol_int" , m_subvol_int[0]);
             } else if ( nsi == nsub) {
                 pp.getarr("subvol_int" , m_subvol_int);
             } else {
                 Abort("There must either be a single value of subvol_int or one for every subdomain");
             }
         }
  
         if (nsr > 0) {
             for (int i = 1; i < nsub; i++) m_subvol_int[i] = -1.0;
             if ( nsr == 1) {
                 m_subvol_per[0] = -1.0;
                 pp.get("subvol_per" , m_subvol_per[0]);
             } else if ( nsr == nsub) {
                 pp.getarr("subvol_per" , m_subvol_per);
             } else {
                 Abort("There must either be a single value of subvol_per or one for every subdomain");
             }
         }
  
         setSubVolVariables("subvol_sampling_vars",subvol3d_var_names);
  
         pp.query("expand_plotvars_to_unif_rr",m_expand_plotvars_to_unif_rr);
  
         pp.query("plot_face_vels",m_plot_face_vels);
  
         if ( (m_plot3d_int_1 > 0 && m_plot3d_per_1 > 0) ||
              (m_plot3d_int_2 > 0 && m_plot3d_per_2 > 0.) ) {
             Abort("Must choose only one of plot_int or plot_per");
         }
         if ( (m_plot2d_int_1 > 0 && m_plot2d_per_1 > 0) ||
              (m_plot2d_int_2 > 0 && m_plot2d_per_2 > 0.) ) {
             Abort("Must choose only one of plot_int or plot_per");
         }
  
         pp.query("profile_int", profile_int);
         pp.query("destag_profiles", destag_profiles);
  
         pp.query("plot_lsm", plot_lsm);
 #ifdef ERF_USE_RRTMGP
         pp.query("plot_rad", plot_rad);
 #endif
         pp.query("profile_rad_int", rad_datalog_int);
  
         pp.query("output_1d_column", output_1d_column);
         pp.query("column_per", column_per);
         pp.query("column_interval", column_interval);
         pp.query("column_loc_x", column_loc_x);
         pp.query("column_loc_y", column_loc_y);
         pp.query("column_file_name", column_file_name);
  
         // Sampler output frequency
         pp.query("line_sampling_per", line_sampling_per);
         pp.query("line_sampling_interval", line_sampling_interval);
         pp.query("plane_sampling_per", plane_sampling_per);
         pp.query("plane_sampling_interval", plane_sampling_interval);
  
         // Specify information about outputting planes of data
         pp.query("output_bndry_planes", output_bndry_planes);
         pp.query("bndry_output_planes_interval", bndry_output_planes_interval);
         pp.query("bndry_output_planes_per", bndry_output_planes_per);
         pp.query("bndry_output_start_time", bndry_output_planes_start_time);
  
         // Specify whether ingest boundary planes of data
         pp.query("input_bndry_planes", input_bndry_planes);
  
         // Query the total width for wrfbdy interior ghost cells
         pp.query("real_width", real_width);
  
         // If using real boundaries, do we extrapolate w (or set to 0)
         pp.query("real_extrap_w", real_extrap_w);
  
         // Query the set and total widths for crse-fine interior ghost cells
         pp.query("cf_width", cf_width);
         pp.query("cf_set_width", cf_set_width);
  
         // AmrMesh iterate on grids?
         bool iterate(true);
         pp_amr.query("iterate_grids",iterate);
         if (!iterate) SetIterateToFalse();
     }
  
 #ifdef ERF_USE_PARTICLES
     readTracersParams();
 #endif
  
     solverChoice.init_params(max_level,pp_prefix);
  
     {
         ParmParse pp_no_prefix;  // Traditionally, max_step and stop_time do not have prefix.
         pp_no_prefix.query("max_step", max_step);
         if (max_step < 0) {
             max_step = std::numeric_limits<int>::max();
         }
  
         std::string start_datetime, stop_datetime;
         if (pp_no_prefix.query("start_datetime", start_datetime)) {
             if (start_datetime.length() == 16) { // YYYY-MM-DD HH:MM
                 start_datetime += ":00"; // add seconds
             }
             if (start_datetime.length() != 19) {
                 Print() << "Got start_datetime = \"" << start_datetime
                     << "\", format should be " << datetime_format << std::endl;
                 exit(0);
             }
             start_time = getEpochTime(start_datetime, datetime_format);
  
 #ifdef ERF_USE_NETCDF
             if (solverChoice.init_type == InitType::WRFInput) {
                 // This is the start time as written in the wrfinput file
                 Real start_time_from_wrfinput = read_start_time_from_wrfinput(0, nc_init_file[0][0]);
                 if (start_time != start_time_from_wrfinput) {
                     amrex::Print() << "start_datetime from inputs file = "       << start_time <<
                                       " does not match SIMULATION START DATE from wrfinput = " <<
                                        start_time_from_wrfinput << std::endl;
                     amrex::Abort();
                 }
             }
 #endif
             Print() << "Start datetime   : " << start_datetime << std::endl;
  
             use_datetime = true;
  
         } else {
  
 #ifdef ERF_USE_NETCDF
             if (solverChoice.init_type == InitType::WRFInput) {
                 // This is the start time as written in the wrfinput file
                 Real start_time_from_wrfinput = read_start_time_from_wrfinput(0, nc_init_file[0][0]);
                 start_time = start_time_from_wrfinput;
  
                 use_datetime = true;
  
                 if (pp_no_prefix.query("start_time", start_time)) {
                     amrex::Print() << "start_time should not be set from inputs file; we are reading SIMULATION START DATE from wrfinput" << std::endl;
                     amrex::Abort();
                 }
             }
 #endif
         }
  
         if (pp_no_prefix.query("stop_datetime", stop_datetime)) {
             if (stop_datetime.length() == 16) { // YYYY-MM-DD HH:MM
                 stop_datetime += ":00"; // add seconds
             }
             if (stop_datetime.length() != 19) {
                 Print() << "Got stop_datetime = \"" << stop_datetime
                     << "\", format should be " << datetime_format << std::endl;
                 exit(0);
             }
  
             stop_time = getEpochTime(stop_datetime, datetime_format);
             Print() << "Stop  datetime : " << start_datetime << std::endl;
  
         } else {
  
             if (pp_no_prefix.query("stop_time", stop_time)) {
                 Print() << "Maximum simulation length based on stop_time: " << stop_time << " s (elapsed) " << std::endl;
                 amrex::Print() <<" Adding stop time " << stop_time << " to start_time " << start_time << std::endl;
                 stop_time += start_time;
             }
         }
     }
  
 #ifndef ERF_USE_NETCDF
     AMREX_ALWAYS_ASSERT_WITH_MESSAGE(( (solverChoice.init_type != InitType::WRFInput) &&
                                        (solverChoice.init_type != InitType::Metgrid ) &&
                                        (solverChoice.init_type != InitType::NCFile  )  ),
                                      "init_type cannot be 'WRFInput', 'MetGrid' or 'NCFile' if we don't build with netcdf!");
 #endif
  
     // Query the canopy model file name
     std::string forestfile;
     solverChoice.do_forest_drag = pp.query("forest_file", forestfile);
     if (solverChoice.do_forest_drag) {
         for (int lev = 0; lev <= max_level; ++lev) {
             m_forest_drag[lev] = std::make_unique<ForestDrag>(forestfile);
         }
     }
  
     // If init from WRFInput or Metgrid make sure a valid file name is present at level 0.
     // We allow for the possibility that finer levels may use native refinement rather than reading from a file
     if ((solverChoice.init_type == InitType::WRFInput) ||
         (solverChoice.init_type == InitType::Metgrid)  ||
         (solverChoice.init_type == InitType::NCFile) ) {
         int num_files = nc_init_file[0].size();
         AMREX_ALWAYS_ASSERT_WITH_MESSAGE(num_files>0, "A file name must be present at level 0 for init type WRFInput, Metgrid or NCFile.");
         for (int j = 0; j < num_files; j++) {
             AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!nc_init_file[0][j].empty(), "Valid file name must be present at level 0 for init type WRFInput, Metgrid or NCFile.");
         } //j
     } // InitType
  
     // What type of land surface model to use
     // NOTE: Must be checked after init_params
     if (solverChoice.lsm_type == LandSurfaceType::SLM) {
         lsm.SetModel<SLM>();
         Print() << "SLM land surface model!\n";
     } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
         lsm.SetModel<MM5>();
         Print() << "MM5 land surface model!\n";
 #ifdef ERF_USE_NOAHMP
     } else if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
         lsm.SetModel<NOAHMP>();
         Print() << "Noah-MP land surface model!\n";
 #endif
     } else if (solverChoice.lsm_type == LandSurfaceType::None) {
         lsm.SetModel<NullSurf>();
         Print() << "Null land surface model!\n";
     } else {
         Abort("Dont know this LandSurfaceType!") ;
     }
  
     if (verbose > 0) {
         solverChoice.display(max_level,pp_prefix);
     }
  
     ParameterSanityChecks();
 }

Here is the call graph for this function:

◆ refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )

private

Function to define the refinement criteria based on user input

 {
     if (max_level > 0)
     {
         ParmParse pp(pp_prefix);
         Vector<std::string> refinement_indicators;
         pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
  
         for (int i=0; i<refinement_indicators.size(); ++i)
         {
             std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
  
             ParmParse ppr(ref_prefix);
             RealBox realbox;
             int lev_for_box;
  
             int num_real_lo = ppr.countval("in_box_lo");
             int num_indx_lo = ppr.countval("in_box_lo_indices");
             int num_real_hi = ppr.countval("in_box_hi");
             int num_indx_hi = ppr.countval("in_box_hi_indices");
             int num_indx_lo_crse = ppr.countval("in_box_lo_indices_crse");
             int num_indx_hi_crse = ppr.countval("in_box_hi_indices_crse");
  
             AMREX_ALWAYS_ASSERT(num_real_lo == num_real_hi);
             AMREX_ALWAYS_ASSERT(num_indx_lo == num_indx_hi);
             AMREX_ALWAYS_ASSERT(num_indx_lo_crse == num_indx_hi_crse);
  
             // Problem low and high (in real not index space) are the same at all levels
             const Real* plo = geom[0].ProbLo();
             const Real* phi = geom[0].ProbHi();
             if ( !((num_real_lo >= AMREX_SPACEDIM-1 && num_indx_lo == 0 && num_indx_lo_crse == 0) ||
                    (num_indx_lo >= AMREX_SPACEDIM-1 && num_real_lo == 0 && num_indx_lo_crse == 0) ||
                    (num_indx_lo ==              0   && num_real_lo == 0 && num_indx_lo_crse == 0) ||
                    (num_indx_lo_crse >= AMREX_SPACEDIM-1 && num_real_lo == 0 && num_indx_lo == 0)
                 ) )
             {
                 amrex::Abort("Must only specify box for refinement using real OR index space with fine/coarse grid indices");
             }
  
             if (num_real_lo > 0) {
                 std::vector<Real> rbox_lo(3), rbox_hi(3);
                 lev_for_box = max_level;
                 ppr.query("max_level",lev_for_box);
                 if (lev_for_box <= max_level)
                 {
                     if (n_error_buf[0] != IntVect::TheZeroVector()) {
                         amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
                     }
  
                     ppr.getarr("in_box_lo",rbox_lo,0,num_real_lo);
                     ppr.getarr("in_box_hi",rbox_hi,0,num_real_hi);
  
                     if (rbox_lo[0] < plo[0]) rbox_lo[0] = plo[0];
                     if (rbox_lo[1] < plo[1]) rbox_lo[1] = plo[1];
                     if (rbox_hi[0] > phi[0]) rbox_hi[0] = phi[0];
                     if (rbox_hi[1] > phi[1]) rbox_hi[1] = phi[1];
                     if (num_real_lo < AMREX_SPACEDIM) {
                         rbox_lo[2] = plo[2];
                         rbox_hi[2] = phi[2];
                     }
  
                     const Box& domain = geom[lev_for_box].Domain();
  
                     realbox = RealBox(&(rbox_lo[0]),&(rbox_hi[0]));
  
                     Print() << "Realbox read in and intersected laterally with domain is " << realbox << std::endl;
  
                     num_boxes_at_level[lev_for_box] += 1;
  
                     int ilo, jlo, klo;
                     int ihi, jhi, khi;
                     const auto* dx  = geom[lev_for_box].CellSize();
                     ilo = static_cast<int>((rbox_lo[0] - plo[0])/dx[0]);
                     jlo = static_cast<int>((rbox_lo[1] - plo[1])/dx[1]);
                     ihi = static_cast<int>((rbox_hi[0] - plo[0])/dx[0]-1);
                     jhi = static_cast<int>((rbox_hi[1] - plo[1])/dx[1]-1);
                     if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                         // Search for k indices corresponding to nominal grid
                         // AGL heights
                         klo = domain.smallEnd(2) - 1;
                         khi = domain.smallEnd(2) - 1;
  
                         if (rbox_lo[2] <= zlevels_stag[lev_for_box][domain.smallEnd(2)])
                         {
                             klo = domain.smallEnd(2);
                         }
                         else
                         {
                             for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
                                 if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
                                     klo = k-1;
                                     break;
                                 }
                             }
                         }
                         AMREX_ASSERT(klo >= domain.smallEnd(2));
  
                         if (rbox_hi[2] >= zlevels_stag[lev_for_box][domain.bigEnd(2)+1])
                         {
                             khi = domain.bigEnd(2);
                         }
                         else
                         {
                             for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
                                 if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
                                     khi = k-1;
                                     break;
                                 }
                             }
                         }
                         AMREX_ASSERT((khi <= domain.bigEnd(2)) && (khi > klo));
  
                         // Need to update realbox because tagging is based on
                         // the initial _un_deformed grid
                         realbox = RealBox(plo[0]+ ilo   *dx[0], plo[1]+ jlo   *dx[1], plo[2]+ klo   *dx[2],
                                           plo[0]+(ihi+1)*dx[0], plo[1]+(jhi+1)*dx[1], plo[2]+(khi+1)*dx[2]);
                     } else {
                         klo = static_cast<int>((rbox_lo[2] - plo[2])/dx[2]);
                         khi = static_cast<int>((rbox_hi[2] - plo[2])/dx[2]-1);
                     }
  
                     Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
                     // Error check for each index
                     if(ilo%ref_ratio[lev_for_box-1][0] != 0){
                         amrex::Print()<< "Requested in_box_lo in x direction = " << rbox_lo[0] << " corresponds to ilo = " << ilo << std::endl;
                         amrex::Print() << "ilo = " << ilo << " is not divisible by ref_ratio in x direction = " << ref_ratio[lev_for_box-1][0] << std::endl;
                         amrex::Error("Adjust in_box_lo in x-direction to be divisible by ref_ratio and try again");
                     }
                     if((ihi+1)%ref_ratio[lev_for_box-1][0] != 0){
                         amrex::Print()<< "Requested in_box_hi in x direction = " << rbox_hi[0] << " corresponds to ihi+1 = " << ihi+1 << std::endl;
                         amrex::Print() << "ihi+1 = " << ihi+1 << " is not divisible by ref_ratio in x direction = " << ref_ratio[lev_for_box-1][0] << std::endl;
                         amrex::Error("Adjust in_box_hi in x-direction to be divisible by ref_ratio and try again");
                     }
                      if(jlo%ref_ratio[lev_for_box-1][1] != 0){
                         amrex::Print()<< "Requested in_box_lo in y direction = " << rbox_lo[1] << " corresponds to jlo = " << jlo << std::endl;
                         amrex::Print() << "jlo = " << jlo << " is not divisible by ref_ratio in y direction = " << ref_ratio[lev_for_box-1][1] << std::endl;
                         amrex::Error("Adjust in_box_lo in y-direction to be divisible by ref_ratio and try again");
                     }
                     if((jhi+1)%ref_ratio[lev_for_box-1][1] != 0){
                         amrex::Print()<< "Requested in_box_hi in y direction = " << rbox_hi[1] << " corresponds to jhi+1 = " << jhi+1 << std::endl;
                         amrex::Print() << "jhi+1 = " << jhi+1 << " is not divisible by ref_ratio in y direction = " << ref_ratio[lev_for_box-1][1] << std::endl;
                         amrex::Error("Adjust in_box_hi in y-direction to be divisible by ref_ratio and try again");
                     }
                     if(klo%ref_ratio[lev_for_box-1][2] != 0){
                         amrex::Print()<< "Requested in_box_lo in z direction = " << rbox_lo[2] << " corresponds to klo = " << klo << std::endl;
                         amrex::Print() << "klo = " << klo << " is not divisible by ref_ratio in z direction = " << ref_ratio[lev_for_box-1][2] << std::endl;
                         amrex::Error("Adjust in_box_lo in z-direction to be divisible by ref_ratio and try again");
                     }
                     if((khi+1)%ref_ratio[lev_for_box-1][2] != 0){
                         amrex::Print()<< "Requested in_box_hi in z direction = " << rbox_hi[2] << " corresponds to khi+1 = " << khi+1 << std::endl;
                         amrex::Print() << "khi+1 = " << khi+1 << " is not divisible by ref_ratio in z direction = " << ref_ratio[lev_for_box-1][2] << std::endl;
                         amrex::Error("Adjust in_box_hi in z-direction to be divisible by ref_ratio and try again");
                     }
  
                     bool using_pbl = (solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYJ      ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNN25   ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNNEDMF ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::YSU      ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MRF);
  
                     if ( using_pbl && ( (rbox_lo[2] > plo[2]) || (rbox_hi[2] < phi[2]) ) ) {
                         amrex::Print() << "PBL models need refinement boxes that go from the bottom to the top of the domain for calculation of PBLH" << std::endl;
                         amrex::Print() << "Please set in_box_lo to geometry.prob_lo in z and in_box_hi to geometry.prob_hi in z and try again" << std::endl;
                         amrex::Abort();
                     }
  
                     boxes_at_level[lev_for_box].push_back(bx);
                     Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                 } // lev
  
                 if (solverChoice.init_type == InitType::WRFInput) {
                     if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                         amrex::Error("Number of boxes doesn't match number of input files");
  
                     }
                 }
  
             } else if (num_indx_lo > 0) {
  
                 std::vector<int> box_lo(3), box_hi(3);
                 ppr.get("max_level",lev_for_box);
                 if (lev_for_box <= max_level)
                 {
                     if (n_error_buf[0] != IntVect::TheZeroVector()) {
                         amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
                     }
  
                     ppr.getarr("in_box_lo_indices",box_lo,0,AMREX_SPACEDIM);
                     ppr.getarr("in_box_hi_indices",box_hi,0,AMREX_SPACEDIM);
  
                     Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
                     amrex::Print() << "BOX " << bx << std::endl;
  
                     const auto* dx  = geom[lev_for_box].CellSize();
                     realbox = RealBox(plo[0]+ box_lo[0]   *dx[0], plo[1]+ box_lo[1]   *dx[1], plo[2]+ box_lo[2]   *dx[2],
                                       plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
  
                     Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
                     num_boxes_at_level[lev_for_box] += 1;
                     if(box_lo[0]%ref_ratio[lev_for_box-1][0] != 0){
                         amrex::Print()<< "Requested ilo in x-direction : " << box_lo[0] << std::endl;
                         amrex::Print() << "ilo = " << box_lo[0] << " is not divisible by ref_ratio in x direction = " << ref_ratio[lev_for_box-1][0] << std::endl;
                         amrex::Error("Adjust in_box_lo_indices in x-direction to be divisible by ref_ratio and try again");
                     }
                     if((box_hi[0]+1)%ref_ratio[lev_for_box-1][0] != 0){
                         amrex::Print()<< "Requested ihi in x-direction : " << box_hi[0] << std::endl;
                         amrex::Print() << "ihi+1 = " << box_hi[0]+1 << " is not divisible by ref_ratio in x direction = " << ref_ratio[lev_for_box-1][0] << std::endl;
                         amrex::Error("Adjust in_box_hi_indices in x-direction to be divisible by ref_ratio and try again");
                     }
                      if(box_lo[1]%ref_ratio[lev_for_box-1][1] != 0){
                         amrex::Print()<< "Requested jlo in y-direction : " << box_lo[1] << std::endl;
                         amrex::Print() << "jlo = " << box_lo[1] << " is not divisible by ref_ratio in y direction = " << ref_ratio[lev_for_box-1][1] << std::endl;
                         amrex::Error("Adjust in_box_lo_indices in y-direction to be divisible by ref_ratio and try again");
                     }
                     if((box_hi[1]+1)%ref_ratio[lev_for_box-1][1] != 0){
                         amrex::Print()<< "Requested jhi in y-direction : " << box_hi[1] << std::endl;
                         amrex::Print() << "jhi+1 = " << box_hi[1]+1 << " is not divisible by ref_ratio in y direction = " << ref_ratio[lev_for_box-1][1] << std::endl;
                         amrex::Error("Adjust in_box_hi_indices in y-direction to be divisible by ref_ratio and try again");
                     }
                     if(box_lo[2]%ref_ratio[lev_for_box-1][2] != 0){
                         amrex::Print()<< "Requested klo in z-direction : " << box_lo[2] << std::endl;
                         amrex::Print() << "klo = " << box_lo[2] << " is not   divisible by ref_ratio in z direction = " << ref_ratio[lev_for_box-1][2] << std::endl;
                         amrex::Error("Adjust in_box_lo_indices in z-direction to be divisible by ref_ratio and try again");
                     }
                     if((box_hi[2]+1)%ref_ratio[lev_for_box-1][2] != 0){
                         amrex::Print()<< "Requested khi in z-direction : " << box_hi[2] << std::endl;
                         amrex::Print() << "khi+1 = " << box_hi[2]+1 << " is not divisible by ref_ratio in z direction = " << ref_ratio[lev_for_box-1][2] << std::endl;
                         amrex::Error("Adjust in_box_hi_indices in z-direction to be divisible by ref_ratio and try again");
                     }
  
                     bool using_pbl = (solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYJ      ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNN25   ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNNEDMF ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::YSU      ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MRF);
  
                     const Box& domain = geom[lev_for_box].Domain();
  
                     if ( using_pbl && ( (box_lo[2] > 0) || (box_hi[2] < domain.bigEnd(2)) ) ) {
                         amrex::Print() << "PBL models need refinement boxes that go from the bottom to the top of the domain for calculation of PBLH" << std::endl;
                         amrex::Print() << "Please set in_box_lo_indices to 0 in z and in_box_hi_indices to amr.n_cell-1 in z and try again" << std::endl;
                         amrex::Abort();
                     }
  
                     boxes_at_level[lev_for_box].push_back(bx);
                     Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                 } // lev
  
                 if (solverChoice.init_type == InitType::WRFInput) {
                     if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                         amrex::Error("Number of boxes doesn't match number of input files");
  
                     }
                 }
             }
             else if (num_indx_lo_crse > 0) {
  
                 std::vector<int> box_lo(3), box_hi(3);
                 ppr.get("max_level",lev_for_box);
                 if (lev_for_box <= max_level)
                 {
                     if (n_error_buf[0] != IntVect::TheZeroVector()) {
                         amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
                     }
  
                     ppr.getarr("in_box_lo_indices_crse",box_lo,0,AMREX_SPACEDIM);
                     ppr.getarr("in_box_hi_indices_crse",box_hi,0,AMREX_SPACEDIM);
  
                     Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
                     amrex::Print() << "BOX " << bx << std::endl;
  
                     const auto* dx  = geom[lev_for_box-1].CellSize();
                     realbox = RealBox(plo[0]+ box_lo[0]   *dx[0], plo[1]+ box_lo[1]   *dx[1], plo[2]+ box_lo[2]   *dx[2],
                                       plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
  
                     Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
                     num_boxes_at_level[lev_for_box] += 1;
                     bool using_pbl = (solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYJ      ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNN25   ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNNEDMF ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::YSU      ||
                                       solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MRF);
  
                     const Box& domain = geom[lev_for_box].Domain();
  
                     if ( using_pbl && ( (box_lo[2] > 0) || (box_hi[2] < domain.bigEnd(2)) ) ) {
                         amrex::Print() << "PBL models need refinement boxes that go from the bottom to the top of the domain for calculation of PBLH" << std::endl;
                         amrex::Print() << "Please set in_box_lo_indices_crse to 0 in z and in_box_hi_indices_crse  to amr.n_cell-1 in z and try again" << std::endl;
                         amrex::Abort();
                     }
  
                     boxes_at_level[lev_for_box].push_back(bx);
                     Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                 } // lev
  
                 if (solverChoice.init_type == InitType::WRFInput) {
                     if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
                         amrex::Error("Number of boxes doesn't match number of input files");
  
                     }
                 }
             }
             AMRErrorTagInfo info;
  
             if (realbox.ok()) {
                 info.SetRealBox(realbox);
             }
             if (ppr.countval("start_time") > 0) {
                 Real ref_min_time; ppr.get("start_time",ref_min_time);
                 info.SetMinTime(ref_min_time);
             }
             if (ppr.countval("end_time") > 0) {
                 Real ref_max_time; ppr.get("end_time",ref_max_time);
                 info.SetMaxTime(ref_max_time);
             }
             if (ppr.countval("max_level") > 0) {
                 int ref_max_level; ppr.get("max_level",ref_max_level);
                 info.SetMaxLevel(ref_max_level);
             }
  
             if (ppr.countval("value_greater")) {
                 int num_val = ppr.countval("value_greater");
                 Vector<Real> value(num_val);
                 ppr.getarr("value_greater",value,0,num_val);
                 std::string field; ppr.get("field_name",field);
                 ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
             }
             else if (ppr.countval("value_less")) {
                 int num_val = ppr.countval("value_less");
                 Vector<Real> value(num_val);
                 ppr.getarr("value_less",value,0,num_val);
                 std::string field; ppr.get("field_name",field);
                 ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
             }
             else if (ppr.countval("adjacent_difference_greater")) {
                 int num_val = ppr.countval("adjacent_difference_greater");
                 Vector<Real> value(num_val);
                 ppr.getarr("adjacent_difference_greater",value,0,num_val);
                 std::string field; ppr.get("field_name",field);
                 ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
             }
             else if (realbox.ok())
             {
                 ref_tags.push_back(AMRErrorTag(info));
             } else if (refinement_indicators[i] != "storm_tracker") {
                 Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
             }
         } // loop over criteria
     } // if max_level > 0
 }

Here is the call graph for this function:

◆ remake_zphys()

void ERF::remake_zphys	(	int	lev,
		std::unique_ptr< amrex::MultiFab > &	temp_zphys_nd
	)

 {
     if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
     {
         if (lev > 0)
         {
             //
             // First interpolate from coarser level
             // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
             //       have been pre-filled - this includes ghost cells both inside and outside
             //       the domain
             //
             InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
                                   IntVect(0,0,0), // do NOT fill ghost cells outside the domain
                                   *z_phys_nd[lev-1], 0, 0, 1,
                                   geom[lev-1], geom[lev],
                                   refRatio(lev-1), &node_bilinear_interp,
                                   domain_bcs_type, BCVars::cons_bc);
  
             // This recomputes the fine values using the bottom terrain at the fine resolution,
             //    and also fills values of z_phys_nd outside the domain
             make_terrain_fitted_coords(lev,geom[lev],*temp_zphys_nd,zlevels_stag[lev],phys_bc_type);
  
             std::swap(temp_zphys_nd, z_phys_nd[lev]);
         } // lev > 0
     } else {
         if (lev > 0)
         {
             //
             // First interpolate from coarser level
             // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
             //       have been pre-filled - this includes ghost cells both inside and outside
             //       the domain
             //
             InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
                                   z_phys_nd[lev]->nGrowVect(), // DO fill ghost cells outside the domain
                                   *z_phys_nd[lev-1], 0, 0, 1,
                                   geom[lev-1], geom[lev],
                                   refRatio(lev-1), &node_bilinear_interp,
                                   domain_bcs_type, BCVars::cons_bc);
  
             std::swap(temp_zphys_nd, z_phys_nd[lev]);
         } // lev > 0
     }
  
     if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
         solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
         //
         // This assumes we have already remade the EBGeometry
         //
         terrain_blanking[lev]->setVal(1.0);
         MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, z_phys_nd[lev]->nGrowVect());
     }
  
     // Compute the min dz and pass to the micro model
     Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
     micro->Set_dzmin(lev, dzmin);
 }

Here is the call graph for this function:

◆ RemakeLevel()

void ERF::RemakeLevel	(	int	lev,
		amrex::Real	time,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

override

 {
     //
     // Note that "time" here is elapsed time
     //
     if (verbose) {
         amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
     }
  
     AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::MovingFittedMesh);
  
     BoxArray            ba_old(vars_new[lev][Vars::cons].boxArray());
     DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
  
     if (verbose) {
         amrex::Print() <<"               OLD BA AT LEVEL " << lev << " " << ba_old << std::endl;
     }
  
     //
     // Re-define subdomain at this level within the domain such that
     // 1) all boxes in a given subdomain are "connected"
     // 2) no boxes in a subdomain touch any boxes in any other subdomain
     //
     if (solverChoice.anelastic[lev] == 1) {
         make_subdomains(ba.simplified_list(), subdomains[lev]);
     }
  
     int     ncomp_cons  = vars_new[lev][Vars::cons].nComp();
     IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
  
     int ngrow_vels = ComputeGhostCells(solverChoice);
  
     Vector<MultiFab> temp_lev_new(Vars::NumTypes);
     Vector<MultiFab> temp_lev_old(Vars::NumTypes);
     MultiFab temp_base_state;
  
     std::unique_ptr<MultiFab> temp_zphys_nd;
  
     //********************************************************************************************
     // This allocates all kinds of things, including but not limited to: solution arrays,
     //      terrain arrays and metrics, and base state.
     // *******************************************************************************************
     init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old, temp_base_state, temp_zphys_nd);
  
     // ********************************************************************************************
     // Build the data structures for terrain-related quantities
     // ********************************************************************************************
     if ( solverChoice.terrain_type == TerrainType::EB ||
          solverChoice.terrain_type == TerrainType::ImmersedForcing ||
          solverChoice.buildings_type == BuildingsType::ImmersedForcing)
     {
         const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
         const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
         if (solverChoice.terrain_type == TerrainType::EB) {
             eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
         } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
                    solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
             eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
         }
     }
     remake_zphys(lev, temp_zphys_nd);
     update_terrain_arrays(lev);
  
     // ********************************************************************************************
     // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
     // Note that this shouldn't be necessary because the fine grid is created by interpolation
     // from the coarse ... but just in case ...
     // ********************************************************************************************
     if ( (SolverChoice::mesh_type != MeshType::ConstantDz) && (solverChoice.coupling_type == CouplingType::TwoWay) ) {
         for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
             average_down(  *detJ_cc[crse_lev+1],   *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
             average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
         }
     }
  
     // ********************************************************************************************
     // Build the data structures for canopy model (depends upon z_phys)
     // ********************************************************************************************
     if (solverChoice.do_forest_drag) {
         m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
     }
  
     // *****************************************************************************************************
     // Create the physbcs objects (after initializing the terrain but before calling FillCoarsePatch
     // *****************************************************************************************************
     make_physbcs(lev);
  
     // ********************************************************************************************
     // Update the base state at this level by interpolation from coarser level AND copy
     //    from previous (pre-regrid) base_state array
     // ********************************************************************************************
     if (lev > 0) {
         Interpolater* mapper = &cell_cons_interp;
  
         Vector<MultiFab*> fmf = {&base_state[lev  ], &base_state[lev  ]};
         Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
         Vector<Real> ftime    = {time, time};
         Vector<Real> ctime    = {time, time};
  
         // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
         FillPatchTwoLevels(temp_base_state, temp_base_state.nGrowVect(), IntVect(0,0,0),
                            time, cmf, ctime, fmf, ftime,
                            0, 0, temp_base_state.nComp(), geom[lev-1], geom[lev],
                            refRatio(lev-1), mapper, domain_bcs_type,
                            BaseBCVars::rho0_bc_comp);
  
         // Impose bc's outside the domain
         (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
  
         // *************************************************************************************************
         // This will fill the temporary MultiFabs with data from vars_new
         // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
         // NOTE: we must create the new base state before calling FillPatch because we will
         //       interpolate perturbational quantities
         // *************************************************************************************************
         FillPatchFineLevel(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
                            &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
                           {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
                            base_state[lev], temp_base_state, false);
     } else {
         temp_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
                                      base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
         temp_lev_new[Vars::cons].ParallelCopy(vars_new[lev][Vars::cons],0,0,ncomp_cons,ngrow_state,ngrow_state);
         temp_lev_new[Vars::xvel].ParallelCopy(vars_new[lev][Vars::xvel],0,0,         1,ngrow_vels,ngrow_vels);
         temp_lev_new[Vars::yvel].ParallelCopy(vars_new[lev][Vars::yvel],0,0,         1,ngrow_vels,ngrow_vels);
  
         temp_lev_new[Vars::zvel].setVal(0.);
         temp_lev_new[Vars::zvel].ParallelCopy(vars_new[lev][Vars::zvel],0,0,         1,
                                               IntVect(ngrow_vels,ngrow_vels,0),IntVect(ngrow_vels,ngrow_vels,0));
     }
  
     // Now swap the pointers since we needed both old and new in the FillPatch
     std::swap(temp_base_state, base_state[lev]);
  
     // ********************************************************************************************
     // Copy from new into old just in case
     // ********************************************************************************************
     MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
     MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0,         1,ngrow_vels);
     MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0,         1,ngrow_vels);
     MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0,         1,IntVect(ngrow_vels,ngrow_vels,0));
  
     // ********************************************************************************************
     // Now swap the pointers
     // ********************************************************************************************
     for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
         std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
         std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
     }
  
     //
     // Note that t_new = time here is elapsed time
     //
     t_new[lev] = time;
     t_old[lev] = time - 1.e200;
  
     // ********************************************************************************************
     // Build the data structures for calculating diffusive/turbulent terms
     // ********************************************************************************************
     update_diffusive_arrays(lev, ba, dm);
  
     //********************************************************************************************
     // Microphysics
     // *******************************************************************************************
     int q_size = micro->Get_Qmoist_Size(lev);
     qmoist[lev].resize(q_size);
     micro->Define(lev, solverChoice);
     if (solverChoice.moisture_type != MoistureType::None)
     {
         micro->Init(lev, vars_new[lev][Vars::cons],
                     grids[lev], Geom(lev), 0.0,
                     z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
     }
     for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
         qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
     }
  
     //********************************************************************************************
     // Radiation
     // *******************************************************************************************
     if (solverChoice.rad_type != RadiationType::None)
     {
         rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
     }
  
     // ********************************************************************************************
     // Initialize the integrator class
     // ********************************************************************************************
     initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
  
     // We need to re-define the FillPatcher if the grids have changed
     if (lev > 0 && cf_width >= 0) {
         bool ba_changed = (ba != ba_old);
         bool dm_changed = (dm != dm_old);
         if (ba_changed || dm_changed) {
           Define_ERFFillPatchers(lev);
         }
     }
  
     // These calls are done in AmrCore::regrid if this is a regrid at lev > 0
     // For a level 0 regrid we must explicitly do them here
     if (lev == 0) {
         // Define grids[lev] to be ba
         SetBoxArray(lev, ba);
  
         // Define dmap[lev] to be dm
         SetDistributionMap(lev, dm);
     }
  
     // ********************************************************************************************
     // Initialize the 2D data structures
     // ********************************************************************************************
     // NOTE: 2D MFs must be filled before SurfaceLayer is defined since SL class uses sst/tsk
     // Clear the 2D arrays
     if (sst_lev[lev][0]) {
         for (int n = 0; n < sst_lev[lev].size(); n++) {
             sst_lev[lev][n].reset();
         }
     }
     if (tsk_lev[lev][0]) {
         for (int n = 0; n < tsk_lev[lev].size(); n++) {
             tsk_lev[lev][n].reset();
         }
     }
     if (lat_m[lev]) {
         lat_m[lev].reset();
     }
     if (lon_m[lev]) {
         lon_m[lev].reset();
     }
     if (sinPhi_m[lev]) {
         sinPhi_m[lev].reset();
     }
     if (cosPhi_m[lev]) {
         cosPhi_m[lev].reset();
     }
  
     //
     // Interpolate the 2D arrays at the lower boundary. We assume that since we created
     //     them by interpolation it is ok just to recreate them by interpolation.
     // Note that ba2d is constructed already in init_stuff, but we have not yet defined dmap[lev]
     //     so we must explicitly pass dm.
     Interp2DArrays(lev,ba2d[lev],dm);
  
     // ********************************************************************************************
     // Update the SurfaceLayer arrays at this level
     // ********************************************************************************************
     if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
         int nlevs = finest_level+1;
         Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
                                      &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
         m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
                                                    mfv_old, Theta_prim[lev], Qv_prim[lev],
                                                    Qr_prim[lev], z_phys_nd[lev],
                                                    Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
                                                    lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
                                                    sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
     }
  
 #ifdef ERF_USE_PARTICLES
     particleData.Redistribute();
 #endif
 }

Here is the call graph for this function:

◆ restart()

void ERF::restart ( )

 {
     auto dRestartTime0 = amrex::second();
  
     ReadCheckpointFile();
  
     if (regrid_level_0_on_restart) {
         //
         // Coarsening before we split the grids ensures that each resulting
         // grid will have an even number of cells in each direction.
         //
         BoxArray new_ba(amrex::coarsen(Geom(0).Domain(),2));
         //
         // Now split up into list of grids within max_grid_size[0] limit.
         //
         new_ba.maxSize(max_grid_size[0]/2);
         //
         // Now refine these boxes back to level 0.
         //
         new_ba.refine(2);
  
         if (refine_grid_layout) {
             ChopGrids(0, new_ba, ParallelDescriptor::NProcs());
         }
  
         if (new_ba != grids[0]) {
             DistributionMapping new_dm(new_ba);
             RemakeLevel(0,t_new[0],new_ba,new_dm);
         }
     }
  
 #ifdef ERF_USE_PARTICLES
     // We call this here without knowing whether the particles have already been initialized or not
     initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,t_new[0]);
 #endif
  
     Real cur_time = t_new[0];
     if (m_check_per    > 0.) {last_check_file_time    = cur_time;}
     if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 = std::floor(cur_time/m_plot2d_per_1) * m_plot2d_per_1;}
     if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 = std::floor(cur_time/m_plot2d_per_2) * m_plot2d_per_2;}
     if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 = std::floor(cur_time/m_plot3d_per_1) * m_plot3d_per_1;}
     if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 = std::floor(cur_time/m_plot3d_per_2) * m_plot3d_per_2;}
  
     if (m_check_int    > 0.) {last_check_file_step    = istep[0];}
     if (m_plot2d_int_1 > 0.) {last_plot2d_file_step_1 = istep[0];}
     if (m_plot2d_int_2 > 0.) {last_plot2d_file_step_2 = istep[0];}
     if (m_plot3d_int_1 > 0.) {last_plot3d_file_step_1 = istep[0];}
     if (m_plot3d_int_2 > 0.) {last_plot3d_file_step_2 = istep[0];}
  
     if (verbose > 0)
     {
         auto dRestartTime = amrex::second() - dRestartTime0;
         ParallelDescriptor::ReduceRealMax(dRestartTime,ParallelDescriptor::IOProcessorNumber());
         amrex::Print() << "Restart time = " << dRestartTime << " seconds." << '\n';
     }
 }

◆ sample_lines()

void ERF::sample_lines	(	int	lev,
		amrex::Real	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters

lev	Current level
time	Current time
cell	IntVect containing the x,y-dimension indices to sample along z
mf	MultiFab from which we sample the data

 {
     int ifile = 0;
  
     const int ncomp = mf.nComp(); // cell-centered state vars
  
     MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
     average_face_to_cellcenter(mf_vels, 0,
                                Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
  
     //
     // Sample the data at a line (in direction "dir") in space
     // In this case we sample in the vertical direction so dir = 2
     // The "k" value of "cell" is ignored
     //
     int dir = 2;
     MultiFab my_line       = get_line_data(mf,              dir, cell);
     MultiFab my_line_vels  = get_line_data(mf_vels,         dir, cell);
     MultiFab my_line_tau11 = get_line_data(*Tau[lev][TauType::tau11], dir, cell);
     MultiFab my_line_tau12 = get_line_data(*Tau[lev][TauType::tau12], dir, cell);
     MultiFab my_line_tau13 = get_line_data(*Tau[lev][TauType::tau13], dir, cell);
     MultiFab my_line_tau22 = get_line_data(*Tau[lev][TauType::tau22], dir, cell);
     MultiFab my_line_tau23 = get_line_data(*Tau[lev][TauType::tau23], dir, cell);
     MultiFab my_line_tau33 = get_line_data(*Tau[lev][TauType::tau33], dir, cell);
  
     for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
     {
         // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
  
         std::ostream& sample_log = SampleLineLog(ifile);
         if (sample_log.good()) {
           sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
           const auto& my_line_arr = my_line[0].const_array();
           const auto& my_line_vels_arr = my_line_vels[0].const_array();
           const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
           const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
           const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
           const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
           const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
           const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
           const Box&  my_box = my_line[0].box();
           const int klo = my_box.smallEnd(2);
           const int khi = my_box.bigEnd(2);
           int i = cell[0];
           int j = cell[1];
           for (int n = 0; n < ncomp; n++) {
               for (int k = klo; k <= khi; k++) {
                   sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
               }
           }
           for (int n = 0; n < AMREX_SPACEDIM; n++) {
               for (int k = klo; k <= khi; k++) {
                   sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
               }
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
           }
           for (int k = klo; k <= khi; k++) {
               sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
           }
           sample_log << std::endl;
         } // if good
     } // mfi
 }

◆ sample_points()

void ERF::sample_points	(	int	lev,
		amrex::Real	time,
		amrex::IntVect	cell,
		amrex::MultiFab &	mf
	)

Utility function for sampling MultiFab data at a specified cell index.

Parameters

lev	Level for the associated MultiFab data
time	Current time
cell	IntVect containing the indexes for the cell where we want to sample
mf	MultiFab from which we wish to sample data

 {
     int ifile = 0;
  
     //
     // Sample the data at a single point in space
     //
     int ncomp = mf.nComp();
     Vector<Real> my_point = get_cell_data(mf, cell);
  
     if (!my_point.empty()) {
  
         // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
  
         std::ostream& sample_log = SamplePointLog(ifile);
         if (sample_log.good()) {
           sample_log << std::setw(datwidth) << time;
           for (int i = 0; i < ncomp; ++i)
           {
               sample_log << std::setw(datwidth) << my_point[i];
           }
           sample_log << std::endl;
         } // if good
     } // only write from processor that holds the cell
 }

◆ SampleLine()

amrex::IntVect& ERF::SampleLine ( int i )

inlineprivate

     {
         return sampleline[i];
     }

◆ SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int i )

inlineprivate

     {
         return *samplelinelog[i];
     }

◆ SampleLineLogName()

std::string ERF::SampleLineLogName ( int i ) const

inlineprivatenoexcept

The filename of the ith samplelinelog file.

1623 { return samplelinelogname[i]; }

◆ SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int i )

inlineprivate

     {
         return samplepoint[i];
     }

◆ SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int i )

inlineprivate

     {
         return *sampleptlog[i];
     }

◆ SamplePointLogName()

std::string ERF::SamplePointLogName ( int i ) const

inlineprivatenoexcept

The filename of the ith sampleptlog file.

1620 { return sampleptlogname[i]; }

◆ setPlotVariables()

void ERF::setPlotVariables	(	const std::string &	pp_plot_var_names,
		amrex::Vector< std::string > &	plot_var_names
	)

private

 {
     ParmParse pp(pp_prefix);
  
     if (pp.contains(pp_plot_var_names.c_str()))
     {
         std::string nm;
  
         int nPltVars = pp.countval(pp_plot_var_names.c_str());
  
         for (int i = 0; i < nPltVars; i++)
         {
             pp.get(pp_plot_var_names.c_str(), nm, i);
  
             // Add the named variable to our list of plot variables
             // if it is not already in the list
             if (!containerHasElement(plot_var_names, nm)) {
                 plot_var_names.push_back(nm);
             }
         }
     } else {
         //
         // The default is to add none of the variables to the list
         //
         plot_var_names.clear();
     }
  
     // Get state variables in the same order as we define them,
     // since they may be in any order in the input list
     Vector<std::string> tmp_plot_names;
  
     for (int i = 0; i < cons_names.size(); ++i) {
         if ( containerHasElement(plot_var_names, cons_names[i]) ) {
             if (solverChoice.moisture_type == MoistureType::None) {
                 if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
                     cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                 {
                     tmp_plot_names.push_back(cons_names[i]);
                 }
             } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
                 if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                 {
                     tmp_plot_names.push_back(cons_names[i]);
                 }
             } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
                         (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
                         (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
                 if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
                     cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                 {
                     tmp_plot_names.push_back(cons_names[i]);
                 }
             } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                         (solverChoice.moisture_type == MoistureType::SAM_NoIce     ) ) { // allow rhoQ1, rhoQ2, rhoQ4
                 if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                 {
                     tmp_plot_names.push_back(cons_names[i]);
                 }
             } else
             {
                 // For moisture_type SAM and Morrison we have all six variables
                 tmp_plot_names.push_back(cons_names[i]);
             }
         }
     }
  
     // check for velocity since it's not in cons_names
     // if we are asked for any velocity component, we will need them all
     if (containerHasElement(plot_var_names, "x_velocity") ||
         containerHasElement(plot_var_names, "y_velocity") ||
         containerHasElement(plot_var_names, "z_velocity")) {
         tmp_plot_names.push_back("x_velocity");
         tmp_plot_names.push_back("y_velocity");
         tmp_plot_names.push_back("z_velocity");
     }
  
     //
     // If the model we are running doesn't have the variable listed in the inputs file,
     //     just ignore it rather than aborting
     //
     for (int i = 0; i < derived_names.size(); ++i) {
         if ( containerHasElement(plot_var_names, derived_names[i]) ) {
             bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing || solverChoice.buildings_type == BuildingsType::ImmersedForcing ) ||
                                (derived_names[i] != "terrain_IB_mask") );
             ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                                (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                                (derived_names[i] != "detJ") );
             ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                                (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                                (derived_names[i] != "z_phys") );
 #ifndef ERF_USE_WINDFARM
             ok_to_add     &= (derived_names[i] != "SMark0" && derived_names[i] != "SMark1");
 #endif
             if (ok_to_add)
             {
                 if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
                     if (derived_names[i] != "qv" && derived_names[i] != "qc"    && derived_names[i] != "qrain"  &&
                         derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                         derived_names[i] != "qt" && derived_names[i] != "qn"    && derived_names[i] != "qp" &&
                         derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                     {
                         tmp_plot_names.push_back(derived_names[i]);
                     }
                 } else if ( (solverChoice.moisture_type == MoistureType::Kessler       ) ||
                             (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                             (solverChoice.moisture_type == MoistureType::SAM_NoIce     ) ) { // allow qv, qc, qrain
                     if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                         derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                     {
                         tmp_plot_names.push_back(derived_names[i]);
                     }
                 } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
                             (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
                             (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
                     if (derived_names[i] != "qrain"  &&
                         derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                         derived_names[i] != "qp" &&
                         derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                     {
                         tmp_plot_names.push_back(derived_names[i]);
                     }
                 } else
                 {
                     // For moisture_type SAM and Morrison we have all moist quantities
                     tmp_plot_names.push_back(derived_names[i]);
                 }
             } // use_terrain?
         } // hasElement
     }
  
 #ifdef ERF_USE_WINDFARM
     for (int i = 0; i < derived_names.size(); ++i) {
         if ( containerHasElement(plot_var_names, derived_names[i]) ) {
             if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP) {
                 if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0") {
                     tmp_plot_names.push_back(derived_names[i]);
                 }
             }
             if( solverChoice.windfarm_type == WindFarmType::SimpleAD or
                 solverChoice.windfarm_type == WindFarmType::GeneralAD ) {
                 if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0" or derived_names[i] == "SMark1") {
                     tmp_plot_names.push_back(derived_names[i]);
                 }
             }
         }
     }
 #endif
  
 #ifdef ERF_USE_PARTICLES
     const auto& particles_namelist( particleData.getNamesUnalloc() );
     for (auto it = particles_namelist.cbegin(); it != particles_namelist.cend(); ++it) {
         std::string tmp( (*it)+"_count" );
         if (containerHasElement(plot_var_names, tmp) ) {
             tmp_plot_names.push_back(tmp);
         }
     }
 #endif
  
     plot_var_names = tmp_plot_names;
 }

Here is the call graph for this function:

◆ setPlotVariables2D()

void ERF::setPlotVariables2D	(	const std::string &	pp_plot_var_names,
		amrex::Vector< std::string > &	plot_var_names
	)

private

 {
     ParmParse pp(pp_prefix);
  
     if (pp.contains(pp_plot_var_names.c_str()))
     {
         std::string nm;
  
         int nPltVars = pp.countval(pp_plot_var_names.c_str());
  
         for (int i = 0; i < nPltVars; i++)
         {
             pp.get(pp_plot_var_names.c_str(), nm, i);
  
             // Add the named variable to our list of plot variables
             // if it is not already in the list
             if (!containerHasElement(plot_var_names, nm)) {
                 plot_var_names.push_back(nm);
             }
         }
     } else {
         //
         // The default is to add none of the variables to the list
         //
         plot_var_names.clear();
     }
  
     // Get state variables in the same order as we define them,
     // since they may be in any order in the input list
     Vector<std::string> tmp_plot_names;
  
     // 2D plot variables
     for (int i = 0; i < derived_names_2d.size(); ++i) {
         if (containerHasElement(plot_var_names, derived_names_2d[i]) ) {
             tmp_plot_names.push_back(derived_names_2d[i]);
         }
     }
  
     plot_var_names = tmp_plot_names;
 }

Here is the call graph for this function:

◆ setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool restarting )

private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters

[in] restarting Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

 {
     // If we are restarting then we haven't read the input_sounding file yet
     //    so we need to read it here
     // TODO: should we store this information in the checkpoint file instead?
     if (restarting) {
         input_sounding_data.resize_arrays();
         for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
             input_sounding_data.read_from_file(geom[0], zlevels_stag[0], n);
         }
     }
  
     const Real* z_inp_sound     = input_sounding_data.z_inp_sound[0].dataPtr();
     const Real* U_inp_sound     = input_sounding_data.U_inp_sound[0].dataPtr();
     const Real* V_inp_sound     = input_sounding_data.V_inp_sound[0].dataPtr();
     const Real* theta_inp_sound = input_sounding_data.theta_inp_sound[0].dataPtr();
     const int   inp_sound_size  = input_sounding_data.size(0);
  
     int refine_fac{1};
     for (int lev = 0; lev <= finest_level; lev++)
     {
         const int klo = geom[lev].Domain().smallEnd(2);
         const int khi = geom[lev].Domain().bigEnd(2);
         const int Nz = khi - klo + 1;
  
         Vector<Real> zcc(Nz);
         Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
                                  zlevels_stag[0].begin()+khi/refine_fac+2);
         expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
 #if 0
         amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
         for (int k = 0; k < zlevels_sub.size(); k++) { amrex::AllPrint() << zlevels_sub[k] << " "; }
         amrex::AllPrint() << " --> ";
         for (int k = 0; k < Nz; k++) { amrex::AllPrint() << zcc[k] << " "; }
         amrex::AllPrint() << std::endl;
 #endif
  
         for (int k = 0; k < Nz; k++)
         {
             h_rayleigh_ptrs[lev][Rayleigh::ubar][k]     = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
             h_rayleigh_ptrs[lev][Rayleigh::vbar][k]     = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
             h_rayleigh_ptrs[lev][Rayleigh::wbar][k]     = Real(0.0);
             h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
         }
  
         // Copy from host version to device version
         Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
                          d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
         Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
                          d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
         Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
                          d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
         Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
                          d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
  
         refine_fac *= ref_ratio[lev][2];
     }
 }

Here is the call graph for this function:

◆ setRecordDataInfo()

void ERF::setRecordDataInfo	(	int	i,
		const std::string &	filename
	)

inlineprivate

     {
         if (amrex::ParallelDescriptor::IOProcessor())
         {
             datalog[i] = std::make_unique<std::fstream>();
             datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
             if (!datalog[i]->good()) {
                 amrex::FileOpenFailed(filename);
             }
         }
         amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
     }

◆ setRecordDerDataInfo()

void ERF::setRecordDerDataInfo	(	int	i,
		const std::string &	filename
	)

inlineprivate

     {
         if (amrex::ParallelDescriptor::IOProcessor())
         {
             der_datalog[i] = std::make_unique<std::fstream>();
             der_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
             if (!der_datalog[i]->good()) {
                 amrex::FileOpenFailed(filename);
             }
         }
         amrex::ParallelDescriptor::Barrier("ERF::setRecordDerDataInfo");
     }

◆ setRecordEnergyDataInfo()

void ERF::setRecordEnergyDataInfo	(	int	i,
		const std::string &	filename
	)

inlineprivate

     {
         if (amrex::ParallelDescriptor::IOProcessor())
         {
             tot_e_datalog[i] = std::make_unique<std::fstream>();
             tot_e_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
             if (!tot_e_datalog[i]->good()) {
                 amrex::FileOpenFailed(filename);
             }
         }
         amrex::ParallelDescriptor::Barrier("ERF::setRecordEnergyDataInfo");
     }

◆ setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo	(	int	i,
		int	lev,
		amrex::IntVect &	cell,
		const std::string &	filename
	)

inlineprivate

     {
         amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
         for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
         {
             const amrex::Box& bx = mfi.validbox();
             if (bx.contains(cell)) {
                 samplelinelog[i] = std::make_unique<std::fstream>();
                 samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                 if (!samplelinelog[i]->good()) {
                     amrex::FileOpenFailed(filename);
                 }
             }
         }
         amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
     }

◆ setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo	(	int	i,
		int	lev,
		amrex::IntVect &	cell,
		const std::string &	filename
	)

inlineprivate

     {
         amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
         for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
         {
             const amrex::Box& bx = mfi.validbox();
             if (bx.contains(cell)) {
                 sampleptlog[i] = std::make_unique<std::fstream>();
                 sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
                 if (!sampleptlog[i]->good()) {
                     amrex::FileOpenFailed(filename);
                 }
             }
         }
         amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
     }

◆ setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool restarting )

private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters

[in] restarting Boolean parameter that indicates whether we are currently restarting from a checkpoint file.

 {
     // If we are restarting then we haven't read the input_sponge file yet
     //    so we need to read it here
     // TODO: should we store this information in the checkpoint file instead?
     if (restarting) {
         input_sponge_data.read_from_file(geom[0], zlevels_stag[0]);
     }
  
     const Real* z_inp_sponge     = input_sponge_data.z_inp_sponge.dataPtr();
     const Real* U_inp_sponge     = input_sponge_data.U_inp_sponge.dataPtr();
     const Real* V_inp_sponge     = input_sponge_data.V_inp_sponge.dataPtr();
     const int   inp_sponge_size  = input_sponge_data.size();
  
     for (int lev = 0; lev <= finest_level; lev++)
     {
         const int khi = geom[lev].Domain().bigEnd()[2];
         Vector<Real> zcc(khi+1);
  
         if (z_phys_cc[lev]) {
             // use_terrain=1
             // calculate the damping strength based on the max height at each k
             reduce_to_max_per_height(zcc, z_phys_cc[lev]);
         } else {
             const auto *const prob_lo = geom[lev].ProbLo();
             const auto *const dx = geom[lev].CellSize();
             for (int k = 0; k <= khi; k++)
             {
                 zcc[k] = prob_lo[2] + (k+0.5) * dx[2];
             }
         }
  
         for (int k = 0; k <= khi; k++)
         {
             h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
             h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
         }
  
         // Copy from host version to device version
         Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
                          d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
         Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
                          d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
     }
 }

Here is the call graph for this function:

◆ setSubVolVariables()

void ERF::setSubVolVariables	(	const std::string &	pp_subvol_var_names,
		amrex::Vector< std::string > &	subvol_var_names
	)

private

 {
     ParmParse pp(pp_prefix);
  
     std::string nm;
  
     int nSubVolVars = pp.countval(pp_subvol_var_names.c_str());
  
     // We pre-populate the list with velocities, but allow these to be over-written
     //    by user input
     if (nSubVolVars == 0)
     {
         subvol_var_names.push_back("x_velocity");
         subvol_var_names.push_back("y_velocity");
         subvol_var_names.push_back("z_velocity");
  
     } else {
         for (int i = 0; i < nSubVolVars; i++)
         {
             pp.get(pp_subvol_var_names.c_str(), nm, i);
  
             // Add the named variable to our list of subvol variables
             // if it is not already in the list
             if (!containerHasElement(subvol_var_names, nm)) {
                 subvol_var_names.push_back(nm);
             }
         }
     }
  
     // Get state variables in the same order as we define them,
     // since they may be in any order in the input list
     Vector<std::string> tmp_plot_names;
  
     for (int i = 0; i < cons_names.size(); ++i) {
         if ( containerHasElement(subvol_var_names, cons_names[i]) ) {
             if (solverChoice.moisture_type == MoistureType::None) {
                 if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
                     cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                 {
                     tmp_plot_names.push_back(cons_names[i]);
                 }
             } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
                 if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                 {
                     tmp_plot_names.push_back(cons_names[i]);
                 }
             } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
                         (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
                         (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
                 if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
                     cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                 {
                     tmp_plot_names.push_back(cons_names[i]);
                 }
             } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                         (solverChoice.moisture_type == MoistureType::SAM_NoIce     ) ) { // allow rhoQ1, rhoQ2, rhoQ4
                 if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
                 {
                     tmp_plot_names.push_back(cons_names[i]);
                 }
             } else
             {
                 // For moisture_type SAM and Morrison we have all six variables
                 tmp_plot_names.push_back(cons_names[i]);
             }
         }
     }
  
     // Check for velocity since it's not in cons_names
     if (containerHasElement(subvol_var_names, "x_velocity")) {
         tmp_plot_names.push_back("x_velocity");
     }
     if (containerHasElement(subvol_var_names, "y_velocity")) {
         tmp_plot_names.push_back("y_velocity");
     }
     if (containerHasElement(subvol_var_names, "z_velocity")) {
         tmp_plot_names.push_back("z_velocity");
     }
  
     //
     // If the model we are running doesn't have the variable listed in the inputs file,
     //     just ignore it rather than aborting
     //
     for (int i = 0; i < derived_subvol_names.size(); ++i) {
         if ( containerHasElement(subvol_var_names, derived_names[i]) ) {
             bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing) ||
                                (derived_names[i] != "terrain_IB_mask") );
             ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                                (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                                (derived_names[i] != "detJ") );
             ok_to_add     &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
                                (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
                                (derived_names[i] != "z_phys") );
             if (ok_to_add)
             {
                 if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
                     if (derived_names[i] != "qv" && derived_names[i] != "qc"    && derived_names[i] != "qrain"  &&
                         derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                         derived_names[i] != "qt" && derived_names[i] != "qn"    && derived_names[i] != "qp" &&
                         derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                     {
                         tmp_plot_names.push_back(derived_names[i]);
                     }
                 } else if ( (solverChoice.moisture_type == MoistureType::Kessler       ) ||
                             (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                             (solverChoice.moisture_type == MoistureType::SAM_NoIce     ) ) { // allow qv, qc, qrain
                     if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                         derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                     {
                         tmp_plot_names.push_back(derived_names[i]);
                     }
                 } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
                             (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
                             (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
                     if (derived_names[i] != "qrain"  &&
                         derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
                         derived_names[i] != "qp" &&
                         derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
                     {
                         tmp_plot_names.push_back(derived_names[i]);
                     }
                 } else
                 {
                     // For moisture_type SAM and Morrison we have all moist quantities
                     tmp_plot_names.push_back(derived_names[i]);
                 }
             } // use_terrain?
         } // hasElement
     }
  
     subvol_var_names = tmp_plot_names;
 }

Here is the call graph for this function:

◆ solve_with_gmres()

void ERF::solve_with_gmres	(	int	lev,
		const amrex::Box &	subdomain,
		amrex::MultiFab &	rhs,
		amrex::MultiFab &	p,
		amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &	fluxes,
		amrex::MultiFab &	ax_sub,
		amrex::MultiFab &	ay_sub,
		amrex::MultiFab &	az_sub,
		amrex::MultiFab &	,
		amrex::MultiFab &	znd_sub
	)

Solve the Poisson equation using FFT-preconditioned GMRES

 {
 #ifdef ERF_USE_FFT
     BL_PROFILE("ERF::solve_with_gmres()");
  
     Real reltol = solverChoice.poisson_reltol;
     Real abstol = solverChoice.poisson_abstol;
  
     auto const dom_lo = lbound(Geom(lev).Domain());
     auto const dom_hi = ubound(Geom(lev).Domain());
  
     auto const sub_lo = lbound(subdomain);
     auto const sub_hi = ubound(subdomain);
  
     auto dx    = Geom(lev).CellSizeArray();
  
     Geometry my_geom;
  
     Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
     if (Geom(lev).isPeriodic(0) && sub_lo.x == dom_lo.x && sub_hi.x == dom_hi.x) { is_per[0] = 1;}
     if (Geom(lev).isPeriodic(1) && sub_lo.y == dom_lo.y && sub_hi.y == dom_hi.y) { is_per[1] = 1;}
  
     int coord_sys = 0;
  
     // If subdomain == domain then we pass Geom(lev) to the FFT solver
     if (subdomain == Geom(lev).Domain()) {
         my_geom.define(Geom(lev).Domain(), Geom(lev).ProbDomain(), coord_sys, is_per);
     } else {
         // else we create a new geometry based only on the subdomain
         // The information in my_geom used by the FFT routines is:
         //   1) my_geom.Domain()
         //   2) my_geom.CellSize()
         //   3) my_geom.isAllPeriodic() / my_geom.periodicity()
         RealBox rb( sub_lo.x   *dx[0],  sub_lo.y   *dx[1],  sub_lo.z   *dx[2],
                    (sub_hi.x+1)*dx[0], (sub_hi.y+1)*dx[1], (sub_hi.z+1)*dx[2]);
         my_geom.define(subdomain, rb, coord_sys, is_per);
     }
  
     amrex::GMRES<MultiFab, TerrainPoisson> gmsolver;
  
     TerrainPoisson tp(my_geom, rhs.boxArray(), rhs.DistributionMap(), domain_bc_type,
                       stretched_dz_d[lev], ax_sub, ay_sub, az_sub, dJ_sub, &znd_sub,
                       solverChoice.use_real_bcs);
  
     gmsolver.define(tp);
  
     gmsolver.setVerbose(mg_verbose);
  
     gmsolver.setRestartLength(50);
  
     tp.usePrecond(true);
  
     gmsolver.solve(phi, rhs, reltol, abstol);
  
     tp.getFluxes(phi, fluxes);
  
     for (MFIter mfi(phi); mfi.isValid(); ++mfi)
     {
         Box xbx = mfi.nodaltilebox(0);
         Box ybx = mfi.nodaltilebox(1);
         const Array4<Real      >& fx_ar = fluxes[0].array(mfi);
         const Array4<Real      >& fy_ar = fluxes[1].array(mfi);
         const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
         const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
         ParallelFor(xbx,ybx,
         [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
         {
             fx_ar(i,j,k) *= mf_ux(i,j,0);
         },
         [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
         {
             fy_ar(i,j,k) *= mf_vy(i,j,0);
         });
     } // mfi
 #else
     amrex::ignore_unused(lev, rhs, phi, fluxes, ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
 #endif
  
     // ****************************************************************************
     // Impose bc's on pprime
     // ****************************************************************************
     ImposeBCsOnPhi(lev, phi, subdomain);
 }

Here is the call graph for this function:

◆ sum_derived_quantities()

void ERF::sum_derived_quantities ( amrex::Real time )

 {
     if (verbose <= 0 || NumDerDataLogs() <= 0) return;
  
     int lev = 0;
  
     AMREX_ALWAYS_ASSERT(lev == 0);
  
     auto& mfx0 = *mapfac[0][MapFacType::m_x];
     auto& mfy0 = *mapfac[0][MapFacType::m_x];
     auto&  dJ0 = *detJ_cc[0];
  
     // ************************************************************************
     // WARNING: we are not filling ghost cells other than periodic outside the domain
     // ************************************************************************
  
     MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
     mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
  
     // Average all three components of velocity (on faces) to the cell center
     average_face_to_cellcenter(mf_cc_vel,0,
                                Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                         &vars_new[lev][Vars::yvel],
                                                         &vars_new[lev][Vars::zvel]});
     mf_cc_vel.FillBoundary(geom[lev].periodicity());
  
     if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
         amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
     }
  
     MultiFab r_wted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
     MultiFab unwted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
     MultiFab     enstrophysq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
     MultiFab        theta_mf(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(unwted_magvelsq, TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
         auto& src_fab = mf_cc_vel[mfi];
  
         auto& dest1_fab = unwted_magvelsq[mfi];
         derived::erf_dermagvelsq(bx, dest1_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
  
         auto& dest2_fab = enstrophysq[mfi];
         derived::erf_derenstrophysq(bx, dest2_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
     }
  
     // Copy the MF holding 1/2(u^2 + v^2 + w^2) into the MF that will hold 1/2 rho (u^2 + v^2 + w^2)d
     MultiFab::Copy(r_wted_magvelsq, unwted_magvelsq, 0, 0, 1, 0);
  
     // Multiply the MF holding 1/2(u^2 + v^2 + w^2) by rho to get  1/2 rho (u^2 + v^2 + w^2)
     MultiFab::Multiply(r_wted_magvelsq, vars_new[lev][Vars::cons], 0, 0, 1, 0);
  
     // Copy the MF holding (rho theta) into "theta_mf"
     MultiFab::Copy(theta_mf, vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, 0);
  
     // Divide (rho theta) by rho to get theta in the MF "theta_mf"
     MultiFab::Divide(theta_mf, vars_new[lev][Vars::cons], Rho_comp, 0, 1, 0);
  
     Real  unwted_avg = volWgtSumMF(lev, unwted_magvelsq, 0, dJ0, mfx0, mfy0, false);
     Real  r_wted_avg = volWgtSumMF(lev, r_wted_magvelsq, 0, dJ0, mfx0, mfy0, false);
     Real enstrsq_avg = volWgtSumMF(lev, enstrophysq,     0, dJ0, mfx0, mfy0, false);
     Real   theta_avg = volWgtSumMF(lev, theta_mf,        0, dJ0, mfx0, mfy0, false);
  
     // Get volume including terrain (consistent with volWgtSumMF routine)
     MultiFab volume(grids[lev], dmap[lev], 1, 0);
     auto const& dx = geom[lev].CellSizeArray();
     Real cell_vol  = dx[0]*dx[1]*dx[2];
     volume.setVal(cell_vol);
     if (SolverChoice::mesh_type != MeshType::ConstantDz) {
         MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
     }
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& tbx  = mfi.tilebox();
         auto dst        = volume.array(mfi);
         const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
         const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
         ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
         });
     }
     Real vol = volume.sum();
  
      unwted_avg /= vol;
      r_wted_avg /= vol;
     enstrsq_avg /= vol;
       theta_avg /= vol;
  
     const int nfoo = 4;
     Real foo[nfoo] = {unwted_avg,r_wted_avg,enstrsq_avg,theta_avg};
 #ifdef AMREX_LAZY
     Lazy::QueueReduction([=]() mutable {
 #endif
     ParallelDescriptor::ReduceRealSum(
         foo, nfoo, ParallelDescriptor::IOProcessorNumber());
  
       if (ParallelDescriptor::IOProcessor()) {
         int i = 0;
         unwted_avg  = foo[i++];
         r_wted_avg  = foo[i++];
         enstrsq_avg = foo[i++];
           theta_avg = foo[i++];
  
         std::ostream& data_log_der = DerDataLog(0);
  
         if (time == 0.0) {
             data_log_der << std::setw(datwidth) << "          time";
             data_log_der << std::setw(datwidth) << "        ke_den";
             data_log_der << std::setw(datwidth) << "         velsq";
             data_log_der << std::setw(datwidth) << "     enstrophy";
             data_log_der << std::setw(datwidth) << "    int_energy";
             data_log_der << std::endl;
         }
         data_log_der << std::setw(datwidth) << std::setprecision(timeprecision) << time;
         data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  <<  unwted_avg;
         data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  <<  r_wted_avg;
         data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  << enstrsq_avg;
         data_log_der << std::setw(datwidth) << std::setprecision(datprecision)  <<   theta_avg;
         data_log_der << std::endl;
  
       } // if IOProcessor
 #ifdef AMREX_LAZY
     }
 #endif
 }

Here is the call graph for this function:

◆ sum_energy_quantities()

void ERF::sum_energy_quantities ( amrex::Real time )

 {
     if ( (verbose <= 0) || (tot_e_datalog.size() < 1) ) { return; }
  
     int lev = 0;
  
     auto& mfx0 = *mapfac[0][MapFacType::m_x];
     auto& mfy0 = *mapfac[0][MapFacType::m_x];
     auto&  dJ0 = *detJ_cc[0];
  
     AMREX_ALWAYS_ASSERT(lev == 0);
  
     bool local = true;
  
     // ************************************************************************
     // WARNING: we are not filling ghost cells other than periodic outside the domain
     // ************************************************************************
  
     MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
     mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
  
     // Average all three components of velocity (on faces) to the cell center
     average_face_to_cellcenter(mf_cc_vel,0,
                                Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                         &vars_new[lev][Vars::yvel],
                                                         &vars_new[lev][Vars::zvel]});
     mf_cc_vel.FillBoundary(geom[lev].periodicity());
  
     if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
         amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
     }
  
     MultiFab tot_mass  (grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
     MultiFab tot_energy(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
  
     auto const& dx = geom[lev].CellSizeArray();
     bool is_moist = (solverChoice.moisture_type != MoistureType::None);
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(tot_mass, TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& bx = mfi.tilebox();
  
         const Array4<Real>& cc_vel_arr     = mf_cc_vel.array(mfi);
         const Array4<Real>& tot_mass_arr   = tot_mass.array(mfi);
         const Array4<Real>& tot_energy_arr = tot_energy.array(mfi);
         const Array4<const Real>& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
         const Array4<const Real>& z_arr    = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) :
                                                                 Array4<const Real>{};
         ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
         {
             Real Qv   = (is_moist) ? cons_arr(i,j,k,RhoQ1_comp) : 0.0;
             Real Qc   = (is_moist) ? cons_arr(i,j,k,RhoQ2_comp) : 0.0;
             Real Qt   = Qv + Qc;
             Real Rhod = cons_arr(i,j,k,Rho_comp);
             Real Rhot = Rhod * (1.0 + Qt);
             Real Temp = getTgivenRandRTh(Rhod, cons_arr(i,j,k,RhoTheta_comp), Qv);
             Real TKE  = 0.5 * ( cc_vel_arr(i,j,k,0)*cc_vel_arr(i,j,k,0)
                               + cc_vel_arr(i,j,k,1)*cc_vel_arr(i,j,k,1)
                               + cc_vel_arr(i,j,k,2)*cc_vel_arr(i,j,k,2) );
             Real zval = (z_arr) ? z_arr(i,j,k) : Real(k)*dx[2];
  
             Real Cv   = Cp_d - R_d;
             Real Cvv  = Cp_v - R_v;
             Real Cpv  = Cp_v;
  
             tot_mass_arr(i,j,k)   = Rhot;
             tot_energy_arr(i,j,k) = Rhod * ( (Cv + Cvv*Qv + Cpv*Qc)*Temp - L_v*Qc
                                            + (1.0 + Qt)*TKE + (1.0 + Qt)*CONST_GRAV*zval );
  
         });
  
     }
  
     Real  tot_mass_avg   = volWgtSumMF(lev, tot_mass  , 0, dJ0, mfx0, mfy0, false, local);
     Real  tot_energy_avg = volWgtSumMF(lev, tot_energy, 0, dJ0, mfx0, mfy0, false, local);
  
     // Get volume including terrain (consistent with volWgtSumMF routine)
     MultiFab volume(grids[lev], dmap[lev], 1, 0);
     Real cell_vol  = dx[0]*dx[1]*dx[2];
     volume.setVal(cell_vol);
     if (SolverChoice::mesh_type != MeshType::ConstantDz) {
         MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
     }
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
     {
         const Box& tbx  = mfi.tilebox();
         auto dst        = volume.array(mfi);
         const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
         const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
         ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
         {
             dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
         });
     }
     Real vol = volume.sum();
  
     // Divide by the volume
     tot_mass_avg   /= vol;
     tot_energy_avg /= vol;
  
     const int nfoo = 2;
     Real foo[nfoo] = {tot_mass_avg,tot_energy_avg};
 #ifdef AMREX_LAZY
     Lazy::QueueReduction([=]() mutable {
 #endif
     ParallelDescriptor::ReduceRealSum(
         foo, nfoo, ParallelDescriptor::IOProcessorNumber());
  
       if (ParallelDescriptor::IOProcessor()) {
         int i = 0;
         tot_mass_avg   = foo[i++];
         tot_energy_avg = foo[i++];
  
         std::ostream& data_log_energy = *tot_e_datalog[0];
  
         if (time == 0.0) {
             data_log_energy << std::setw(datwidth) << "          time";
             data_log_energy << std::setw(datwidth) << "      tot_mass";
             data_log_energy << std::setw(datwidth) << "    tot_energy";
             data_log_energy << std::endl;
         }
         data_log_energy << std::setw(datwidth) << std::setprecision(timeprecision) << time;
         data_log_energy << std::setw(datwidth) << std::setprecision(datprecision)  << tot_mass_avg;
         data_log_energy << std::setw(datwidth) << std::setprecision(datprecision)  << tot_energy_avg;
         data_log_energy << std::endl;
  
       } // if IOProcessor
 #ifdef AMREX_LAZY
     }
 #endif
 }

Here is the call graph for this function:

◆ sum_integrated_quantities()

void ERF::sum_integrated_quantities ( amrex::Real time )

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters

time	Current time

 {
     BL_PROFILE("ERF::sum_integrated_quantities()");
  
     if (verbose <= 0)
       return;
  
     // Single level sum
     Real mass_sl;
  
     // Multilevel sums
     Real mass_ml = 0.0;
     Real rhth_ml = 0.0;
     Real scal_ml = 0.0;
     Real mois_ml = 0.0;
  
     bool local = true;
  
     auto& mfx0 = *mapfac[0][MapFacType::m_x];
     auto& mfy0 = *mapfac[0][MapFacType::m_x];
     auto&  dJ0 = *detJ_cc[0];
  
     mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,dJ0,mfx0,mfy0,false,local);
  
     for (int lev = 0; lev <= finest_level; lev++) {
         auto& mfx = *mapfac[lev][MapFacType::m_x];
         auto& mfy = *mapfac[lev][MapFacType::m_x];
         auto&  dJ = *detJ_cc[lev];
         mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,dJ,mfx,mfy,true);
     }
  
     Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,dJ0,mfx0,mfy0,false);
     Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,dJ0,mfx0,mfy0,false);
     Real mois_sl = 0.0;
     if (solverChoice.moisture_type != MoistureType::None) {
         int n_qstate_moist = micro->Get_Qstate_Moist_Size();
         for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
             mois_sl += volWgtSumMF(0,vars_new[0][Vars::cons],RhoQ1_comp+qoff,dJ0,mfx0,mfy0,false);
         }
     }
  
     for (int lev = 0; lev <= finest_level; lev++) {
         auto& mfx = *mapfac[lev][MapFacType::m_x];
         auto& mfy = *mapfac[lev][MapFacType::m_x];
         auto&  dJ = *detJ_cc[lev];
         rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,dJ,mfx,mfy,true);
         scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,dJ,mfx,mfy,true);
         if (solverChoice.moisture_type != MoistureType::None) {
             int n_qstate_moist = micro->Get_Qstate_Moist_Size();
             for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
                 mois_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoQ1_comp+qoff,dJ,mfx,mfy,false);
             }
         }
     }
  
     Gpu::HostVector<Real> h_avg_ustar; h_avg_ustar.resize(1);
     Gpu::HostVector<Real> h_avg_tstar; h_avg_tstar.resize(1);
     Gpu::HostVector<Real> h_avg_olen; h_avg_olen.resize(1);
     if ((m_SurfaceLayer != nullptr) && (NumDataLogs() > 0)) {
         Box domain = geom[0].Domain();
         int zdir = 2;
         h_avg_ustar = sumToLine(*m_SurfaceLayer->get_u_star(0),0,1,domain,zdir);
         h_avg_tstar = sumToLine(*m_SurfaceLayer->get_t_star(0),0,1,domain,zdir);
         h_avg_olen  = sumToLine(*m_SurfaceLayer->get_olen(0)  ,0,1,domain,zdir);
  
         // Divide by the total number of cells we are averaging over
         Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
         h_avg_ustar[0] /= area_z;
         h_avg_tstar[0] /= area_z;
         h_avg_olen[0]  /= area_z;
  
     } else {
         h_avg_ustar[0] = 0.;
         h_avg_tstar[0] = 0.;
         h_avg_olen[0]  = 0.;
     }
  
     const int nfoo = 8;
     Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mois_sl,mass_ml,rhth_ml,scal_ml,mois_ml};
 #ifdef AMREX_LAZY
     Lazy::QueueReduction([=]() mutable {
 #endif
     ParallelDescriptor::ReduceRealSum(
         foo, nfoo, ParallelDescriptor::IOProcessorNumber());
  
       if (ParallelDescriptor::IOProcessor()) {
         int i = 0;
         mass_sl = foo[i++];
         rhth_sl = foo[i++];
         scal_sl = foo[i++];
         mois_sl = foo[i++];
         mass_ml = foo[i++];
         rhth_ml = foo[i++];
         scal_ml = foo[i++];
         mois_ml = foo[i++];
  
         Print() << '\n';
         Print() << "TIME= " << std::setw(datwidth) << std::setprecision(timeprecision) << std::left << time << '\n';
         if (finest_level ==  0) {
 #if 1
            Print() << " MASS       = " << mass_sl << '\n';
 #else
            Print() << " PERT MASS  = " << mass_sl << '\n';
 #endif
            Print() << " RHO THETA  = " << rhth_sl << '\n';
            if (solverChoice.transport_scalar) { Print() << " RHO SCALAR = " << scal_sl << '\n'; }
            if (solverChoice.moisture_type != MoistureType::None) { Print() << " RHO QTOTAL = " << mois_sl << '\n'; }
         } else {
 #if 1
            Print() << " MASS       SL/ML = " << mass_sl << " " << mass_ml << '\n';
 #else
            Print() << " PERT MASS  SL/ML = " << mass_sl << " " << mass_ml << '\n';
 #endif
            Print() << " RHO THETA  SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
            if (solverChoice.transport_scalar) { Print() << " RHO SCALAR SL/ML = " << scal_sl << " " << scal_ml << '\n'; }
            if (solverChoice.moisture_type != MoistureType::None) { Print() << " RHO QTOTAL SL/ML = " << mois_sl << " " << mois_ml << '\n'; }
         }
  
         // The first data log only holds scalars
         if (NumDataLogs() > 0)
         {
             int n_d = 0;
             std::ostream& data_log1 = DataLog(n_d);
             if (data_log1.good()) {
                 if (time == 0.0) {
                     data_log1 << std::setw(datwidth) << "          time";
                     data_log1 << std::setw(datwidth) << "          u_star";
                     data_log1 << std::setw(datwidth) << "          t_star";
                     data_log1 << std::setw(datwidth) << "          olen";
                     data_log1 << std::endl;
                 } // time = 0
  
               // Write the quantities at this time
               data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time;
               data_log1 << std::setw(datwidth) << std::setprecision(datprecision)  << h_avg_ustar[0];
               data_log1 << std::setw(datwidth) << std::setprecision(datprecision)  << h_avg_tstar[0];
               data_log1 << std::setw(datwidth) << std::setprecision(datprecision)  << h_avg_olen[0];
               data_log1 << std::endl;
             } // if good
         } // loop over i
       } // if IOProcessor
 #ifdef AMREX_LAZY
     });
 #endif
  
     // This is just an alias for convenience
     int lev = 0;
     if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
         for (int i = 0; i < NumSamplePoints(); ++i)
         {
             sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
         }
     }
     if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
         for (int i = 0; i < NumSampleLines(); ++i)
         {
             sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
         }
     }
 }

◆ SurfaceDataInterpolation()

void ERF::SurfaceDataInterpolation	(	const int	nlevs,
		const amrex::Real	time,
		amrex::Vector< std::unique_ptr< amrex::MultiFab >> &	z_phys_nd,
		bool	regrid_forces_file_read
	)

 {
  
     static amrex::Vector<Real> next_read_forecast_time;
     static amrex::Vector<Real> last_read_forecast_time;
  
     const int nlevs = a_z_phys_nd.size();
  
     Real hindcast_data_interval = solverChoice.hindcast_data_interval_in_hrs*3600.0;
  
     // Initialize static vectors once
     if (next_read_forecast_time.empty()) {
         next_read_forecast_time.resize(nlevs, -1.0);
         last_read_forecast_time.resize(nlevs, -1.0);
         Print() << "Initializing the time vector values here by " << lev << std::endl;
     }
  
     if (next_read_forecast_time[lev] < 0.0) {
         int next_multiple = static_cast<int>(time / hindcast_data_interval);
         next_read_forecast_time[lev] = next_multiple * hindcast_data_interval;
         last_read_forecast_time[lev] = next_read_forecast_time[lev];
     }
  
     if (time >= next_read_forecast_time[lev] or regrid_forces_file_read) {
  
         Print() << "Data reading happening at level " << lev << std::endl;
  
         std::string folder = solverChoice.hindcast_surface_data_dir;
  
         // Check if folder exists and is a directory
         if (!fs::exists(folder) || !fs::is_directory(folder)) {
             throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
         }
  
         std::vector<std::string> bin_files;
  
         for (const auto& entry : fs::directory_iterator(folder)) {
             if (!entry.is_regular_file()) continue;
  
             std::string fname = entry.path().filename().string();
             if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
                 bin_files.push_back(entry.path().string());
             }
         }
         std::sort(bin_files.begin(), bin_files.end());
  
     // Check if no .bin files were found
         if (bin_files.empty()) {
             throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
         }
  
         std::string filename1, filename2;
  
         int idx1 = static_cast<int>(time / hindcast_data_interval);
         int idx2 = static_cast<int>(time / hindcast_data_interval)+1;
         Print() << "Reading surface data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
  
         if (idx2 >= static_cast<int>(bin_files.size())) {
             throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
         }
  
         filename1 = bin_files[idx1];
         filename2 = bin_files[idx2];
  
         FillSurfaceStateMultiFabs(lev, filename1, surface_state_1);
         FillSurfaceStateMultiFabs(lev, filename2, surface_state_2);
  
          // Create the time-interpolated forecast state
         //CreateForecastStateMultiFabs(forecast_state_interp);
         if(!regrid_forces_file_read){
             last_read_forecast_time[lev] = next_read_forecast_time[lev];
             next_read_forecast_time[lev] += hindcast_data_interval;
             Print() << "Next forecast time getting updated here " << std::endl;
         }
     }
  
     Real prev_read_time = last_read_forecast_time[lev];
     Real alpha1 = 1.0 - (time - prev_read_time)/hindcast_data_interval;
     Real alpha2 = 1.0 - alpha1;
  
     amrex::Print()<< "The values of alpha1 and alpha2 are " << alpha1 << " "<< alpha2 <<std::endl;
  
     if (alpha1 < 0.0 || alpha1 > 1.0 ||
     alpha2 < 0.0 || alpha2 > 1.0)
     {
         std::stringstream ss;
         ss << "Interpolation weights for hindcast files are incorrect: "
         << "alpha1 = " << alpha1 << ", alpha2 = " << alpha2;
        Abort(ss.str());
     }
  
     /*MultiFab& mf_surf_interp   = surface_state_interp[lev];
  
     // Fill the time-interpolated forecast states
     MultiFab::LinComb(surface_state_interp[lev],
                       alpha1, surface_state_1[lev], 0,
                       alpha2, surface_state_2[lev], 0,
                       0, mf_surf_interp.nComp(), mf_surf_interp.nGrow());
  
     std::string pltname = "plt_interp_surface";
     Vector<std::string> varnames_plot_mf = {"ls_mask", "SST"};
  
     const MultiFab& src = vars_new[0][0];
  
     MultiFab plot_mf(src.boxArray(),
                      src.DistributionMap(),
                      2, 0);
  
     plot_mf.setVal(0.0);
  
     for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
         const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
         const Array4<Real> &surf_mf_arr = surface_state_1[0].array(mfi);
  
         const Box& bx = mfi.validbox();
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
             plot_mf_arr(i,j,k,0) = surf_mf_arr(i,j,0);
             plot_mf_arr(i,j,k,1) = surf_mf_arr(i,j,1);
         });
     }
  
     WriteSingleLevelPlotfile(
             pltname,
             plot_mf,
             varnames_plot_mf,
             geom[0],
             time,
             0 // level
     );*/
 }

◆ timeStep()

void ERF::timeStep	(	int	lev,
		amrex::Real	time,
		int	iteration
	)

private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	time	start time for time advance
[in]	iteration	time step counter

 {
     //
     // We need to FillPatch the coarse level before assessing whether to regrid
     // We have not done the swap yet so we fill the "new" which will become the "old"
     //
     MultiFab& S_new = vars_new[lev][Vars::cons];
     MultiFab& U_new = vars_new[lev][Vars::xvel];
     MultiFab& V_new = vars_new[lev][Vars::yvel];
     MultiFab& W_new = vars_new[lev][Vars::zvel];
  
 #ifdef ERF_USE_NETCDF
     //
     // Since we now only read in a subset of the time slices in wrfbdy and
     //     wrflowinp, we need to check whether it's time to read in more.
     //
     bool use_moist = (solverChoice.moisture_type != MoistureType::None);
     if (solverChoice.use_real_bcs && (lev==0))
     {
         int ntimes = bdy_data_xlo.size();
         Real time_since_start_bdy = time + start_time - start_bdy_time;
         int n_time_old = std::min(static_cast<int>( (time_since_start_bdy        ) /  bdy_time_interval), ntimes-1);
         int n_time_new = std::min(static_cast<int>( (time_since_start_bdy+dt[lev]) /  bdy_time_interval), ntimes-1);
  
         for (int itime = 0; itime < ntimes; itime++)
         {
             /*
             if (bdy_data_xlo[itime].size() > 0) {
                 amrex::Print() << "HAVE  BDY DATA AT TIME " << itime << std::endl;
             } else {
                 amrex::Print() << " NO   BDY DATA AT TIME " << itime << std::endl;
             }
             */
  
             bool clear_itime = (itime < n_time_old);
  
             if (clear_itime && bdy_data_xlo[itime].size() > 0) {
                 bdy_data_xlo[itime].clear();
                 bdy_data_xhi[itime].clear();
                 bdy_data_ylo[itime].clear();
                 bdy_data_yhi[itime].clear();
                 //amrex::Print() << "CLEAR BDY DATA AT TIME " << itime << std::endl;
             }
  
             bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
             //if (need_itime) { amrex::Print()  << "NEED  BDY DATA AT TIME " << itime << std::endl; }
  
             if (bdy_data_xlo[itime].size() == 0 && need_itime) {
                 read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
                                  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
                                  real_width);
  
                 convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
                                         *mf_MUB, *mf_C1H, *mf_C2H,
                                         vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel], vars_new[lev][Vars::cons],
                                         geom[lev], use_moist);
            }
         } // itime
     } // use_real_bcs && lev == 0
  
     if (!nc_low_file.empty() && (lev==0))
     {
         int ntimes = low_data_zlo.size();
         Real time_since_start_low = time + start_time - start_low_time;
         int n_time_old = std::min(static_cast<int>( (time_since_start_low        ) /  low_time_interval), ntimes-1);
         int n_time_new = std::min(static_cast<int>( (time_since_start_low+dt[lev]) /  low_time_interval), ntimes-1);
  
         for (int itime = 0; itime < ntimes; itime++)
         {
             /*
             if (low_data_zlo[itime].size() > 0) {
                 amrex::Print() << "HAVE  LOW DATA AT TIME " << itime << std::endl;
             } else {
                 amrex::Print() << " NO   LOW DATA AT TIME " << itime << std::endl;
             }
             */
  
             bool clear_itime = (itime < n_time_old);
  
             if (clear_itime && low_data_zlo[itime].size() > 0) {
                 low_data_zlo[itime].clear();
                 //amrex::Print() << "CLEAR LOW DATA AT TIME " << itime << std::endl;
             }
  
             bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
             //if (need_itime) { amrex::Print()  << "NEED  LOW DATA AT TIME " << itime << std::endl; }
  
             if (low_data_zlo[itime].size() == 0 && need_itime) {
                 read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
  
                 update_sst_tsk(itime, geom[lev], ba2d[lev],
                                sst_lev[lev], tsk_lev[lev],
                                m_SurfaceLayer, low_data_zlo,
                                S_new, *mf_PSFC[lev],
                                solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
             }
         } // itime
     } // have nc_low_file && lev == 0
 #endif
  
     //
     // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
     //
     if (lev == 0) {
         FillPatchCrseLevel(lev, time, {&S_new, &U_new, &V_new, &W_new});
     } else if (lev < finest_level) {
         FillPatchFineLevel(lev, time, {&S_new, &U_new, &V_new, &W_new},
                            {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                            base_state[lev], base_state[lev]);
     }
  
     if (regrid_int > 0)  // We may need to regrid
     {
         // help keep track of whether a level was already regridded
         // from a coarser level call to regrid
         static Vector<int> last_regrid_step(max_level+1, 0);
  
         // regrid changes level "lev+1" so we don't regrid on max_level
         // also make sure we don't regrid fine levels again if
         // it was taken care of during a coarser regrid
         if (lev < max_level)
         {
             if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
             {
                 // regrid could add newly refine levels (if finest_level < max_level)
                 // so we save the previous finest level index
                 int old_finest = finest_level;
  
                 regrid(lev, time);
  
 #ifdef ERF_USE_PARTICLES
                 if (finest_level != old_finest) {
                     particleData.Redistribute();
                 }
 #endif
  
                 // mark that we have regridded this level already
                 for (int k = lev; k <= finest_level; ++k) {
                     last_regrid_step[k] = istep[k];
                 }
  
                 // if there are newly created levels, set the time step
                 for (int k = old_finest+1; k <= finest_level; ++k) {
                     dt[k] = dt[k-1] / static_cast<Real>(nsubsteps[k]);
                 }
             } // if
         } // lev
     }
  
     // Update what we call "old" and "new" time
     t_old[lev] = t_new[lev];
     t_new[lev] += dt[lev];
  
     if (Verbose()) {
         amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
         amrex::Print() << std::setprecision(timeprecision)
                        << "ADVANCE from elapsed time = " << t_old[lev] << " to " << t_new[lev]
                        << " with dt = " << dt[lev] << std::endl;
     }
  
 #ifdef ERF_USE_WW3_COUPLING
     amrex::Print() <<  " About to call send_to_ww3 from ERF_Timestep" << std::endl;
     send_to_ww3(lev);
     amrex::Print() <<  " About to call read_waves from ERF_Timestep"  << std::endl;
     read_waves(lev);
     //send_to_ww3(lev);
     //read_waves(lev);
     //send_to_ww3(lev);
 #endif
  
     // Advance a single level for a single time step
     Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
  
     ++istep[lev];
  
     if (Verbose()) {
         amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
         amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
     }
  
     if (lev < finest_level)
     {
         // recursive call for next-finer level
         for (int i = 1; i <= nsubsteps[lev+1]; ++i)
         {
             Real strt_time_for_fine = time + (i-1)*dt[lev+1];
             timeStep(lev+1, strt_time_for_fine, i);
         }
     }
  
     if (verbose && lev == 0 && solverChoice.moisture_type != MoistureType::None) {
         amrex::Print() << "Cloud fraction " << time << "  " << cloud_fraction(time) << std::endl;
     }
 }

◆ turbPert_amplitude()

void ERF::turbPert_amplitude ( const int lev )

private

 {
     // Accessing data
     auto& lev_new = vars_new[lev];
  
     // Creating local data
     int ncons = lev_new[Vars::cons].nComp();
     MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
  
     // Defining BoxArray type
     auto m_ixtype = cons_data.boxArray().ixType();
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
         const Box &bx  = mfi.validbox();
         const auto &cons_pert_arr = cons_data.array(mfi); // Address of perturbation array
         const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
  
         turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cons_pert_arr, pert_cell);
     } // mfi
 }

Here is the call graph for this function:

◆ turbPert_update()

void ERF::turbPert_update	(	const int	lev,
		const amrex::Real	dt
	)

private

 {
     // Accessing data
     auto& lev_new = vars_new[lev];
  
     // Create aliases to state data to pass to calc_tpi_update
     int ncons = lev_new[Vars::cons].nComp();
     MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
     MultiFab xvel_data(lev_new[Vars::xvel], make_alias, 0, 1);
     MultiFab yvel_data(lev_new[Vars::yvel], make_alias, 0, 1);
  
     // Computing perturbation update time
     turbPert.calc_tpi_update(lev, local_dt, xvel_data, yvel_data, cons_data);
  
     Print() << "Successfully initialized turbulent perturbation update time and amplitude with type: "<< turbPert.pt_type <<"\n";
 }

◆ update_diffusive_arrays()

void ERF::update_diffusive_arrays	(	int	lev,
		const amrex::BoxArray &	ba,
		const amrex::DistributionMapping &	dm
	)

private

 {
     // ********************************************************************************************
     // Diffusive terms
     // ********************************************************************************************
     bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None);
     bool l_use_kturb   = solverChoice.turbChoice[lev].use_kturb;
     bool l_use_diff    = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
                            l_use_kturb );
     bool l_need_SmnSmn = solverChoice.turbChoice[lev].use_keqn;
     bool l_use_moist   = (  solverChoice.moisture_type != MoistureType::None  );
     bool l_rotate      = (  solverChoice.use_rotate_surface_flux  );
  
     bool l_implicit_diff = (solverChoice.vert_implicit_fac[0] > 0 ||
                             solverChoice.vert_implicit_fac[1] > 0 ||
                             solverChoice.vert_implicit_fac[2] > 0);
  
     BoxArray ba12 = convert(ba, IntVect(1,1,0));
     BoxArray ba13 = convert(ba, IntVect(1,0,1));
     BoxArray ba23 = convert(ba, IntVect(0,1,1));
  
     Tau[lev].resize(9);
     Tau_corr[lev].resize(3);
  
     if (l_use_diff) {
         //
         // NOTE: We require ghost cells in the vertical when allowing grids that don't
         //       cover the entire vertical extent of the domain at this level
         //
         for (int i = 0; i < 3; i++) {
             Tau[lev][i] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
         }
         Tau[lev][TauType::tau12] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
         Tau[lev][TauType::tau13] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
         Tau[lev][TauType::tau23] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
         Tau[lev][TauType::tau12]->setVal(0.);
         Tau[lev][TauType::tau13]->setVal(0.);
         Tau[lev][TauType::tau23]->setVal(0.);
         if (l_use_terrain) {
             Tau[lev][TauType::tau21] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
             Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
             Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
             Tau[lev][TauType::tau21]->setVal(0.);
             Tau[lev][TauType::tau31]->setVal(0.);
             Tau[lev][TauType::tau32]->setVal(0.);
         } else if (l_implicit_diff) {
             Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
             Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
             Tau[lev][TauType::tau31]->setVal(0.);
             Tau[lev][TauType::tau32]->setVal(0.);
         } else {
             Tau[lev][TauType::tau21] = nullptr;
             Tau[lev][TauType::tau31] = nullptr;
             Tau[lev][TauType::tau32] = nullptr;
         }
  
         if (l_implicit_diff && solverChoice.implicit_momentum_diffusion)
         {
             Tau_corr[lev][0] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); // Tau31
             Tau_corr[lev][1] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); // Tau32
             Tau_corr[lev][0]->setVal(0.);
             Tau_corr[lev][1]->setVal(0.);
 #ifdef ERF_IMPLICIT_W
             Tau_corr[lev][2] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) ); // Tau33
             Tau_corr[lev][2]->setVal(0.);
 #else
             Tau_corr[lev][2] = nullptr;
 #endif
         } else {
             Tau_corr[lev][0] = nullptr;
             Tau_corr[lev][1] = nullptr;
             Tau_corr[lev][2] = nullptr;
         }
  
         SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
         SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
         SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
         SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba  , dm, 1, IntVect(1,1,1) );
         SFS_hfx1_lev[lev]->setVal(0.);
         SFS_hfx2_lev[lev]->setVal(0.);
         SFS_hfx3_lev[lev]->setVal(0.);
         SFS_diss_lev[lev]->setVal(0.);
         if (l_use_moist) {
             SFS_q1fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
             SFS_q2fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
             SFS_q1fx3_lev[lev]->setVal(0.0);
             SFS_q2fx3_lev[lev]->setVal(0.0);
             if (l_rotate) {
                 SFS_q1fx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
                 SFS_q1fx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
                 SFS_q1fx1_lev[lev]->setVal(0.0);
                 SFS_q1fx2_lev[lev]->setVal(0.0);
             } else {
                 SFS_q1fx1_lev[lev] = nullptr;
                 SFS_q1fx2_lev[lev] = nullptr;
             }
         } else {
             SFS_q1fx1_lev[lev] = nullptr;
             SFS_q1fx2_lev[lev] = nullptr;
             SFS_q1fx3_lev[lev] = nullptr;
             SFS_q2fx3_lev[lev] = nullptr;
         }
     } else {
         for (int i = 0; i < 9; i++) {
             Tau[lev][i] = nullptr;
         }
         SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
         SFS_diss_lev[lev] = nullptr;
     }
  
     if (l_use_kturb) {
         eddyDiffs_lev[lev] = std::make_unique<MultiFab>(ba, dm, EddyDiff::NumDiffs, 2);
         eddyDiffs_lev[lev]->setVal(0.0);
         if(l_need_SmnSmn) {
             SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
         } else {
             SmnSmn_lev[lev] = nullptr;
         }
     } else {
         eddyDiffs_lev[lev] = nullptr;
         SmnSmn_lev[lev]    = nullptr;
     }
 }

◆ update_terrain_arrays()

void ERF::update_terrain_arrays ( int lev )

 {
     if (SolverChoice::mesh_type == MeshType::StretchedDz ||
         SolverChoice::mesh_type == MeshType::VariableDz) {
         make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
         make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
         make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
     } else { // MeshType::ConstantDz
         if (SolverChoice::terrain_type == TerrainType::EB) {
             const auto& ebfact = *eb[lev]->get_const_factory();
             const MultiFab& volfrac = ebfact.getVolFrac();
             detJ_cc[lev] = std::make_unique<MultiFab>(volfrac, amrex::make_alias, 0, volfrac.nComp());
         }
     }
 }

Here is the call graph for this function:

◆ volWgtColumnSum()

void ERF::volWgtColumnSum	(	int	lev,
		const amrex::MultiFab &	mf,
		int	comp,
		amrex::MultiFab &	mf_2d,
		const amrex::MultiFab &	dJ
	)

 {
     BL_PROFILE("ERF::volWgtSumColumnMF()");
  
     mf_2d.setVal(0.);
  
     // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
     // m is the map scale factor at cell centers
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(mf_to_be_summed, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box& bx   = mfi.tilebox();
         const auto  dst_arr = mf_2d.array(mfi);
         const auto  src_arr = mf_to_be_summed.array(mfi);
         if (SolverChoice::mesh_type == MeshType::ConstantDz) {
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 amrex::HostDevice::Atomic::Add(&dst_arr(i,j,0),src_arr(i,j,k,comp));
             });
         } else {
             const auto& dJ_arr = dJ.const_array(mfi);
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 amrex::HostDevice::Atomic::Add(&dst_arr(i,j,0),src_arr(i,j,k,comp)*dJ_arr(i,j,k));
             });
         }
     } // mfi
  
     auto const& dx = geom[lev].CellSizeArray();
  
     mf_2d.mult(dx[2]);
 }

Here is the call graph for this function:

◆ volWgtSumMF()

Real ERF::volWgtSumMF	(	int	lev,
		const amrex::MultiFab &	mf,
		int	comp,
		const amrex::MultiFab &	dJ,
		const amrex::MultiFab &	mfx,
		const amrex::MultiFab &	mfy,
		bool	finemask,
		bool	local = `true`
	)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters

lev	Current level
mf_to_be_summed	: MultiFab on which we do the volume weighted sum
dJ	: volume weighting due to metric terms
mfmx	: map factor in x-direction at cell centers
mfmy	: map factor in y-direction at cell centers
comp	: Index of the component we want to sum
finemask	: If a finer level is available, determines whether we mask fine data
local	: Boolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)

 {
     BL_PROFILE("ERF::volWgtSumMF()");
  
     Real sum = 0.0;
     MultiFab tmp(mf_to_be_summed.boxArray(), mf_to_be_summed.DistributionMap(), 1, 0);
  
     // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
     // m is the map scale factor at cell centers
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
     for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const Box& bx   = mfi.tilebox();
         const auto  dst_arr = tmp.array(mfi);
         const auto  src_arr = mf_to_be_summed.array(mfi);
         const auto& mfx_arr = mfmx.const_array(mfi);
         const auto& mfy_arr = mfmy.const_array(mfi);
  
         if (SolverChoice::terrain_type != TerrainType::EB) {
             if (SolverChoice::mesh_type == MeshType::ConstantDz) {
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     dst_arr(i,j,k,0) = src_arr(i,j,k,comp) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                 });
             } else {
                 const auto&  dJ_arr = dJ.const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
                 {
                     dst_arr(i,j,k,0) = src_arr(i,j,k,comp) * dJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
                 });
             }
         } else {
             const auto&  dJ_arr = dJ.const_array(mfi);
             ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
             {
                 dst_arr(i,j,k,0) = src_arr(i,j,k,comp) * dJ_arr(i,j,k);
             });
         }
  
     } // mfi
  
     if (lev < finest_level && finemask) {
         MultiFab::Multiply(tmp, *fine_mask[lev+1].get(), 0, 0, 1, 0);
     }
  
     // If local = true  then "sum" will be the sum only over the FABs on each rank
     // If local = false then "sum" will be the sum over the whole MultiFab, and will be broadcast to all ranks
     sum = tmp.sum(0,local);
  
     auto const& dx = geom[lev].CellSizeArray();
  
     sum *= dx[0]*dx[1]*dx[2];
  
     return sum;
 }

Here is the call graph for this function:

◆ WeatherDataInterpolation()

void ERF::WeatherDataInterpolation	(	const int	nlevs,
		const amrex::Real	time,
		amrex::Vector< std::unique_ptr< amrex::MultiFab >> &	z_phys_nd,
		bool	regrid_forces_file_read
	)

 {
  
     static amrex::Vector<Real> next_read_forecast_time;
     static amrex::Vector<Real> last_read_forecast_time;
  
     const int nlevs = a_z_phys_nd.size();
  
     Real hindcast_data_interval = solverChoice.hindcast_data_interval_in_hrs*3600.0;
  
     // Initialize static vectors once
     if (next_read_forecast_time.empty()) {
         next_read_forecast_time.resize(nlevs, -1.0);
         last_read_forecast_time.resize(nlevs, -1.0);
         Print() << "Initializing the time vector values here by " << lev << std::endl;
     }
  
     if (next_read_forecast_time[lev] < 0.0) {
         int next_multiple = static_cast<int>(time / hindcast_data_interval);
         next_read_forecast_time[lev] = next_multiple * hindcast_data_interval;
         last_read_forecast_time[lev] = next_read_forecast_time[lev];
     }
  
     if (time >= next_read_forecast_time[lev] or regrid_forces_file_read) {
  
         Print() << "Data reading happening at level " << lev << std::endl;
  
         std::string folder = solverChoice.hindcast_boundary_data_dir;
  
         // Check if folder exists and is a directory
         if (!fs::exists(folder) || !fs::is_directory(folder)) {
             throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
         }
  
         std::vector<std::string> bin_files;
  
         for (const auto& entry : fs::directory_iterator(folder)) {
             if (!entry.is_regular_file()) continue;
  
             std::string fname = entry.path().filename().string();
             if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
                 bin_files.push_back(entry.path().string());
             }
         }
         std::sort(bin_files.begin(), bin_files.end());
  
     // Check if no .bin files were found
         if (bin_files.empty()) {
             throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
         }
  
         std::string filename1, filename2;
  
         int idx1 = static_cast<int>(time / hindcast_data_interval);
         int idx2 = static_cast<int>(time / hindcast_data_interval)+1;
         Print() << "Reading weather data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
  
         if (idx2 >= static_cast<int>(bin_files.size())) {
             throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
         }
  
         filename1 = bin_files[idx1];
         filename2 = bin_files[idx2];
  
         FillForecastStateMultiFabs(lev, filename1, a_z_phys_nd[lev], forecast_state_1);
         FillForecastStateMultiFabs(lev, filename2, a_z_phys_nd[lev], forecast_state_2);
  
          // Create the time-interpolated forecast state
         //CreateForecastStateMultiFabs(forecast_state_interp);
         if(!regrid_forces_file_read){
             last_read_forecast_time[lev] = next_read_forecast_time[lev];
             next_read_forecast_time[lev] += hindcast_data_interval;
             Print() << "Next forecast time getting updated here " << std::endl;
         }
     }
  
     Real prev_read_time = last_read_forecast_time[lev];
     Real alpha1 = 1.0 - (time - prev_read_time)/hindcast_data_interval;
     Real alpha2 = 1.0 - alpha1;
  
     amrex::Print()<< "The values of alpha1 and alpha2 are " << alpha1 << " "<< alpha2 <<std::endl;
  
     if (alpha1 < 0.0 || alpha1 > 1.0 ||
     alpha2 < 0.0 || alpha2 > 1.0)
     {
         std::stringstream ss;
         ss << "Interpolation weights for hindcast files are incorrect: "
         << "alpha1 = " << alpha1 << ", alpha2 = " << alpha2;
        Abort(ss.str());
     }
  
     MultiFab& erf_mf_cons   = forecast_state_interp[lev][Vars::cons];
     MultiFab& erf_mf_xvel   = forecast_state_interp[lev][Vars::xvel];
     MultiFab& erf_mf_yvel   = forecast_state_interp[lev][Vars::yvel];
     //MultiFab& erf_mf_zvel   = forecast_state_interp[0][Vars::zvel];
     MultiFab& erf_mf_latlon = forecast_state_interp[lev][4];
  
     // Fill the time-interpolated forecast states
     MultiFab::LinComb(forecast_state_interp[lev][Vars::cons],
                       alpha1, forecast_state_1[lev][Vars::cons], 0,
                       alpha2, forecast_state_2[lev][Vars::cons], 0,
                       0, erf_mf_cons.nComp(), forecast_state_interp[lev][Vars::cons].nGrow());
     MultiFab::LinComb(forecast_state_interp[lev][Vars::xvel],
                       alpha1, forecast_state_1[lev][Vars::xvel], 0,
                       alpha2, forecast_state_2[lev][Vars::xvel], 0,
                       0, erf_mf_xvel.nComp(), forecast_state_interp[lev][Vars::xvel].nGrow());
     MultiFab::LinComb(forecast_state_interp[lev][Vars::yvel],
                       alpha1, forecast_state_1[lev][Vars::yvel], 0,
                       alpha2, forecast_state_2[lev][Vars::yvel], 0,
                       0, erf_mf_yvel.nComp(), forecast_state_interp[lev][Vars::yvel].nGrow());
     MultiFab::LinComb(forecast_state_interp[lev][4],
                       alpha1, forecast_state_1[lev][4], 0,
                       alpha2, forecast_state_2[lev][4], 0,
                       0, erf_mf_latlon.nComp(), forecast_state_interp[lev][4].nGrow());
  
     /*Vector<std::string> varnames_plot_mf = {
     "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
     }; // Customize variable names
  
     std::string pltname = "plt_interp";
  
     MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
                      10, 0);
  
     plot_mf.setVal(0.0);
  
     for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
         const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
         const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
         const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
         const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
         const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
         const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
  
         const Box& bx = mfi.validbox();
  
         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
             plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
             plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
             plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
             plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
             plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
  
             plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/2.0;
             plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/2.0;
             plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/2.0;
  
             plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
             plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
         });
     }
  
  
     WriteSingleLevelPlotfile(
             pltname,
             plot_mf,
             varnames_plot_mf,
             geom[0],
             time,
             0 // level
         );*/
 }

◆ Write2DPlotFile()

void ERF::Write2DPlotFile	(	int	which,
		PlotFileType	plotfile_type,
		amrex::Vector< std::string >	plot_var_names
	)

 {
     const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
     const int ncomp_mf = varnames.size();
  
     if (ncomp_mf == 0) return;
  
     // Vector of MultiFabs for cell-centered data
     Vector<MultiFab> mf(finest_level+1);
     for (int lev = 0; lev <= finest_level; ++lev) {
         mf[lev].define(ba2d[lev], dmap[lev], ncomp_mf, 0);
     }
  
  
     // **********************************************************************************************
     // (Effectively) 2D arrays
     // **********************************************************************************************
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         // Make sure getPgivenRTh and getTgivenRandRTh don't fail
         if (check_for_nans) {
             check_for_negative_theta(vars_new[lev][Vars::cons]);
         }
  
         int mf_comp = 0;
  
         // Set all components to zero in case they aren't defined below
         mf[lev].setVal(0.0);
  
         // Expose domain khi and klo at each level
         int klo = geom[lev].Domain().smallEnd(2);
         int khi = geom[lev].Domain().bigEnd(2);
  
         if (containerHasElement(plot_var_names, "z_surf")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<const Real>& z_phys_arr = z_phys_nd[lev]->const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i, j, k, mf_comp) = Compute_Z_AtWFace(i, j, 0, z_phys_arr);
                 });
             }
             mf_comp++;
         }
  
         if (containerHasElement(plot_var_names, "landmask")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<const int>& lmask_arr = lmask_lev[lev][0]->const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i, j, k, mf_comp) = lmask_arr(i, j, 0);
                 });
             }
             mf_comp++;
         }
  
         if (containerHasElement(plot_var_names, "mapfac")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<Real>& mf_m   = mapfac[lev][MapFacType::m_x]->array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
                 });
             }
             mf_comp++;
         }
  
         if (containerHasElement(plot_var_names, "lat_m")) {
             if (lat_m[lev]) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = lat_m[lev]->array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = data(i,j,0);
                     });
                 }
             }
             mf_comp++;
         } // lat_m
  
         if (containerHasElement(plot_var_names, "lon_m")) {
             if (lon_m[lev]) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = lon_m[lev]->array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = data(i,j,0);
                     });
                 }
             } else {
                 mf[lev].setVal(0.0,mf_comp,1,0);
             }
  
             mf_comp++;
  
         } // lon_m
  
         ///////////////////////////////////////////////////////////////////////
         // These quantities are diagnosed by the surface layer
         if (containerHasElement(plot_var_names, "u_star")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (m_SurfaceLayer) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& ustar  = m_SurfaceLayer->get_u_star(lev)->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // ustar
  
         if (containerHasElement(plot_var_names, "w_star")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (m_SurfaceLayer) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& ustar  = m_SurfaceLayer->get_w_star(lev)->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // wstar
  
         if (containerHasElement(plot_var_names, "t_star")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (m_SurfaceLayer) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& ustar  = m_SurfaceLayer->get_t_star(lev)->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // tstar
  
         if (containerHasElement(plot_var_names, "q_star")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (m_SurfaceLayer) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& ustar  = m_SurfaceLayer->get_q_star(lev)->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // qstar
  
         if (containerHasElement(plot_var_names, "Olen")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (m_SurfaceLayer) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& ustar  = m_SurfaceLayer->get_olen(lev)->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // Olen
  
         if (containerHasElement(plot_var_names, "pblh")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (m_SurfaceLayer) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& ustar  = m_SurfaceLayer->get_pblh(lev)->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // pblh
  
         if (containerHasElement(plot_var_names, "t_surf")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (m_SurfaceLayer) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& tsurf  = m_SurfaceLayer->get_t_surf(lev)->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = tsurf(i, j, 0);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // tsurf
  
         if (containerHasElement(plot_var_names, "q_surf")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (m_SurfaceLayer) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& ustar  = m_SurfaceLayer->get_q_surf(lev)->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // qsurf
  
         if (containerHasElement(plot_var_names, "z0")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (m_SurfaceLayer) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& ustar  = m_SurfaceLayer->get_z0(lev)->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = ustar(i, j, 0);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // z0
  
         if (containerHasElement(plot_var_names, "OLR")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (solverChoice.rad_type != RadiationType::None) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& olr    = rad_fluxes[lev]->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                         derdat(i, j, k, mf_comp) = olr(i, j, khi, 2);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // OLR
  
         if (containerHasElement(plot_var_names, "sens_flux")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (SFS_hfx3_lev[lev]) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& hfx_arr = SFS_hfx3_lev[lev]->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                         derdat(i, j, k, mf_comp) = hfx_arr(i, j, klo);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // sens_flux
  
         if (containerHasElement(plot_var_names, "laten_flux")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             if (SFS_hfx3_lev[lev]) {
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const auto& derdat = mf[lev].array(mfi);
                     const auto& qfx_arr = SFS_q1fx3_lev[lev]->const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                         derdat(i, j, k, mf_comp) = qfx_arr(i, j, klo);
                     });
                 }
             } else {
                 mf[lev].setVal(-999,mf_comp,1,0);
             }
             mf_comp++;
         } // laten_flux
  
         if (containerHasElement(plot_var_names, "surf_pres")) {
             bool moist = (solverChoice.moisture_type != MoistureType::None);
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const auto& derdat   = mf[lev].array(mfi);
                 const auto& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     auto rt = cons_arr(i,j,klo,RhoTheta_comp);
                     auto qv = (moist) ? cons_arr(i,j,klo,RhoQ1_comp)/cons_arr(i,j,klo,Rho_comp)
                                       : 0.0;
                     derdat(i, j, k, mf_comp) = getPgivenRTh(rt, qv);
                 });
             }
             mf_comp++;
         } // surf_pres
  
         if (containerHasElement(plot_var_names, "integrated_qv")) {
             MultiFab mf_qv_int(mf[lev],make_alias,mf_comp,1);
             if (solverChoice.moisture_type != MoistureType::None) {
                 volWgtColumnSum(lev, vars_new[lev][Vars::cons], RhoQ1_comp, mf_qv_int, *detJ_cc[lev]);
             } else {
                 mf_qv_int.setVal(0.);
             }
             mf_comp++;
         }
     } // lev
  
     std::string plotfilename;
     if (which == 1) {
        plotfilename = Concatenate(plot2d_file_1, istep[0], file_name_digits);
     } else if (which == 2) {
        plotfilename = Concatenate(plot2d_file_2, istep[0], file_name_digits);
     }
  
     Vector<Geometry> my_geom(finest_level+1);
  
     Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
     if (geom[0].isPeriodic(0)) { is_per[0] = 1;}
     if (geom[0].isPeriodic(1)) { is_per[1] = 1;}
  
     int coord_sys = 0;
  
     for (int lev = 0; lev <= finest_level; lev++)
     {
         Box slab = makeSlab(geom[lev].Domain(),2,0);
         auto const slab_lo = lbound(slab);
         auto const slab_hi = ubound(slab);
  
         // Create a new geometry based only on the 2D slab
         Real dz = geom[lev].CellSize(2);
         RealBox rb = geom[lev].ProbDomain();
         rb.setLo(2,  slab_lo.z   *dz);
         rb.setHi(2, (slab_hi.z+1)*dz);
         my_geom[lev].define(slab, rb, coord_sys, is_per);
     }
  
     if (plotfile_type == PlotFileType::Amrex)
     {
         Print() << "Writing 2D native plotfile " << plotfilename << "\n";
         WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                 GetVecOfConstPtrs(mf),
                                 varnames, my_geom, t_new[0], istep, refRatio());
         writeJobInfo(plotfilename);
  
 #ifdef ERF_USE_NETCDF
     } else if (plotfile_type == PlotFileType::Netcdf) {
          int lev   = 0;
          int l_which = 0;
          const Real* p_lo = my_geom[lev].ProbLo();
          const Real* p_hi = my_geom[lev].ProbHi();
          const auto dx    = my_geom[lev].CellSize();
          writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
                          {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],dx[2]}, {dx[0],dx[1],dx[2]},
                          my_geom[lev].Domain(), t_new[0], start_bdy_time);
 #endif
     } else {
         // Here we assume the plotfile_type is PlotFileType::None
         Print() << "Writing no 2D plotfile since plotfile_type is none" << std::endl;
     }
 }

Here is the call graph for this function:

◆ Write3DPlotFile()

void ERF::Write3DPlotFile	(	int	which,
		PlotFileType	plotfile_type,
		amrex::Vector< std::string >	plot_var_names
	)

 {
     auto dPlotTime0 = amrex::second();
  
     const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
     const int ncomp_mf = varnames.size();
  
     int ncomp_cons = vars_new[0][Vars::cons].nComp();
  
     if (ncomp_mf == 0) return;
  
     // We Fillpatch here because some of the derived quantities require derivatives
     //     which require ghost cells to be filled.  We do not need to call FillPatcher
     //     because we don't need to set interior fine points.
     // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
  
     // Level 0 FillPatch
     FillPatchCrseLevel(0, t_new[0], {&vars_new[0][Vars::cons], &vars_new[0][Vars::xvel],
                        &vars_new[0][Vars::yvel], &vars_new[0][Vars::zvel]});
  
     for (int lev = 1; lev <= finest_level; ++lev) {
         bool fillset = false;
         FillPatchFineLevel(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
                            &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
                           {&vars_new[lev][Vars::cons], &rU_new[lev], &rV_new[lev], &rW_new[lev]},
                            base_state[lev], base_state[lev], fillset);
     }
  
     // Get qmoist pointers if using moisture
     bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
     for (int lev = 0; lev <= finest_level; ++lev) {
         for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
             qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
         }
     }
  
     // Vector of MultiFabs for cell-centered data
     Vector<MultiFab> mf(finest_level+1);
     for (int lev = 0; lev <= finest_level; ++lev) {
         mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
     }
  
     // Vector of MultiFabs for nodal data
     Vector<MultiFab> mf_nd(finest_level+1);
     if ( SolverChoice::mesh_type != MeshType::ConstantDz) {
         for (int lev = 0; lev <= finest_level; ++lev) {
             BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
             mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
             mf_nd[lev].setVal(0.);
         }
     }
  
     // Vector of MultiFabs for face-centered velocity
     Vector<MultiFab> mf_u(finest_level+1);
     Vector<MultiFab> mf_v(finest_level+1);
     Vector<MultiFab> mf_w(finest_level+1);
     if (m_plot_face_vels) {
         for (int lev = 0; lev <= finest_level; ++lev) {
             BoxArray grid_stag_u(grids[lev]); grid_stag_u.surroundingNodes(0);
             BoxArray grid_stag_v(grids[lev]); grid_stag_v.surroundingNodes(1);
             BoxArray grid_stag_w(grids[lev]); grid_stag_w.surroundingNodes(2);
             mf_u[lev].define(grid_stag_u, dmap[lev], 1, 0);
             mf_v[lev].define(grid_stag_v, dmap[lev], 1, 0);
             mf_w[lev].define(grid_stag_w, dmap[lev], 1, 0);
             MultiFab::Copy(mf_u[lev],vars_new[lev][Vars::xvel],0,0,1,0);
             MultiFab::Copy(mf_v[lev],vars_new[lev][Vars::yvel],0,0,1,0);
             MultiFab::Copy(mf_w[lev],vars_new[lev][Vars::zvel],0,0,1,0);
         }
     }
  
     // Array of MultiFabs for cell-centered velocity
     Vector<MultiFab> mf_cc_vel(finest_level+1);
  
     if (containerHasElement(plot_var_names, "x_velocity" ) ||
         containerHasElement(plot_var_names, "y_velocity" ) ||
         containerHasElement(plot_var_names, "z_velocity" ) ||
         containerHasElement(plot_var_names, "magvel"     ) ||
         containerHasElement(plot_var_names, "vorticity_x") ||
         containerHasElement(plot_var_names, "vorticity_y") ||
         containerHasElement(plot_var_names, "vorticity_z") ) {
  
         for (int lev = 0; lev <= finest_level; ++lev) {
             mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
             mf_cc_vel[lev].setVal(-1.e20);
             average_face_to_cellcenter(mf_cc_vel[lev],0,
                                        Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
                                                                 &vars_new[lev][Vars::yvel],
                                                                 &vars_new[lev][Vars::zvel]});
         } // lev
     } // if (vel or vort)
  
     // We need ghost cells if computing vorticity
     if ( containerHasElement(plot_var_names, "vorticity_x")||
          containerHasElement(plot_var_names, "vorticity_y") ||
          containerHasElement(plot_var_names, "vorticity_z") )
     {
         amrex::Interpolater* mapper = &cell_cons_interp;
         for (int lev = 1; lev <= finest_level; ++lev)
         {
             Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
             Vector<Real> ftime    = {t_new[lev], t_new[lev]};
             Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
             Vector<Real> ctime    = {t_new[lev], t_new[lev]};
  
             FillBdyCCVels(mf_cc_vel,lev-1);
  
             // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
             FillPatchTwoLevels(mf_cc_vel[lev], mf_cc_vel[lev].nGrowVect(), IntVect(0,0,0),
                                t_new[lev], cmf, ctime, fmf, ftime,
                                0, 0, mf_cc_vel[lev].nComp(), geom[lev-1], geom[lev],
                                refRatio(lev-1), mapper, domain_bcs_type,
                                BaseBCVars::rho0_bc_comp);
         } // lev
         FillBdyCCVels(mf_cc_vel);
     } // if (vort)
  
  
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         // Make sure getPgivenRTh and getTgivenRandRTh don't fail
         if (check_for_nans) {
             check_for_negative_theta(vars_new[lev][Vars::cons]);
         }
  
         int mf_comp = 0;
  
         BoxArray ba(vars_new[lev][Vars::cons].boxArray());
         DistributionMapping dm = vars_new[lev][Vars::cons].DistributionMap();
  
         // First, copy any of the conserved state variables into the output plotfile
         for (int i = 0; i < cons_names.size(); ++i) {
             if (containerHasElement(plot_var_names, cons_names[i])) {
                 MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,0);
                 mf_comp++;
             }
         }
  
         // Next, check for velocities
         if (containerHasElement(plot_var_names, "x_velocity")) {
             MultiFab::Copy(mf[lev], mf_cc_vel[lev], 0, mf_comp, 1, 0);
             mf_comp += 1;
         }
         if (containerHasElement(plot_var_names, "y_velocity")) {
             MultiFab::Copy(mf[lev], mf_cc_vel[lev], 1, mf_comp, 1, 0);
             mf_comp += 1;
         }
         if (containerHasElement(plot_var_names, "z_velocity")) {
             MultiFab::Copy(mf[lev], mf_cc_vel[lev], 2, mf_comp, 1, 0);
             mf_comp += 1;
         }
  
         // Create multifabs for HSE and pressure fields used to derive other quantities
         MultiFab  r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
         MultiFab  p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
         MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
  
         MultiFab pressure;
  
         if (solverChoice.anelastic[lev] == 0) {
             if (containerHasElement(plot_var_names, "pressure")     ||
                 containerHasElement(plot_var_names, "pert_pres")    ||
                 containerHasElement(plot_var_names, "dpdx")         ||
                 containerHasElement(plot_var_names, "dpdy")         ||
                 containerHasElement(plot_var_names, "dpdz")         ||
                 containerHasElement(plot_var_names, "eq_pot_temp")  ||
                 containerHasElement(plot_var_names, "qsat"))
             {
                 int ng = (containerHasElement(plot_var_names, "dpdx") || containerHasElement(plot_var_names, "dpdy") ||
                           containerHasElement(plot_var_names, "dpdz")) ? 1 : 0;
  
                 // Allocate space for pressure
                 pressure.define(ba,dm,1,ng);
  
                 if (ng > 0) {
                     // Default to p_hse as a way of filling ghost cells at domain boundaries
                     MultiFab::Copy(pressure,p_hse,0,0,1,1);
                 }
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& gbx = mfi.growntilebox(IntVect(ng,ng,0));
  
                     const Array4<Real      >& p_arr = pressure.array(mfi);
                     const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                     const int ncomp = vars_new[lev][Vars::cons].nComp();
  
                     ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
                         const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
                         p_arr(i, j, k) = getPgivenRTh(rhotheta,qv_for_p);
                     });
                } // mfi
                pressure.FillBoundary(geom[lev].periodicity());
             } // compute compressible pressure
         } // not anelastic
         else {
             if (containerHasElement(plot_var_names, "dpdx")         ||
                 containerHasElement(plot_var_names, "dpdy")         ||
                 containerHasElement(plot_var_names, "dpdz")         ||
                 containerHasElement(plot_var_names, "eq_pot_temp")  ||
                 containerHasElement(plot_var_names, "qsat"))
             {
                 // Copy p_hse into pressure if using anelastic
                 pressure.define(ba,dm,1,0);
                 MultiFab::Copy(pressure,p_hse,0,0,1,0);
             }
         }
  
         // Finally, check for any derived quantities and compute them, inserting
         // them into our output multifab
         auto calculate_derived = [&](const std::string& der_name,
                                      MultiFab& src_mf,
                                      decltype(derived::erf_dernull)& der_function)
         {
             if (containerHasElement(plot_var_names, der_name)) {
                 MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     auto& dfab = dmf[mfi];
                     auto& sfab = src_mf[mfi];
                     der_function(bx, dfab, 0, 1, sfab, Geom(lev), t_new[0], nullptr, lev);
                 }
  
                 mf_comp++;
             }
         };
  
         // *****************************************************************************************
         // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
         //       defined in ERF.H
         // *****************************************************************************************
  
         calculate_derived("soundspeed",  vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
         if (use_moisture) {
             calculate_derived("temp",        vars_new[lev][Vars::cons], derived::erf_dermoisttemp);
         } else {
             calculate_derived("temp",        vars_new[lev][Vars::cons], derived::erf_dertemp);
         }
         calculate_derived("theta",       vars_new[lev][Vars::cons], derived::erf_dertheta);
         calculate_derived("KE",          vars_new[lev][Vars::cons], derived::erf_derKE);
         calculate_derived("scalar",      vars_new[lev][Vars::cons], derived::erf_derscalar);
         calculate_derived("vorticity_x", mf_cc_vel[lev]           , derived::erf_dervortx);
         calculate_derived("vorticity_y", mf_cc_vel[lev]           , derived::erf_dervorty);
         calculate_derived("vorticity_z", mf_cc_vel[lev]           , derived::erf_dervortz);
         calculate_derived("magvel"     , mf_cc_vel[lev]           , derived::erf_dermagvel);
  
         if (containerHasElement(plot_var_names, "divU"))
         {
             // TODO TODO TODO  -- we need to convert w to omega here!!
             MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
             Array<MultiFab const*, AMREX_SPACEDIM> u;
             u[0] = &(vars_new[lev][Vars::xvel]);
             u[1] = &(vars_new[lev][Vars::yvel]);
             u[2] = &(vars_new[lev][Vars::zvel]);
             compute_divergence (lev, dmf, u, geom[lev]);
             mf_comp += 1;
         }
  
         if (containerHasElement(plot_var_names, "pres_hse"))
         {
             MultiFab::Copy(mf[lev],p_hse,0,mf_comp,1,0);
             mf_comp += 1;
         }
         if (containerHasElement(plot_var_names, "dens_hse"))
         {
             MultiFab::Copy(mf[lev],r_hse,0,mf_comp,1,0);
             mf_comp += 1;
         }
         if (containerHasElement(plot_var_names, "theta_hse"))
         {
             MultiFab::Copy(mf[lev],th_hse,0,mf_comp,1,0);
             mf_comp += 1;
         }
  
         if (containerHasElement(plot_var_names, "pressure"))
         {
             if (solverChoice.anelastic[lev] == 1) {
                 MultiFab::Copy(mf[lev], p_hse, 0, mf_comp, 1, 0);
             } else {
                 MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
             }
  
             mf_comp += 1;
         }
  
         if (containerHasElement(plot_var_names, "pert_pres"))
         {
             if (solverChoice.anelastic[lev] == 1) {
                 MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0);
             } else {
                 MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
                 MultiFab::Subtract(mf[lev],p_hse,0,mf_comp,1,IntVect{0});
             }
             mf_comp += 1;
         }
  
         if (containerHasElement(plot_var_names, "pert_dens"))
         {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat  = mf[lev].array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
                 });
             }
             mf_comp ++;
         }
  
         if (containerHasElement(plot_var_names, "eq_pot_temp"))
         {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat  = mf[lev].array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 const Array4<Real const>& p_arr = pressure.const_array(mfi);
                 const int ncomp = vars_new[lev][Vars::cons].nComp();
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
                     Real qc = (use_moisture && (ncomp > RhoQ2_comp)) ? S_arr(i,j,k,RhoQ2_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
                     Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                     Real fac = Cp_d + Cp_l*(qv + qc);
                     Real pv = erf_esatw(T)*100.0;
  
                     derdat(i, j, k, mf_comp) = T*std::pow((p_arr(i,j,k) - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
                 });
             }
             mf_comp ++;
         }
  
  
         if (containerHasElement(plot_var_names, "VPD"))
         {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat  = mf[lev].array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                 const Array4<Real const>& p_arr = pressure.const_array(mfi);
                 const int ncomp = vars_new[lev][Vars::cons].nComp();
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                 {
                     const Real qv       = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
  
                     const Real T        = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                     const Real e_sat = 100.0 * erf_esatw_cc(T);
  
                     const Real P     = p_arr(i,j,k);
                     const Real e_act = P * qv / (Real(0.622) + qv);
  
                     derdat(i,j,k,mf_comp) = std::max(Real(0.0), e_sat - e_act) * Real(0.001);
                 });
             }
             mf_comp ++;
         }
  
 #ifdef ERF_USE_WINDFARM
         if ( containerHasElement(plot_var_names, "num_turb") and
              (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
               solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
         {
             MultiFab::Copy(mf[lev],Nturb[lev],0,mf_comp,1,0);
             mf_comp ++;
         }
  
         if ( containerHasElement(plot_var_names, "SMark0") and
              (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
               solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
         {
             MultiFab::Copy(mf[lev],SMark[lev],0,mf_comp,1,0);
             mf_comp ++;
         }
  
         if (containerHasElement(plot_var_names, "SMark1") and
            (solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD))
         {
             MultiFab::Copy(mf[lev],SMark[lev],1,mf_comp,1,0);
             mf_comp ++;
         }
 #endif
  
         // **********************************************************************************************
         // Allocate space if we are computing any pressure gradients
         // **********************************************************************************************
  
         Vector<MultiFab> gradp_temp;  gradp_temp.resize(AMREX_SPACEDIM);
         if (containerHasElement(plot_var_names, "dpdx")       ||
             containerHasElement(plot_var_names, "dpdy")       ||
             containerHasElement(plot_var_names, "dpdz")       ||
             containerHasElement(plot_var_names, "pres_hse_x") ||
             containerHasElement(plot_var_names, "pres_hse_y"))
         {
             gradp_temp[GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp_temp[GpVars::gpx].setVal(0.);
             gradp_temp[GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp_temp[GpVars::gpy].setVal(0.);
             gradp_temp[GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp_temp[GpVars::gpz].setVal(0.);
         }
  
         // **********************************************************************************************
         // These are based on computing gradient of full pressure
         // **********************************************************************************************
  
         if (solverChoice.anelastic[lev] == 0) {
             if ( (containerHasElement(plot_var_names, "dpdx")) ||
                  (containerHasElement(plot_var_names, "dpdy")) ||
                  (containerHasElement(plot_var_names, "dpdz")) ) {
                 compute_gradp(pressure, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
                               get_eb(lev), gradp_temp, solverChoice);
             }
         }
  
         if (containerHasElement(plot_var_names, "dpdx"))
         {
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real      >&   derdat  = mf[lev].array(mfi);
                 const Array4<Real const>&   gpx_arr = (solverChoice.anelastic[lev] == 1) ?
                       gradp[lev][GpVars::gpx].array(mfi) : gradp_temp[GpVars::gpx].array(mfi);
                 const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
                 });
             }
             mf_comp ++;
         } // dpdx
         if (containerHasElement(plot_var_names, "dpdy"))
         {
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real      >&   derdat  = mf[lev].array(mfi);
                 const Array4<Real const>&   gpy_arr = (solverChoice.anelastic[lev] == 1) ?
                       gradp[lev][GpVars::gpy].array(mfi) : gradp_temp[GpVars::gpy].array(mfi);
                 const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
                 });
             }
             mf_comp ++;
         } // dpdy
         if (containerHasElement(plot_var_names, "dpdz"))
         {
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real      >&  derdat  = mf[lev].array(mfi);
                 const Array4<Real const>&  gpz_arr = (solverChoice.anelastic[lev] == 1) ?
                       gradp[lev][GpVars::gpz].array(mfi) : gradp_temp[GpVars::gpz].array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     derdat(i ,j ,k, mf_comp) = 0.5 * (gpz_arr(i,j,k+1) + gpz_arr(i,j,k));
                 });
             }
             mf_comp ++;
         } // dpdz
  
         // **********************************************************************************************
         // These are based on computing gradient of basestate pressure
         // **********************************************************************************************
  
         if ( (containerHasElement(plot_var_names, "pres_hse_x")) ||
              (containerHasElement(plot_var_names, "pres_hse_y")) ) {
             compute_gradp(p_hse, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
                           get_eb(lev), gradp_temp, solverChoice);
         }
  
         if (containerHasElement(plot_var_names, "pres_hse_x"))
         {
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real      >&  derdat  = mf[lev].array(mfi);
                 const Array4<Real const>&  gpx_arr = gradp_temp[0].array(mfi);
                 const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
                 });
             }
             mf_comp += 1;
         } // pres_hse_x
  
         if (containerHasElement(plot_var_names, "pres_hse_y"))
         {
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real      >&  derdat  = mf[lev].array(mfi);
                 const Array4<Real const>&  gpy_arr = gradp_temp[1].array(mfi);
                 const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                     derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
                 });
             }
             mf_comp += 1;
         } // pres_hse_y
  
         // **********************************************************************************************
         // Metric terms
         // **********************************************************************************************
  
         if (SolverChoice::mesh_type != MeshType::ConstantDz) {
             if (containerHasElement(plot_var_names, "z_phys"))
             {
                 MultiFab::Copy(mf[lev],*z_phys_cc[lev],0,mf_comp,1,0);
                 mf_comp ++;
             }
  
             if (containerHasElement(plot_var_names, "detJ"))
             {
                 MultiFab::Copy(mf[lev],*detJ_cc[lev],0,mf_comp,1,0);
                 mf_comp ++;
             }
         } // use_terrain
  
         if (containerHasElement(plot_var_names, "mapfac")) {
             amrex::Print() << "You are plotting a 3D version of mapfac; we suggest using the 2D plotfile instead" << std::endl;
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<Real>& mf_m   = mapfac[lev][MapFacType::m_x]->array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                    derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
                 });
             }
             mf_comp ++;
         }
  
         if (containerHasElement(plot_var_names, "lat_m")) {
             amrex::Print() << "You are plotting a 3D version of lat_m; we suggest using the 2D plotfile instead" << std::endl;
             if (lat_m[lev]) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = lat_m[lev]->array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = data(i,j,0);
                     });
                 }
             } else {
                 mf[lev].setVal(0.0,mf_comp,1,0);
             }
             mf_comp++;
         } // lat_m
  
         if (containerHasElement(plot_var_names, "lon_m")) {
             amrex::Print() << "You are plotting a 3D version of lon_m; we suggest using the 2D plotfile instead" << std::endl;
             if (lon_m[lev]) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = lon_m[lev]->array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
                        derdat(i, j, k, mf_comp) = data(i,j,0);
                     });
                 }
             } else {
                 mf[lev].setVal(0.0,mf_comp,1,0);
             }
             mf_comp++;
         } // lon_m
  
         if (solverChoice.time_avg_vel) {
             if (containerHasElement(plot_var_names, "u_t_avg")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                     const Real norm = t_avg_cnt[lev];
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         derdat(i ,j ,k, mf_comp) = data(i,j,k,0) / norm;
                     });
                 }
                 mf_comp ++;
             }
  
             if (containerHasElement(plot_var_names, "v_t_avg")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                     const Real norm = t_avg_cnt[lev];
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         derdat(i ,j ,k, mf_comp) = data(i,j,k,1) / norm;
                     });
                 }
                 mf_comp ++;
             }
  
             if (containerHasElement(plot_var_names, "w_t_avg")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                     const Real norm = t_avg_cnt[lev];
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         derdat(i ,j ,k, mf_comp) = data(i,j,k,2) / norm;
                     });
                 }
                 mf_comp ++;
             }
  
             if (containerHasElement(plot_var_names, "umag_t_avg")) {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat = mf[lev].array(mfi);
                     const Array4<Real>& data   = vel_t_avg[lev]->array(mfi);
                     const Real norm = t_avg_cnt[lev];
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         derdat(i ,j ,k, mf_comp) = data(i,j,k,3) / norm;
                     });
                 }
                 mf_comp ++;
             }
         }
  
         if (containerHasElement(plot_var_names, "nut")) {
             MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
             MultiFab cmf(vars_new[lev][Vars::cons], make_alias, 0, 1); // to provide rho only
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 auto       prim = dmf[mfi].array();
                 auto const cons = cmf[mfi].const_array();
                 auto const diff = (*eddyDiffs_lev[lev])[mfi].const_array();
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                 {
                     const Real rho = cons(i, j, k, Rho_comp);
                     const Real Kmv = diff(i, j, k, EddyDiff::Mom_v);
                     prim(i,j,k) = Kmv / rho;
                 });
             }
  
             mf_comp++;
         }
  
         if (containerHasElement(plot_var_names, "Kmv")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_v,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "Kmh")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "Khv")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_v,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "Khh")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "Lturb")) {
             MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Turb_lengthscale,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "walldist")) {
             MultiFab::Copy(mf[lev],*walldist[lev],0,mf_comp,1,0);
             mf_comp ++;
         }
         if (containerHasElement(plot_var_names, "diss")) {
             MultiFab::Copy(mf[lev],*SFS_diss_lev[lev],0,mf_comp,1,0);
             mf_comp ++;
         }
  
         // TODO: The size of the q variables can vary with different
         //       moisture models. Therefore, certain components may
         //       reside at different indices. For example, Kessler is
         //       warm but precipitating. This puts qp at index 3.
         //       However, SAM is cold and precipitating so qp is index 4.
         //       Need to built an external enum struct or a better pathway.
  
         // NOTE: Protect against accessing non-existent data
         if (use_moisture) {
             int n_qstate_moist   = micro->Get_Qstate_Moist_Size();
  
             // Moist density
             if(containerHasElement(plot_var_names, "moist_density"))
             {
                 int n_start = RhoQ1_comp; // qv
                 int n_end   = RhoQ2_comp; // qc
                 if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
                 MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
                 for (int n_comp(n_start); n_comp <= n_end; ++n_comp) {
                     MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                 }
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qv") && (n_qstate_moist >= 1))
             {
                 MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ1_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qc") && (n_qstate_moist >= 2))
             {
                 MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ2_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qi") && (n_qstate_moist >= 4))
             {
                 MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ3_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qrain") && (n_qstate_moist >= 3))
             {
                 int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
                 MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], n_start , mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qsnow") && (n_qstate_moist >= 5))
             {
                 MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ5_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons],   Rho_comp, mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if(containerHasElement(plot_var_names, "qgraup") && (n_qstate_moist >= 6))
             {
                 MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], RhoQ6_comp, mf_comp, 1, 0);
                 MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons],   Rho_comp, mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             // Precipitating + non-precipitating components
             //--------------------------------------------------------------------------
             if(containerHasElement(plot_var_names, "qt"))
             {
                 int n_start = RhoQ1_comp; // qv
                 int n_end   = n_start + n_qstate_moist;
                 MultiFab::Copy(mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
                 for (int n_comp(n_start+1); n_comp < n_end; ++n_comp) {
                     MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                 }
                 MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             // Non-precipitating components
             //--------------------------------------------------------------------------
             if (containerHasElement(plot_var_names, "qn"))
             {
                 int n_start = RhoQ1_comp; // qv
                 int n_end   = RhoQ2_comp; // qc
                 if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
                 MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
                 for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
                     MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                 }
                 MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             // Precipitating components
             //--------------------------------------------------------------------------
             if(containerHasElement(plot_var_names, "qp") && (n_qstate_moist >= 3))
             {
                 int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
                 int n_end   = ncomp_cons - 1;
                 MultiFab::Copy(  mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
                 for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
                     MultiFab::Add(  mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
                 }
                 MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp  , mf_comp, 1, 0);
                 mf_comp += 1;
             }
  
             if (containerHasElement(plot_var_names, "qsat"))
             {
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
                 for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
                 {
                     const Box& bx = mfi.tilebox();
                     const Array4<Real>& derdat  = mf[lev].array(mfi);
                     const Array4<Real const>& p_arr = pressure.array(mfi);
                     const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                     {
                         Real qv = S_arr(i,j,k,RhoQ1_comp) / S_arr(i,j,k,Rho_comp);
                         Real T  = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
                         Real p  = p_arr(i,j,k) * Real(0.01);
                         erf_qsatw(T, p, derdat(i,j,k,mf_comp));
                     });
                 }
                 mf_comp ++;
             }
  
             if ( (solverChoice.moisture_type == MoistureType::Kessler) ||
                  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
                  (solverChoice.moisture_type == MoistureType::SAM_NoIce) )
             {
                 int offset = (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ? 5 : 0;
                 if (containerHasElement(plot_var_names, "rain_accum"))
                 {
                     MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
                     mf_comp += 1;
                 }
                 if (containerHasElement(plot_var_names, "rel_humidity")) {
                     Print() << "Warning: plot variable \"rel_humidity\" is not available with Kessler moisture model.\n";
                     mf[lev].setVal(0.0, mf_comp, 1, 0);
                     mf_comp += 1;
                 }
             }
             else if ( (solverChoice.moisture_type == MoistureType::SAM) ||
                       (solverChoice.moisture_type == MoistureType::Morrison) )
             {
                 int offset = (solverChoice.moisture_type == MoistureType::Morrison) ? 5 : 0;
                 if (containerHasElement(plot_var_names, "rain_accum"))
                 {
                     MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
                     mf_comp += 1;
                 }
                 if (containerHasElement(plot_var_names, "snow_accum"))
                 {
                     MultiFab::Copy(mf[lev],*(qmoist[lev][offset+1]),0,mf_comp,1,0);
                     mf_comp += 1;
                 }
                 if (containerHasElement(plot_var_names, "graup_accum"))
                 {
                     MultiFab::Copy(mf[lev],*(qmoist[lev][offset+2]),0,mf_comp,1,0);
                     mf_comp += 1;
                 }
                 if (containerHasElement(plot_var_names, "rel_humidity")) {
                     Print() << "Warning: plot variable \"rel_humidity\" is not available with SAM moisture model.\n";
                     mf[lev].setVal(0.0, mf_comp, 1, 0);
                     mf_comp += 1;
                 }
             }
             else if(solverChoice.moisture_type == MoistureType::SuperDroplets) {
                 if (containerHasElement(plot_var_names, "rain_accum")) {
                     MultiFab::Copy(mf[lev],*(qmoist[lev][6]),0,mf_comp,1,0);
                     mf_comp += 1;
                 }
                 if (containerHasElement(plot_var_names, "rel_humidity")) {
                     MultiFab::Copy(mf[lev],*(qmoist[lev][5]),0,mf_comp,1,0);
                     mf_comp += 1;
                 }
                 if (containerHasElement(plot_var_names, "condensation_rate")) {
                     MultiFab::Copy(mf[lev],*(qmoist[lev][3]),0,mf_comp,1,0);
                     mf_comp += 1;
                 }
             }
  
             if (containerHasElement(plot_var_names, "reflectivity")) {
                 if (solverChoice.moisture_type == MoistureType::Morrison) {
  
                     for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                         const Box& bx = mfi.tilebox();
                         const Array4<Real>& derdat  = mf[lev].array(mfi);
                         const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
                         ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
  
                             Real rho = S_arr(i,j,k,Rho_comp);
                             Real qv  = std::max(0.0,S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp));
                             Real qpr = std::max(0.0,S_arr(i,j,k,RhoQ4_comp)/S_arr(i,j,k,Rho_comp));
                             Real qps = std::max(0.0,S_arr(i,j,k,RhoQ5_comp)/S_arr(i,j,k,Rho_comp));
                             Real qpg = std::max(0.0,S_arr(i,j,k,RhoQ6_comp)/S_arr(i,j,k,Rho_comp));
  
                             Real temp  = getTgivenRandRTh(S_arr(i,j,k,Rho_comp),
                                                      S_arr(i,j,k,RhoTheta_comp),
                                                      qv);
                             derdat(i, j, k, mf_comp) = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
                                                                                 1, 1, 1, 1) ;
                         });
                     }
                     mf_comp ++;
                 }
             }
  
            if (solverChoice.moisture_type == MoistureType::Morrison) {
                 if (containerHasElement(plot_var_names, "max_reflectivity")) {
                     for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                         const Box& bx = mfi.tilebox();
  
                         const Array4<Real>& derdat  = mf[lev].array(mfi);
                         const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
  
                         // collapse to i,j box (ignore vertical for now)
                         Box b2d = bx;
                         b2d.setSmall(2,0);
                         b2d.setBig(2,0);
  
                         ParallelFor(b2d, [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept {
  
                             Real max_dbz = -1.0e30;
  
                             // find max reflectivity over k
                             for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
                                 Real rho = S_arr(i,j,k,Rho_comp);
                                 Real qv  = std::max(0.0, S_arr(i,j,k,RhoQ1_comp)/rho);
                                 Real qpr = std::max(0.0, S_arr(i,j,k,RhoQ4_comp)/rho);
                                 Real qps = std::max(0.0, S_arr(i,j,k,RhoQ5_comp)/rho);
                                 Real qpg = std::max(0.0, S_arr(i,j,k,RhoQ6_comp)/rho);
  
                                 Real temp = getTgivenRandRTh(rho, S_arr(i,j,k,RhoTheta_comp), qv);
  
                                 Real dbz = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
                                                                         1, 1, 1, 1);
                                 max_dbz = amrex::max(max_dbz, dbz);
                             }
  
                             // store max_dbz into *all* levels for this (i,j)
                             for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
                                 derdat(i, j, k, mf_comp) = max_dbz;
                             }
                         });
                     }
                     mf_comp++;
                 }
             }
         } // use_moisture
  
         if (containerHasElement(plot_var_names, "terrain_IB_mask"))
         {
             MultiFab* terrain_blank = terrain_blanking[lev].get();
             MultiFab::Copy(mf[lev],*terrain_blank,0,mf_comp,1,0);
             mf_comp ++;
         }
  
         if (containerHasElement(plot_var_names, "volfrac")) {
             if ( solverChoice.terrain_type == TerrainType::EB ||
                  solverChoice.terrain_type == TerrainType::ImmersedForcing)
             {
                 MultiFab::Copy(mf[lev], EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
             } else {
                 mf[lev].setVal(1.0, mf_comp, 1, 0);
             }
             mf_comp += 1;
         }
  
 #ifdef ERF_COMPUTE_ERROR
         // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
         if (containerHasElement(plot_var_names, "xvel_err") ||
             containerHasElement(plot_var_names, "yvel_err") ||
             containerHasElement(plot_var_names, "zvel_err"))
         {
             //
             // Moving terrain ANALYTICAL
             //
             Real H           = geom[lev].ProbHi()[2];
             Real Ampl        = 0.16;
             Real wavelength  = 100.;
             Real kp          = 2. * PI / wavelength;
             Real g           = CONST_GRAV;
             Real omega       = std::sqrt(g * kp);
             Real omega_t     = omega * t_new[lev];
  
             const auto dx = geom[lev].CellSizeArray();
  
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.validbox();
                 Box xbx(bx); xbx.surroundingNodes(0);
                 const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
                 const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
  
                 const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
  
                 ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     Real x = i * dx[0];
                     Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
  
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
                     z -= z_base;
  
                     Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
  
                     xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
                 });
  
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     Real x   = (i + 0.5) * dx[0];
                     Real z   = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
  
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
                     z -= z_base;
  
                     Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
  
                     zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
                 });
             }
  
             MultiFab temp_mf(mf[lev].boxArray(), mf[lev].DistributionMap(), AMREX_SPACEDIM, 0);
             average_face_to_cellcenter(temp_mf,0,
                 Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
  
             if (containerHasElement(plot_var_names, "xvel_err")) {
                 MultiFab::Copy(mf[lev],temp_mf,0,mf_comp,1,0);
                 mf_comp += 1;
             }
             if (containerHasElement(plot_var_names, "yvel_err")) {
                 MultiFab::Copy(mf[lev],temp_mf,1,mf_comp,1,0);
                 mf_comp += 1;
             }
             if (containerHasElement(plot_var_names, "zvel_err")) {
                 MultiFab::Copy(mf[lev],temp_mf,2,mf_comp,1,0);
                 mf_comp += 1;
             }
  
             // Now restore the velocities to what they were
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.validbox();
                 Box xbx(bx); xbx.surroundingNodes(0);
  
                 const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
                 const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
  
                 const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
  
                 ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     Real x = i * dx[0];
                     Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
  
                     z -= z_base;
  
                     Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                     xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
                 });
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     Real x   = (i + 0.5) * dx[0];
                     Real z   = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
  
                     z -= z_base;
                     Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
  
                     zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
                 });
             }
         } // end xvel_err, yvel_err, zvel_err
  
         if (containerHasElement(plot_var_names, "pp_err"))
         {
             // Moving terrain ANALYTICAL
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& bx = mfi.tilebox();
                 const Array4<Real>& derdat = mf[lev].array(mfi);
                 const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
                 const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
  
                 const auto dx = geom[lev].CellSizeArray();
                 const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
                 const Array4<Real const>&  p_arr = pressure.const_array(mfi);
                 const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
  
                 Real H           = geom[lev].ProbHi()[2];
                 Real Ampl        = 0.16;
                 Real wavelength  = 100.;
                 Real kp          = 2. * PI / wavelength;
                 Real g           = CONST_GRAV;
                 Real omega       = std::sqrt(g * kp);
                 Real omega_t     = omega * t_new[lev];
  
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
                 {
                     derdat(i, j, k, mf_comp) = p_arr(i,j,k) - p0_arr(i,j,k);
  
                     Real rho_hse     = r0_arr(i,j,k);
  
                     Real x   = (i + 0.5) * dx[0];
                     Real z   = 0.125 * ( z_nd(i,j,k  ) + z_nd(i+1,j,k  ) + z_nd(i,j+1,k  ) + z_nd(i+1,j+1,k  )
                                         +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
                     Real z_base = Ampl * std::sin(kp * x - omega_t);
  
                     z -= z_base;
                     Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
                     Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
                                               std::sin(kp * x - omega_t) * r0_arr(i,j,k);
  
                     derdat(i,j,k,mf_comp) -= pprime_exact;
                 });
             }
             mf_comp += 1;
         }
 #endif
  
         if (solverChoice.rad_type != RadiationType::None) {
             if (containerHasElement(plot_var_names, "qsrc_sw")) {
                 MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 0, mf_comp, 1, 0);
                 mf_comp += 1;
             }
             if (containerHasElement(plot_var_names, "qsrc_lw")) {
                 MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 1, mf_comp, 1, 0);
                 mf_comp += 1;
             }
         }
  
         // *****************************************************************************************
         // End of derived variables corresponding to "derived_names" in ERF.H
         //
         // Particles and microphysics can provide additional outputs, which are handled below.
         // *****************************************************************************************
  
 #ifdef ERF_USE_PARTICLES
         const auto& particles_namelist( particleData.getNames() );
  
         if (containerHasElement(plot_var_names, "tracer_particles_count")) {
             if (particles_namelist.size() == 0) {
                 MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
                 temp_dat.setVal(0);
                 MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                 mf_comp += 1;
             } else {
                 for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++) {
                     if (containerHasElement(plot_var_names, std::string(particles_namelist[i]+"_count"))) {
                         MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
                         temp_dat.setVal(0);
                         if (particleData.HasSpecies(particles_namelist[i])) {
                             particleData[particles_namelist[i]]->Increment(temp_dat, lev);
                         }
                         MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                         mf_comp += 1;
                     }
                 }
             }
         }
  
         Vector<std::string> particle_mesh_plot_names(0);
         particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
  
         for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
             std::string plot_var_name(particle_mesh_plot_names[i]);
             if (containerHasElement(plot_var_names, plot_var_name) ) {
                 MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
                 temp_dat.setVal(0);
                 particleData.GetMeshPlotVar(plot_var_name, temp_dat, lev);
                 MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                 mf_comp += 1;
             }
         }
 #endif
  
         {
             Vector<std::string> microphysics_plot_names;
             micro->GetPlotVarNames(microphysics_plot_names);
             for (auto& plot_name : microphysics_plot_names) {
                 if (containerHasElement(plot_var_names, plot_name)) {
                     MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
                     temp_dat.setVal(0);
                     micro->GetPlotVar(plot_name, temp_dat, lev);
                     MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
                     mf_comp += 1;
                 }
             }
         }
  
  
     }
  
     if (solverChoice.terrain_type == TerrainType::EB)
     {
         for (int lev = 0; lev <= finest_level; ++lev) {
             EB_set_covered(mf[lev], 0.0);
         }
     }
  
     // Fill terrain distortion MF (nu_nd)
     if (SolverChoice::mesh_type != MeshType::ConstantDz) {
         for (int lev(0); lev <= finest_level; ++lev) {
             MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
             Real dz = Geom()[lev].CellSizeArray()[2];
             for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                 const Box& bx = mfi.tilebox();
                 Array4<Real> mf_arr = mf_nd[lev].array(mfi);
                 ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                 {
                     mf_arr(i,j,k,2) -= k * dz;
                 });
             }
         }
     }
  
     std::string plotfilename;
     std::string plotfilenameU;
     std::string plotfilenameV;
     std::string plotfilenameW;
  
     if (which == 1) {
        if (use_real_time_in_pltname) {
            const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
            plotfilename = plot3d_file_1+"_"+getTimestamp(start_time+t_new[0], dt_format,false);
        } else {
            plotfilename  = Concatenate(plot3d_file_1, istep[0], file_name_digits);
        }
        plotfilenameU = Concatenate(plot3d_file_1+"U", istep[0], file_name_digits);
        plotfilenameV = Concatenate(plot3d_file_1+"V", istep[0], file_name_digits);
        plotfilenameW = Concatenate(plot3d_file_1+"W", istep[0], file_name_digits);
     } else if (which == 2) {
        if (use_real_time_in_pltname) {
            const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
            plotfilename = plot3d_file_2+"_"+getTimestamp(start_time+t_new[0], dt_format,false);
        } else {
            plotfilename  = Concatenate(plot3d_file_2, istep[0], file_name_digits);
        }
        plotfilenameU = Concatenate(plot3d_file_2+"U", istep[0], file_name_digits);
        plotfilenameV = Concatenate(plot3d_file_2+"V", istep[0], file_name_digits);
        plotfilenameW = Concatenate(plot3d_file_2+"W", istep[0], file_name_digits);
     }
  
     // LSM writes it's own data
     if (which==1 && plot_lsm) {
         lsm.Plot_Lsm_Data(t_new[0], istep, refRatio());
     }
  
 #ifdef ERF_USE_RRTMGP
     /*
     // write additional RRTMGP data
     // TODO: currently single level only
     if (which==1 && plot_rad) {
         rad[0]->writePlotfile(plot_file_1, t_new[0], istep[0]);
     }
     */
 #endif
  
     // Single level
     if (finest_level == 0)
     {
         if (plotfile_type == PlotFileType::Amrex)
         {
             Print() << "Writing native 3D plotfile " << plotfilename << "\n";
             if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                 WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                    GetVecOfConstPtrs(mf),
                                                    GetVecOfConstPtrs(mf_nd),
                                                    varnames,
                                                    Geom(), t_new[0], istep, refRatio());
             } else {
                 WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                         GetVecOfConstPtrs(mf),
                                         varnames,
                                         Geom(), t_new[0], istep, refRatio());
             }
             writeJobInfo(plotfilename);
  
             if (m_plot_face_vels) {
                 Print() << "Writing face velocities" << std::endl;
                 WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
                                         GetVecOfConstPtrs(mf_u),
                                         {"x_velocity_stag"},
                                         Geom(), t_new[0], istep, refRatio());
                 WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
                                         GetVecOfConstPtrs(mf_v),
                                         {"y_velocity_stag"},
                                         Geom(), t_new[0], istep, refRatio());
                 WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
                                         GetVecOfConstPtrs(mf_w),
                                         {"z_velocity_stag"},
                                         Geom(), t_new[0], istep, refRatio());
             }
  
 #ifdef ERF_USE_PARTICLES
             particleData.writePlotFile(plotfilename);
 #endif
 #ifdef ERF_USE_NETCDF
         } else if (plotfile_type == PlotFileType::Netcdf) {
              AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
              int lev   = 0;
              int l_which = 0;
              const Real* p_lo = geom[lev].ProbLo();
              const Real* p_hi = geom[lev].ProbHi();
              const auto dx    = geom[lev].CellSize();
              writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
                              {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
                              geom[lev].Domain(), t_new[0], start_bdy_time);
 #endif
         } else {
             // Here we assume the plotfile_type is PlotFileType::None
             Print() << "Writing no 3D plotfile since plotfile_type is none" << std::endl;
         }
  
     } else { // Multilevel
  
         if (plotfile_type == PlotFileType::Amrex) {
  
             int lev0 = 0;
             int desired_ratio = std::max(std::max(ref_ratio[lev0][0],ref_ratio[lev0][1]),ref_ratio[lev0][2]);
             bool any_ratio_one = ( ( (ref_ratio[lev0][0] == 1) || (ref_ratio[lev0][1] == 1) ) ||
                                      (ref_ratio[lev0][2] == 1) );
             for (int lev = 1; lev < finest_level; lev++) {
                 any_ratio_one = any_ratio_one ||
                                      ( ( (ref_ratio[lev][0] == 1) || (ref_ratio[lev][1] == 1) ) ||
                                          (ref_ratio[lev][2] == 1) );
             }
  
             if (any_ratio_one && m_expand_plotvars_to_unif_rr)
             {
                 Vector<IntVect>   r2(finest_level);
                 Vector<Geometry>  g2(finest_level+1);
                 Vector<MultiFab> mf2(finest_level+1);
  
                 mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
  
                 // Copy level 0 as is
                 MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
  
                 // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
                 Array<int,AMREX_SPACEDIM> periodicity =
                              {Geom()[lev0].isPeriodic(0),Geom()[lev0].isPeriodic(1),Geom()[lev0].isPeriodic(2)};
                 g2[lev0].define(Geom()[lev0].Domain(),&(Geom()[lev0].ProbDomain()),0,periodicity.data());
  
                 r2[0] = IntVect(desired_ratio/ref_ratio[lev0][0],
                                 desired_ratio/ref_ratio[lev0][1],
                                 desired_ratio/ref_ratio[lev0][2]);
  
                 for (int lev = 1; lev <= finest_level; ++lev) {
                     if (lev > 1) {
                         r2[lev-1][0] = r2[lev-2][0] * desired_ratio / ref_ratio[lev-1][0];
                         r2[lev-1][1] = r2[lev-2][1] * desired_ratio / ref_ratio[lev-1][1];
                         r2[lev-1][2] = r2[lev-2][2] * desired_ratio / ref_ratio[lev-1][2];
                     }
  
                     mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
  
                     // Set the new problem domain
                     Box d2(Geom()[lev].Domain());
                     d2.refine(r2[lev-1]);
  
                     g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
                 }
  
                 //
                 // We need to make a temporary that is the size of ncomp_mf
                 // in order to not get an out of bounds error
                 // even though the values will not be used
                 //
                 Vector<BCRec> temp_domain_bcs_type;
                 temp_domain_bcs_type.resize(ncomp_mf);
  
                 //
                 // Do piecewise constant interpolation of mf into mf2
                 //
                 for (int lev = 1; lev <= finest_level; ++lev) {
                     Interpolater* mapper_c = &pc_interp;
                     InterpFromCoarseLevel(mf2[lev], t_new[lev], mf[lev],
                                           0, 0, ncomp_mf,
                                           geom[lev], g2[lev],
                                           null_bc_for_fill, 0, null_bc_for_fill, 0,
                                           r2[lev-1], mapper_c, temp_domain_bcs_type, 0);
                 }
  
                 // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
                 Vector<IntVect> rr(finest_level);
                 for (int lev = 0; lev < finest_level; ++lev) {
                     rr[lev] = IntVect(desired_ratio);
                 }
  
                Print() << "Writing 3D plotfile " << plotfilename << "\n";
                if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                    WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                       GetVecOfConstPtrs(mf2),
                                                       GetVecOfConstPtrs(mf_nd),
                                                       varnames,
                                                       g2, t_new[0], istep, rr);
                } else {
                    WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                            GetVecOfConstPtrs(mf2), varnames,
                                            g2, t_new[0], istep, rr);
                }
  
             } else {
                if (SolverChoice::mesh_type != MeshType::ConstantDz) {
                     WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
                                                        GetVecOfConstPtrs(mf),
                                                        GetVecOfConstPtrs(mf_nd),
                                                        varnames,
                                                        geom, t_new[0], istep, ref_ratio);
                 } else {
                     WriteMultiLevelPlotfile(plotfilename, finest_level+1,
                                             GetVecOfConstPtrs(mf), varnames,
                                             geom, t_new[0], istep, ref_ratio);
                 }
                 if (m_plot_face_vels) {
                     Print() << "Writing face velocities" << std::endl;
                     WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
                                             GetVecOfConstPtrs(mf_u),
                                             {"x_velocity_stag"},
                                             geom, t_new[0], istep, ref_ratio);
                     WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
                                             GetVecOfConstPtrs(mf_v),
                                             {"y_velocity_stag"},
                                             geom, t_new[0], istep, ref_ratio);
                     WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
                                             GetVecOfConstPtrs(mf_w),
                                             {"z_velocity_stag"},
                                             geom, t_new[0], istep, ref_ratio);
                 }
             } // ref_ratio test
  
             writeJobInfo(plotfilename);
  
 #ifdef ERF_USE_PARTICLES
             particleData.writePlotFile(plotfilename);
 #endif
  
 #ifdef ERF_USE_NETCDF
         } else if (plotfile_type == PlotFileType::Netcdf) {
              AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
              for (int lev = 0; lev <= finest_level; ++lev) {
                  for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
                      Box bounding_region = (lev == 0)  ? geom[lev].Domain() : boxes_at_level[lev][which_box];
                      const Real* p_lo = geom[lev].ProbLo();
                      const Real* p_hi = geom[lev].ProbHi();
                      const auto dx    = geom[lev].CellSizeArray();
                      writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
                                      {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
                                      bounding_region, t_new[0], start_bdy_time);
                  }
              }
 #endif
         }
     } // end multi-level
  
     if (verbose > 0)
     {
         auto dPlotTime = amrex::second() - dPlotTime0;
         ParallelDescriptor::ReduceRealMax(dPlotTime,ParallelDescriptor::IOProcessorNumber());
         amrex::Print() << "3DPlotfile write time = " << dPlotTime << " seconds." << '\n';
     }
 }

Here is the call graph for this function:

◆ write_1D_profiles()

void ERF::write_1D_profiles ( amrex::Real time )

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters

time	Current time

 {
     BL_PROFILE("ERF::write_1D_profiles()");
  
     if (NumDataLogs() > 1)
     {
         // Define the 1d arrays we will need
         Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
         Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
         Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
         Gpu::HostVector<Real> h_avg_wthv;
         Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
         Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
         Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
         Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
         Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
         Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
  
         if (NumDataLogs() > 1) {
             derive_diag_profiles(time,
                                  h_avg_u, h_avg_v, h_avg_w,
                                  h_avg_rho, h_avg_th, h_avg_ksgs,
                                  h_avg_Kmv, h_avg_Khv,
                                  h_avg_qv, h_avg_qc, h_avg_qr,
                                  h_avg_wqv, h_avg_wqc, h_avg_wqr,
                                  h_avg_qi, h_avg_qs, h_avg_qg,
                                  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
                                  h_avg_wthv);
         }
  
         if (NumDataLogs() > 3 && time > 0.) {
             derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                    h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                    h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
                                    h_avg_sgsdiss);
         }
  
         int hu_size =  h_avg_u.size();
  
         auto const& dx = geom[0].CellSizeArray();
         if (ParallelDescriptor::IOProcessor()) {
             if (NumDataLogs() > 1) {
                 std::ostream& data_log1 = DataLog(1);
                 if (data_log1.good()) {
                   // Write the quantities at this time
                   for (int k = 0; k < hu_size; k++) {
                       Real z;
                       if (zlevels_stag[0].size() > 1) {
                           z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                       } else {
                           z = (k + 0.5)* dx[2];
                       }
                       data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_u[k]   << " " << h_avg_v[k]   << " " << h_avg_w[k]     << " "
                                 << h_avg_rho[k] << " " << h_avg_th[k]  << " " << h_avg_ksgs[k] << " "
                                 << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
                                 << h_avg_qv[k]  << " " << h_avg_qc[k]  << " " << h_avg_qr[k]    << " "
                                 << h_avg_qi[k]  << " " << h_avg_qs[k]  << " " << h_avg_qg[k]
                                 << std::endl;
                   } // loop over z
                 } // if good
             } // NumDataLogs
  
             if (NumDataLogs() > 2) {
                 std::ostream& data_log2 = DataLog(2);
                 if (data_log2.good()) {
                   // Write the perturbational quantities at this time
                   for (int k = 0; k < hu_size; k++) {
                       Real z;
                       if (zlevels_stag[0].size() > 1) {
                           z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                       } else {
                           z = (k + 0.5)* dx[2];
                       }
                       Real thv = h_avg_th[k] * (1 + 0.61*h_avg_qv[k] - h_avg_qc[k] - h_avg_qr[k]);
                       data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]  << " "
                                 << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]  << " "
                                 << h_avg_uw[k]   - h_avg_u[k]*h_avg_w[k]  << " "
                                 << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]  << " "
                                 << h_avg_vw[k]   - h_avg_v[k]*h_avg_w[k]  << " "
                                 << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]  << " "
                                 << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k] << " "
                                 << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k] << " "
                                 << h_avg_wth[k]  - h_avg_w[k]*h_avg_th[k] << " "
                                 << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
                                 // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                 //                        -   <u_i u_i> * <u_j>
                                 //                        - 2*<u_i> * <u_i u_j>
                                 //                        + 2*<u_i>*<u_i> * <u_j>
                                 << h_avg_uiuiu[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
                                  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
                                    << " " // (u'_i u'_i)u'
                                 << h_avg_uiuiv[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
                                  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
                                    << " " // (u'_i u'_i)v'
                                 << h_avg_uiuiw[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
                                  - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                    << " " // (u'_i u'_i)w'
                                 << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k] << " "
                                 << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k] << " "
                                 << h_avg_pw[k]   - h_avg_p[k]*h_avg_w[k] << " "
                                 << h_avg_wqv[k]  - h_avg_qv[k]*h_avg_w[k] << " "
                                 << h_avg_wqc[k]  - h_avg_qc[k]*h_avg_w[k] << " "
                                 << h_avg_wqr[k]  - h_avg_qr[k]*h_avg_w[k] << " "
                                 << h_avg_wthv[k] - h_avg_w[k]*thv
                                 << std::endl;
                   } // loop over z
                 } // if good
             } // NumDataLogs
  
             if (NumDataLogs() > 3 && time > 0.) {
                 std::ostream& data_log3 = DataLog(3);
                 if (data_log3.good()) {
                   // Write the average stresses
                   for (int k = 0; k < hu_size; k++) {
                       Real z;
                       if (zlevels_stag[0].size() > 1) {
                           z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
                       } else {
                           z = (k + 0.5)* dx[2];
                       }
                       data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_tau11[k]  << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                 << h_avg_tau22[k]  << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                 << h_avg_sgshfx[k] << " "
                                 << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
                                 << h_avg_sgsdiss[k]
                                 << std::endl;
                   } // loop over z
                 } // if good
             } // if (NumDataLogs() > 3)
         } // if IOProcessor
     } // if (NumDataLogs() > 1)
 }

◆ write_1D_profiles_stag()

void ERF::write_1D_profiles_stag ( amrex::Real time )

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Quantities are output at their native grid locations. Therefore, w and associated flux quantities <(•)'w'>, tau13, and tau23 (where '•' includes u, v, p, theta, ...) will be output at staggered heights (i.e., coincident with z faces) rather than cell-center heights to avoid performing additional averaging. Unstaggered (i.e., cell-centered) quantities are output alongside staggered quantities at the lower cell faces in the log file; these quantities will have a zero value at the big end, corresponding to k=Nz+1.

The structure of file should follow ERF_Write1DProfiles.cpp

Parameters

time	Current time

 {
     BL_PROFILE("ERF::write_1D_profiles()");
  
     if (NumDataLogs() > 1)
     {
         // Define the 1d arrays we will need
         Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
         Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
         Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
         Gpu::HostVector<Real> h_avg_wthv;
         Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
         Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
         Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
         Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
         Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
         Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
  
         if (NumDataLogs() > 1) {
             derive_diag_profiles_stag(time,
                                       h_avg_u, h_avg_v, h_avg_w,
                                       h_avg_rho, h_avg_th, h_avg_ksgs,
                                       h_avg_Kmv, h_avg_Khv,
                                       h_avg_qv, h_avg_qc, h_avg_qr,
                                       h_avg_wqv, h_avg_wqc, h_avg_wqr,
                                       h_avg_qi, h_avg_qs, h_avg_qg,
                                       h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
                                       h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
                                       h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
                                       h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
                                       h_avg_wthv);
         }
  
         if (NumDataLogs() > 3 && time > 0.) {
             derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
                                         h_avg_tau22, h_avg_tau23, h_avg_tau33,
                                         h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
                                         h_avg_sgsdiss);
         }
  
         int unstag_size =  h_avg_w.size() - 1; // _un_staggered heights
  
         auto const& dx = geom[0].CellSizeArray();
         if (ParallelDescriptor::IOProcessor()) {
             if (NumDataLogs() > 1) {
                 std::ostream& data_log1 = DataLog(1);
                 if (data_log1.good()) {
                   // Write the quantities at this time
                   for (int k = 0; k < unstag_size; k++) {
                       Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                       data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_u[k]   << " " << h_avg_v[k]   << " " << h_avg_w[k]     << " "
                                 << h_avg_rho[k] << " " << h_avg_th[k]  << " " << h_avg_ksgs[k] << " "
                                 << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
                                 << h_avg_qv[k]  << " " << h_avg_qc[k]  << " " << h_avg_qr[k]    << " "
                                 << h_avg_qi[k]  << " " << h_avg_qs[k]  << " " << h_avg_qg[k]
                                 << std::endl;
                   } // loop over z
                   // Write top face values
                   Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                   data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                             << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                             << 0 << " " << 0 << " " << h_avg_w[unstag_size] << " "
                             << 0 << " " << 0 << " " << 0 << " " // rho, theta, ksgs
                             << 0 << " " << 0 << " "             // Kmv, Khv
                             << 0 << " " << 0 << " " << 0 << " " // qv, qc, qr
                             << 0 << " " << 0 << " " << 0        // qi, qs, qg
                             << std::endl;
                 } // if good
             } // NumDataLogs
  
             if (NumDataLogs() > 2) {
                 std::ostream& data_log2 = DataLog(2);
                 if (data_log2.good()) {
                   // Write the perturbational quantities at this time
                   // For surface values (k=0), assume w = uw = vw = ww = 0
                   Real w_cc  = h_avg_w[1] / 2;  // w at first cell center
                   Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
                   Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
                   Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
                   data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                             << std::setw(datwidth) << std::setprecision(datprecision) << 0 << " "
                             << h_avg_uu[0]   - h_avg_u[0]*h_avg_u[0]   << " " // u'u'
                             << h_avg_uv[0]   - h_avg_u[0]*h_avg_v[0]   << " " // u'v'
                             << 0                                       << " " // u'w'
                             << h_avg_vv[0]   - h_avg_v[0]*h_avg_v[0]   << " " // v'v'
                             << 0                                       << " " // v'w'
                             << 0                                       << " " // w'w'
                             << h_avg_uth[0]  - h_avg_u[0]*h_avg_th[0]  << " " // u'th'
                             << h_avg_vth[0]  - h_avg_v[0]*h_avg_th[0]  << " " // v'th'
                             << 0                                       << " " // w'th'
                             << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
                             << h_avg_uiuiu[0]
                              - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
                              - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
                              + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
                                << " " // (u'_i u'_i)u'
                             << h_avg_uiuiv[0]
                              - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
                              - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
                              + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
                                << " " // (u'_i u'_i)v'
                             << 0 << " " // (u'_i u'_i)w'
                             << h_avg_pu[0]   - h_avg_p[0]*h_avg_u[0]   << " " // p'u'
                             << h_avg_pv[0]   - h_avg_p[0]*h_avg_v[0]   << " " // p'v'
                             << 0                                       << " " // p'w'
                             << 0                                       << " " // qv'w'
                             << 0                                       << " " // qc'w'
                             << 0                                       << " " // qr'w'
                             << 0                                              // thv'w'
                             << std::endl;
  
                   // For internal values, interpolate scalar quantities to faces
                   for (int k = 1; k < unstag_size; k++) {
                       Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                       Real uface  = 0.5*(h_avg_u[k]  + h_avg_u[k-1]);
                       Real vface  = 0.5*(h_avg_v[k]  + h_avg_v[k-1]);
                       Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]);
                       Real pface  = 0.5*(h_avg_p[k]  + h_avg_p[k-1]);
                       Real qvface = 0.5*(h_avg_qv[k] + h_avg_qv[k-1]);
                       Real qcface = 0.5*(h_avg_qc[k] + h_avg_qc[k-1]);
                       Real qrface = 0.5*(h_avg_qr[k] + h_avg_qr[k-1]);
                       Real uuface = 0.5*(h_avg_uu[k] + h_avg_uu[k-1]);
                       Real vvface = 0.5*(h_avg_vv[k] + h_avg_vv[k-1]);
                       Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
                       w_cc   = 0.5*(h_avg_w[k-1]  + h_avg_w[k]);
                       uw_cc  = 0.5*(h_avg_uw[k-1] + h_avg_uw[k]);
                       vw_cc  = 0.5*(h_avg_vw[k-1] + h_avg_vw[k]);
                       ww_cc  = 0.5*(h_avg_ww[k-1] + h_avg_ww[k]);
                       data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_uu[k]   - h_avg_u[k]*h_avg_u[k]   << " " // u'u'
                                 << h_avg_uv[k]   - h_avg_u[k]*h_avg_v[k]   << " " // u'v'
                                 << h_avg_uw[k]   -      uface*h_avg_w[k]   << " " // u'w'
                                 << h_avg_vv[k]   - h_avg_v[k]*h_avg_v[k]   << " " // v'v'
                                 << h_avg_vw[k]   -      vface*h_avg_w[k]   << " " // v'w'
                                 << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k]   << " " // w'w'
                                 << h_avg_uth[k]  - h_avg_u[k]*h_avg_th[k]  << " " // u'th'
                                 << h_avg_vth[k]  - h_avg_v[k]*h_avg_th[k]  << " " // v'th'
                                 << h_avg_wth[k]  - h_avg_w[k]*thface       << " " // w'th'
                                 << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
                                 // Note: <u'_i u'_i u'_j> =   <u_i u_i u_j>
                                 //                        -   <u_i u_i> * <u_j>
                                 //                        - 2*<u_i> * <u_i u_j>
                                 //                        + 2*<u_i>*<u_i> * <u_j>
                                 << h_avg_uiuiu[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
                                  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
                                    << " " // cell-centered (u'_i u'_i)u'
                                 << h_avg_uiuiv[k]
                                  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
                                  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
                                  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
                                    << " " // cell-centered (u'_i u'_i)v'
                                 << h_avg_uiuiw[k]
                                  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                                  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                                  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                    << " " // face-centered (u'_i u'_i)w'
                                 << h_avg_pu[k]   - h_avg_p[k]*h_avg_u[k]   << " " // cell-centered p'u'
                                 << h_avg_pv[k]   - h_avg_p[k]*h_avg_v[k]   << " " // cell-centered p'v'
                                 << h_avg_pw[k]   -      pface*h_avg_w[k]   << " " // face-centered p'w'
                                 << h_avg_wqv[k]  -     qvface*h_avg_w[k]   << " "
                                 << h_avg_wqc[k]  -     qcface*h_avg_w[k]   << " "
                                 << h_avg_wqr[k]  -     qrface*h_avg_w[k]   << " "
                                 << h_avg_wthv[k] -    thvface*h_avg_w[k]
                                 << std::endl;
                   } // loop over z
  
                   // Write top face values, extrapolating scalar quantities
                   const int k = unstag_size;
                   Real uface  = 1.5*h_avg_u[k-1]  - 0.5*h_avg_u[k-2];
                   Real vface  = 1.5*h_avg_v[k-1]  - 0.5*h_avg_v[k-2];
                   Real thface = 1.5*h_avg_th[k-1] - 0.5*h_avg_th[k-2];
                   Real pface  = 1.5*h_avg_p[k-1]  - 0.5*h_avg_p[k-2];
                   Real qvface = 1.5*h_avg_qv[k-1] - 0.5*h_avg_qv[k-2];
                   Real qcface = 1.5*h_avg_qc[k-1] - 0.5*h_avg_qc[k-2];
                   Real qrface = 1.5*h_avg_qr[k-1] - 0.5*h_avg_qr[k-2];
                   Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2];
                   Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2];
                   Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
                   Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                   data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                             << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                             << 0                                     << " " // u'u'
                             << 0                                     << " " // u'v'
                             << h_avg_uw[k]   -      uface*h_avg_w[k] << " " // u'w'
                             << 0                                     << " " // v'v'
                             << h_avg_vw[k]   -      vface*h_avg_w[k] << " " // v'w'
                             << h_avg_ww[k]   - h_avg_w[k]*h_avg_w[k] << " " // w'w'
                             << 0                                     << " " // u'th'
                             << 0                                     << " " // v'th'
                             << h_avg_wth[k]  -     thface*h_avg_w[k] << " " // w'th'
                             << 0                                     << " " // th'th'
                             << 0                                     << " " // (u'_i u'_i)u'
                             << 0                                     << " " // (u'_i u'_i)v'
                             << h_avg_uiuiw[k]
                              - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
                              - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
                              + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
                                << " " // (u'_i u'_i)w'
                             << 0                                     << " " // pu'
                             << 0                                     << " " // pv'
                             << h_avg_pw[k]   -      pface*h_avg_w[k] << " " // pw'
                             << h_avg_wqv[k]  -     qvface*h_avg_w[k] << " "
                             << h_avg_wqc[k]  -     qcface*h_avg_w[k] << " "
                             << h_avg_wqr[k]  -     qrface*h_avg_w[k] << " "
                             << h_avg_wthv[k] -    thvface*h_avg_w[k]
                             << std::endl;
                 } // if good
             } // NumDataLogs
  
             if (NumDataLogs() > 3 && time > 0.) {
                 std::ostream& data_log3 = DataLog(3);
                 if (data_log3.good()) {
                   // Write the average stresses
                   for (int k = 0; k < unstag_size; k++) {
                       Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
                       data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                                 << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                                 << h_avg_tau11[k]  << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
                                 << h_avg_tau22[k]  << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
                                 << h_avg_sgshfx[k] << " "
                                 << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
                                 << h_avg_sgsdiss[k]
                                 << std::endl;
                   } // loop over z
                   // Write top face values
                   Real NANval = 0.0;
                   Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
                   data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
                             << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
                             << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
                             << NANval << " " << h_avg_tau23[unstag_size] << " " << NANval << " "
                             << h_avg_sgshfx[unstag_size] << " "
                             << h_avg_sgsq1fx[unstag_size] << " " << h_avg_sgsq2fx[unstag_size] << " "
                             << NANval
                             << std::endl;
                 } // if good
             } // if (NumDataLogs() > 3)
         } // if IOProcessor
     } // if (NumDataLogs() > 1)
 }

◆ writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream & os )

static

 {
     std::string PrettyLine = std::string(78, '=') + "\n";
     std::string OtherLine = std::string(78, '-') + "\n";
     std::string SkipSpace = std::string(8, ' ');
  
     // build information
     os << PrettyLine;
     os << " ERF Build Information\n";
     os << PrettyLine;
  
     os << "build date:    " << buildInfoGetBuildDate() << "\n";
     os << "build machine: " << buildInfoGetBuildMachine() << "\n";
     os << "build dir:     " << buildInfoGetBuildDir() << "\n";
     os << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
  
     os << "\n";
  
     os << "COMP:          " << buildInfoGetComp() << "\n";
     os << "COMP version:  " << buildInfoGetCompVersion() << "\n";
  
     os << "C++ compiler:  " << buildInfoGetCXXName() << "\n";
     os << "C++ flags:     " << buildInfoGetCXXFlags() << "\n";
  
     os << "\n";
  
     os << "Link flags:    " << buildInfoGetLinkFlags() << "\n";
     os << "Libraries:     " << buildInfoGetLibraries() << "\n";
  
     os << "\n";
  
     for (int n = 1; n <= buildInfoGetNumModules(); n++) {
       os << buildInfoGetModuleName(n) << ": "
          << buildInfoGetModuleVal(n) << "\n";
     }
  
     os << "\n";
     const char* githash1 = buildInfoGetGitHash(1);
     const char* githash2 = buildInfoGetGitHash(2);
     if (strlen(githash1) > 0) {
       os << "ERF       git hash: " << githash1 << "\n";
     }
     if (strlen(githash2) > 0) {
       os << "AMReX       git hash: " << githash2 << "\n";
     }
  
     const char* buildgithash = buildInfoGetBuildGitHash();
     const char* buildgitname = buildInfoGetBuildGitName();
     if (strlen(buildgithash) > 0) {
       os << buildgitname << " git hash: " << buildgithash << "\n";
     }
  
     os << "\n";
     os << " ERF Compile time variables: \n";
  
     os << "\n";
     os << " ERF Defines: \n";
 #ifdef _OPENMP
     os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
        << std::endl;
 #else
     os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
        << std::endl;
 #endif
  
 #ifdef MPI_VERSION
     os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
        << MPI_VERSION << std::endl;
 #else
     os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
        << "UNDEFINED" << std::endl;
 #endif
  
 #ifdef MPI_SUBVERSION
     os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
        << MPI_SUBVERSION << std::endl;
 #else
     os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
        << "UNDEFINED" << std::endl;
 #endif
  
     os << "\n\n";
 }

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ WriteCheckpointFile()

void ERF::WriteCheckpointFile ( ) const

ERF function for writing a checkpoint file.

 {
     auto dCheckTime0 = amrex::second();
  
     // chk00010            write a checkpoint file with this root directory
     // chk00010/Header     this contains information you need to save (e.g., finest_level, t_new, etc.) and also
     //                     the BoxArrays at each level
     // chk00010/Level_0/
     // chk00010/Level_1/
     // etc.                these subdirectories will hold the MultiFab data at each level of refinement
  
     // checkpoint file name, e.g., chk00010
     const std::string& checkpointname = Concatenate(check_file,istep[0],file_name_digits);
  
     Print() << "Writing native checkpoint " << checkpointname << "\n";
  
     const int nlevels = finest_level+1;
  
     // ---- prebuild a hierarchy of directories
     // ---- dirName is built first.  if dirName exists, it is renamed.  then build
     // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
     // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
     // ---- after all directories are built
     // ---- ParallelDescriptor::IOProcessor() creates the directories
     PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
  
     int ncomp_cons = vars_new[0][Vars::cons].nComp();
  
     // write Header file
     if (ParallelDescriptor::IOProcessor()) {
  
        std::string HeaderFileName(checkpointname + "/Header");
        VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
        std::ofstream HeaderFile;
        HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
        HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                                std::ofstream::trunc |
                                                std::ofstream::binary);
        if(! HeaderFile.good()) {
            FileOpenFailed(HeaderFileName);
        }
  
        HeaderFile.precision(17);
  
        // write out title line
        HeaderFile << "Checkpoint file for ERF\n";
  
        // write out finest_level
        HeaderFile << finest_level << "\n";
  
        // write the number of components
        // for each variable we store
  
        // conservative, cell-centered vars
        HeaderFile << ncomp_cons << "\n";
  
        // x-velocity on faces
        HeaderFile << 1 << "\n";
  
        // y-velocity on faces
        HeaderFile << 1 << "\n";
  
        // z-velocity on faces
        HeaderFile << 1 << "\n";
  
        // write out array of istep
        for (int i = 0; i < istep.size(); ++i) {
            HeaderFile << istep[i] << " ";
        }
        HeaderFile << "\n";
  
        // write out array of dt
        for (int i = 0; i < dt.size(); ++i) {
            HeaderFile << dt[i] << " ";
        }
        HeaderFile << "\n";
  
        // write out array of t_new
        for (int i = 0; i < t_new.size(); ++i) {
            HeaderFile << t_new[i] << " ";
        }
        HeaderFile << "\n";
  
        // write the BoxArray at each level
        for (int lev = 0; lev <= finest_level; ++lev) {
            boxArray(lev).writeOn(HeaderFile);
            HeaderFile << '\n';
        }
  
        // Write separate file that tells how many components we have of the base state
        std::string BaseStateFileName(checkpointname + "/num_base_state_comps");
        std::ofstream BaseStateFile;
        BaseStateFile.open(BaseStateFileName.c_str(), std::ofstream::out   |
                                                      std::ofstream::trunc |
                                                      std::ofstream::binary);
        if(! BaseStateFile.good()) {
            FileOpenFailed(BaseStateFileName);
        } else {
            // write out number of components in base state
            BaseStateFile << BaseState::num_comps      << "\n";
            BaseStateFile << base_state[0].nGrowVect() << "\n";
        }
    }
  
     // write the MultiFab data to, e.g., chk00010/Level_0/
     // Here we make copies of the MultiFab with no ghost cells
     for (int lev = 0; lev <= finest_level; ++lev)
     {
         MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
         MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
         VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
  
         MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
         MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
         VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
  
         MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
         MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
         VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
  
         MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
         MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
         VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
  
         if (solverChoice.anelastic[lev] == 1) {
             MultiFab ppinc(grids[lev],dmap[lev],1,0);
             MultiFab::Copy(ppinc,pp_inc[lev],0,0,1,0);
             VisMF::Write(ppinc, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PP_Inc"));
  
             MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
             MultiFab::Copy(gpx,gradp[lev][GpVars::gpx],0,0,1,0);
             VisMF::Write(gpx, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpx"));
  
             MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
             MultiFab::Copy(gpy,gradp[lev][GpVars::gpy],0,0,1,0);
             VisMF::Write(gpy, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpy"));
  
             MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
             MultiFab::Copy(gpz,gradp[lev][GpVars::gpz],0,0,1,0);
             VisMF::Write(gpz, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpz"));
         }
  
         // Note that we write the ghost cells of the base state (unlike above)
         IntVect ng_base = base_state[lev].nGrowVect();
         int  ncomp_base = base_state[lev].nComp();
         MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
         MultiFab::Copy(base,base_state[lev],0,0,ncomp_base,ng_base);
         VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
  
         if (SolverChoice::mesh_type != MeshType::ConstantDz)  {
             // Note that we also write the ghost cells of z_phys_nd
             IntVect ng = z_phys_nd[lev]->nGrowVect();
             MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
             MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
             VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
         }
  
         // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
         // Write the moisture model restart variables
         std::vector<int> qmoist_indices;
         std::vector<std::string> qmoist_names;
         micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
         int qmoist_nvar = qmoist_indices.size();
         for (int var = 0; var < qmoist_nvar; var++) {
             const int ncomp  = 1;
             IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
             MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
             MultiFab::Copy(moist_vars,*(qmoist[lev][qmoist_indices[var]]),0,0,ncomp,ng_moist);
             VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", qmoist_names[var]));
         }
  
 #if defined(ERF_USE_WINDFARM)
         if(solverChoice.windfarm_type == WindFarmType::Fitch or
            solverChoice.windfarm_type == WindFarmType::EWP or
            solverChoice.windfarm_type == WindFarmType::SimpleAD){
             IntVect ng_turb = Nturb[lev].nGrowVect();
             MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng_turb);
             MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng_turb);
             VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
         }
 #endif
  
         // Write the LSM data
         if (solverChoice.lsm_type != LandSurfaceType::None) {
             for (int ivar(0); ivar<lsm_data[lev].size(); ++ivar) {
                 BoxArray ba = lsm_data[lev][ivar]->boxArray();
                 DistributionMapping dm = lsm_data[lev][ivar]->DistributionMap();
                 IntVect ng = lsm_data[lev][ivar]->nGrowVect();
                 int nvar   = lsm_data[lev][ivar]->nComp();
                 MultiFab lsm_vars(ba,dm,nvar,ng);
                 MultiFab::Copy(lsm_vars,*(lsm_data[lev][ivar]),0,0,nvar,ng);
                 VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmData" + std::to_string(ivar)));
             }
             for (int iflux(0); iflux<lsm_flux[lev].size(); ++iflux) {
                 BoxArray ba = lsm_flux[lev][iflux]->boxArray();
                 DistributionMapping dm = lsm_flux[lev][iflux]->DistributionMap();
                 IntVect ng = lsm_flux[lev][iflux]->nGrowVect();
                 int nvar   = lsm_flux[lev][iflux]->nComp();
                 MultiFab lsm_vars(ba,dm,nvar,ng);
                 MultiFab::Copy(lsm_vars,*(lsm_flux[lev][iflux]),0,0,nvar,ng);
                 VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmFlux" + std::to_string(iflux)));
             }
         }
  
         IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
         MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
         MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_x],0,0,1,ng);
         VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_mx"));
  
 #if 0
         if (MapFacType::m_x != MapFacType::m_y) {
             MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_y],0,0,1,ng);
             VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_my"));
         }
 #endif
  
         ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
         MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
         MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_x],0,0,1,ng);
         VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_ux"));
  
 #if 0
         if (MapFacType::m_x != MapFacType::m_y) {
             MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_y],0,0,1,ng);
             VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_uy"));
         }
 #endif
  
         ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
         MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
         MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_x],0,0,1,ng);
         VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vx"));
  
 #if 0
         if (MapFacType::m_x != MapFacType::m_y) {
             MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_y],0,0,1,ng);
             VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vy"));
         }
 #endif
  
         if (m_SurfaceLayer)  {
             amrex::Print() << "Writing SurfaceLayer variables at level " << lev << std::endl;
             ng = IntVect(1,1,0);
             MultiFab   m_var(ba2d[lev],dmap[lev],1,ng);
             MultiFab* src = nullptr;
  
             // U*
             src = m_SurfaceLayer->get_u_star(lev);
             MultiFab::Copy(m_var,*src,0,0,1,ng);
             VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Ustar"));
  
             // W*
             src = m_SurfaceLayer->get_w_star(lev);
             MultiFab::Copy(m_var,*src,0,0,1,ng);
             VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Wstar"));
  
             // T*
             src = m_SurfaceLayer->get_t_star(lev);
             MultiFab::Copy(m_var,*src,0,0,1,ng);
             VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Tstar"));
  
             // Q*
             src = m_SurfaceLayer->get_q_star(lev);
             MultiFab::Copy(m_var,*src,0,0,1,ng);
             VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qstar"));
  
             // Olen
             src = m_SurfaceLayer->get_olen(lev);
             MultiFab::Copy(m_var,*src,0,0,1,ng);
             VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Olen"));
  
             // Qsurf
             src = m_SurfaceLayer->get_q_surf(lev);
             MultiFab::Copy(m_var,*src,0,0,1,ng);
             VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qsurf"));
  
             // PBLH
             src = m_SurfaceLayer->get_pblh(lev);
             MultiFab::Copy(m_var,*src,0,0,1,ng);
             VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PBLH"));
  
             // Z0
             src = m_SurfaceLayer->get_z0(lev);
             MultiFab::Copy(m_var,*src,0,0,1,ng);
             VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z0"));
         }
  
         if (sst_lev[lev][0]) {
             int ntimes = 1;
             ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
             MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
             for (int nt(0); nt<ntimes; ++nt) {
                 MultiFab::Copy(sst_at_t,*sst_lev[lev][nt],0,0,1,ng);
                 VisMF::Write(sst_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
                                                              "SST_" + std::to_string(nt)));
             }
         }
  
         if (tsk_lev[lev][0]) {
             int ntimes = 1;
             ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
             MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
             for (int nt(0); nt<ntimes; ++nt) {
                 MultiFab::Copy(tsk_at_t,*tsk_lev[lev][nt],0,0,1,ng);
                 VisMF::Write(tsk_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
                                                              "TSK_" + std::to_string(nt)));
             }
         }
  
         {
             int ntimes = 1;
             ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
             MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
             for (int nt(0); nt<ntimes; ++nt) {
                 for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
                     const Box& bx = mfi.growntilebox();
                     Array4<int>  const& src_arr = lmask_lev[lev][nt]->array(mfi);
                     Array4<Real> const& dst_arr = lmask_at_t.array(mfi);
                     ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                     {
                         dst_arr(i,j,k) = Real(src_arr(i,j,k));
                     });
                 }
                 VisMF::Write(lmask_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
                                                               "LMASK_" + std::to_string(nt)));
             }
         }
  
         IntVect ngv = ng; ngv[2] = 0;
  
         // Write lat/lon if it exists
         if (lat_m[lev] && lon_m[lev] && solverChoice.has_lat_lon) {
             amrex::Print() << "Writing Lat/Lon variables at level " << lev << std::endl;
             MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
             MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
             MultiFab::Copy(lat,*lat_m[lev],0,0,1,ngv);
             MultiFab::Copy(lon,*lon_m[lev],0,0,1,ngv);
             VisMF::Write(lat, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LAT"));
             VisMF::Write(lon, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LON"));
         }
  
  
 #ifdef ERF_USE_NETCDF
         // Write sinPhi and cosPhi if it exists
         if (cosPhi_m[lev] && sinPhi_m[lev] && solverChoice.variable_coriolis) {
             amrex::Print() << "Writing Coriolis factors at level " << lev << std::endl;
             MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
             MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
             MultiFab::Copy(sphi,*sinPhi_m[lev],0,0,1,ngv);
             MultiFab::Copy(cphi,*cosPhi_m[lev],0,0,1,ngv);
             VisMF::Write(sphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SinPhi"));
             VisMF::Write(cphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "CosPhi"));
         }
  
         if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
             if (lev == 0) {
                 amrex::Print() << "Writing C1H/C2H/MUB variables at level " << lev << std::endl;
                 MultiFab tmp1d(ba1d[0],dmap[0],1,0);
  
                 MultiFab::Copy(tmp1d,*mf_C1H,0,0,1,0);
                 VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C1H"));
  
                 MultiFab::Copy(tmp1d,*mf_C2H,0,0,1,0);
                 VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C2H"));
  
                 MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
  
                 MultiFab::Copy(tmp2d,*mf_MUB,0,0,1,mf_MUB->nGrowVect());
                 VisMF::Write(tmp2d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MUB"));
             }
         }
 #endif
     } // for lev
  
 #ifdef ERF_USE_PARTICLES
    particleData.Checkpoint(checkpointname);
 #endif
  
 #if 0
 #ifdef ERF_USE_NETCDF
    // Write bdy_data files
    if ( ParallelDescriptor::IOProcessor() &&
         ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
          solverChoice.use_real_bcs )
    {
        // Vector dimensions
        int num_time = bdy_data_xlo.size();
        int num_var  = bdy_data_xlo[0].size();
  
        // Open header file and write to it
        std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
        bdy_h_file << std::setprecision(1) << std::fixed;
        bdy_h_file << num_time << "\n";
        bdy_h_file << num_var  << "\n";
        bdy_h_file << start_bdy_time << "\n";
        bdy_h_file << bdy_time_interval << "\n";
        bdy_h_file << real_width << "\n";
        for (int ivar(0); ivar<num_var; ++ivar) {
            bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
            bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
            bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
            bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
        }
  
        // Open data file and write to it
        std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
        for (int itime(0); itime<num_time; ++itime) {
            if (bdy_data_xlo[itime].size() > 0) {
                for (int ivar(0); ivar<num_var; ++ivar) {
                    bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
                    bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
                    bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
                    bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
                }
            }
        }
    }
 #endif
 #endif
  
     if (verbose > 0)
     {
         auto dCheckTime = amrex::second() - dCheckTime0;
         ParallelDescriptor::ReduceRealMax(dCheckTime,ParallelDescriptor::IOProcessorNumber());
         amrex::Print() << "Checkpoint write time = " << dCheckTime << " seconds." << '\n';
     }
 }

Here is the call graph for this function:

◆ WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain	(	std::ostream &	HeaderFile,
		int	nlevels,
		const amrex::Vector< amrex::BoxArray > &	bArray,
		const amrex::Vector< std::string > &	varnames,
		const amrex::Vector< amrex::Geometry > &	my_geom,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const amrex::Vector< amrex::IntVect > &	my_ref_ratio,
		const std::string &	versionName,
		const std::string &	levelPrefix,
		const std::string &	mfPrefix
	)		const

 {
     AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
     AMREX_ALWAYS_ASSERT(nlevels <= my_ref_ratio.size()+1);
     AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
  
     HeaderFile.precision(17);
  
     // ---- this is the generic plot file type name
     HeaderFile << versionName << '\n';
  
     HeaderFile << varnames.size() << '\n';
  
     for (int ivar = 0; ivar < varnames.size(); ++ivar) {
         HeaderFile << varnames[ivar] << "\n";
     }
     HeaderFile << AMREX_SPACEDIM << '\n';
     HeaderFile << my_time << '\n';
     HeaderFile << finest_level << '\n';
     for (int i = 0; i < AMREX_SPACEDIM; ++i) {
         HeaderFile << my_geom[0].ProbLo(i) << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i < AMREX_SPACEDIM; ++i) {
         HeaderFile << my_geom[0].ProbHi(i) << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i < finest_level; ++i) {
         HeaderFile << my_ref_ratio[i][0] << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i <= finest_level; ++i) {
         HeaderFile << my_geom[i].Domain() << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i <= finest_level; ++i) {
         HeaderFile << level_steps[i] << ' ';
     }
     HeaderFile << '\n';
     for (int i = 0; i <= finest_level; ++i) {
         for (int k = 0; k < AMREX_SPACEDIM; ++k) {
             HeaderFile << my_geom[i].CellSize()[k] << ' ';
         }
         HeaderFile << '\n';
     }
     HeaderFile << (int) my_geom[0].Coord() << '\n';
     HeaderFile << "0\n";
  
     for (int level = 0; level <= finest_level; ++level) {
         HeaderFile << level << ' ' << bArray[level].size() << ' ' << my_time << '\n';
         HeaderFile << level_steps[level] << '\n';
  
         const IntVect& domain_lo = my_geom[level].Domain().smallEnd();
         for (int i = 0; i < bArray[level].size(); ++i)
         {
             // Need to shift because the RealBox ctor we call takes the
             // physical location of index (0,0,0).  This does not affect
             // the usual cases where the domain index starts with 0.
             const Box& b = shift(bArray[level][i], -domain_lo);
             RealBox loc = RealBox(b, my_geom[level].CellSize(), my_geom[level].ProbLo());
             for (int n = 0; n < AMREX_SPACEDIM; ++n) {
                 HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
             }
         }
  
         HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
     }
     HeaderFile << "1" << "\n";
     HeaderFile << "3" << "\n";
     HeaderFile << "amrexvec_nu_x" << "\n";
     HeaderFile << "amrexvec_nu_y" << "\n";
     HeaderFile << "amrexvec_nu_z" << "\n";
     std::string mf_nodal_prefix = "Nu_nd";
     for (int level = 0; level <= finest_level; ++level) {
         HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
     }
 }

Here is the call graph for this function:

◆ writeJobInfo()

void ERF::writeJobInfo ( const std::string & dir ) const

 {
     // job_info file with details about the run
     std::ofstream jobInfoFile;
     std::string FullPathJobInfoFile = dir;
     FullPathJobInfoFile += "/job_info";
     jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out);
  
     std::string PrettyLine = "==================================================="
       "============================\n";
     std::string OtherLine = "----------------------------------------------------"
       "----------------------------\n";
     std::string SkipSpace = "        ";
  
     // job information
     jobInfoFile << PrettyLine;
     jobInfoFile << " ERF Job Information\n";
     jobInfoFile << PrettyLine;
  
     jobInfoFile << "inputs file: " << inputs_name << "\n\n";
  
     jobInfoFile << "number of MPI processes: "
                 << ParallelDescriptor::NProcs() << "\n";
 #ifdef _OPENMP
     jobInfoFile << "number of threads:       " << omp_get_max_threads() << "\n";
 #endif
  
     jobInfoFile << "\n";
     jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
                 << getCPUTime() / 3600.0;
  
     jobInfoFile << "\n\n";
  
     if (use_datetime) {
         const std::string dt_format = "%Y-%m-%d %H:%M:%S"; // ISO 8601 standard
         jobInfoFile << "Simulation time: " << getTimestamp(start_time+t_new[0], dt_format) << "\n";
         jobInfoFile << "\n\n";
     }
  
     // plotfile information
     jobInfoFile << PrettyLine;
     jobInfoFile << " Plotfile Information\n";
     jobInfoFile << PrettyLine;
  
     time_t now = time(nullptr);
  
     // Convert now to tm struct for local timezone
     tm* localtm = localtime(&now);
     jobInfoFile << "output data / time: " << asctime(localtm);
  
     std::string currentDir = FileSystem::CurrentPath();
     jobInfoFile << "output dir:         " << currentDir << "\n";
  
     jobInfoFile << "\n\n";
  
     // build information
     jobInfoFile << PrettyLine;
     jobInfoFile << " Build Information\n";
     jobInfoFile << PrettyLine;
  
     jobInfoFile << "build date:    " << buildInfoGetBuildDate() << "\n";
     jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
     jobInfoFile << "build dir:     " << buildInfoGetBuildDir() << "\n";
     jobInfoFile << "AMReX dir:     " << buildInfoGetAMReXDir() << "\n";
  
     jobInfoFile << "\n";
  
     jobInfoFile << "COMP:          " << buildInfoGetComp() << "\n";
     jobInfoFile << "COMP version:  " << buildInfoGetCompVersion() << "\n";
  
     jobInfoFile << "\n";
  
     for (int n = 1; n <= buildInfoGetNumModules(); n++) {
       jobInfoFile << buildInfoGetModuleName(n) << ": "
                   << buildInfoGetModuleVal(n) << "\n";
     }
  
     jobInfoFile << "\n";
  
     const char* githash1 = buildInfoGetGitHash(1);
     const char* githash2 = buildInfoGetGitHash(2);
     if (strlen(githash1) > 0) {
       jobInfoFile << "ERF       git hash: " << githash1 << "\n";
     }
     if (strlen(githash2) > 0) {
       jobInfoFile << "AMReX       git hash: " << githash2 << "\n";
     }
  
     const char* buildgithash = buildInfoGetBuildGitHash();
     const char* buildgitname = buildInfoGetBuildGitName();
     if (strlen(buildgithash) > 0) {
       jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
     }
  
     jobInfoFile << "\n\n";
  
     // grid information
     jobInfoFile << PrettyLine;
     jobInfoFile << " Grid Information\n";
     jobInfoFile << PrettyLine;
  
     int f_lev = finest_level;
  
     for (int i = 0; i <= f_lev; i++) {
       jobInfoFile << " level: " << i << "\n";
       jobInfoFile << "   number of boxes = " << grids[i].size() << "\n";
       jobInfoFile << "   maximum zones   = ";
       for (int n = 0; n < AMREX_SPACEDIM; n++) {
         jobInfoFile << geom[i].Domain().length(n) << " ";
       }
       jobInfoFile << "\n\n";
     }
  
     jobInfoFile << " Boundary conditions\n";
  
     jobInfoFile << "   -x: " << domain_bc_type[0] << "\n";
     jobInfoFile << "   +x: " << domain_bc_type[3] << "\n";
     jobInfoFile << "   -y: " << domain_bc_type[1] << "\n";
     jobInfoFile << "   +y: " << domain_bc_type[4] << "\n";
     jobInfoFile << "   -z: " << domain_bc_type[2] << "\n";
     jobInfoFile << "   +z: " << domain_bc_type[5] << "\n";
  
     jobInfoFile << "\n\n";
  
     // runtime parameters
     jobInfoFile << PrettyLine;
     jobInfoFile << " Inputs File Parameters\n";
     jobInfoFile << PrettyLine;
  
     ParmParse::dumpTable(jobInfoFile, true);
     jobInfoFile.close();
 }

Here is the call graph for this function:

◆ WriteLinePlot()

void ERF::WriteLinePlot	(	const std::string &	filename,
		amrex::Vector< std::array< amrex::Real, 2 >> &	points_xy
	)

 {
     std::ofstream ofs(filename);
     if (!ofs.is_open()) {
         amrex::Print() << "Error: Could not open file " << filename << " for writing.\n";
         return;
     }
  
     ofs << std::setprecision(10) << std::scientific;
     ofs << "# x y\n";
  
     for (const auto& p : points_xy) {
         ofs << p[0] << " " << p[1] << "\n";
     }
  
     ofs.close();
  
     amrex::Print() << "Line plot data written to " << filename << "\n";
 }

◆ WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain	(	const std::string &	plotfilename,
		int	nlevels,
		const amrex::Vector< const amrex::MultiFab * > &	mf,
		const amrex::Vector< const amrex::MultiFab * > &	mf_nd,
		const amrex::Vector< std::string > &	varnames,
		const amrex::Vector< amrex::Geometry > &	my_geom,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const amrex::Vector< amrex::IntVect > &	my_ref_ratio,
		const std::string &	versionName = `"HyperCLaw-V1.1"`,
		const std::string &	levelPrefix = `"Level_"`,
		const std::string &	mfPrefix = `"Cell"`,
		const amrex::Vector< std::string > &	extra_dirs = `amrex::Vector<std::string>()`
	)		const

 {
     BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
  
     AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
     AMREX_ALWAYS_ASSERT(nlevels <= rr.size()+1);
     AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
     AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
  
     bool callBarrier(false);
     PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
     if (!extra_dirs.empty()) {
         for (const auto& d : extra_dirs) {
             const std::string ed = plotfilename+"/"+d;
             PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
         }
     }
     ParallelDescriptor::Barrier();
  
     if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
         Vector<BoxArray> boxArrays(nlevels);
         for(int level(0); level < boxArrays.size(); ++level) {
             boxArrays[level] = mf[level]->boxArray();
         }
  
         auto f = [=]() {
             VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
             std::string HeaderFileName(plotfilename + "/Header");
             std::ofstream HeaderFile;
             HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
             HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                                     std::ofstream::trunc |
                                                     std::ofstream::binary);
             if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
             WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
                                                   my_geom, time, level_steps, rr, versionName,
                                                   levelPrefix, mfPrefix);
         };
  
         if (AsyncOut::UseAsyncOut()) {
             AsyncOut::Submit(std::move(f));
         } else {
             f();
         }
     }
  
     std::string mf_nodal_prefix = "Nu_nd";
     for (int level = 0; level <= finest_level; ++level)
     {
         if (AsyncOut::UseAsyncOut()) {
             VisMF::AsyncWrite(*mf[level],
                               MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
                               true);
             VisMF::AsyncWrite(*mf_nd[level],
                               MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
                               true);
         } else {
             const MultiFab* data;
             std::unique_ptr<MultiFab> mf_tmp;
             if (mf[level]->nGrowVect() != 0) {
                 mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
                                                     mf[level]->DistributionMap(),
                                                     mf[level]->nComp(), 0, MFInfo(),
                                                     mf[level]->Factory());
                 MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
                 data = mf_tmp.get();
             } else {
                 data = mf[level];
             }
             VisMF::Write(*data        , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
             VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
         }
     }
 }

Here is the call graph for this function:

◆ WriteMyEBSurface()

void ERF::WriteMyEBSurface ( )

◆ writeNow()

bool ERF::writeNow	(	const amrex::Real	cur_time,
		const int	nstep,
		const int	plot_int,
		const amrex::Real	plot_per,
		const amrex::Real	dt_0,
		amrex::Real &	last_file_time
	)

 {
     bool write_now = false;
  
     if ( plot_int > 0) {
  
         write_now = (nstep % plot_int == 0);
  
     } else if (plot_per > 0.0) {
  
         amrex::Print() << "CUR NEXT PER " << cur_time << " " << next_file_time << " " << plot_per << std::endl;
  
         // Only write now if nstep newly matches the number of elapsed periods
         write_now = (cur_time > (next_file_time - Real(0.1)*dt_0));
     }
  
     return write_now;
 }

◆ WriteSubvolume()

void ERF::WriteSubvolume	(	int	isub,
		amrex::Vector< std::string >	subvol_var_names
	)

 {
     ParmParse pp("erf.subvol");
  
     Vector<Real> origin;
     Vector< int> ncell;
     Vector<Real> delta;
  
     // **************************************************************
     // Read in the origin, number of cells in each dir, and resolution
     // **************************************************************
  
     int lev_for_sub = 0;
     int offset = isub * AMREX_SPACEDIM;
  
     pp.getarr("origin",origin,offset,AMREX_SPACEDIM);
     pp.getarr("nxnynz", ncell,offset,AMREX_SPACEDIM);
     pp.getarr("dxdydz", delta,offset,AMREX_SPACEDIM);
  
     bool found = false;
     for (int i = 0; i <= finest_level; i++) {
         if (!found) {
             if (almostEqual(delta[offset+0],geom[i].CellSize(0)) &&
                 almostEqual(delta[offset+1],geom[i].CellSize(1)) &&
                 almostEqual(delta[offset+2],geom[i].CellSize(2)) ) {
  
                 amrex::Print() << "WriteSubvolume:Resolution specified matches that of level " << i << std::endl;
                 found = true;
                 lev_for_sub = i;
             }
         }
     }
  
     if (!found) {
         amrex::Abort("Resolution specified for subvol does not match the resolution of any of the levels.");
     }
  
  
     // **************************************************************
     // Now that we know which level we're at, we can figure out which (i,j,k) the origin corresponds to
     // Note we use 1.0001 as a fudge factor since the division of two reals --> integer will do a floor
     // **************************************************************
     int i0 = static_cast<int>((origin[offset+0] - geom[lev_for_sub].ProbLo(0)) * 1.0001 / delta[offset+0]);
     int j0 = static_cast<int>((origin[offset+1] - geom[lev_for_sub].ProbLo(1)) * 1.0001 / delta[offset+1]);
     int k0 = static_cast<int>((origin[offset+2] - geom[lev_for_sub].ProbLo(2)) * 1.0001 / delta[offset+2]);
  
     found = false;
     if (almostEqual(geom[lev_for_sub].ProbLo(0)+i0*delta[offset+0],origin[offset+0]) &&
         almostEqual(geom[lev_for_sub].ProbLo(1)+j0*delta[offset+1],origin[offset+1]) &&
         almostEqual(geom[lev_for_sub].ProbLo(2)+k0*delta[offset+2],origin[offset+2]) )
     {
         amrex::Print() << "WriteSubvolume:Specified origin is the lower left corner of cell " << IntVect(i0,j0,k0) << std::endl;
         found = true;
     }
  
     if (!found) {
         amrex::Abort("Origin specified does not correspond to a node at this level.");
     }
  
     Box domain(geom[lev_for_sub].Domain());
  
     Box bx(IntVect(i0,j0,k0),IntVect(i0+ncell[offset+0]-1,j0+ncell[offset+1]-1,k0+ncell[offset+2]-1));
     amrex::Print() << "WriteSubvolume:Box requested is " << bx << std::endl;
  
     if (!domain.contains(bx))
     {
         amrex::Abort("WriteSubvolume:Box requested is larger than the existing domain");
     }
  
     Vector<int> cs(AMREX_SPACEDIM);
     int count = pp.countval("chunk_size");
     if (count > 0) {
         pp.queryarr("chunk_size",cs,0,AMREX_SPACEDIM);
     } else {
         cs[0] = max_grid_size[0][0];
         cs[1] = max_grid_size[0][1];
         cs[2] = max_grid_size[0][2];
     }
     IntVect chunk_size(cs[0],cs[1],cs[2]);
  
     BoxArray ba(bx);
     ba.maxSize(chunk_size);
  
     amrex::Print() << "WriteSubvolume:BoxArray is " << ba << std::endl;
  
     Vector<std::string> varnames;
     varnames.insert(varnames.end(), subvol_var_names.begin(), subvol_var_names.end());
  
     int ncomp_mf = subvol_var_names.size();
  
     DistributionMapping dm(ba);
  
     MultiFab mf(ba, dm, ncomp_mf, 0);
  
     int mf_comp = 0;
  
     // *****************************************************************************************
  
     // First, copy any of the conserved state variables into the output plotfile
     for (int i = 0; i < cons_names.size(); ++i) {
         if (containerHasElement(subvol_var_names, cons_names[i])) {
             mf.ParallelCopy(vars_new[lev_for_sub][Vars::cons],i,mf_comp,1,1,0);
             mf_comp++;
         }
     }
  
     // *****************************************************************************************
  
     if (containerHasElement(subvol_var_names, "x_velocity") ||
         containerHasElement(subvol_var_names, "y_velocity") ||
         containerHasElement(subvol_var_names, "z_velocity"))
     {
         MultiFab mf_cc_vel(grids[lev_for_sub], dmap[lev_for_sub], AMREX_SPACEDIM, 0);
         average_face_to_cellcenter(mf_cc_vel,0,
                                    Array<const MultiFab*,3>{&vars_new[lev_for_sub][Vars::xvel],
                                                             &vars_new[lev_for_sub][Vars::yvel],
                                                             &vars_new[lev_for_sub][Vars::zvel]});
         if (containerHasElement(subvol_var_names, "x_velocity")) {
             mf.ParallelCopy(mf_cc_vel,0,mf_comp,1,0,0);
             mf_comp++;
         }
         if (containerHasElement(subvol_var_names, "y_velocity")) {
             mf.ParallelCopy(mf_cc_vel,1,mf_comp,1,0,0);
             mf_comp++;
         }
         if (containerHasElement(subvol_var_names, "z_velocity")) {
             mf.ParallelCopy(mf_cc_vel,2,mf_comp,1,0,0);
             mf_comp++;
         }
     }
  
     // *****************************************************************************************
  
     // Finally, check for any derived quantities and compute them, inserting
     // them into our output multifab
     auto calculate_derived = [&](const std::string& der_name,
                                  MultiFab& src_mf,
                                  decltype(derived::erf_dernull)& der_function)
     {
         if (containerHasElement(subvol_var_names, der_name)) {
             MultiFab dmf(src_mf.boxArray(), src_mf.DistributionMap(), 1, 0);
 #ifdef _OPENMP
 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
 #endif
             for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
             {
                 const Box& tbx = mfi.tilebox();
                 auto& dfab = dmf[mfi];
                 auto& sfab = src_mf[mfi];
                 der_function(tbx, dfab, 0, 1, sfab, Geom(lev_for_sub), t_new[0], nullptr, lev_for_sub);
             }
             mf.ParallelCopy(dmf,0,mf_comp,1,0,0);
             mf_comp++;
         }
     };
  
     // *****************************************************************************************
     // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
     //       defined in ERF.H
     // *****************************************************************************************
  
     calculate_derived("soundspeed",  vars_new[lev_for_sub][Vars::cons], derived::erf_dersoundspeed);
     if (solverChoice.moisture_type != MoistureType::None) {
         calculate_derived("temp",    vars_new[lev_for_sub][Vars::cons], derived::erf_dermoisttemp);
     } else {
         calculate_derived("temp",    vars_new[lev_for_sub][Vars::cons], derived::erf_dertemp);
     }
     calculate_derived("theta",       vars_new[lev_for_sub][Vars::cons], derived::erf_dertheta);
     calculate_derived("KE",          vars_new[lev_for_sub][Vars::cons], derived::erf_derKE);
     calculate_derived("scalar",      vars_new[lev_for_sub][Vars::cons], derived::erf_derscalar);
  
     // *****************************************************************************************
  
     Real time = t_new[lev_for_sub];
  
     std::string sf = subvol_file + "_" + std::to_string(isub);
     std::string subvol_filename;
  
     if (use_real_time_in_pltname) {
         const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
         subvol_filename = sf + getTimestamp(start_time+time, dt_format);
     } else {
        subvol_filename = Concatenate(sf + "_", istep[0], file_name_digits);
     }
  
     amrex::Print() <<"Writing subvolume into " << subvol_filename << std::endl;
     WriteSingleLevelPlotfile(subvol_filename,mf,varnames,geom[lev_for_sub],time,istep[0]);
  
 }

Here is the call graph for this function:

◆ WriteVTKPolyline()

void ERF::WriteVTKPolyline	(	const std::string &	filename,
		amrex::Vector< std::array< amrex::Real, 2 >> &	points_xy
	)

 {
     std::ofstream vtkfile(filename);
     if (!vtkfile.is_open()) {
         std::cerr << "Error: Cannot open file " << filename << std::endl;
         return;
     }
  
     int num_points = points_xy.size();
     if (num_points == 0) {
         vtkfile << "# vtk DataFile Version 3.0\n";
         vtkfile << "Storm Track\n";
         vtkfile << "ASCII\n";
         vtkfile << "DATASET POLYDATA\n";
         vtkfile << "POINTS " << num_points << " float\n";
         vtkfile.close();
         return;
     }
     if (num_points < 2) {
         points_xy.push_back(points_xy[0]);
     }
     num_points = points_xy.size();
  
     vtkfile << "# vtk DataFile Version 3.0\n";
     vtkfile << "Storm Track\n";
     vtkfile << "ASCII\n";
     vtkfile << "DATASET POLYDATA\n";
  
     // Write points (Z=0 assumed)
     vtkfile << "POINTS " << num_points << " float\n";
     for (const auto& pt : points_xy) {
         vtkfile << pt[0] << " " << pt[1] << " 10000.0\n";
     }
  
     // Write polyline connectivity
     vtkfile << "LINES 1 " << num_points + 1 << "\n";
     vtkfile << num_points << " ";
     for (int i = 0; i < num_points; ++i) {
         vtkfile << i << " ";
     }
     vtkfile << "\n";
  
     vtkfile.close();
 }

Member Data Documentation

◆ advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg

private

Referenced by getAdvFluxReg().

◆ avg_xmom

amrex::Vector<amrex::MultiFab> ERF::avg_xmom

private

◆ avg_ymom

amrex::Vector<amrex::MultiFab> ERF::avg_ymom

private

◆ avg_zmom

amrex::Vector<amrex::MultiFab> ERF::avg_zmom

private

◆ ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax

private

◆ ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src

private

◆ ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay

private

◆ ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src

private

◆ az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az

private

◆ az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src

private

◆ ba1d

amrex::Vector<amrex::BoxArray> ERF::ba1d

private

◆ ba2d

amrex::Vector<amrex::BoxArray> ERF::ba2d

private

◆ base_state

amrex::Vector<amrex::MultiFab> ERF::base_state

private

◆ base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new

private

◆ bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1

staticprivate

◆ bndry_output_planes_per

Real ERF::bndry_output_planes_per = -1.0

staticprivate

◆ bndry_output_planes_start_time

Real ERF::bndry_output_planes_start_time = 0.0

staticprivate

◆ boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level

private

◆ cf_set_width

int ERF::cf_set_width {0}

private

◆ cf_width

int ERF::cf_width {0}

private

◆ cfl

Real ERF::cfl = 0.8

staticprivate

◆ change_max

Real ERF::change_max = 1.1

staticprivate

◆ check_file

std::string ERF::check_file {"chk"}

private

◆ check_for_nans

int ERF::check_for_nans = 0

staticprivate

◆ column_file_name

std::string ERF::column_file_name = "column_data.nc"

staticprivate

◆ column_interval

int ERF::column_interval = -1

staticprivate

◆ column_loc_x

Real ERF::column_loc_x = 0.0

staticprivate

◆ column_loc_y

Real ERF::column_loc_y = 0.0

staticprivate

◆ column_per

Real ERF::column_per = -1.0

staticprivate

◆ cons_names

const amrex::Vector<std::string> ERF::cons_names

private

Initial value:

{"density", "rhotheta", "rhoKE", "rhoadv_0",
                                                     "rhoQ1", "rhoQ2", "rhoQ3",
                                                     "rhoQ4", "rhoQ5", "rhoQ6"}

◆ cosPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::cosPhi_m

private

◆ d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density

private

◆ d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure

private

◆ d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc

private

◆ d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv

private

◆ d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature

private

◆ d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs

private

◆ d_sinesq_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_ptrs

private

◆ d_sinesq_stag_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_stag_ptrs

private

◆ d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs

private

◆ d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos

private

◆ d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos

private

◆ d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid

private

◆ datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog

private

Referenced by DataLog(), NumDataLogs(), and setRecordDataInfo().

◆ datalogname

amrex::Vector<std::string> ERF::datalogname

private

Referenced by DataLogName().

◆ datetime_format

const std::string ERF::datetime_format = "%Y-%m-%d %H:%M:%S"

private

◆ datprecision

const int ERF::datprecision = 6

private

◆ datwidth

const int ERF::datwidth = 14

private

◆ der_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::der_datalog

private

Referenced by DerDataLog(), NumDerDataLogs(), and setRecordDerDataInfo().

◆ der_datalogname

amrex::Vector<std::string> ERF::der_datalogname

private

Referenced by DerDataLogName().

◆ derived_names

const amrex::Vector<std::string> ERF::derived_names

private

◆ derived_names_2d

const amrex::Vector<std::string> ERF::derived_names_2d

private

Initial value:

{
        "z_surf", "landmask", "mapfac", "lat_m", "lon_m",
        "u_star", "w_star", "t_star", "q_star", "Olen", "pblh",
        "t_surf", "q_surf", "z0", "OLR", "sens_flux", "laten_flux",
        "surf_pres", "integrated_qv"
    }

◆ derived_subvol_names

const amrex::Vector<std::string> ERF::derived_subvol_names {"soundspeed", "temp", "theta", "KE", "scalar"}

private

◆ destag_profiles

bool ERF::destag_profiles = true

private

◆ detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc

private

◆ detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new

private

◆ detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src

private

◆ domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type

private

◆ domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type

private

◆ domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d

private

◆ dt

amrex::Vector<amrex::Real> ERF::dt

private

◆ dt_max

Real ERF::dt_max = 1.0e9

staticprivate

◆ dt_max_initial

Real ERF::dt_max_initial = 2.0e100

staticprivate

◆ dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio

private

◆ dz_min

amrex::Vector<amrex::Real> ERF::dz_min

private

◆ eb

amrex::Vector<std::unique_ptr<eb_> > ERF::eb

private

Referenced by EBFactory(), and get_eb().

◆ eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev

private

◆ file_name_digits

int ERF::file_name_digits = 5

private

◆ fine_mask

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::fine_mask

private

◆ finished_wave

bool ERF::finished_wave = false

private

◆ fixed_dt

amrex::Vector<amrex::Real> ERF::fixed_dt

private

◆ fixed_fast_dt

amrex::Vector<amrex::Real> ERF::fixed_fast_dt

private

◆ fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0

staticprivate

◆ forecast_state_1

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_1

◆ forecast_state_2

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_2

◆ forecast_state_interp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_interp

◆ FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c

private

◆ FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u

private

◆ FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v

private

◆ FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w

private

◆ gradp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::gradp

private

◆ h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density

private

◆ h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure

private

◆ h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc

private

◆ h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv

private

◆ h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature

private

◆ h_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs

private

◆ h_sinesq_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_ptrs

private

◆ h_sinesq_stag_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_stag_ptrs

private

◆ h_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs

private

◆ h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos

private

◆ h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos

private

◆ h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid

private

◆ have_read_nc_init_file

Vector< Vector< int > > ERF::have_read_nc_init_file = {{0}}

staticprivate

◆ hurricane_eye_track_latlon

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_latlon

◆ hurricane_eye_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_xy

◆ hurricane_maxvel_vs_time

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_maxvel_vs_time

◆ hurricane_minpressure_vs_time

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_minpressure_vs_time

◆ hurricane_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_track_xy

◆ hurricane_tracker_circle

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_tracker_circle

◆ Hwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave

private

◆ Hwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave_onegrid

private

◆ init_shrink

Real ERF::init_shrink = 1.0

staticprivate

◆ input_bndry_planes

int ERF::input_bndry_planes = 0

staticprivate

◆ input_sounding_data

InputSoundingData ERF::input_sounding_data

private

◆ input_sponge_data

InputSpongeData ERF::input_sponge_data

private

◆ interpolation_type

StateInterpType ERF::interpolation_type

staticprivate

◆ istep

amrex::Vector<int> ERF::istep

private

◆ lagged_delta_rt

amrex::Vector<amrex::MultiFab> ERF::lagged_delta_rt

private

◆ land_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::land_type_lev

private

◆ last_check_file_step

int ERF::last_check_file_step = -1

staticprivate

◆ last_check_file_time

Real ERF::last_check_file_time = 0.0

staticprivate

◆ last_plot2d_file_step_1

int ERF::last_plot2d_file_step_1 = -1

staticprivate

◆ last_plot2d_file_step_2

int ERF::last_plot2d_file_step_2 = -1

staticprivate

◆ last_plot2d_file_time_1

Real ERF::last_plot2d_file_time_1 = 0.0

staticprivate

◆ last_plot2d_file_time_2

Real ERF::last_plot2d_file_time_2 = 0.0

staticprivate

◆ last_plot3d_file_step_1

int ERF::last_plot3d_file_step_1 = -1

staticprivate

◆ last_plot3d_file_step_2

int ERF::last_plot3d_file_step_2 = -1

staticprivate

◆ last_plot3d_file_time_1

Real ERF::last_plot3d_file_time_1 = 0.0

staticprivate

◆ last_plot3d_file_time_2

Real ERF::last_plot3d_file_time_2 = 0.0

staticprivate

◆ last_subvol_step

amrex::Vector<int> ERF::last_subvol_step

private

◆ last_subvol_time

amrex::Vector<amrex::Real> ERF::last_subvol_time

private

◆ lat_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lat_m

private

◆ line_sampler

std::unique_ptr<LineSampler> ERF::line_sampler = nullptr

private

◆ line_sampling_interval

int ERF::line_sampling_interval = -1

private

◆ line_sampling_per

amrex::Real ERF::line_sampling_per = -1.0

private

◆ lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev

private

◆ lon_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lon_m

private

◆ lsm

LandSurface ERF::lsm

private

◆ lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data

private

◆ lsm_data_name

amrex::Vector<std::string> ERF::lsm_data_name

private

◆ lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux

private

◆ lsm_flux_name

amrex::Vector<std::string> ERF::lsm_flux_name

private

◆ Lwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave

private

◆ Lwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave_onegrid

private

◆ m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_extdir_vals

private

◆ m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_neumann_vals

private

◆ m_check_int

int ERF::m_check_int = -1

private

◆ m_check_per

amrex::Real ERF::m_check_per = -1.0

private

◆ m_expand_plotvars_to_unif_rr

bool ERF::m_expand_plotvars_to_unif_rr = false

private

◆ m_forest_drag

amrex::Vector<std::unique_ptr<ForestDrag> > ERF::m_forest_drag

private

◆ m_plot2d_int_1

int ERF::m_plot2d_int_1 = -1

private

◆ m_plot2d_int_2

int ERF::m_plot2d_int_2 = -1

private

◆ m_plot2d_per_1

amrex::Real ERF::m_plot2d_per_1 = -1.0

private

◆ m_plot2d_per_2

amrex::Real ERF::m_plot2d_per_2 = -1.0

private

◆ m_plot3d_int_1

int ERF::m_plot3d_int_1 = -1

private

◆ m_plot3d_int_2

int ERF::m_plot3d_int_2 = -1

private

◆ m_plot3d_per_1

amrex::Real ERF::m_plot3d_per_1 = -1.0

private

◆ m_plot3d_per_2

amrex::Real ERF::m_plot3d_per_2 = -1.0

private

◆ m_plot_face_vels

bool ERF::m_plot_face_vels = false

private

◆ m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr

private

◆ m_subvol_int

amrex::Vector<int> ERF::m_subvol_int

private

◆ m_subvol_per

amrex::Vector<amrex::Real> ERF::m_subvol_per

private

◆ m_SurfaceLayer

std::unique_ptr<SurfaceLayer> ERF::m_SurfaceLayer = nullptr

private

◆ m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr

private

◆ mapfac

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::mapfac

private

◆ max_step

int ERF::max_step = -1

private

◆ metgrid_basic_linear

bool ERF::metgrid_basic_linear {false}

private

◆ metgrid_debug_dry

bool ERF::metgrid_debug_dry {false}

private

◆ metgrid_debug_isothermal

bool ERF::metgrid_debug_isothermal {false}

private

◆ metgrid_debug_msf

bool ERF::metgrid_debug_msf {false}

private

◆ metgrid_debug_psfc

bool ERF::metgrid_debug_psfc {false}

private

◆ metgrid_debug_quiescent

bool ERF::metgrid_debug_quiescent {false}

private

◆ metgrid_force_sfc_k

int ERF::metgrid_force_sfc_k {6}

private

◆ metgrid_interp_theta

bool ERF::metgrid_interp_theta {false}

private

◆ metgrid_order

int ERF::metgrid_order {2}

private

◆ metgrid_proximity

amrex::Real ERF::metgrid_proximity {500.0}

private

◆ metgrid_retain_sfc

bool ERF::metgrid_retain_sfc {false}

private

◆ metgrid_use_below_sfc

bool ERF::metgrid_use_below_sfc {true}

private

◆ metgrid_use_sfc

bool ERF::metgrid_use_sfc {true}

private

◆ mf_C1H

std::unique_ptr<amrex::MultiFab> ERF::mf_C1H

private

◆ mf_C2H

std::unique_ptr<amrex::MultiFab> ERF::mf_C2H

private

◆ mf_MUB

std::unique_ptr<amrex::MultiFab> ERF::mf_MUB

private

◆ mf_PSFC

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_PSFC

private

◆ mg_verbose

int ERF::mg_verbose = 0

staticprivate

◆ micro

std::unique_ptr<Microphysics> ERF::micro

private

◆ mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem

private

◆ nc_bdy_file

std::string ERF::nc_bdy_file

staticprivate

◆ nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}

staticprivate

◆ nc_low_file

std::string ERF::nc_low_file

staticprivate

◆ ng_dens_hse

int ERF::ng_dens_hse

staticprivate

◆ ng_pres_hse

int ERF::ng_pres_hse

staticprivate

◆ nsubsteps

amrex::Vector<int> ERF::nsubsteps

private

◆ num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level

private

◆ num_files_at_level

amrex::Vector<int> ERF::num_files_at_level

private

◆ output_1d_column

int ERF::output_1d_column = 0

staticprivate

◆ output_bndry_planes

int ERF::output_bndry_planes = 0

staticprivate

◆ pert_interval

int ERF::pert_interval = -1

staticprivate

◆ phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type

private

◆ physbcs_base

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_base> > ERF::physbcs_base

private

◆ physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons

private

◆ physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u

private

◆ physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v

private

◆ physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w

private

◆ plane_sampler

std::unique_ptr<PlaneSampler> ERF::plane_sampler = nullptr

private

◆ plane_sampling_interval

int ERF::plane_sampling_interval = -1

private

◆ plane_sampling_per

amrex::Real ERF::plane_sampling_per = -1.0

private

◆ plot2d_file_1

std::string ERF::plot2d_file_1 {"plt2d_1_"}

private

◆ plot2d_file_2

std::string ERF::plot2d_file_2 {"plt2d_2_"}

private

◆ plot2d_var_names_1

amrex::Vector<std::string> ERF::plot2d_var_names_1

private

◆ plot2d_var_names_2

amrex::Vector<std::string> ERF::plot2d_var_names_2

private

◆ plot3d_file_1

std::string ERF::plot3d_file_1 {"plt_1_"}

private

◆ plot3d_file_2

std::string ERF::plot3d_file_2 {"plt_2_"}

private

◆ plot3d_var_names_1

amrex::Vector<std::string> ERF::plot3d_var_names_1

private

◆ plot3d_var_names_2

amrex::Vector<std::string> ERF::plot3d_var_names_2

private

◆ plot_file_on_restart

bool ERF::plot_file_on_restart = true

staticprivate

◆ plot_lsm

bool ERF::plot_lsm = false

private

◆ plot_rad

bool ERF::plot_rad = false

private

◆ plotfile2d_type_1

PlotFileType ERF::plotfile2d_type_1 = PlotFileType::None

staticprivate

◆ plotfile2d_type_2

PlotFileType ERF::plotfile2d_type_2 = PlotFileType::None

staticprivate

◆ plotfile3d_type_1

PlotFileType ERF::plotfile3d_type_1 = PlotFileType::None

staticprivate

◆ plotfile3d_type_2

PlotFileType ERF::plotfile3d_type_2 = PlotFileType::None

staticprivate

◆ pp_inc

amrex::Vector<amrex::MultiFab> ERF::pp_inc

private

◆ pp_prefix

std::string ERF::pp_prefix {"erf"}

◆ previousCPUTimeUsed

Real ERF::previousCPUTimeUsed = 0.0

staticprivate

Referenced by getCPUTime().

◆ prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr

private

◆ profile_int

int ERF::profile_int = -1

private

◆ qheating_rates

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::qheating_rates

private

◆ qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist

private

◆ Qr_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qr_prim

private

◆ Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim

private

◆ rad

amrex::Vector<std::unique_ptr<IRadiation> > ERF::rad

private

◆ rad_datalog_int

int ERF::rad_datalog_int = -1

private

◆ rad_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rad_fluxes

private

◆ real_extrap_w

bool ERF::real_extrap_w {true}

private

◆ real_width

int ERF::real_width {0}

private

◆ ref_tags

Vector< AMRErrorTag > ERF::ref_tags

staticprivate

◆ regrid_int

int ERF::regrid_int = -1

private

◆ regrid_level_0_on_restart

bool ERF::regrid_level_0_on_restart = false

private

◆ restart_chkfile

std::string ERF::restart_chkfile = ""

private

◆ rhoqt_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rhoqt_src

private

◆ rhotheta_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rhotheta_src

private

◆ rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new

private

◆ rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old

private

◆ rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new

private

◆ rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old

private

◆ rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new

private

◆ rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old

private

◆ sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline

private

Referenced by NumSampleLines(), and SampleLine().

◆ samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog

private

Referenced by NumSampleLineLogs(), SampleLineLog(), and setRecordSampleLineInfo().

◆ samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname

private

Referenced by SampleLineLogName().

◆ samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint

private

Referenced by NumSamplePoints(), and SamplePoint().

◆ sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog

private

Referenced by NumSamplePointLogs(), SamplePointLog(), and setRecordSamplePointInfo().

◆ sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname

private

Referenced by SamplePointLogName().

◆ SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev

private

◆ SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev

private

◆ SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev

private

◆ SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev

private

◆ SFS_q1fx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx1_lev

private

◆ SFS_q1fx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx2_lev

private

◆ SFS_q1fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx3_lev

private

◆ SFS_q2fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q2fx3_lev

private

◆ sinPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sinPhi_m

private

◆ SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev

private

◆ soil_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::soil_type_lev

private

◆ solar_zenith

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::solar_zenith

private

◆ solverChoice

SolverChoice ERF::solverChoice

staticprivate

◆ sponge_type

std::string ERF::sponge_type

staticprivate

◆ sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev

private

◆ start_time

Real ERF::start_time = 0.0

staticprivate

◆ startCPUTime

Real ERF::startCPUTime = 0.0

staticprivate

Referenced by getCPUTime().

◆ stop_time

Real ERF::stop_time = std::numeric_limits<amrex::Real>::max()

staticprivate

◆ stretched_dz_d

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::stretched_dz_d

private

◆ stretched_dz_h

amrex::Vector<amrex::Vector<amrex::Real> > ERF::stretched_dz_h

private

◆ sub_cfl

Real ERF::sub_cfl = 1.0

staticprivate

◆ subdomains

amrex::Vector<amrex::Vector<amrex::BoxArray> > ERF::subdomains

private

◆ subvol3d_var_names

amrex::Vector<std::string> ERF::subvol3d_var_names

private

◆ subvol_file

std::string ERF::subvol_file {"subvol"}

private

◆ sum_interval

int ERF::sum_interval = -1

staticprivate

◆ sum_per

Real ERF::sum_per = -1.0

staticprivate

◆ surface_state_1

amrex::Vector<amrex::MultiFab> ERF::surface_state_1

◆ surface_state_2

amrex::Vector<amrex::MultiFab> ERF::surface_state_2

◆ surface_state_interp

amrex::Vector<amrex::MultiFab> ERF::surface_state_interp

◆ sw_lw_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sw_lw_fluxes

private

◆ t_avg_cnt

amrex::Vector<amrex::Real> ERF::t_avg_cnt

private

◆ t_new

amrex::Vector<amrex::Real> ERF::t_new

private

◆ t_old

amrex::Vector<amrex::Real> ERF::t_old

private

◆ Tau

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau

private

◆ Tau_corr

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau_corr

private

◆ terrain_blanking

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking

private

◆ th_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::th_bc_data

private

◆ Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim

private

◆ thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce

private

◆ thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce

private

◆ thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce

private

◆ timeprecision

const int ERF::timeprecision = 13

private

◆ tot_e_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::tot_e_datalog

private

Referenced by setRecordEnergyDataInfo().

◆ tot_e_datalogname

amrex::Vector<std::string> ERF::tot_e_datalogname

private

◆ tsk_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::tsk_lev

private

◆ turbPert

TurbulentPerturbation ERF::turbPert

private

◆ urb_frac_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::urb_frac_lev

private

◆ use_datetime

bool ERF::use_datetime = false

private

◆ use_fft

bool ERF::use_fft = false

staticprivate

◆ use_real_time_in_pltname

bool ERF::use_real_time_in_pltname = false

private

◆ vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new

private

◆ vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old

private

◆ vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg

private

◆ verbose

int ERF::verbose = 0

staticprivate

◆ walldist

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::walldist

private

◆ weather_forecast_data_1

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_1

◆ weather_forecast_data_2

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_2

◆ xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask

private

◆ xvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::xvel_bc_data

private

◆ yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask

private

◆ yvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::yvel_bc_data

private

◆ z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc

private

◆ z_phys_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc_src

private

◆ z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd

private

◆ z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new

private

◆ z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src

private

◆ z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk

private

◆ zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask

private

◆ zlevels_stag

amrex::Vector<amrex::Vector<amrex::Real> > ERF::zlevels_stag

private

◆ zmom_crse_rhs

amrex::Vector<amrex::MultiFab> ERF::zmom_crse_rhs

private

◆ zvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::zvel_bc_data

private

The documentation for this class was generated from the following files:

Source/ERF.H
Source/BoundaryConditions/ERF_BoundaryConditionsBndryReg.cpp
Source/BoundaryConditions/ERF_FillBdyCCVels.cpp
Source/BoundaryConditions/ERF_FillCoarsePatch.cpp
Source/BoundaryConditions/ERF_FillIntermediatePatch.cpp
Source/BoundaryConditions/ERF_FillPatch.cpp
Source/ERF.cpp
Source/ERF_MakeNewArrays.cpp
Source/ERF_MakeNewLevel.cpp
Source/ERF_Tagging.cpp
Source/Initialization/ERF_Init1D.cpp
Source/Initialization/ERF_InitBCs.cpp
Source/Initialization/ERF_InitCustomPertState.cpp
Source/Initialization/ERF_InitFromHSE.cpp
Source/Initialization/ERF_InitFromInputSounding.cpp
Source/Initialization/ERF_InitGeowind.cpp
Source/Initialization/ERF_InitImmersedForcing.cpp
Source/Initialization/ERF_InitRayleigh.cpp
Source/Initialization/ERF_InitSponge.cpp
Source/Initialization/ERF_InitTurbPert.cpp
Source/IO/ERF_Checkpoint.cpp
Source/IO/ERF_ConsoleIO.cpp
Source/IO/ERF_Plotfile.cpp
Source/IO/ERF_TrackerOutput.cpp
Source/IO/ERF_Write1DProfiles.cpp
Source/IO/ERF_Write1DProfiles_stag.cpp
Source/IO/ERF_WriteJobInfo.cpp
Source/IO/ERF_WriteScalarProfiles.cpp
Source/IO/ERF_WriteSubvolume.cpp
Source/LinearSolvers/ERF_ComputeDivergence.cpp
Source/LinearSolvers/ERF_ImposeBCsOnPhi.cpp
Source/LinearSolvers/ERF_PoissonSolve.cpp
Source/LinearSolvers/ERF_PoissonSolve_tb.cpp
Source/LinearSolvers/ERF_PoissonWallDist.cpp
Source/LinearSolvers/ERF_SolveWithGMRES.cpp
Source/TimeIntegration/ERF_Advance.cpp
Source/TimeIntegration/ERF_AdvanceDycore.cpp
Source/TimeIntegration/ERF_AdvanceLSM.cpp
Source/TimeIntegration/ERF_AdvanceMicrophysics.cpp
Source/TimeIntegration/ERF_AdvanceRadiation.cpp
Source/TimeIntegration/ERF_ComputeTimestep.cpp
Source/TimeIntegration/ERF_TimeStep.cpp
Source/Utils/ERF_AverageDown.cpp
Source/Utils/ERF_MakeSubdomains.cpp
Source/Utils/ERF_SurfaceDataInterpolation.cpp
Source/Utils/ERF_VolWgtSum.cpp
Source/Utils/ERF_WeatherDataInterpolation.cpp

Public Member Functions

Static Public Member Functions

Public Attributes

Private Member Functions

Static Private Member Functions

Private Attributes

Static Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ ERF() [1/3]

◆ ~ERF()

◆ ERF() [2/3]

◆ ERF() [3/3]

Member Function Documentation

◆ Advance()

◆ advance_dycore()

◆ advance_lsm()

◆ advance_microphysics()

◆ advance_radiation()

◆ appendPlotVariables()

◆ apply_gaussian_smoothing_to_perturbations()

◆ AverageDown()

◆ AverageDownTo()

◆ build_fine_mask()

◆ check_for_low_temp()

◆ check_for_negative_theta()

◆ check_state_for_nans()

◆ check_vels_for_nans()

◆ ClearLevel()

◆ cloud_fraction()

◆ compute_divergence()

◆ ComputeDt()

◆ ComputeGhostCells()

◆ Construct_ERFFillPatchers()

◆ create_random_perturbations()

◆ DataLog()

◆ DataLogName()

◆ Define_ERFFillPatchers()

◆ DerDataLog()

◆ DerDataLogName()

◆ derive_diag_profiles()

◆ derive_diag_profiles_stag()

◆ derive_stress_profiles()

◆ derive_stress_profiles_stag()

◆ derive_upwp()

◆ EBFactory()

◆ erf_enforce_hse()

◆ ERF_shared()

◆ ErrorEst()

◆ estTimeStep()

◆ Evolve()

◆ fill_from_bndryregs()

◆ fill_rhs()

◆ FillBdyCCVels()

◆ FillCoarsePatch()

◆ FillForecastStateMultiFabs()

◆ FillIntermediatePatch()

◆ FillPatchCrseLevel()

◆ FillPatchFineLevel()

◆ FillSurfaceStateMultiFabs()

◆ FindInitialEye()

◆ get_eb()

◆ getAdvFluxReg()

◆ getCPUTime()

◆ GotoNextLine()

◆ HurricaneTracker()

◆ ImposeBCsOnPhi()

◆ init1DArrays()

◆ init_bcs()

◆ init_custom()

◆ init_Dirichlet_bc_data()

◆ init_from_hse()

◆ init_from_input_sounding()

◆ init_geo_wind_profile()

◆ init_immersed_forcing()

◆ init_only()

◆ init_phys_bcs()

◆ init_stuff()

◆ init_thin_body()

◆ init_zphys()