ERF
Energy Research and Forecasting: An Atmospheric Modeling Code
Public Member Functions | Static Public Member Functions | Public Attributes | Private Member Functions | Static Private Member Functions | Private Attributes | Static Private Attributes | List of all members
ERF Class Reference

#include <ERF.H>

Inheritance diagram for ERF:
Inheritance graph
[legend]
Collaboration diagram for ERF:
Collaboration graph
[legend]

Public Member Functions

 ERF ()
 
 ~ERF () override
 
void ERF_shared ()
 
 ERF (ERF &&) noexcept=delete
 
ERF & operator= (ERF &&other) noexcept=delete
 
 ERF (const ERF &other)=delete
 
ERF & operator= (const ERF &other)=delete
 
void Evolve ()
 
double EvolveOneStep (double time, double dt_request)
 
void ConfigureDriverAtmosToOceanCoupling (bool use_coupling_driver, bool use_two_way_coupling, bool use_state_contract)
 
void SetDriverAtmosToOceanStateContract (bool use_state_contract)
 
void GetOceanToAtmosSurfaceLayout (amrex::BoxArray &ba, amrex::DistributionMapping &dm)
 
void PackAtmosphericStates (amrex::Vector< amrex::MultiFab * > &states, double time)
 
void ApplyOceanSurfaceState (const amrex::Vector< amrex::MultiFab * > &state, double time)
 
void ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override
 
void read_box_for_refinement (std::string &ref_prefix, int &lev_for_box, amrex::RealBox &real_box)
 
void update_box_for_refinement (std::string &ref_prefix, int &lev_for_box, amrex::RealBox &real_box, const double time)
 
void HurricaneTracker (int lev, double time, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::TagBoxArray *tags=nullptr)
 
bool FindInitialEye (int lev, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::Real &eye_x, amrex::Real &eye_y)
 
std::string MakeVTKFilename (int nstep)
 
std::string MakeVTKFilename_TrackerCircle (int nstep)
 
std::string MakeVTKFilename_EyeTracker_xy (int nstep)
 
std::string MakeFilename_EyeTracker_latlon (int nstep)
 
std::string MakeFilename_EyeTracker_maxvel (int nstep)
 
std::string MakeFilename_EyeTracker_minpressure (int nstep)
 
void WriteVTKPolyline (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
 
void WriteLinePlot (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
 
void HurricaneEyeTracker (const SolverChoice &solverChoice)
 
void HurricaneEyeTrackerInitial (const SolverChoice &sc, const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, const amrex::Real &hurricane_eye_latitude, const amrex::Real &hurricane_eye_longitude)
 
void HurricaneEyeTrackerNotInitial (const SolverChoice &sc, const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, MoistureType moisture_type)
 
void ComputeGlobalMinLocation (const SolverChoice &sc, const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, amrex::Real *d_val_min_ptr, int *d_i_min_ptr, int *d_j_min_ptr, amrex::Real &global_val_min, int &global_i_min, int &global_j_min)
 
void HurricaneMaxVelTracker (const amrex::Geometry &geom, const amrex::MultiFab &mf_cc_vel, const double &time)
 
void HurricaneMinPressureTracker (MoistureType moisture_type, const amrex::Geometry &geom, const amrex::MultiFab &mf_cons_var, const double &time)
 
void HurricaneTrackerCircle ()
 
void ReadStormTrackerRestart ()
 
void InitData ()
 
void InitData_pre ()
 
void InitData_post ()
 
void Interp2DArrays (int lev, const amrex::BoxArray &my_ba2d, const amrex::DistributionMapping &my_dm)
 
void WriteMyEBSurface ()
 
void compute_divergence (int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
 
void project_initial_velocity (int lev, double time, double dt)
 
void project_momenta (int lev, double l_time, double l_dt, amrex::Vector< amrex::MultiFab > &vars)
 
void project_velocity_tb (int lev, double dt, amrex::Vector< amrex::MultiFab > &vars)
 
void poisson_wall_dist (int lev)
 
void make_subdomains (const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
 
void solve_with_gmres (int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &az_sub, amrex::MultiFab &, amrex::MultiFab &znd_sub)
 
void ImposeBCsOnPhi (int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
 
void init_only (int lev, double time)
 
void restart ()
 
void check_state_for_nans (amrex::MultiFab const &S)
 
void check_vels_for_nans (amrex::MultiFab const &xvel, amrex::MultiFab const &yvel, amrex::MultiFab const &zvel)
 
void check_for_negative_theta (amrex::MultiFab &S)
 
void check_for_low_temp (amrex::MultiFab &S)
 
void compute_max_pressure_gradient_diagnostic (int lev)
 
void check_mesh_type (int lev)
 
bool writeNow (double cur_time, const int nstep, const int plot_int, const double plot_per, const double dt_0, double &last_file_time)
 
void post_timestep (int nstep, double time, double dt_lev)
 
void WriteAtIntermediateTime (int nstep, double time)
 
void WriteAtFinalTime ()
 
void sum_integrated_quantities (double time)
 
void sum_derived_quantities (double time)
 
void sum_energy_quantities (double time)
 
void write_1D_profiles (double time)
 
void write_1D_profiles_stag (double time)
 
amrex::Real cloud_fraction (double time)
 
void FillBdyCCVels (amrex::MultiFab &mf_cc_vel, amrex::Geometry &lev_geom)
 
void sample_points (int lev, double time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void sample_lines (int lev, double time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void derive_diag_profiles (double time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_diag_profiles_stag (double time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
void derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
amrex::Real volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, const amrex::MultiFab &dJ, const amrex::MultiFab &mfx, const amrex::MultiFab &mfy, bool finemask, bool local=true)
 
void volWgtColumnSum (int lev, const amrex::MultiFab &mf, int comp, amrex::MultiFab &mf_2d, const amrex::MultiFab &dJ)
 
void MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void ClearLevel (int lev) override
 
void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
double estTimeStep (int lev, long &dt_fast_ratio) const
 
void advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, double dt, double time)
 
void advance_microphysics (int lev, amrex::MultiFab &cons_in, const double &dt_advance, const int &iteration, const double &time)
 
void advance_lsm (int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const double &time, const double &dt_advance)
 
void advance_radiation (int lev, amrex::MultiFab &cons_in, const double &dt_advance)
 
void compute_native_shoc_tendencies (int lev, amrex::MultiFab *cons, amrex::MultiFab *xvel, amrex::MultiFab *yvel, amrex::MultiFab *zvel, amrex::Real *w_subsid, amrex::MultiFab *tau13, amrex::MultiFab *tau23, amrex::MultiFab *hfx3, amrex::MultiFab *qfx3, amrex::MultiFab *eddyDiffs, amrex::MultiFab *z_phys_nd, const double &dt_advance)
 
void build_fine_mask (int lev, amrex::MultiFab &fine_mask)
 
void MakeHorizontalAverages ()
 
void MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)
 
void derive_upwp (amrex::Vector< amrex::Real > &h_havg)
 
void Write3DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
 
void Write2DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
 
void WriteSubvolume (int isub, amrex::Vector< std::string > subvol_var_names)
 
void WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, double time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
 
void WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, double time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
 
void erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
 
void init_from_input_sounding (int lev)
 
void init_immersed_forcing (int lev)
 
void input_sponge (int lev)
 
void init_from_hse (int lev)
 
void init_thin_body (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void FillForecastStateMultiFabs (const int lev, const std::string &filename, const std::unique_ptr< amrex::MultiFab > &z_phys_nd, amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
 
void FillSurfaceStateMultiFabs (const int lev, const std::string &filename, amrex::Vector< amrex::MultiFab > &surface_state)
 
void WeatherDataInterpolation (const int nlevs, const double time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)
 
void SurfaceDataInterpolation (const int nlevs, const double time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)
 
void create_background_state_for_ensemble (int lev, amrex::MultiFab &mf_cc_pert, amrex::MultiFab &cons_pert, amrex::MultiFab &xvel_pert, amrex::MultiFab &yvel_pert, amrex::MultiFab &zvel_pert)
 
void create_random_perturbations (const int lev, amrex::MultiFab &mf_cc_pert)
 
void apply_gaussian_smoothing_to_perturbations (const int lev, amrex::MultiFab &mf_cc_pert)
 
void ComputeAndWriteEnsemblePerturbations ()
 
void PerformDataAssimilation (int da_iter)
 
void init_custom (int lev)
 
void fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, double time)
 
void MakeEBGeometry ()
 
void make_eb_box ()
 
void make_eb_regular ()
 
void AverageDownTo (int crse_lev, int scomp, int ncomp, bool do_perturbational_and_momenta=true)
 
void AverageDownMoistStateTo (int crse_lev)
 Conservation-preserving fine→coarse average of moist state components (RhoTheta and the contiguous moist q range) using detJ/mfac weighting, reusing AverageDownTo with perturbational/momenta handling disabled. More...
 
void WriteCheckpointFile () const
 
void ReadCheckpointFile ()
 
void ReadVelsOnlyFromCheckpointFile (int lev_to_fill, std::string &chkfile)
 
void ReadCheckpointFileSurfaceLayer ()
 
void init_zphys (int lev, double elapsed_time)
 
void remake_zphys (int lev, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
 
void update_terrain_arrays (int lev)
 
void writeJobInfo (const std::string &dir, erf_provenance::ArtifactType artifact_type, int artifact_step, double artifact_time_seconds) const
 

Static Public Member Functions

static bool is_it_time_for_action (int nstep, double time, double dt, int action_interval, double action_per)
 
static void writeBuildInfo (std::ostream &os)
 
static void print_banner (MPI_Comm, std::ostream &)
 
static void print_usage (MPI_Comm, std::ostream &)
 
static void print_error (MPI_Comm, const std::string &msg)
 
static void print_summary (std::ostream &)
 
static void print_tpls (std::ostream &)
 

Public Attributes

amrex::Vector< std::array< amrex::Real, 2 > > hurricane_track_xy
 
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_eye_track_xy
 
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_eye_track_latlon
 
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_maxvel_vs_time
 
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_minpressure_vs_time
 
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_tracker_circle
 
amrex::Vector< amrex::MultiFab > weather_forecast_data_1
 
amrex::Vector< amrex::MultiFab > weather_forecast_data_2
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_1
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_2
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_interp
 
amrex::Vector< amrex::MultiFab > surface_state_1
 
amrex::Vector< amrex::MultiFab > surface_state_2
 
amrex::Vector< amrex::MultiFab > surface_state_interp
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > bckgnd_state
 
std::string pp_prefix {"erf"}
 
bool m_driver_has_atm2ocn_coupling = false
 
bool m_driver_uses_two_way_coupling = false
 
bool m_driver_atm2ocn_uses_state_contract = true
 

Private Member Functions

void ReadParameters ()
 
void ParameterSanityChecks ()
 
void AverageDown ()
 
void update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void Construct_ERFFillPatchers (int lev)
 
void Define_ERFFillPatchers (int lev)
 
void init1DArrays ()
 
void init_bcs ()
 
void init_phys_bcs (bool &rho_read, bool &read_prim_theta)
 
void init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
 
void turbPert_update (const int lev, const double dt)
 
void turbPert_amplitude (const int lev)
 
void turbPert_amplitude_w (const int lev)
 
void initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
 
void make_physbcs (int lev)
 
void initializeMicrophysics (const int &)
 
void FillPatchCrseLevel (int lev, double time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
 
void FillPatchFineLevel (int lev, double time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
 
void FillIntermediatePatch (int lev, double time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
 
void FillCoarsePatch (int lev, double time)
 
void timeStep (int lev, double time, int iteration)
 
void Advance (int lev, double time, double dt_lev, int iteration, int ncycle)
 
void initHSE ()
 Initialize HSE. More...
 
void initHSE (int lev)
 
void initRayleigh_at_level (const int &lev)
 Initialize Rayleigh damping profiles at a level. More...
 
void initSponge ()
 Initialize sponge profiles. More...
 
void setRayleighRefFromSounding (bool restarting)
 Set Rayleigh mean profiles from input sounding. More...
 
void setSpongeRefFromSounding (bool restarting)
 Set sponge mean profiles from input sounding. More...
 
void ComputeDt (int step=-1, double cur_time_d=0.0)
 
std::string PlotFileName (int lev) const
 
void setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void setPlotVariables2D (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void setSubVolVariables (const std::string &pp_subvol_var_names, amrex::Vector< std::string > &subvol_var_names)
 
void init_Dirichlet_bc_data (const std::string input_file)
 
void InitializeFromFile ()
 
void InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)
 
void post_update (amrex::MultiFab &state_mf, double time, const amrex::Geometry &geom)
 
void fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, double time, const amrex::Geometry &geom)
 
void init_geo_wind_profile (const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
 
void refinement_criteria_setup ()
 
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg (int lev)
 
AMREX_FORCE_INLINE std::ostream & DataLog (int i)
 
AMREX_FORCE_INLINE std::ostream & DerDataLog (int i)
 
AMREX_FORCE_INLINE int NumDataLogs () noexcept
 
AMREX_FORCE_INLINE int NumDerDataLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SamplePointLog (int i)
 
AMREX_FORCE_INLINE int NumSamplePointLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SampleLineLog (int i)
 
AMREX_FORCE_INLINE int NumSampleLineLogs () noexcept
 
amrex::IntVect & SamplePoint (int i)
 
AMREX_FORCE_INLINE int NumSamplePoints () noexcept
 
amrex::IntVect & SampleLine (int i)
 
AMREX_FORCE_INLINE int NumSampleLines () noexcept
 
void setRecordDataInfo (int i, const std::string &filename)
 
void setRecordDerDataInfo (int i, const std::string &filename)
 
void setRecordEnergyDataInfo (int i, const std::string &filename)
 
void setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
void setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
std::string DataLogName (int i) const noexcept
 The filename of the ith datalog file. More...
 
std::string DerDataLogName (int i) const noexcept
 
std::string SamplePointLogName (int i) const noexcept
 The filename of the ith sampleptlog file. More...
 
std::string SampleLineLogName (int i) const noexcept
 The filename of the ith samplelinelog file. More...
 
eb_ const & get_eb (int lev) const noexcept
 
amrex::EBFArrayBoxFactory const & EBFactory (int lev) const noexcept
 

Static Private Member Functions

static amrex::Vector< std::string > PlotFileVarNames (amrex::Vector< std::string > plot_var_names)
 
static void GotoNextLine (std::istream &is)
 
static AMREX_FORCE_INLINE int ComputeGhostCells (const SolverChoice &sc)
 
static double getCPUTime ()
 
static int nghost_eb_basic ()
 
static int nghost_eb_volume ()
 
static int nghost_eb_full ()
 

Private Attributes

amrex::Vector< std::unique_ptr< amrex::MultiFab > > lat_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lon_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
 
InputSoundingData input_sounding_data
 
InputSpongeData input_sponge_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
 
std::unique_ptr< ProblemBase > prob = nullptr
 
amrex::Vector< int > num_boxes_at_level
 
amrex::Vector< int > num_files_at_level
 
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
 
amrex::Vector< int > istep
 
amrex::Vector< int > nsubsteps
 
amrex::Vector< double > t_new
 
amrex::Vector< double > t_old
 
amrex::Vector< double > dt
 
amrex::Vector< long > dt_mri_ratio
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > gradp
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
 
amrex::Vector< double > t_avg_cnt
 
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
 
amrex::Vector< amrex::MultiFab > pp_inc
 
amrex::Vector< amrex::MultiFab > lagged_delta_rt
 
amrex::Vector< amrex::MultiFab > avg_xmom
 
amrex::Vector< amrex::MultiFab > avg_ymom
 
amrex::Vector< amrex::MultiFab > avg_zmom
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
 
amrex::Vector< amrex::MultiFab > rU_old
 
amrex::Vector< amrex::MultiFab > rU_new
 
amrex::Vector< amrex::MultiFab > rV_old
 
amrex::Vector< amrex::MultiFab > rV_new
 
amrex::Vector< amrex::MultiFab > rW_old
 
amrex::Vector< amrex::MultiFab > rW_new
 
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
 
std::unique_ptr< Microphysics > micro
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
 
LandSurface lsm
 
amrex::Vector< std::string > lsm_data_name
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
 
amrex::Vector< std::string > lsm_flux_name
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
 
amrex::Vector< std::unique_ptr< IRadiation > > rad
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > qheating_rates
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > rad_fluxes
 
amrex::Vector< std::unique_ptr< ShocDriver > > native_shoc_driver
 
bool plot_rad = false
 
int rad_datalog_int = -1
 
int cf_width {0}
 
int cf_set_width {0}
 
amrex::Vector< ERFFillPatcher > FPr_c
 
amrex::Vector< ERFFillPatcher > FPr_u
 
amrex::Vector< ERFFillPatcher > FPr_v
 
amrex::Vector< ERFFillPatcher > FPr_w
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau_corr
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > land_type_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > soil_type_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > urb_frac_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
 
amrex::Vector< amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > > Tau_EB
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > hfx3_EB
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > qfx3_EB
 
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > mapfac
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > fine_mask
 
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
 
amrex::Vector< amrex::MultiFab > base_state
 
amrex::Vector< amrex::MultiFab > base_state_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
 
bool finished_wave = false
 
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
 
amrex::Vector< amrex::BCRec > domain_bcs_type
 
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
 
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_extdir_vals
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_neumann_vals
 
amrex::Array< bool, AMREX_SPACEDIM *2 > m_bc_nonreflecting = {{false}}
 
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
 
amrex::Vector< int > last_subvol_step
 
amrex::Vector< double > last_subvol_time
 
const int datwidth = 14
 
const int datprecision = 6
 
const int timeprecision = 13
 
int max_step = -1
 
bool use_datetime = false
 
const std::string datetime_format = "%Y-%m-%d %H:%M:%S"
 
std::string restart_chkfile = ""
 
erf_provenance::ExecutionProvenance execution_provenance
 
amrex::Vector< double > fixed_dt
 
amrex::Vector< double > fixed_fast_dt
 
int regrid_int = -1
 
bool regrid_level_0_on_restart = false
 
std::string plot3d_file_1 {"plt_1_"}
 
std::string plot3d_file_2 {"plt_2_"}
 
std::string plot2d_file_1 {"plt2d_1_"}
 
std::string plot2d_file_2 {"plt2d_2_"}
 
std::string subvol_file {"subvol"}
 
bool m_expand_plotvars_to_unif_rr = false
 
int m_plot3d_int_1 = -1
 
int m_plot3d_int_2 = -1
 
int m_plot2d_int_1 = -1
 
int m_plot2d_int_2 = -1
 
amrex::Vector< int > m_subvol_int
 
amrex::Vector< double > m_subvol_per
 
double m_plot3d_per_1 = -1.0
 
double m_plot3d_per_2 = -1.0
 
double m_plot2d_per_1 = -1.0
 
double m_plot2d_per_2 = -1.0
 
bool m_plot_face_vels = false
 
bool plot_lsm = false
 
int profile_int = -1
 
bool destag_profiles = true
 
std::string check_file {"chk"}
 
int m_check_int = -1
 
double m_check_per = -1.0
 
amrex::Vector< std::string > subvol3d_var_names
 
amrex::Vector< std::string > plot3d_var_names_1
 
amrex::Vector< std::string > plot3d_var_names_2
 
amrex::Vector< std::string > plot2d_var_names_1
 
amrex::Vector< std::string > plot2d_var_names_2
 
const amrex::Vector< std::string > cons_names
 
const amrex::Vector< std::string > derived_names
 
const amrex::Vector< std::string > derived_subvol_names {"soundspeed", "temp", "theta", "KE", "scalar"}
 
TurbulentPerturbation turbPert
 
int file_name_digits = 5
 
bool use_real_time_in_pltname = false
 
int real_width {0}
 
bool real_extrap_w {true}
 
bool metgrid_debug_quiescent {false}
 
bool metgrid_debug_isothermal {false}
 
bool metgrid_debug_dry {false}
 
bool metgrid_debug_psfc {false}
 
bool metgrid_debug_msf {false}
 
bool metgrid_interp_theta {false}
 
bool metgrid_basic_linear {false}
 
bool metgrid_use_below_sfc {true}
 
bool metgrid_use_sfc {true}
 
bool metgrid_retain_sfc {false}
 
amrex::Real metgrid_proximity {amrex::Real(500.0)}
 
int metgrid_order {2}
 
int metgrid_force_sfc_k {6}
 
bool write_erfbdy {false}
 
bool use_erfbdy {false}
 
std::string erfbdy_file {"erfbdy"}
 
int nvars_erfbdy {0}
 
amrex::Vector< amrex::BoxArray > ba1d
 
amrex::Vector< amrex::BoxArray > ba2d
 
std::unique_ptr< amrex::MultiFab > wrf_C1H
 
std::unique_ptr< amrex::MultiFab > wrf_C2H
 
std::unique_ptr< amrex::MultiFab > wrf_MUB
 
std::unique_ptr< amrex::MultiFab > wrf_RDNW
 
std::unique_ptr< amrex::MultiFab > wrf_PHB
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_PSFC
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > rhotheta_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > rhoqt_src
 
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
 
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
 
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
 
amrex::Vector< amrex::Vector< amrex::Real > > h_sinesq_ptrs
 
amrex::Vector< amrex::Vector< amrex::Real > > h_sinesq_stag_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_sinesq_ptrs
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_sinesq_stag_ptrs
 
amrex::Vector< amrex::Real > h_havg_density
 
amrex::Vector< amrex::Real > h_havg_temperature
 
amrex::Vector< amrex::Real > h_havg_pressure
 
amrex::Vector< amrex::Real > h_havg_qv
 
amrex::Vector< amrex::Real > h_havg_qc
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_density
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_temperature
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_pressure
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qv
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qc
 
std::unique_ptr< WriteBndryPlanes > m_w2d = nullptr
 
std::unique_ptr< ReadBndryPlanes > m_r2d = nullptr
 
std::unique_ptr< SurfaceLayer > m_SurfaceLayer = nullptr
 
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
 
amrex::Vector< amrex::Vector< amrex::BoxArray > > subdomains
 
amrex::Vector< amrex::Real > dz_min
 
int line_sampling_interval = -1
 
int plane_sampling_interval = -1
 
amrex::Real line_sampling_per = -one
 
amrex::Real plane_sampling_per = -one
 
std::unique_ptr< LineSampler > line_sampler = nullptr
 
std::unique_ptr< PlaneSampler > plane_sampler = nullptr
 
amrex::Vector< std::unique_ptr< std::fstream > > datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
 
amrex::Vector< std::string > datalogname
 
amrex::Vector< std::string > der_datalogname
 
amrex::Vector< std::string > tot_e_datalogname
 
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
 
amrex::Vector< std::string > sampleptlogname
 
amrex::Vector< amrex::IntVect > samplepoint
 
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
 
amrex::Vector< std::string > samplelinelogname
 
amrex::Vector< amrex::IntVect > sampleline
 
amrex::Vector< std::unique_ptr< eb_ > > eb
 

Static Private Attributes

static int last_plot3d_file_step_1 = -1
 
static int last_plot3d_file_step_2 = -1
 
static int last_plot2d_file_step_1 = -1
 
static int last_plot2d_file_step_2 = -1
 
static int last_check_file_step = -1
 
static double last_plot3d_file_time_1 = 0.0
 
static double last_plot3d_file_time_2 = 0.0
 
static double last_plot2d_file_time_1 = 0.0
 
static double last_plot2d_file_time_2 = 0.0
 
static double last_check_file_time = 0.0
 
static bool plot_file_on_restart = true
 
static double start_time = 0.0
 
static double stop_time = std::numeric_limits<double>::max()
 
static amrex::Real cfl = Real(0.8)
 
static amrex::Real sub_cfl = one
 
static amrex::Real init_shrink = one
 
static amrex::Real change_max = Real(1.1)
 
static double dt_max_initial = static_cast<double>(bogus_large_value)
 
static double dt_max = 1.0e9
 
static int fixed_mri_dt_ratio = 0
 
static SolverChoice solverChoice
 
static int verbose = 0
 
static int mg_verbose = 0
 
static bool use_fft = false
 
static int check_for_nans = 0
 
static int sum_interval = -1
 
static int pert_interval = -1
 
static double sum_per = -1.0
 
static PlotFileType plotfile3d_type_1 = PlotFileType::None
 
static PlotFileType plotfile3d_type_2 = PlotFileType::None
 
static PlotFileType plotfile2d_type_1 = PlotFileType::None
 
static PlotFileType plotfile2d_type_2 = PlotFileType::None
 
static StateInterpType interpolation_type
 
static amrex::Vector< amrex::Vector< std::string > > nc_init_file = {{""}}
 
static amrex::Vector< amrex::Vector< int > > have_read_nc_init_file = {{0}}
 
static std::string nc_bdy_file
 
static std::string nc_low_file
 
static int output_1d_column = 0
 
static int column_interval = -1
 
static double column_per = -1.0
 
static amrex::Real column_loc_x = zero
 
static amrex::Real column_loc_y = zero
 
static std::string column_file_name = "column_data.nc"
 
static int output_bndry_planes = 0
 
static int bndry_output_planes_interval = -1
 
static double bndry_output_planes_per = -1.0
 
static double bndry_output_planes_start_time = 0.0
 
static int input_bndry_planes = 0
 
static int ng_dens_hse
 
static int ng_pres_hse
 
static amrex::Vector< amrex::AMRErrorTag > ref_tags
 
static double startCPUTime = 0.0
 
static double previousCPUTimeUsed = 0.0
 

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

◆ ERF() [1/3]

ERF::ERF ( )
22 {
23  int fix_random_seed = 0;
24  ParmParse pp("erf"); pp.query("fix_random_seed", fix_random_seed);
25  // Note that the value of 1024UL is not significant -- the point here is just to set the
26  // same seed for all MPI processes for the purpose of regression testing
27  if (fix_random_seed) {
28  Print() << "Fixing the random seed" << std::endl;
29  InitRandom(1024UL, ParallelDescriptor::NProcs(), 1024UL);
30  }
31 
32  ERF_shared();
33 }
pp
ParmParse pp("prob")
ERF::ERF_shared
void ERF_shared()
Definition: ERF_Constructors.cpp:61
Here is the call graph for this function:

◆ ~ERF()

ERF::~ERF ( )
overridedefault

◆ ERF() [2/3]

ERF::ERF ( ERF &&  )
deletenoexcept

◆ ERF() [3/3]

ERF::ERF ( const ERF &  other)
delete

Member Function Documentation

◆ Advance()

void ERF::Advance ( int  lev,
double  time,
double  dt_lev,
int  iteration,
int  ncycle 
)
private

Function that advances the solution at one level for a single time step – this does some preliminaries then calls erf_advance

Parameters
[in]levlevel of refinement (coarsest level is 0)
[in]timestart time for time advance
[in]dt_levtime step for this time advance
21 {
22  BL_PROFILE("ERF::Advance()");
23 
24  // We must swap the pointers so the previous step's "new" is now this step's "old"
25  std::swap(vars_old[lev], vars_new[lev]);
26 
27  MultiFab& S_old = vars_old[lev][Vars::cons];
28  MultiFab& S_new = vars_new[lev][Vars::cons];
29 
30  MultiFab& U_old = vars_old[lev][Vars::xvel];
31  MultiFab& V_old = vars_old[lev][Vars::yvel];
32  MultiFab& W_old = vars_old[lev][Vars::zvel];
33 
34  MultiFab& U_new = vars_new[lev][Vars::xvel];
35  MultiFab& V_new = vars_new[lev][Vars::yvel];
36  MultiFab& W_new = vars_new[lev][Vars::zvel];
37 
38  // We need to set these because otherwise in the first call to erf_advance we may
39  // read uninitialized data on ghost values in setting the bc's on the velocities
40  U_new.setVal(bogus_large_value,U_new.nGrowVect());
41  V_new.setVal(bogus_large_value,V_new.nGrowVect());
42  W_new.setVal(bogus_large_value,W_new.nGrowVect());
43 
44  //
45  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
46  // If lev == 0 we have already FillPatched this in ERF::TimeStep
47  //
48  if (lev > 0) {
49  // Set ghost cells to bogus values so they aren't uninitialized
50  W_old.setBndry(bogus_large_value);
51  FillPatchFineLevel(lev, time, {&S_old, &U_old, &V_old, &W_old},
52  {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]},
53  base_state[lev], base_state[lev]);
54  }
55 
56  //
57  // So we must convert the fillpatched to momenta, including the ghost values
58  //
59  const MultiFab* c_vfrac = nullptr;
60  if (solverChoice.terrain_type == TerrainType::EB) {
61  c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
62  }
63 
64  VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
65  V_old, rV_old[lev].nGrowVect(),
66  W_old, rW_old[lev].nGrowVect(),
67  S_old, rU_old[lev], rV_old[lev], rW_old[lev],
68  Geom(lev).Domain(),
69  domain_bcs_type, c_vfrac);
70 
71  // Update the inflow perturbation update time and amplitude
72  if (solverChoice.use_perturbation(lev))
73  {
74  turbPert.calc_tpi_update(lev, dt_lev, U_old, V_old, S_old);
75  }
76 
77  // If PerturbationType::Direct or CPM is selected, directly add the computed perturbation
78  // on the conserved field
79  if (solverChoice.use_direct_perturbation(lev))
80  {
81  if (solverChoice.use_wvel_perturbation(lev)) { // CPM_W
82  auto m_ixtype = W_old.boxArray().ixType();
83  for (MFIter mfi(W_old,TileNoZ()); mfi.isValid(); ++mfi) {
84  Box bx = mfi.tilebox();
85  const Array4<Real> &cell_data = W_old.array(mfi);
86  const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
87  turbPert.apply_tpi(lev, bx, -1, m_ixtype, cell_data, pert_cell);
88  }
89  } else {
90  auto m_ixtype = S_old.boxArray().ixType(); // Conserved term
91  for (MFIter mfi(S_old,TileNoZ()); mfi.isValid(); ++mfi) {
92  Box bx = mfi.tilebox();
93  const Array4<Real> &cell_data = S_old.array(mfi);
94  const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
95  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cell_data, pert_cell);
96  }
97  }
98  }
99 
100  // configure SurfaceLayer params if needed
101  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
102  if (m_SurfaceLayer) {
103  IntVect ng = Theta_prim[lev]->nGrowVect();
104  MultiFab::Copy( *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
105  MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp , 0, 1, ng);
106  if (solverChoice.moisture_type != MoistureType::None) {
107  ng = Qv_prim[lev]->nGrowVect();
108 
109  MultiFab::Copy( *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
110  MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp , 0, 1, ng);
111 
112  if (solverChoice.moisture_indices.qr > -1) {
113  MultiFab::Copy( *Qr_prim[lev], S_old, solverChoice.moisture_indices.qr, 0, 1, ng);
114  MultiFab::Divide(*Qr_prim[lev], S_old, Rho_comp , 0, 1, ng);
115  } else {
116  Qr_prim[lev]->setVal(0);
117  }
118  }
119  // NOTE: std::swap above causes the field ptrs to be out of date.
120  // Reassign the field ptrs for MAC avg computation.
121  m_SurfaceLayer->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim, Qr_prim);
122  m_SurfaceLayer->update_pblh(lev, vars_old, z_phys_cc[lev].get(),
123  solverChoice.moisture_indices);
124 
125 #ifdef ERF_USE_NETCDF
126  double elapsed_time_since_start_low = time + (start_time - start_low_time);
127 #else
128  double elapsed_time_since_start_low = time;
129 #endif
130  m_SurfaceLayer->update_fluxes(lev, time, elapsed_time_since_start_low,
131  S_old, z_phys_nd[lev], walldist[lev]);
132  }
133  }
134 
135 #if defined(ERF_USE_WINDFARM)
136  // **************************************************************************************
137  // Update the windfarm sources
138  // **************************************************************************************
139  if (solverChoice.windfarm_type != WindFarmType::None) {
140  advance_windfarm(Geom(lev), dt_lev, S_old,
141  U_old, V_old, W_old, vars_windfarm[lev],
142  Nturb[lev], SMark[lev], time);
143  }
144 
145 #endif
146 
147  // **************************************************************************************
148  // Update the radiation sources with the "old" state
149  // **************************************************************************************
150  advance_radiation(lev, S_old, dt_lev);
151 
152  // **************************************************************************************
153  // Update the "old" state using SHOC
154  // **************************************************************************************
155  if (solverChoice.turbChoice[lev].uses_shoc_family()) {
156  // Get SFC fluxes from SurfaceLayer
157  if (m_SurfaceLayer) {
158  Vector<const MultiFab*> mfs = {&S_old, &U_old, &V_old, &W_old};
159  m_SurfaceLayer->impose_SurfaceLayer_bcs(lev, mfs, Tau[lev],
160  SFS_hfx1_lev[lev].get() , SFS_hfx2_lev[lev].get() , SFS_hfx3_lev[lev].get(),
161  SFS_q1fx1_lev[lev].get(), SFS_q1fx2_lev[lev].get(), SFS_q1fx3_lev[lev].get(),
162  z_phys_nd[lev].get());
163  }
164 
165  // Apply SHOC before the dycore so it sees a coherent state.
166  Real* w_sub = (solverChoice.custom_w_subsidence) ? d_w_subsid[lev].data() : nullptr;
167  if (solverChoice.turbChoice[lev].uses_eamxx_shoc()) {
168 #ifdef ERF_USE_EAMXX_SHOC
169  compute_shoc_tendencies(lev, &S_old, &U_old, &V_old, &W_old, w_sub,
170  Tau[lev][TauType::tau13].get(), Tau[lev][TauType::tau23].get(),
171  SFS_hfx3_lev[lev].get() , SFS_q1fx3_lev[lev].get() ,
172  eddyDiffs_lev[lev].get() , z_phys_nd[lev].get() ,
173  dt_lev);
174 #endif
175  } else if (solverChoice.turbChoice[lev].uses_native_shoc()) {
176  compute_native_shoc_tendencies(lev, &S_old, &U_old, &V_old, &W_old, w_sub,
177  Tau[lev][TauType::tau13].get(), Tau[lev][TauType::tau23].get(),
178  SFS_hfx3_lev[lev].get() , SFS_q1fx3_lev[lev].get() ,
179  eddyDiffs_lev[lev].get() , z_phys_nd[lev].get() ,
180  dt_lev);
181 
182  if (native_shoc_driver[lev] && native_shoc_driver[lev]->uses_state_update()) {
183  // Native SHOC updates the old-time state before the dycore reads it.
184  // Re-fill the updated state, velocities, and momenta now so the
185  // pre-dycore checks and strain calculation see coherent fields.
186  Vector<MultiFab*> mfs_vel = {&S_old, &U_old, &V_old, &W_old};
187  if (lev == 0) {
188  FillPatchCrseLevel(lev, time, mfs_vel, false);
189  VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
190  V_old, rV_old[lev].nGrowVect(),
191  W_old, rW_old[lev].nGrowVect(),
192  S_old, rU_old[lev], rV_old[lev], rW_old[lev],
193  Geom(lev).Domain(),
194  domain_bcs_type, c_vfrac);
195  } else {
196  Vector<MultiFab*> mfs_mom = {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]};
197  FillPatchFineLevel(lev, time, mfs_vel, mfs_mom,
198  base_state[lev], base_state[lev],
199  true, false);
200  }
201  }
202  }
203  }
204 
205  const BoxArray& ba = S_old.boxArray();
206  const DistributionMapping& dm = S_old.DistributionMap();
207 
208  int nvars = S_old.nComp();
209 
210  // Source array for conserved cell-centered quantities -- this will be filled
211  // in the call to make_sources in ERF_TI_slow_rhs_pre.H
212  MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0);
213 
214  // Source arrays for momenta -- these will be filled
215  // in the call to make_mom_sources in ERF_TI_slow_rhs_pre.H
216  BoxArray ba_x(ba); ba_x.surroundingNodes(0);
217  MultiFab xmom_source(ba_x,dm,1,1); xmom_source.setVal(0);
218 
219  BoxArray ba_y(ba); ba_y.surroundingNodes(1);
220  MultiFab ymom_source(ba_y,dm,1,1); ymom_source.setVal(0);
221 
222  BoxArray ba_z(ba); ba_z.surroundingNodes(2);
223  MultiFab zmom_source(ba_z,dm,1,1); zmom_source.setVal(0);
224  MultiFab buoyancy(ba_z,dm,1,1); buoyancy.setVal(0);
225 
226  amrex::Vector<MultiFab> state_old;
227  amrex::Vector<MultiFab> state_new;
228 
229  // **************************************************************************************
230  // Here we define state_old and state_new which are to be advanced
231  // **************************************************************************************
232  // Initial solution
233  // Note that "old" and "new" here are relative to each RK stage.
234  state_old.push_back(MultiFab(S_old , amrex::make_alias, 0, nvars)); // cons
235  state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0, 1)); // xmom
236  state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0, 1)); // ymom
237  state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0, 1)); // zmom
238 
239  // Final solution
240  // state_new at the end of the last RK stage holds the t^{n+1} data
241  state_new.push_back(MultiFab(S_new , amrex::make_alias, 0, nvars)); // cons
242  state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0, 1)); // xmom
243  state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0, 1)); // ymom
244  state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0, 1)); // zmom
245 
246  // **************************************************************************************
247  // Tests on the reasonableness of the solution before the dycore
248  // **************************************************************************************
249  // Test for NaNs after dycore
250  if (check_for_nans > 1) {
251  if (verbose > 1) {
252  amrex::Print() << "Testing old state and vels for NaNs before dycore" << std::endl;
253  }
254  check_state_for_nans(S_old);
255  check_vels_for_nans(rU_old[lev],rV_old[lev],rW_old[lev]);
256  }
257 
258  // We only test on low temp if we have a moisture model because we are protecting against
259  // the test on low temp inside the moisture models
260  if (solverChoice.moisture_type != MoistureType::None) {
261  if (verbose > 1) {
262  amrex::Print() << "Testing on low temperature before dycore" << std::endl;
263  }
264  check_for_low_temp(S_old);
265  } else {
266  if (verbose > 1) {
267  amrex::Print() << "Testing on negative temperature before dycore" << std::endl;
268  }
269  check_for_negative_theta(S_old);
270  }
271 
272  // **************************************************************************************
273  // Update the dycore
274  // **************************************************************************************
275  advance_dycore(lev, state_old, state_new,
276  U_old, V_old, W_old,
277  U_new, V_new, W_new,
278  cc_source, xmom_source, ymom_source, zmom_source, buoyancy,
279  Geom(lev), dt_lev, time);
280 
281  // **************************************************************************************
282  // Tests on the reasonableness of the solution after the dycore
283  // **************************************************************************************
284  // Test for NaNs after dycore
285  if (check_for_nans > 0) {
286  if (verbose > 1) {
287  amrex::Print() << "Testing new state and vels for NaNs after dycore" << std::endl;
288  }
289  check_state_for_nans(S_new);
290  check_vels_for_nans(rU_new[lev],rV_new[lev],rW_new[lev]);
291  }
292 
293  // We only test on low temp if we have a moisture model because we are protecting against
294  // the test on low temp inside the moisture models
295  if (solverChoice.moisture_type != MoistureType::None) {
296  if (verbose > 1) {
297  amrex::Print() << "Testing on low temperature after dycore" << std::endl;
298  }
299  check_for_low_temp(S_new);
300  } else {
301  // Otherwise we will test on negative (rhotheta) coming out of the dycore
302  if (verbose > 1) {
303  amrex::Print() << "Testing on negative temperature after dycore" << std::endl;
304  }
305  check_for_negative_theta(S_new);
306  }
307 
308  // **************************************************************************************
309  // Update the microphysics (moisture)
310  // **************************************************************************************
311  if (!solverChoice.moisture_tight_coupling)
312  {
313  advance_microphysics(lev, S_new, dt_lev, iteration, time);
314 
315  // Test for NaNs after microphysics
316  if (check_for_nans > 0) {
317  amrex::Print() << "Testing new state for NaNs after advance_microphysics" << std::endl;
318  check_state_for_nans(S_new);
319  }
320  }
321 
322  // **************************************************************************************
323  // Update the land surface model
324  // **************************************************************************************
325  double time_at_end_of_step = time+dt_lev;
326  advance_lsm(lev, S_new, U_new, V_new, time_at_end_of_step, dt_lev);
327 
328 #ifdef ERF_USE_PARTICLES
329  // **************************************************************************************
330  // Update the particle positions
331  // **************************************************************************************
332  evolveTracers(lev, dt_lev, vars_new, z_phys_nd);
333 #endif
334 
335  // ***********************************************************************************************
336  // Impose domain boundary conditions here so that in FillPatching the fine data we won't
337  // need to re-fill these
338  // ***********************************************************************************************
339  if (lev < finest_level) {
340  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
341  (*physbcs_cons[lev])(vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
342  0,vars_new[lev][Vars::cons].nComp(),
343  vars_new[lev][Vars::cons].nGrowVect(),time,BCVars::cons_bc,true);
344  (*physbcs_u[lev])(vars_new[lev][Vars::xvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
345  ngvect_vels,time,BCVars::xvel_bc,true);
346  (*physbcs_v[lev])(vars_new[lev][Vars::yvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
347  ngvect_vels,time,BCVars::yvel_bc,true);
348  (*physbcs_w[lev])(vars_new[lev][Vars::zvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
349  ngvect_vels,time,BCVars::zvel_bc,true);
350  }
351 
352  // **************************************************************************************
353  // Register old and new coarse data if we are at a level less than the finest level
354  // **************************************************************************************
355  if (lev < finest_level) {
356  if (cf_width > 0) {
357  // We must fill the ghost cells of these so that the parallel copy works correctly
358  state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
359  state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
360  FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
361  {time, time+dt_lev});
362  }
363 
364  if (cf_width >= 0) {
365  // We must fill the ghost cells of these so that the parallel copy works correctly
366  state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
367  state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
368  FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
369  {time, time+dt_lev});
370 
371  state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
372  state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
373  FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
374  {time, time+dt_lev});
375 
376  state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
377  state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
378  FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
379  {time, time+dt_lev});
380  }
381 
382  //
383  // Now create a MultiFab that holds (S_new - S_old) / dt from the coarse level interpolated
384  // on to the coarse/fine boundary at the fine resolution
385  //
386  Interpolater* mapper_f = &face_cons_linear_interp;
387 
388  // PhysBCFunctNoOp null_bc;
389  // MultiFab tempx(vars_new[lev+1][Vars::xvel].boxArray(),vars_new[lev+1][Vars::xvel].DistributionMap(),1,0);
390  // tempx.setVal(0);
391  // xmom_crse_rhs[lev+1].setVal(0);
392  // FPr_u[lev].FillSet(tempx , time , null_bc, domain_bcs_type);
393  // FPr_u[lev].FillSet(xmom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
394  // MultiFab::Subtract(xmom_crse_rhs[lev+1],tempx,0,0,1,IntVect{0});
395  // xmom_crse_rhs[lev+1].mult(one/dt_lev,0,1,0);
396 
397  // MultiFab tempy(vars_new[lev+1][Vars::yvel].boxArray(),vars_new[lev+1][Vars::yvel].DistributionMap(),1,0);
398  // tempy.setVal(0);
399  // ymom_crse_rhs[lev+1].setVal(0);
400  // FPr_v[lev].FillSet(tempy , time , null_bc, domain_bcs_type);
401  // FPr_v[lev].FillSet(ymom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
402  // MultiFab::Subtract(ymom_crse_rhs[lev+1],tempy,0,0,1,IntVect{0});
403  // ymom_crse_rhs[lev+1].mult(one/dt_lev,0,1,0);
404 
405  MultiFab temp_state(zmom_crse_rhs[lev+1].boxArray(),zmom_crse_rhs[lev+1].DistributionMap(),1,0);
406  InterpFromCoarseLevel(temp_state, IntVect{0}, IntVect{0}, state_old[IntVars::zmom], 0, 0, 1,
407  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
408  InterpFromCoarseLevel(zmom_crse_rhs[lev+1], IntVect{0}, IntVect{0}, state_new[IntVars::zmom], 0, 0, 1,
409  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
410  MultiFab::Subtract(zmom_crse_rhs[lev+1],temp_state,0,0,1,IntVect{0});
411 
412  Real inv_dt = static_cast<Real>(one/dt_lev);
413  zmom_crse_rhs[lev+1].mult(inv_dt,0,1,0);
414  }
415 
416  // ***********************************************************************************************
417  // Update the time averaged velocities if they are requested
418  // ***********************************************************************************************
419  if (solverChoice.time_avg_vel) {
420  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
421  }
422 }
bogus_large_value
constexpr amrex::Real bogus_large_value
Definition: ERF_Constants.H:26
one
constexpr amrex::Real one
Definition: ERF_Constants.H:9
tau23
@ tau23
Definition: ERF_DataStruct.H:32
tau13
@ tau13
Definition: ERF_DataStruct.H:32
nvars
@ nvars
Definition: ERF_DataStruct.H:98
Rho_comp
#define Rho_comp
Definition: ERF_IndexDefines.H:36
RhoTheta_comp
#define RhoTheta_comp
Definition: ERF_IndexDefines.H:37
RhoQ1_comp
#define RhoQ1_comp
Definition: ERF_IndexDefines.H:42
ERF_BC::surface_layer
@ surface_layer
get
pp get("wavelength", wavelength)
Real
amrex::Real Real
Definition: ERF_ShocInterface.H:19
TileNoZ
AMREX_FORCE_INLINE amrex::IntVect TileNoZ()
Definition: ERF_TileNoZ.H:11
Time_Avg_Vel_atCC
void Time_Avg_Vel_atCC(double dt_d, double &t_avg_cnt, MultiFab *vel_t_avg, MultiFab &xvel, MultiFab &yvel, MultiFab &zvel)
Definition: ERF_TimeAvgVel.cpp:9
VelocityToMomentum
void VelocityToMomentum(const amrex::MultiFab &xvel_in, const amrex::IntVect &xvel_ngrow, const amrex::MultiFab &yvel_in, const amrex::IntVect &yvel_ngrow, const amrex::MultiFab &zvel_in, const amrex::IntVect &zvel_ngrow, const amrex::MultiFab &cons_in, amrex::MultiFab &xmom_out, amrex::MultiFab &ymom_out, amrex::MultiFab &zmom_out, const amrex::Box &domain, const amrex::Vector< amrex::BCRec > &domain_bcs_type_h, const amrex::MultiFab *c_vfrac=nullptr)
ERF::rU_new
amrex::Vector< amrex::MultiFab > rU_new
Definition: ERF.H:962
ERF::walldist
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
Definition: ERF.H:1074
ERF::check_vels_for_nans
void check_vels_for_nans(amrex::MultiFab const &xvel, amrex::MultiFab const &yvel, amrex::MultiFab const &zvel)
Definition: ERF.cpp:2853
ERF::FPr_u
amrex::Vector< ERFFillPatcher > FPr_u
Definition: ERF.H:1014
ERF::SFS_q1fx3_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
Definition: ERF.H:1037
ERF::vars_new
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
Definition: ERF.H:927
ERF::SFS_hfx3_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
Definition: ERF.H:1035
ERF::native_shoc_driver
amrex::Vector< std::unique_ptr< ShocDriver > > native_shoc_driver
Definition: ERF.H:1001
ERF::FPr_v
amrex::Vector< ERFFillPatcher > FPr_v
Definition: ERF.H:1015
ERF::SFS_hfx1_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
Definition: ERF.H:1035
ERF::get_eb
eb_ const & get_eb(int lev) const noexcept
Definition: ERF.H:1757
ERF::physbcs_cons
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
Definition: ERF.H:949
ERF::dt
amrex::Vector< double > dt
Definition: ERF.H:921
ERF::advance_lsm
void advance_lsm(int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const double &time, const double &dt_advance)
Definition: ERF_AdvanceLSM.cpp:5
ERF::z_phys_cc
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
Definition: ERF.H:1051
ERF::eddyDiffs_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
Definition: ERF.H:1021
ERF::solverChoice
static SolverChoice solverChoice
Definition: ERF.H:1295
ERF::FPr_c
amrex::Vector< ERFFillPatcher > FPr_c
Definition: ERF.H:1013
ERF::Tau
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
Definition: ERF.H:1019
ERF::vel_t_avg
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
Definition: ERF.H:934
ERF::verbose
static int verbose
Definition: ERF.H:1330
ERF::physbcs_w
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
Definition: ERF.H:952
ERF::base_state
amrex::Vector< amrex::MultiFab > base_state
Definition: ERF.H:1085
ERF::Qv_prim
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
Definition: ERF.H:957
ERF::t_avg_cnt
amrex::Vector< double > t_avg_cnt
Definition: ERF.H:935
ERF::SFS_q1fx2_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
Definition: ERF.H:1037
ERF::rV_new
amrex::Vector< amrex::MultiFab > rV_new
Definition: ERF.H:964
ERF::domain_bcs_type
amrex::Vector< amrex::BCRec > domain_bcs_type
Definition: ERF.H:1101
ERF::Qr_prim
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
Definition: ERF.H:958
ERF::physbcs_u
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
Definition: ERF.H:950
ERF::rU_old
amrex::Vector< amrex::MultiFab > rU_old
Definition: ERF.H:961
ERF::Theta_prim
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
Definition: ERF.H:956
ERF::check_for_nans
static int check_for_nans
Definition: ERF.H:1334
ERF::physbcs_v
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
Definition: ERF.H:951
ERF::advance_microphysics
void advance_microphysics(int lev, amrex::MultiFab &cons_in, const double &dt_advance, const int &iteration, const double &time)
Definition: ERF_AdvanceMicrophysics.cpp:5
ERF::check_state_for_nans
void check_state_for_nans(amrex::MultiFab const &S)
Definition: ERF.cpp:2786
ERF::z_phys_nd
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
Definition: ERF.H:1050
ERF::rW_new
amrex::Vector< amrex::MultiFab > rW_new
Definition: ERF.H:966
ERF::FillPatchCrseLevel
void FillPatchCrseLevel(int lev, double time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
Definition: ERF_FillPatch.cpp:290
ERF::zmom_crse_rhs
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
Definition: ERF.H:970
ERF::advance_radiation
void advance_radiation(int lev, amrex::MultiFab &cons_in, const double &dt_advance)
Definition: ERF_AdvanceRadiation.cpp:5
ERF::check_for_low_temp
void check_for_low_temp(amrex::MultiFab &S)
Definition: ERF.cpp:2880
ERF::turbPert
TurbulentPerturbation turbPert
Definition: ERF.H:1298
ERF::rW_old
amrex::Vector< amrex::MultiFab > rW_old
Definition: ERF.H:965
ERF::check_for_negative_theta
void check_for_negative_theta(amrex::MultiFab &S)
Definition: ERF.cpp:2915
ERF::m_SurfaceLayer
std::unique_ptr< SurfaceLayer > m_SurfaceLayer
Definition: ERF.H:1470
ERF::d_w_subsid
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
Definition: ERF.H:1421
ERF::FPr_w
amrex::Vector< ERFFillPatcher > FPr_w
Definition: ERF.H:1016
ERF::SFS_hfx2_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
Definition: ERF.H:1035
ERF::start_time
static double start_time
Definition: ERF.H:1160
ERF::cf_width
int cf_width
Definition: ERF.H:1011
ERF::compute_native_shoc_tendencies
void compute_native_shoc_tendencies(int lev, amrex::MultiFab *cons, amrex::MultiFab *xvel, amrex::MultiFab *yvel, amrex::MultiFab *zvel, amrex::Real *w_subsid, amrex::MultiFab *tau13, amrex::MultiFab *tau23, amrex::MultiFab *hfx3, amrex::MultiFab *qfx3, amrex::MultiFab *eddyDiffs, amrex::MultiFab *z_phys_nd, const double &dt_advance)
Definition: ERF_ShocCoupling.cpp:6
ERF::SFS_q1fx1_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
Definition: ERF.H:1037
ERF::phys_bc_type
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
Definition: ERF.H:1117
ERF::advance_dycore
void advance_dycore(int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, double dt, double time)
Definition: ERF_AdvanceDycore.cpp:38
ERF::rV_old
amrex::Vector< amrex::MultiFab > rV_old
Definition: ERF.H:963
ERF::FillPatchFineLevel
void FillPatchFineLevel(int lev, double time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
Definition: ERF_FillPatch.cpp:20
ERF::vars_old
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
Definition: ERF.H:928
eb_::get_const_factory
const std::unique_ptr< amrex::EBFArrayBoxFactory > & get_const_factory() const noexcept
Definition: ERF_EB.H:46
BCVars::zvel_bc
@ zvel_bc
Definition: ERF_IndexDefines.H:104
BCVars::yvel_bc
@ yvel_bc
Definition: ERF_IndexDefines.H:103
BCVars::cons_bc
@ cons_bc
Definition: ERF_IndexDefines.H:86
BCVars::xvel_bc
@ xvel_bc
Definition: ERF_IndexDefines.H:102
IntVars::ymom
@ ymom
Definition: ERF_IndexDefines.H:195
IntVars::cons
@ cons
Definition: ERF_IndexDefines.H:193
IntVars::zmom
@ zmom
Definition: ERF_IndexDefines.H:196
IntVars::xmom
@ xmom
Definition: ERF_IndexDefines.H:194
MicVar_Morr::ng
@ ng
Definition: ERF_Morrison.H:49
Vars::xvel
@ xvel
Definition: ERF_IndexDefines.H:176
Vars::cons
@ cons
Definition: ERF_IndexDefines.H:175
Vars::zvel
@ zvel
Definition: ERF_IndexDefines.H:178
Vars::yvel
@ yvel
Definition: ERF_IndexDefines.H:177
MoistureComponentIndices::qr
int qr
Definition: ERF_DataStruct.H:110
SolverChoice::moisture_tight_coupling
bool moisture_tight_coupling
Definition: ERF_DataStruct.H:1443
SolverChoice::custom_w_subsidence
bool custom_w_subsidence
Definition: ERF_DataStruct.H:1344
SolverChoice::use_wvel_perturbation
bool use_wvel_perturbation(int lev) const
Definition: ERF_DataStruct.H:1389
SolverChoice::use_perturbation
bool use_perturbation(int lev) const
Definition: ERF_DataStruct.H:1369
SolverChoice::use_direct_perturbation
bool use_direct_perturbation(int lev) const
Definition: ERF_DataStruct.H:1377
SolverChoice::turbChoice
amrex::Vector< TurbChoice > turbChoice
Definition: ERF_DataStruct.H:1251
SolverChoice::moisture_type
MoistureType moisture_type
Definition: ERF_DataStruct.H:1424
SolverChoice::terrain_type
static TerrainType terrain_type
Definition: ERF_DataStruct.H:1230
SolverChoice::windfarm_type
WindFarmType windfarm_type
Definition: ERF_DataStruct.H:1425
SolverChoice::moisture_indices
MoistureComponentIndices moisture_indices
Definition: ERF_DataStruct.H:1441
SolverChoice::time_avg_vel
bool time_avg_vel
Definition: ERF_DataStruct.H:1367
TurbulentPerturbation::pb_cell
amrex::Vector< amrex::MultiFab > pb_cell
Definition: ERF_TurbPertStruct.H:699
TurbulentPerturbation::calc_tpi_update
void calc_tpi_update(const int lev, const double dt, amrex::MultiFab &mf_xvel, amrex::MultiFab &mf_yvel, amrex::MultiFab &mf_cons)
Definition: ERF_TurbPertStruct.H:270
TurbulentPerturbation::apply_tpi
void apply_tpi(const int &lev, const amrex::Box &vbx, const int &comp, const amrex::IndexType &m_ixtype, const amrex::Array4< amrex::Real > &src_arr, const amrex::Array4< amrex::Real const > &pert_cell)
Definition: ERF_TurbPertStruct.H:373
Here is the call graph for this function:

◆ advance_dycore()

void ERF::advance_dycore ( int  level,
amrex::Vector< amrex::MultiFab > &  state_old,
amrex::Vector< amrex::MultiFab > &  state_new,
amrex::MultiFab &  xvel_old,
amrex::MultiFab &  yvel_old,
amrex::MultiFab &  zvel_old,
amrex::MultiFab &  xvel_new,
amrex::MultiFab &  yvel_new,
amrex::MultiFab &  zvel_new,
amrex::MultiFab &  source,
amrex::MultiFab &  xmom_src,
amrex::MultiFab &  ymom_src,
amrex::MultiFab &  zmom_src,
amrex::MultiFab &  buoyancy,
amrex::Geometry  fine_geom,
double  dt,
double  time 
)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters
[in]levellevel of refinement (coarsest level is 0)
[in]state_oldold-time conserved variables
[in]state_newnew-time conserved variables
[in]xvel_oldold-time x-component of velocity
[in]yvel_oldold-time y-component of velocity
[in]zvel_oldold-time z-component of velocity
[in]xvel_newnew-time x-component of velocity
[in]yvel_newnew-time y-component of velocity
[in]zvel_newnew-time z-component of velocity
[in]cc_srcsource term for conserved variables
[in]xmom_srcsource term for x-momenta
[in]ymom_srcsource term for y-momenta
[in]zmom_srcsource term for z-momenta
[in]fine_geomcontainer for geometry information at current level
[in]dt_advancetime step for this time advance
[in]old_timeold time for this time advance
48 {
49  BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
50 
51  const Box& domain = fine_geom.Domain();
52 
53  DiffChoice dc = solverChoice.diffChoice;
54  TurbChoice tc = solverChoice.turbChoice[level];
55 
56  MultiFab r_hse (base_state[level], make_alias, BaseState::r0_comp , 1);
57  MultiFab p_hse (base_state[level], make_alias, BaseState::p0_comp , 1);
58  MultiFab pi_hse(base_state[level], make_alias, BaseState::pi0_comp, 1);
59 
60  // These pointers are used in the MRI utility functions
61  MultiFab* r0 = &r_hse;
62  MultiFab* p0 = &p_hse;
63  MultiFab* pi0 = &pi_hse;
64 
65  MultiFab* rhotheta_src_ptr = solverChoice.custom_rhotheta_forcing ? rhotheta_src[level].get() : nullptr;
66  MultiFab* rhoqt_src_ptr = solverChoice.custom_moisture_forcing ? rhoqt_src[level].get() : nullptr;
67  Real* dptr_wbar_sub = solverChoice.custom_w_subsidence ? d_w_subsid[level].data() : nullptr;
68 
69  // Turbulent Perturbation Pointer
70  //Real* dptr_rhotheta_src = solverChoice.pert_type ? d_rhotheta_src[level].data() : nullptr;
71 
72  Vector<Real*> d_rayleigh_ptrs_at_lev;
73  d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
74  d_rayleigh_ptrs_at_lev[Rayleigh::ubar] = solverChoice.dampingChoice.rayleigh_damp_U ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
75  d_rayleigh_ptrs_at_lev[Rayleigh::vbar] = solverChoice.dampingChoice.rayleigh_damp_V ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
76  d_rayleigh_ptrs_at_lev[Rayleigh::wbar] = solverChoice.dampingChoice.rayleigh_damp_W ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
77  d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.dampingChoice.rayleigh_damp_T ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
78 
79  bool use_rayleigh =
80  (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
81  solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T);
82  Real* d_sinesq_at_lev = (use_rayleigh) ? d_sinesq_ptrs[level].data() : nullptr;
83  Real* d_sinesq_stag_at_lev = (use_rayleigh) ? d_sinesq_stag_ptrs[level].data() : nullptr;
84 
85  Vector<Real*> d_sponge_ptrs_at_lev;
86  if (SpongeChoice::sponge_type == SpongeType::Input_Sponge)
87  {
88  d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
89  d_sponge_ptrs_at_lev[Sponge::ubar_sponge] = d_sponge_ptrs[level][Sponge::ubar_sponge].data();
90  d_sponge_ptrs_at_lev[Sponge::vbar_sponge] = d_sponge_ptrs[level][Sponge::vbar_sponge].data();
91  }
92 
93  bool l_use_terrain_fitted_coords = (solverChoice.mesh_type != MeshType::ConstantDz);
94  bool l_use_kturb = tc.use_kturb;
95  bool l_use_diff = ( (dc.molec_diff_type != MolecDiffType::None) ||
96  l_use_kturb );
97 
98  const bool use_SurfLayer = (m_SurfaceLayer != nullptr);
99  const MultiFab* z_0 = (use_SurfLayer) ? m_SurfaceLayer->get_z0(level) : nullptr;
100 
101  const BoxArray& ba = state_old[IntVars::cons].boxArray();
102  const BoxArray& ba_z = zvel_old.boxArray();
103  const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
104 
105  int num_prim = state_old[IntVars::cons].nComp() - 1;
106 
107  MultiFab S_prim (ba , dm, num_prim, state_old[IntVars::cons].nGrowVect());
108  MultiFab pi_stage (ba , dm, 1, 1);
109  MultiFab fast_coeffs(ba_z, dm, 5, 0);
110 
111  MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
112  MultiFab* SmnSmn = SmnSmn_lev[level].get();
113 
114  // **************************************************************************************
115  // Compute strain for use in slow RHS and Smagorinsky model
116  // **************************************************************************************
117  {
118  BL_PROFILE("erf_advance_strain");
119  if (l_use_diff) {
120 
121  const BCRec* bc_ptr_h = domain_bcs_type.data();
122  const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
123 
124 #ifdef _OPENMP
125 #pragma omp parallel if (Gpu::notInLaunchRegion())
126 #endif
127  for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
128  {
129  Box bxcc = mfi.growntilebox(IntVect(1,1,0));
130  Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
131  Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
132  Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
133 
134  if (bxcc.smallEnd(2) != domain.smallEnd(2)) {
135  bxcc.growLo(2,1);
136  tbxxy.growLo(2,1);
137  tbxxz.growLo(2,1);
138  tbxyz.growLo(2,1);
139  }
140 
141  if (bxcc.bigEnd(2) != domain.bigEnd(2)) {
142  bxcc.growHi(2,1);
143  tbxxy.growHi(2,1);
144  tbxxz.growHi(2,1);
145  tbxyz.growHi(2,1);
146  }
147 
148  const Array4<const Real> & u = xvel_old.array(mfi);
149  const Array4<const Real> & v = yvel_old.array(mfi);
150  const Array4<const Real> & w = zvel_old.array(mfi);
151 
152  Array4<Real> tau11 = Tau[level][TauType::tau11].get()->array(mfi);
153  Array4<Real> tau22 = Tau[level][TauType::tau22].get()->array(mfi);
154  Array4<Real> tau33 = Tau[level][TauType::tau33].get()->array(mfi);
155  Array4<Real> tau12 = Tau[level][TauType::tau12].get()->array(mfi);
156  Array4<Real> tau13 = Tau[level][TauType::tau13].get()->array(mfi);
157  Array4<Real> tau23 = Tau[level][TauType::tau23].get()->array(mfi);
158 
159  Array4<Real> tau21 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau21].get()->array(mfi) : Array4<Real>{};
160  Array4<Real> tau31 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau31].get()->array(mfi) : Array4<Real>{};
161  Array4<Real> tau32 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau32].get()->array(mfi) : Array4<Real>{};
162  const Array4<const Real>& z_nd = z_phys_nd[level]->const_array(mfi);
163 
164  const Array4<const Real> mf_mx = mapfac[level][MapFacType::m_x]->const_array(mfi);
165  const Array4<const Real> mf_ux = mapfac[level][MapFacType::u_x]->const_array(mfi);
166  const Array4<const Real> mf_vx = mapfac[level][MapFacType::v_x]->const_array(mfi);
167  const Array4<const Real> mf_my = mapfac[level][MapFacType::m_y]->const_array(mfi);
168  const Array4<const Real> mf_uy = mapfac[level][MapFacType::u_y]->const_array(mfi);
169  const Array4<const Real> mf_vy = mapfac[level][MapFacType::v_y]->const_array(mfi);
170 
171  // We update Tau_corr[level] in erf_make_tau_terms, not here
172  Array4<Real> no_tau_corr_update_here{};
173 
174  if (solverChoice.mesh_type == MeshType::StretchedDz) {
175  ComputeStrain_S(bxcc, tbxxy, tbxxz, tbxyz, domain,
176  u, v, w,
177  tau11, tau22, tau33,
178  tau12, tau21,
179  tau13, tau31,
180  tau23, tau32,
181  stretched_dz_d[level], dxInv,
182  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
183  no_tau_corr_update_here, no_tau_corr_update_here);
184  } else if (l_use_terrain_fitted_coords) {
185  ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
186  u, v, w,
187  tau11, tau22, tau33,
188  tau12, tau21,
189  tau13, tau31,
190  tau23, tau32,
191  z_nd, detJ_cc[level]->const_array(mfi), dxInv,
192  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
193  no_tau_corr_update_here, no_tau_corr_update_here);
194  } else {
195  if (solverChoice.terrain_type == TerrainType::EB) {
196  ComputeStrain_EB(mfi, bxcc, tbxxy, tbxxz, tbxyz, domain,
197  u, v, w,
198  tau11, tau22, tau33,
199  tau12, tau13, tau23,
200  dxInv,
201  bc_ptr_h,
202  get_eb(level),
203  no_tau_corr_update_here, no_tau_corr_update_here);
204  } else {
205  ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
206  u, v, w,
207  tau11, tau22, tau33,
208  tau12, tau13, tau23,
209  dxInv,
210  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h,
211  no_tau_corr_update_here, no_tau_corr_update_here);
212  }
213  }
214  } // mfi
215  } // l_use_diff
216  } // profile
217 
218 #include "ERF_TI_utils.H"
219 
220  // Additional SFS quantities, calculated once per timestep
221  MultiFab* Hfx1 = SFS_hfx1_lev[level].get();
222  MultiFab* Hfx2 = SFS_hfx2_lev[level].get();
223  MultiFab* Hfx3 = SFS_hfx3_lev[level].get();
224  MultiFab* Q1fx1 = SFS_q1fx1_lev[level].get();
225  MultiFab* Q1fx2 = SFS_q1fx2_lev[level].get();
226  MultiFab* Q1fx3 = SFS_q1fx3_lev[level].get();
227  MultiFab* Q2fx3 = SFS_q2fx3_lev[level].get();
228  MultiFab* Diss = SFS_diss_lev[level].get();
229 
230  MultiFab* Hfx3_EB = nullptr;
231  if (solverChoice.terrain_type == TerrainType::EB) {
232  Hfx3_EB = hfx3_EB[level].get();
233  }
234 
235  // *************************************************************************
236  // Calculate cell-centered eddy viscosity & diffusivities
237  //
238  // Notes -- we fill all the data in ghost cells before calling this so
239  // that we can fill the eddy viscosity in the ghost regions and
240  // not have to call a boundary filler on this data itself
241  //
242  // LES - updates both horizontal and vertical eddy viscosity components
243  // PBL - only updates vertical eddy viscosity components so horizontal
244  // components come from the LES model or are left as zero.
245  // *************************************************************************
246  if (l_use_kturb)
247  {
248  // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
249  bool l_use_moisture = ( solverChoice.moisture_type != MoistureType::None );
250  const BCRec* bc_ptr_h = domain_bcs_type.data();
251  ComputeTurbulentViscosity(dt_advance, xvel_old, yvel_old,Tau[level],
252  state_old[IntVars::cons],
253  *walldist[level].get(),
254  *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
255  fine_geom, mapfac[level],
256  z_phys_nd[level], z_phys_cc[level], solverChoice,
257  m_SurfaceLayer, z_0, l_use_terrain_fitted_coords,
258  l_use_moisture, level,
259  bc_ptr_h,
260  get_eb(level),
261  false, // vert_only
262  qheating_rates[level].get());
263  }
264 
265  // ***********************************************************************************************
266  // Update user-defined source terms -- these are defined once per time step (not per RK stage)
267  // ***********************************************************************************************
268  if (solverChoice.custom_rhotheta_forcing) {
269  prob->update_rhotheta_sources(old_time,
270  rhotheta_src_ptr,
271  fine_geom, z_phys_cc[level]);
272  }
273 
274  if (solverChoice.custom_moisture_forcing) {
275  prob->update_rhoqt_sources(old_time,
276  rhoqt_src_ptr,
277  fine_geom, z_phys_cc[level]);
278  }
279 
280  if (solverChoice.custom_geostrophic_profile) {
281  prob->update_geostrophic_profile(old_time,
282  h_u_geos[level], d_u_geos[level],
283  h_v_geos[level], d_v_geos[level],
284  fine_geom, z_phys_cc[level]);
285  }
286 
287  if (solverChoice.custom_w_subsidence) {
288  prob->update_w_subsidence(old_time,
289  h_w_subsid[level], d_w_subsid[level],base_state[level],
290  fine_geom, z_phys_nd[level]);
291  }
292 
293  // ***********************************************************************************************
294  // Convert old velocity available on faces to old momentum on faces to be used in time integration
295  // ***********************************************************************************************
296  MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
297 
298  //
299  // This is an optimization since we won't need more than one ghost
300  // cell of momentum in the integrator if not using numerical diffusion
301  //
302  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
303  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
304  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
305 
306  const MultiFab* c_vfrac = nullptr;
307  if (solverChoice.terrain_type == TerrainType::EB) {
308  c_vfrac = &((get_eb(level).get_const_factory())->getVolFrac());
309  }
310 
311  VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
312  state_old[IntVars::xmom],
313  state_old[IntVars::ymom],
314  state_old[IntVars::zmom],
315  domain, domain_bcs_type, c_vfrac);
316 
317  MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
318  MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
319  MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
320 
321  bool fast_only = false;
322  bool vel_and_mom_synced = true;
323 
324  apply_bcs(state_old, old_time,
325  state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
326  fast_only, vel_and_mom_synced);
327 
328  cons_to_prim(state_old[IntVars::cons], S_prim, state_old[IntVars::cons].nGrow());
329 
330  make_pi_stage(state_old[IntVars::cons]);
331 
332  // ***********************************************************************************************
333  // Define a new MultiFab that holds q_total and fill it by summing the moisture components --
334  // to be used in buoyancy calculation and as part of the inertial weighting in the
335  // ***********************************************************************************************
336 
337  const bool l_eb_terrain = (solverChoice.terrain_type == TerrainType::EB);
338  MultiFab qt(grids[level], dmap[level], 1, (l_eb_terrain) ? 2 : 1);
339  qt.setVal(0);
340 
341 #include "ERF_TI_no_substep_fun.H"
342 #include "ERF_TI_substep_fun.H"
343 #include "ERF_TI_slow_rhs_pre.H"
344 #include "ERF_TI_slow_rhs_post.H"
345 
346  // ***************************************************************************************
347  // Setup the integrator and integrate for a single timestep
348  // **************************************************************************************
349  MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
350 
351  // Define rhs and 'post update' utility function that is called after calculating
352  // any state data (e.g. at RK stages or at the end of a timestep)
353  mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
354  mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
355 
356  mri_integrator.set_acoustic_substepping(acoustic_substepping_fun);
357  mri_integrator.set_slow_fast_timestep_ratio(fixed_mri_dt_ratio > 0 ? fixed_mri_dt_ratio : dt_mri_ratio[level]);
358  mri_integrator.set_no_substep(no_substep_fun);
359 
360  mri_integrator.advance(state_old, state_new, old_time, dt_advance);
361 
362  if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
363 }
ComputeStrain_EB
void ComputeStrain_EB(const MFIter &mfi, Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau13, Array4< Real > &tau23, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const BCRec *bc_ptr, const eb_ &ebfact, Array4< Real > &tau13i, Array4< Real > &tau23i)
Definition: ERF_ComputeStrain_EB.cpp:28
ComputeStrain_N
void ComputeStrain_N(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau13, Array4< Real > &tau23, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr, Array4< Real > &tau13i, Array4< Real > &tau23i)
Definition: ERF_ComputeStrain_N.cpp:31
ComputeStrain_S
void ComputeStrain_S(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau21, Array4< Real > &tau13, Array4< Real > &tau31, Array4< Real > &tau23, Array4< Real > &tau32, const Gpu::DeviceVector< Real > &stretched_dz_d, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr, Array4< Real > &tau13i, Array4< Real > &tau23i)
Definition: ERF_ComputeStrain_S.cpp:39
ComputeStrain_T
void ComputeStrain_T(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau21, Array4< Real > &tau13, Array4< Real > &tau31, Array4< Real > &tau23, Array4< Real > &tau32, const Array4< const Real > &z_nd, const Array4< const Real > &detJ, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr, Array4< Real > &tau13i, Array4< Real > &tau23i)
Definition: ERF_ComputeStrain_T.cpp:39
ComputeTurbulentViscosity
void ComputeTurbulentViscosity(double dt, const MultiFab &xvel, const MultiFab &yvel, Vector< std::unique_ptr< MultiFab >> &Tau_lev, MultiFab &cons_in, const MultiFab &wdist, MultiFab &eddyViscosity, MultiFab &Hfx1, MultiFab &Hfx2, MultiFab &Hfx3, MultiFab &Diss, const Geometry &geom, Vector< std::unique_ptr< MultiFab >> &mapfac, const std::unique_ptr< MultiFab > &z_phys_nd, const std::unique_ptr< MultiFab > &z_phys_cc, const SolverChoice &solverChoice, std::unique_ptr< SurfaceLayer > &SurfLayer, const MultiFab *z_0, const bool &use_terrain_fitted_coords, const bool &use_moisture, int level, const BCRec *bc_ptr, const eb_ &ebfact, bool vert_only, const MultiFab *qheating_rates)
Definition: ERF_ComputeTurbulentViscosity.cpp:792
tau12
@ tau12
Definition: ERF_DataStruct.H:32
tau33
@ tau33
Definition: ERF_DataStruct.H:32
tau22
@ tau22
Definition: ERF_DataStruct.H:32
tau11
@ tau11
Definition: ERF_DataStruct.H:32
tau32
@ tau32
Definition: ERF_DataStruct.H:32
tau31
@ tau31
Definition: ERF_DataStruct.H:32
tau21
@ tau21
Definition: ERF_DataStruct.H:32
ubar
@ ubar
Definition: ERF_DataStruct.H:98
wbar
@ wbar
Definition: ERF_DataStruct.H:98
vbar
@ vbar
Definition: ERF_DataStruct.H:98
thetabar
@ thetabar
Definition: ERF_DataStruct.H:98
nvars_sponge
@ nvars_sponge
Definition: ERF_DataStruct.H:103
vbar_sponge
@ vbar_sponge
Definition: ERF_DataStruct.H:103
ubar_sponge
@ ubar_sponge
Definition: ERF_DataStruct.H:103
v_x
@ v_x
Definition: ERF_DataStruct.H:24
u_y
@ u_y
Definition: ERF_DataStruct.H:25
v_y
@ v_y
Definition: ERF_DataStruct.H:25
m_y
@ m_y
Definition: ERF_DataStruct.H:25
u_x
@ u_x
Definition: ERF_DataStruct.H:24
m_x
@ m_x
Definition: ERF_DataStruct.H:24
MolecDiffType::None
@ None
dxInv
amrex::GpuArray< Real, AMREX_SPACEDIM > dxInv
Definition: ERF_InitCustomPertVels_ParticleTests.H:17
w
Real w
Definition: ERF_Plotfile2DInterpolator.cpp:19
ERF_TI_no_substep_fun.H
no_substep_fun
auto no_substep_fun
Definition: ERF_TI_no_substep_fun.H:4
ERF_TI_slow_rhs_post.H
slow_rhs_fun_post
auto slow_rhs_fun_post
Definition: ERF_TI_slow_rhs_post.H:3
ERF_TI_slow_rhs_pre.H
slow_rhs_fun_pre
auto slow_rhs_fun_pre
Definition: ERF_TI_slow_rhs_pre.H:6
ERF_TI_substep_fun.H
acoustic_substepping_fun
auto acoustic_substepping_fun
Definition: ERF_TI_substep_fun.H:6
ERF_TI_utils.H
make_pi_stage
auto make_pi_stage
Definition: ERF_TI_utils.H:4
apply_bcs
auto apply_bcs
Definition: ERF_TI_utils.H:34
z_0
Real z_0
Definition: ERF_UpdateWSubsidence_Bomex.H:10
cons_to_prim
void cons_to_prim(const MultiFab &cons_state, MultiFab &S_prim, int ng)
Definition: ERF_Utils.cpp:6
ERF::mapfac
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > mapfac
Definition: ERF.H:1077
ERF::mri_integrator_mem
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
Definition: ERF.H:937
ERF::d_sinesq_stag_ptrs
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_sinesq_stag_ptrs
Definition: ERF.H:1452
ERF::h_w_subsid
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
Definition: ERF.H:1420
ERF::detJ_cc
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
Definition: ERF.H:1053
ERF::hfx3_EB
amrex::Vector< std::unique_ptr< amrex::MultiFab > > hfx3_EB
Definition: ERF.H:1043
ERF::d_sponge_ptrs
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
Definition: ERF.H:1448
ERF::dt_mri_ratio
amrex::Vector< long > dt_mri_ratio
Definition: ERF.H:922
ERF::SFS_q2fx3_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
Definition: ERF.H:1038
ERF::prob
std::unique_ptr< ProblemBase > prob
Definition: ERF.H:909
ERF::stretched_dz_d
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
Definition: ERF.H:1083
ERF::SFS_diss_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
Definition: ERF.H:1036
ERF::d_sinesq_ptrs
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_sinesq_ptrs
Definition: ERF.H:1451
ERF::d_v_geos
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
Definition: ERF.H:1427
ERF::h_v_geos
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
Definition: ERF.H:1426
ERF::rhotheta_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > > rhotheta_src
Definition: ERF.H:1417
ERF::h_u_geos
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
Definition: ERF.H:1423
ERF::SmnSmn_lev
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
Definition: ERF.H:1022
ERF::rhoqt_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > > rhoqt_src
Definition: ERF.H:1418
ERF::d_u_geos
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
Definition: ERF.H:1424
ERF::fixed_mri_dt_ratio
static int fixed_mri_dt_ratio
Definition: ERF.H:1184
ERF::d_rayleigh_ptrs
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
Definition: ERF.H:1447
ERF::qheating_rates
amrex::Vector< std::unique_ptr< amrex::MultiFab > > qheating_rates
Definition: ERF.H:995
MRISplitIntegrator
Definition: ERF_MRI.H:16
MRISplitIntegrator::set_slow_rhs_post
void set_slow_rhs_post(std::function< void(T &, T &, T &, T &, const double, const double, const double, const int)> F)
Definition: ERF_MRI.H:138
MRISplitIntegrator::set_slow_rhs_pre
void set_slow_rhs_pre(std::function< void(T &, T &, T &, const double, const double, const double, const int)> F)
Definition: ERF_MRI.H:134
MRISplitIntegrator::set_no_substep
void set_no_substep(std::function< void(T &, T &, T &, const double, const double, int)> F)
Definition: ERF_MRI.H:161
MRISplitIntegrator::advance
double advance(T &S_old, T &S_new, double time, const double time_step)
Definition: ERF_MRI.H:171
MRISplitIntegrator::set_acoustic_substepping
void set_acoustic_substepping(std::function< void(int, int, int, T &, const T &, T &, T &, const double, const double, const amrex::Real, const double, const double)> F)
Definition: ERF_MRI.H:143
MRISplitIntegrator::set_slow_fast_timestep_ratio
void set_slow_fast_timestep_ratio(const int timestep_ratio=1)
Definition: ERF_MRI.H:151
BaseState::pi0_comp
@ pi0_comp
Definition: ERF_IndexDefines.H:75
BaseState::p0_comp
@ p0_comp
Definition: ERF_IndexDefines.H:74
BaseState::r0_comp
@ r0_comp
Definition: ERF_IndexDefines.H:73
MicVar_Kess::qt
@ qt
Definition: ERF_Kessler.H:29
module_model_constants::p0
real(c_double), parameter p0
Definition: ERF_module_model_constants.F90:40
mp_wsm6::r0
real(kind=kind_phys), parameter, private r0
Definition: ERF_module_mp_wsm6.F90:21
DampingChoice::rayleigh_damp_V
bool rayleigh_damp_V
Definition: ERF_DampingStruct.H:85
DampingChoice::rayleigh_damp_T
bool rayleigh_damp_T
Definition: ERF_DampingStruct.H:87
DampingChoice::rayleigh_damp_W
bool rayleigh_damp_W
Definition: ERF_DampingStruct.H:86
DampingChoice::rayleigh_damp_U
bool rayleigh_damp_U
Definition: ERF_DampingStruct.H:84
DiffChoice
Definition: ERF_DiffStruct.H:19
DiffChoice::molec_diff_type
MolecDiffType molec_diff_type
Definition: ERF_DiffStruct.H:84
SolverChoice::mesh_type
static MeshType mesh_type
Definition: ERF_DataStruct.H:1239
SolverChoice::dampingChoice
DampingChoice dampingChoice
Definition: ERF_DataStruct.H:1249
SolverChoice::diffChoice
DiffChoice diffChoice
Definition: ERF_DataStruct.H:1248
SolverChoice::custom_rhotheta_forcing
bool custom_rhotheta_forcing
Definition: ERF_DataStruct.H:1342
SolverChoice::custom_geostrophic_profile
bool custom_geostrophic_profile
Definition: ERF_DataStruct.H:1347
SolverChoice::use_num_diff
bool use_num_diff
Definition: ERF_DataStruct.H:1420
SolverChoice::custom_moisture_forcing
bool custom_moisture_forcing
Definition: ERF_DataStruct.H:1343
SpongeChoice::sponge_type
static SpongeType sponge_type
Definition: ERF_SpongeStruct.H:90
TurbChoice
Definition: ERF_TurbStruct.H:82
TurbChoice::use_kturb
bool use_kturb
Definition: ERF_TurbStruct.H:577
Here is the call graph for this function:

◆ advance_lsm()

void ERF::advance_lsm ( int  lev,
amrex::MultiFab &  cons_in,
amrex::MultiFab &  xvel_in,
amrex::MultiFab &  yvel_in,
const double &  time,
const double &  dt_advance 
)
11 {
12  if (solverChoice.lsm_type != LandSurfaceType::None) {
13  if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
14  // Typed surface-precip accumulation sources from the active microphysics
15  // scheme: borrowed const views of the scheme-native cumulative accumulators
16  // plus their native->kg/m^2 (== water-equivalent mm) conversion factors.
17  // Noah-MP forms the water-equivalent interval precip (RAINBL / MP_RAINNC /
18  // MP_SNOW / MP_GRAUP / SR) from these, so schemes with differing accumulator
19  // semantics or units (e.g. SAM's density-scaled snow/graupel) are handled
20  // correctly. Empty when moisture/precip is off -> land model runs precip-free.
21  const SurfacePrecipAccumulationSources precip_sources =
22  micro ? micro->Get_Surface_Precip_Accumulation_Ptrs(lev)
23  : SurfacePrecipAccumulationSources{};
24  lsm.Advance(lev, cons_in, xvel_in, yvel_in,
25  SFS_hfx3_lev[lev].get(), SFS_q1fx3_lev[lev].get(),
26  precip_sources,
27  time, dt_advance, istep[0], lsm.Get_LSM_Update_Status(0));
28  } else {
29  lsm.Advance(lev, dt_advance);
30  }
31  }
32 }
ERF::micro
std::unique_ptr< Microphysics > micro
Definition: ERF.H:972
ERF::lsm
LandSurface lsm
Definition: ERF.H:988
ERF::istep
amrex::Vector< int > istep
Definition: ERF.H:915
LandSurface::Advance
void Advance(const int &lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, amrex::MultiFab *hfx3_out, amrex::MultiFab *qfx3_out, const SurfacePrecipAccumulationSources &precip_sources, const double &time_d, const double &dt_advance_d, const int &nstep, const bool updated_lev0)
Definition: ERF_LandSurface.H:61
LandSurface::Get_LSM_Update_Status
bool Get_LSM_Update_Status(const int &lev)
Definition: ERF_LandSurface.H:188
SolverChoice::lsm_type
LandSurfaceType lsm_type
Definition: ERF_DataStruct.H:1427
SurfacePrecipAccumulationSources
Definition: ERF_SurfacePrecipitation.H:21
Here is the call graph for this function:

◆ advance_microphysics()

void ERF::advance_microphysics ( int  lev,
amrex::MultiFab &  cons_in,
const double &  dt_advance,
const int &  iteration,
const double &  time 
)
10 {
11  if (solverChoice.moisture_type != MoistureType::None) {
12  if (lev > 0) {
13  MultiFab& U_new = vars_new[lev][Vars::xvel];
14  MultiFab& V_new = vars_new[lev][Vars::yvel];
15  MultiFab& W_new = vars_new[lev][Vars::zvel];
16  FillPatchFineLevel(lev, time + dt_advance,
17  {&cons, &U_new, &V_new, &W_new},
18  {&cons, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
19  base_state[lev], base_state[lev]);
20  } else {
21  cons.FillBoundary(geom[lev].periodicity());
22  }
23  micro->Update_Micro_Vars_Lev(lev, cons);
24  micro->Advance(lev, static_cast<Real>(dt_advance), iteration,
25  static_cast<Real>(time), solverChoice, vars_new, z_phys_nd, phys_bc_type);
26  micro->Update_State_Vars_Lev(lev, cons, *z_phys_nd[lev]);
27 
28  // Sync cons[lev-1] covered cells with the moist state just written
29  // to cons[lev]. Without this, the next sub-cycle's FillPatchFineLevel
30  // for level lev would pull stale (latent-heat-less) values from
31  // cons[lev-1]'s coarse cells via cell-conservative interpolation,
32  // causing an artificial outward heat/q flux across the lev/(lev-1)
33  // boundary that drains the bubble interior.
34  if (lev > 0 && solverChoice.coupling_type == CouplingType::TwoWay &&
35  Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
36  AverageDownMoistStateTo(lev - 1);
37  }
38  }
39 }
ERF::AverageDownMoistStateTo
void AverageDownMoistStateTo(int crse_lev)
Conservation-preserving fine→coarse average of moist state components (RhoTheta and the contiguous mo...
Definition: ERF_AverageDown.cpp:243
Microphysics::modelType
static MoistureModelType modelType(const MoistureType a_moisture_type)
query if a specified moisture model is Eulerian or Lagrangian
Definition: ERF_Microphysics.H:110
SolverChoice::coupling_type
CouplingType coupling_type
Definition: ERF_DataStruct.H:1423
Here is the call graph for this function:

◆ advance_radiation()

void ERF::advance_radiation ( int  lev,
amrex::MultiFab &  cons_in,
const double &  dt_advance 
)
8 {
9  if (solverChoice.rad_type != RadiationType::None) {
10 #ifdef ERF_USE_NETCDF
11  MultiFab *lat_ptr = lat_m[lev].get();
12  MultiFab *lon_ptr = lon_m[lev].get();
13 #else
14  MultiFab *lat_ptr = nullptr;
15  MultiFab *lon_ptr = nullptr;
16 #endif
17  // T surf from SurfaceLayer if we have it
18  MultiFab* t_surf = (m_SurfaceLayer) ? m_SurfaceLayer->get_t_surf(lev) : nullptr;
19 
20  // RRTMGP inputs names and pointers
21  Vector<std::string> lsm_input_names = rad[lev]->get_lsm_input_varnames();
22  Vector<MultiFab*> lsm_input_ptrs(lsm_input_names.size(),nullptr);
23  for (int i(0); i<lsm_input_ptrs.size(); ++i) {
24  int varIdx = lsm.Get_DataIdx(lev,lsm_input_names[i]);
25  if (varIdx >= 0) { lsm_input_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx); }
26  }
27 
28  // RRTMGP output names and pointers
29  Vector<std::string> lsm_output_names = rad[lev]->get_lsm_output_varnames();
30  Vector<MultiFab*> lsm_output_ptrs(lsm_output_names.size(),nullptr);
31  for (int i(0); i<lsm_output_ptrs.size(); ++i) {
32  int varIdx = lsm.Get_DataIdx(lev,lsm_output_names[i]);
33  if (varIdx >= 0) { lsm_output_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx); }
34  }
35 
36  // Force radiation update to sync with lsm?
37  bool lsm_updated = (lev==0) ? lsm.Get_LSM_Update_Status(lev) : false;
38 
39  // Enter radiation class driver
40  double time_for_rad = t_old[lev] + start_time;
41  rad[lev]->Run(lev, istep[lev], time_for_rad, dt_advance,
42  cons.boxArray(), geom[lev], &(cons),
43  lmask_lev[lev][0].get(), t_surf,
44  lsm_input_ptrs, lsm_output_ptrs,
45  qheating_rates[lev].get(), rad_fluxes[lev].get(),
46  z_phys_nd[lev].get() , lat_ptr, lon_ptr,
47  lsm_updated);
48  }
49 }
ERF::lmask_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
Definition: ERF.H:1027
ERF::rad
amrex::Vector< std::unique_ptr< IRadiation > > rad
Definition: ERF.H:994
ERF::t_old
amrex::Vector< double > t_old
Definition: ERF.H:920
ERF::lon_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lon_m
Definition: ERF.H:872
ERF::lat_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lat_m
Definition: ERF.H:872
ERF::rad_fluxes
amrex::Vector< std::unique_ptr< amrex::MultiFab > > rad_fluxes
Definition: ERF.H:996
LandSurface::Get_DataIdx
int Get_DataIdx(const int &lev, std::string &varname)
Definition: ERF_LandSurface.H:151
LandSurface::Get_Data_Ptr
amrex::MultiFab * Get_Data_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:107
LsmVar_OceanSurf::t_surf
@ t_surf
Definition: ERF_OceanSurf.H:14
SolverChoice::rad_type
RadiationType rad_type
Definition: ERF_DataStruct.H:1428
Here is the call graph for this function:

◆ appendPlotVariables()

void ERF::appendPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
177 {
178  ParmParse pp(pp_prefix);
179 
180  Vector<std::string> plot_var_names(0);
181  if (pp.contains(pp_plot_var_names.c_str())) {
182  std::string nm;
183  int nPltVars = pp.countval(pp_plot_var_names.c_str());
184  for (int i = 0; i < nPltVars; i++) {
185  pp.get(pp_plot_var_names.c_str(), nm, i);
186  // Add the named variable to our list of plot variables
187  // if it is not already in the list
188  if (!containerHasElement(plot_var_names, nm)) {
189  plot_var_names.push_back(nm);
190  }
191  }
192  }
193 
194  Vector<std::string> tmp_plot_names(0);
195 #ifdef ERF_USE_PARTICLES
196  Vector<std::string> particle_mesh_plot_names;
197  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
198  if (particle_mesh_plot_names.size() > 0) {
199  static bool first_call = true;
200  if (first_call) {
201  Print() << "ParticleData: the following additional Eulerian variables are available to plot:\n";
202  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
203  Print() << " " << particle_mesh_plot_names[i] << "\n";
204  }
205  first_call = false;
206  }
207  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
208  std::string tmp(particle_mesh_plot_names[i]);
209  if (containerHasElement(plot_var_names, tmp) ) {
210  tmp_plot_names.push_back(tmp);
211  }
212  }
213  }
214 #endif
215 
216  {
217  Vector<std::string> microphysics_plot_names;
218  micro->GetPlotVarNames(microphysics_plot_names);
219  if (microphysics_plot_names.size() > 0) {
220  static bool first_call = true;
221  if (first_call) {
222  Print() << getEnumNameString(solverChoice.moisture_type)
223  << ": the following additional variables are available to plot:\n";
224  for (int i = 0; i < microphysics_plot_names.size(); i++) {
225  Print() << " " << microphysics_plot_names[i] << "\n";
226  }
227  first_call = false;
228  }
229  for (auto& plot_name : microphysics_plot_names) {
230  if (containerHasElement(plot_var_names, plot_name)) {
231  tmp_plot_names.push_back(plot_name);
232  }
233  }
234  }
235  }
236 
237  for (int i = 0; i < tmp_plot_names.size(); i++) {
238  a_plot_var_names.push_back( tmp_plot_names[i] );
239  }
240 
241  // Finally, check to see if we found all the requested variables
242  for (const auto& plot_name : plot_var_names) {
243  if (!containerHasElement(a_plot_var_names, plot_name)) {
244  if (amrex::ParallelDescriptor::IOProcessor()) {
245  Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
246  }
247  }
248  }
249 }
containerHasElement
bool containerHasElement(const V &iterable, const T &query)
Definition: ERF_Container.H:5
ERF::pp_prefix
std::string pp_prefix
Definition: ERF.H:637
WSM6SedCellScratch::tmp
@ tmp
Definition: ERF_AdvanceWSM6.cpp:114
Here is the call graph for this function:

◆ apply_gaussian_smoothing_to_perturbations()

void ERF::apply_gaussian_smoothing_to_perturbations ( const int  lev,
amrex::MultiFab &  mf_cc_pert 
)
88 {
89  const Geometry& gm = geom[lev];
90  const Real dx = gm.CellSize(0);
91  const Real dy = gm.CellSize(1);
92 
93  const Real dmesh = std::min(dx, dy);
94 
95  // ---- User choice ----
96  const Real sigma = solverChoice.ens_pert_correlated_radius;
97  const int r = static_cast<int>(3.0 * sigma / dmesh);
98 
99  const int ncomp = mf_cc_pert.nComp();
100 
101  // ---- Precompute Gaussian weights ----
102  const int wsize = 2*r + 1;
103  Vector<Real> w_host(wsize * wsize);
104 
105  Real Z = zero;
106  for (int m = -r; m <= r; ++m) {
107  for (int n = -r; n <= r; ++n) {
108  Real val = std::exp(-(m*m*dx*dx + n*n*dy*dy)
109  /(two*sigma*sigma));
110  w_host[(m+r)*wsize + (n+r)] = val;
111  Z += val;
112  }
113  }
114 
115  for (auto& v : w_host) {
116  v /= Z;
117  }
118 
119  Gpu::DeviceVector<Real> w_dev(w_host.size());
120  Gpu::copy(Gpu::hostToDevice, w_host.begin(), w_host.end(), w_dev.begin());
121 
122  Real const* w = w_dev.data();
123 
124  // ---- Create a grown copy (for stencil access) ----
125  IntVect ngrow_big(AMREX_D_DECL(r, r, 0));
126 
127  MultiFab mf_copy(mf_cc_pert.boxArray(),
128  mf_cc_pert.DistributionMap(),
129  ncomp, ngrow_big);
130 
131  mf_copy.ParallelCopy(mf_cc_pert,
132  0, 0, ncomp,
133  IntVect(0), ngrow_big,
134  gm.periodicity());
135 
136  // ---- Apply smoothing ----
137  for (MFIter mfi(mf_cc_pert, TilingIfNotGPU()); mfi.isValid(); ++mfi)
138  {
139  const Box& bx = mfi.tilebox();
140 
141  auto const& in = mf_copy.const_array(mfi);
142  auto const& out = mf_cc_pert.array(mfi);
143 
144  ParallelFor(bx, ncomp,
145  [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
146  {
147  Real sum = zero;
148 
149  for (int m = -r; m <= r; ++m) {
150  for (int nn = -r; nn <= r; ++nn) {
151  Real wij = w[(m+r)*wsize + (nn+r)];
152  sum += wij * in(i+m, j+nn, k, n);
153  }
154  }
155 
156  out(i,j,k,n) = sum;
157  });
158  }
159  NormalizeMultiFabRMS_PerComponent(mf_cc_pert);
160 }
two
constexpr amrex::Real two
Definition: ERF_Constants.H:10
zero
constexpr amrex::Real zero
Definition: ERF_Constants.H:8
out
struct @29 out
in
struct @29 in
dy
const Real dy
Definition: ERF_InitCustomPert_ABL.H:24
dx
const Real dx
Definition: ERF_InitCustomPert_ABL.H:23
sigma
amrex::Real sigma
Definition: ERF_InitCustomPert_DataAssimilation_ISV.H:11
NormalizeMultiFabRMS_PerComponent
void NormalizeMultiFabRMS_PerComponent(MultiFab &mf_cc_pert)
Definition: ERF_InitForEnsemble.cpp:40
ParallelFor
ParallelFor(grown_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) { qrcuten_arr(i, j, k)=Real(0);qscuten_arr(i, j, k)=Real(0);qicuten_arr(i, j, k)=Real(0);})
SolverChoice::ens_pert_correlated_radius
amrex::Real ens_pert_correlated_radius
Definition: ERF_DataStruct.H:1483
Here is the call graph for this function:

◆ ApplyOceanSurfaceState()

void ERF::ApplyOceanSurfaceState ( const amrex::Vector< amrex::MultiFab * > &  state,
double  time 
)
510 {
511  if (solverChoice.lsm_type != LandSurfaceType::OceanSurf) {
512  return;
513  }
514 
515  if (!state.empty() && state[0] != nullptr && lsm.Get_Data_Ptr(0, 0) != nullptr) {
516  auto* dst = lsm.Get_Data_Ptr(0, 0);
517  amrex::MultiFab src_remapped(dst->boxArray(), dst->DistributionMap(), 1, 0);
518  src_remapped.ParallelCopy(*state[0], 0, 0, 1);
519  const auto lsm_geom = lsm.Get_Lsm_Geom(0);
520  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
521  lsm_geom.isPeriodic(0) == Geom(0).isPeriodic(0) &&
522  lsm_geom.isPeriodic(1) == Geom(0).isPeriodic(1) &&
523  lsm_geom.isPeriodic(2) == Geom(0).isPeriodic(2),
524  "OceanSurf t_surf geometry lost ERF periodic flags.");
525  dst->ParallelCopy(src_remapped, 0, 0, 1);
526  dst->FillBoundary(lsm_geom.periodicity());
527 
528  const amrex::Real src_min = state[0]->min(0);
529  const amrex::Real src_max = state[0]->max(0);
530  const amrex::Real dst_min = dst->min(0);
531  const amrex::Real dst_max = dst->max(0);
532 
533  if (amrex::ParallelDescriptor::IOProcessor()) {
534  amrex::Print() << "OceanSurf apply at t=" << time
535  << " s from SST slab: source min/max = "
536  << src_min << " / " << src_max
537  << " K, cache min/max = "
538  << dst_min << " / " << dst_max
539  << " K" << std::endl;
540  }
541 
542  if (dst_min < amrex::Real(260.0) || dst_max > amrex::Real(320.0)) {
543  amrex::Warning("OceanSurf t_surf is outside the expected [260, 320] K range");
544  }
545  }
546 }
LandSurface::Get_Lsm_Geom
amrex::Geometry Get_Lsm_Geom(const int &lev)
Definition: ERF_LandSurface.H:124
Here is the call graph for this function:

◆ AverageDown()

void ERF::AverageDown ( )
private
17 {
18  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
19 
20  int src_comp, num_comp;
21  for (int lev = finest_level-1; lev >= 0; --lev)
22  {
23  // If anelastic we don't average down rho because rho == rho0.
24  if (solverChoice.anelastic[lev]) {
25  src_comp = 1;
26  } else {
27  src_comp = 0;
28  }
29  num_comp = vars_new[0][Vars::cons].nComp() - src_comp;
30  AverageDownTo(lev,src_comp,num_comp);
31  }
32 }
AMREX_ALWAYS_ASSERT
AMREX_ALWAYS_ASSERT(bx.length()[2]==khi+1)
ERF::AverageDownTo
void AverageDownTo(int crse_lev, int scomp, int ncomp, bool do_perturbational_and_momenta=true)
Definition: ERF_AverageDown.cpp:36
SolverChoice::anelastic
amrex::Vector< int > anelastic
Definition: ERF_DataStruct.H:1257
Here is the call graph for this function:

◆ AverageDownMoistStateTo()

void ERF::AverageDownMoistStateTo ( int  crse_lev)

Conservation-preserving fine→coarse average of moist state components (RhoTheta and the contiguous moist q range) using detJ/mfac weighting, reusing AverageDownTo with perturbational/momenta handling disabled.

244 {
245  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
246  AMREX_ALWAYS_ASSERT(crse_lev >= 0 && crse_lev < finest_level);
247 
248  AverageDownTo(crse_lev, RhoTheta_comp, 1, false);
249 
250  const int n_moist = (micro) ? micro->Get_Qstate_Size() : 0;
251  if (n_moist > 0) {
252  AverageDownTo(crse_lev, RhoQ1_comp, n_moist, false);
253  }
254 }
Here is the call graph for this function:

◆ AverageDownTo()

void ERF::AverageDownTo ( int  crse_lev,
int  scomp,
int  ncomp,
bool  do_perturbational_and_momenta = true 
)
37 {
38  if (do_perturbational_and_momenta) {
39  if (solverChoice.anelastic[crse_lev]) {
40  AMREX_ALWAYS_ASSERT(scomp == 1);
41  } else {
42  AMREX_ALWAYS_ASSERT(scomp == 0);
43  }
44  AMREX_ALWAYS_ASSERT(ncomp == vars_new[crse_lev][Vars::cons].nComp() - scomp);
45  } else {
46  AMREX_ALWAYS_ASSERT(scomp >= 0);
47  AMREX_ALWAYS_ASSERT(ncomp > 0);
48  AMREX_ALWAYS_ASSERT(scomp + ncomp <= vars_new[crse_lev][Vars::cons].nComp());
49  }
50  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
51 
52  // ******************************************************************************************
53  // First do cell-centered quantities
54  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
55  // m is the map scale factor at cell centers
56  // Here we multiply (rho S) by detJ and divide (rho S) by m^2 before average down
57  // ******************************************************************************************
58  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
59  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
60  const Box& bx = mfi.tilebox();
61  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
62  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
63  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
64  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
65  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
66  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
67  {
68  cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
69  });
70  } else {
71  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
72  {
73  cons_arr(i,j,k,scomp+n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
74  });
75  }
76  } // mfi
77  } // lev
78 
79  int fine_lev = crse_lev+1;
80 
81  if (do_perturbational_and_momenta && interpolation_type == StateInterpType::Perturbational) {
82  // Make the fine rho and (rho theta) be perturbational
83  MultiFab::Divide(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
84  Rho_comp,RhoTheta_comp,1,IntVect{0});
85  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
86  BaseState::r0_comp,Rho_comp,1,IntVect{0});
87  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
88  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
89 
90  // Make the crse rho and (rho theta) be perturbational
91  MultiFab::Divide(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
92  Rho_comp,RhoTheta_comp,1,IntVect{0});
93  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
94  BaseState::r0_comp,Rho_comp,1,IntVect{0});
95  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
96  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
97  }
98 
99  if (SolverChoice::terrain_type != TerrainType::EB) {
100  average_down(vars_new[crse_lev+1][Vars::cons],vars_new[crse_lev ][Vars::cons],
101  scomp, ncomp, refRatio(crse_lev));
102  } else {
103  // const auto dx = geom[fine_lev].CellSize();
104  // Setting cell_vol to the exact value may cause round-off errors in volume average.
105  // const Real cell_vol = dx[0]*dx[1]*dx[2];
106  constexpr Real cell_vol = one;
107  const BoxArray& ba = vars_new[fine_lev][IntVars::cons].boxArray();
108  const DistributionMapping& dm = vars_new[fine_lev][IntVars::cons].DistributionMap();
109  MultiFab vol_fine(ba, dm, 1, 0);
110  vol_fine.setVal(cell_vol);
111  EB_average_down(vars_new[fine_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
112  vol_fine, *detJ_cc[fine_lev],
113  scomp, ncomp, refRatio(crse_lev));
114  }
115 
116  if (do_perturbational_and_momenta && interpolation_type == StateInterpType::Perturbational) {
117  // Restore the fine data to what it was
118  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
119  BaseState::r0_comp,Rho_comp,1,IntVect{0});
120  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
121  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
122  MultiFab::Multiply(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
123  Rho_comp,RhoTheta_comp,1,IntVect{0});
124 
125  // Make the crse data be full values not perturbational
126  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
127  BaseState::r0_comp,Rho_comp,1,IntVect{0});
128  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
129  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
130  MultiFab::Multiply(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
131  Rho_comp,RhoTheta_comp,1,IntVect{0});
132  }
133 
134  vars_new[crse_lev][Vars::cons].FillBoundary(geom[crse_lev].periodicity());
135 
136  // ******************************************************************************************
137  // Here we multiply (rho S) by m^2 and divide by detJ after average down
138  // ******************************************************************************************
139  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
140  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
141  const Box& bx = mfi.tilebox();
142  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
143  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
144  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
145  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
146  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
147  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
148  {
149  cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
150  });
151  } else { // MeshType::ConstantDz
152  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
153  {
154  cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
155  });
156  }
157  } // mfi
158  } // lev
159 
160  // Fill EB covered cells by old values
161  // (This won't be needed because EB_average_down copies the covered value.)
162  if (SolverChoice::terrain_type == TerrainType::EB) {
163  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
164  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
165  const Box& bx = mfi.tilebox();
166  const Array4< Real> cons_new = vars_new[lev][Vars::cons].array(mfi);
167  const Array4<const Real> cons_old = vars_old[lev][Vars::cons].array(mfi);
168  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
169  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
170  {
171  if (detJ_arr(i,j,k) == zero) {
172  cons_new(i,j,k,scomp+n) = cons_old(i,j,k,scomp+n);
173  }
174  });
175  } // mfi
176  } // lev
177  }
178 
179  if (!do_perturbational_and_momenta) return;
180 
181  // ******************************************************************************************
182  // Now average down momenta.
183  // Note that vars_new holds velocities not momenta, but we want to do conservative
184  // averaging so we first convert to momentum, then average down, then convert
185  // back to velocities -- only on the valid region
186  // ******************************************************************************************
187  for (int lev = crse_lev; lev <= crse_lev+1; lev++)
188  {
189  // FillBoundary for density so we can go back and forth between velocity and momentum
190  vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
191 
192  const MultiFab* c_vfrac = nullptr;
193  if (SolverChoice::terrain_type == TerrainType::EB) {
194  c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
195  }
196 
197  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
198  vars_new[lev][Vars::yvel], IntVect(0,0,0),
199  vars_new[lev][Vars::zvel], IntVect(0,0,0),
200  vars_new[lev][Vars::cons],
201  rU_new[lev],
202  rV_new[lev],
203  rW_new[lev],
204  Geom(lev).Domain(),
205  domain_bcs_type,
206  c_vfrac);
207  }
208 
209  if (SolverChoice::terrain_type != TerrainType::EB) {
210  average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
211  average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
212  average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
213  } else {
214  EB_average_down_faces({&rU_new[crse_lev+1], &rV_new[crse_lev+1], &rW_new[crse_lev+1]},
215  {&rU_new[crse_lev], &rV_new[crse_lev], &rW_new[crse_lev]},
216  refRatio(crse_lev), 0);
217  }
218 
219  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
220 
221  const MultiFab* c_vfrac = nullptr;
222  if (SolverChoice::terrain_type == TerrainType::EB) {
223  c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
224  }
225 
226  MomentumToVelocity(vars_new[lev][Vars::xvel],
227  vars_new[lev][Vars::yvel],
228  vars_new[lev][Vars::zvel],
229  vars_new[lev][Vars::cons],
230  rU_new[lev],
231  rV_new[lev],
232  rW_new[lev],
233  Geom(lev).Domain(),
234  domain_bcs_type,
235  c_vfrac);
236  }
237 }
MomentumToVelocity
void MomentumToVelocity(MultiFab &xvel, MultiFab &yvel, MultiFab &zvel, const MultiFab &density, const MultiFab &xmom_in, const MultiFab &ymom_in, const MultiFab &zmom_in, const Box &domain, const Vector< BCRec > &domain_bcs_type_h, const MultiFab *c_vfrac)
Definition: ERF_MomentumToVelocity.cpp:25
ERF::interpolation_type
static StateInterpType interpolation_type
Definition: ERF.H:1350
BaseState::th0_comp
@ th0_comp
Definition: ERF_IndexDefines.H:76
Here is the call graph for this function:

◆ build_fine_mask()

void ERF::build_fine_mask ( int  lev,
amrex::MultiFab &  fine_mask 
)

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters
levelFine level index which masks underlying coarser data
126 {
127  // Mask for zeroing covered cells
128  AMREX_ASSERT(level > 0);
129 
130  BoxArray cba = grids[level-1];
131  DistributionMapping cdm = dmap[level-1];
132 
133  BoxArray fba = fine_mask_lev.boxArray();
134 
135  iMultiFab ifine_mask_lev = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
136 
137  const auto fma = fine_mask_lev.arrays();
138  const auto ifma = ifine_mask_lev.arrays();
139  ParallelFor(fine_mask_lev, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
140  {
141  fma[bno](i,j,k) = ifma[bno](i,j,k);
142  });
143 }
Here is the call graph for this function:

◆ check_for_low_temp()

void ERF::check_for_low_temp ( amrex::MultiFab &  S)
2881 {
2882  // *****************************************************************************
2883  // Test for low temp (low is defined as beyond the microphysics range of validity)
2884  // *****************************************************************************
2885  //
2886  // This value is defined in erf_dtesati in Source/Utils/ERF_MicrophysicsUtils.H
2887  Real t_low = Real(273.16) - Real(85.);
2888  //
2889  for (MFIter mfi(S); mfi.isValid(); ++mfi)
2890  {
2891  Box bx = mfi.tilebox();
2892  const Array4<Real> &s_arr = S.array(mfi);
2893  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
2894  {
2895  const Real rho = s_arr(i, j, k, Rho_comp);
2896  const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
2897  const Real qv = s_arr(i, j, k, RhoQ1_comp) / rho;
2898 
2899  Real temp = getTgivenRandRTh(rho, rhotheta, qv);
2900 
2901  if (temp < t_low) {
2902 #ifdef AMREX_USE_GPU
2903  AMREX_DEVICE_PRINTF("Temperature too low in cell: %d %d %d %e \n", i,j,k,temp);
2904 #else
2905  printf("Temperature too low in cell: %d %d %d \n", i,j,k);
2906  printf("Based on temp / rhotheta / rho / qv %e %e %e %e \n", temp,rhotheta,rho,qv);
2907 #endif
2908  Abort();
2909  }
2910  });
2911  }
2912 }
getTgivenRandRTh
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getTgivenRandRTh(const amrex::Real rho, const amrex::Real rhotheta, const amrex::Real qv=amrex::Real(0))
Definition: ERF_EOS.H:46
rho
rho
Definition: ERF_InitCustomPert_Bubble.H:107
MicVar_Kess::qv
@ qv
Definition: ERF_Kessler.H:30
Here is the call graph for this function:

◆ check_for_negative_theta()

void ERF::check_for_negative_theta ( amrex::MultiFab &  S)
2916 {
2917  // *****************************************************************************
2918  // Test for negative (rho theta)
2919  // *****************************************************************************
2920  for (MFIter mfi(S); mfi.isValid(); ++mfi)
2921  {
2922  Box bx = mfi.tilebox();
2923  const Array4<Real> &s_arr = S.array(mfi);
2924  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
2925  {
2926  const Real rho = s_arr(i, j, k, Rho_comp);
2927  const Real rhotheta = s_arr(i, j, k, RhoTheta_comp);
2928 
2929  if (rho <= zero) {
2930 #ifdef AMREX_USE_GPU
2931  AMREX_DEVICE_PRINTF("Rho is negative at %d %d %d %e \n", i,j,k,rho);
2932 #else
2933  printf("Rho is negative at %d %d %d %e \n", i,j,k,rho);
2934  Abort("Bad rho in check_for_negative_theta");
2935 #endif
2936  }
2937 
2938  if (rhotheta <= zero) {
2939 #ifdef AMREX_USE_GPU
2940  AMREX_DEVICE_PRINTF("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
2941 #else
2942  printf("RhoTheta is negative at %d %d %d %e \n", i,j,k,rhotheta);
2943  Abort("Bad theta in check_for_negative_theta");
2944 #endif
2945  }
2946 
2947  });
2948  } // mfi
2949 }
Here is the call graph for this function:

◆ check_mesh_type()

void ERF::check_mesh_type ( int  lev)
2954 {
2955  if (SolverChoice::mesh_type == MeshType::VariableDz) {
2956  MultiFab z_slab(convert(ba2d[lev],IntVect(1,1,1)),dmap[lev],1,0);
2957  int klo = geom[lev].Domain().smallEnd(2);
2958  for (MFIter mfi(z_slab); mfi.isValid(); ++mfi) {
2959  Box nbx = mfi.tilebox();
2960  Array4<Real const> const& z_arr = z_phys_nd[lev]->const_array(mfi);
2961  Array4<Real > const& z_slab_arr = z_slab.array(mfi);
2962  ParallelFor(nbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
2963  {
2964  z_slab_arr(i,j,k) = z_arr(i,j,klo);
2965  });
2966  }
2967  Real z_min = z_slab.min(0);
2968  Real z_max = z_slab.max(0);
2969 
2970  auto dz = geom[lev].CellSize()[2];
2971  if (z_max - z_min < Real(1.e-8) * dz) {
2972  SolverChoice::set_mesh_type(MeshType::StretchedDz);
2973  if (verbose > 0) {
2974  amrex::Print() << "Resetting mesh type to StretchedDz since terrain is flat" << std::endl;
2975  }
2976  }
2977  }
2978 }
ERF::ba2d
amrex::Vector< amrex::BoxArray > ba2d
Definition: ERF.H:1386
WSM6SedCellScratch::dz
@ dz
Definition: ERF_AdvanceWSM6.cpp:104
SolverChoice::set_mesh_type
static void set_mesh_type(MeshType new_mesh_type)
Definition: ERF_DataStruct.H:1242
Here is the call graph for this function:

◆ check_state_for_nans()

void ERF::check_state_for_nans ( amrex::MultiFab const &  S)
2787 {
2788  amrex::Gpu::DeviceScalar<int> d_found(0);
2789 
2790  // comp, i, j, k
2791  amrex::Gpu::DeviceVector<int> d_info(4, -1);
2792 
2793  for (MFIter mfi(S,TilingIfNotGPU()); mfi.isValid(); ++mfi)
2794  {
2795  const Box& bx = mfi.tilebox();
2796  auto const& s_arr = S.const_array(mfi);
2797 
2798  const int ncomp = S.nComp();
2799 
2800  int* found = d_found.dataPtr();
2801  int* info = d_info.dataPtr();
2802 
2803  ParallelFor(bx,
2804  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
2805  {
2806  // Somebody already found a NaN
2807  if (*found) return;
2808 
2809  for (int n = 0; n < ncomp; ++n)
2810  {
2811  Real val = s_arr(i,j,k,n);
2812 
2813  if (val != val) // NaN test
2814  {
2815  // Only one thread wins
2816  if (amrex::Gpu::Atomic::CAS(found,0,1) == 0)
2817  {
2818  info[0] = n;
2819  info[1] = i;
2820  info[2] = j;
2821  info[3] = k;
2822  }
2823  return;
2824  }
2825  }
2826  });
2827  }
2828 
2829  amrex::Gpu::streamSynchronize();
2830 
2831  if (d_found.dataValue())
2832  {
2833  amrex::Vector<int> h_info(4);
2834  amrex::Print() << "Found flag = " << d_found.dataValue() << "\n";
2835 
2836  amrex::Gpu::copy(
2837  amrex::Gpu::deviceToHost,
2838  d_info.begin(),
2839  d_info.end(),
2840  h_info.begin());
2841 
2842  std::cout << "NaN found in component " << h_info[0]
2843  << " at (i,j,k) = ("
2844  << h_info[1] << ", "
2845  << h_info[2] << ", "
2846  << h_info[3] << ")\n";
2847 
2848  amrex::Abort("NaN detected in state");
2849  }
2850 }
Here is the call graph for this function:

◆ check_vels_for_nans()

void ERF::check_vels_for_nans ( amrex::MultiFab const &  xvel,
amrex::MultiFab const &  yvel,
amrex::MultiFab const &  zvel 
)
2854 {
2855  //
2856  // Test at the end of every full timestep whether the solution data contains NaNs
2857  //
2858  bool any_have_nans = false;
2859  if (xvel.contains_nan(0,1,0))
2860  {
2861  amrex::Print() << "x-velocity contains NaNs " << '\n';
2862  any_have_nans = true;
2863  }
2864  if (yvel.contains_nan(0,1,0))
2865  {
2866  amrex::Print() << "y-velocity contains NaNs" << '\n';
2867  any_have_nans = true;
2868  }
2869  if (zvel.contains_nan(0,1,0))
2870  {
2871  amrex::Print() << "z-velocity contains NaNs" << '\n';
2872  any_have_nans = true;
2873  }
2874  if (any_have_nans) {
2875  exit(0);
2876  }
2877 }

◆ ClearLevel()

void ERF::ClearLevel ( int  lev)
override
841 {
842  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
843  vars_new[lev][var_idx].clear();
844  vars_old[lev][var_idx].clear();
845  }
846 
847  base_state[lev].clear();
848 
849  rU_new[lev].clear();
850  rU_old[lev].clear();
851  rV_new[lev].clear();
852  rV_old[lev].clear();
853  rW_new[lev].clear();
854  rW_old[lev].clear();
855 
856  if (lev > 0) {
857  zmom_crse_rhs[lev].clear();
858  }
859 
860  if ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) {
861  pp_inc[lev].clear();
862  }
863  if (solverChoice.anelastic[lev] == 0) {
864  lagged_delta_rt[lev].clear();
865  }
866  avg_xmom[lev].clear();
867  avg_ymom[lev].clear();
868  avg_zmom[lev].clear();
869 
870  // Clears the integrator memory
871  mri_integrator_mem[lev].reset();
872 
873  // Clears the physical boundary condition routines
874  physbcs_cons[lev].reset();
875  physbcs_u[lev].reset();
876  physbcs_v[lev].reset();
877  physbcs_w[lev].reset();
878  physbcs_base[lev].reset();
879 
880  // Clears the flux register array (only allocated for TwoWay coupling)
881  if (advflux_reg[lev]) {
882  advflux_reg[lev]->reset();
883  }
884 
885  // Clears the 2D arrays
886  if (sst_lev[lev][0]) {
887  for (int n = 0; n < sst_lev[lev].size(); n++) {
888  sst_lev[lev][n].reset();
889  }
890  }
891  if (tsk_lev[lev][0]) {
892  for (int n = 0; n < tsk_lev[lev].size(); n++) {
893  tsk_lev[lev][n].reset();
894  }
895  }
896  if (lat_m[lev]) {
897  lat_m[lev].reset();
898  }
899  if (lon_m[lev]) {
900  lon_m[lev].reset();
901  }
902  if (sinPhi_m[lev]) {
903  sinPhi_m[lev].reset();
904  }
905  if (cosPhi_m[lev]) {
906  cosPhi_m[lev].reset();
907  }
908 
909 #ifdef ERF_USE_FFT
910  // Clear any FFT solvers built at this level
911  if (m_3D_poisson.size() > lev) {
912  for (int n = 0; n < m_3D_poisson[lev].size(); n++) {
913  m_3D_poisson[lev][n].reset();
914  }
915  }
916  if (m_2D_poisson.size() > lev) {
917  for (int n = 0; n < m_2D_poisson[lev].size(); n++) {
918  m_2D_poisson[lev][n].reset();
919  }
920  }
921 #endif
922 }
ERF::avg_xmom
amrex::Vector< amrex::MultiFab > avg_xmom
Definition: ERF.H:944
ERF::pp_inc
amrex::Vector< amrex::MultiFab > pp_inc
Definition: ERF.H:940
ERF::lagged_delta_rt
amrex::Vector< amrex::MultiFab > lagged_delta_rt
Definition: ERF.H:943
ERF::sst_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
Definition: ERF.H:1025
ERF::advflux_reg
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
Definition: ERF.H:1096
ERF::sinPhi_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
Definition: ERF.H:874
ERF::cosPhi_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
Definition: ERF.H:874
ERF::avg_ymom
amrex::Vector< amrex::MultiFab > avg_ymom
Definition: ERF.H:945
ERF::physbcs_base
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
Definition: ERF.H:953
ERF::tsk_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
Definition: ERF.H:1026
ERF::avg_zmom
amrex::Vector< amrex::MultiFab > avg_zmom
Definition: ERF.H:946
Vars::NumTypes
@ NumTypes
Definition: ERF_IndexDefines.H:179
SolverChoice::project_initial_velocity
amrex::Vector< int > project_initial_velocity
Definition: ERF_DataStruct.H:1259

◆ cloud_fraction()

Real ERF::cloud_fraction ( double  time)
458 {
459  BL_PROFILE("ERF::cloud_fraction()");
460 
461  int lev = 0;
462  // This holds all of qc
463  MultiFab qc(vars_new[lev][Vars::cons],make_alias,RhoQ2_comp,1);
464 
465  int direction = 2; // z-direction
466  Box const& domain = geom[lev].Domain();
467 
468  auto const& qc_arr = qc.const_arrays();
469 
470  // qc_2d is an BaseFab<int> holding the max value over the column
471  auto qc_2d = ReduceToPlane<ReduceOpMax,int>(direction, domain, qc,
472  [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) -> int
473  {
474  if (qc_arr[box_no](i,j,k) > 0) {
475  return 1;
476  } else {
477  return 0;
478  }
479  });
480 
481  auto* p = qc_2d.dataPtr();
482 
483  Long numpts = qc_2d.numPts();
484 
485  AMREX_ASSERT(numpts < Long(std::numeric_limits<int>::max));
486 
487 #if 1
488  if (ParallelDescriptor::UseGpuAwareMpi()) {
489  ParallelDescriptor::ReduceIntMax(p,static_cast<int>(numpts));
490  } else {
491  Gpu::PinnedVector<int> hv(numpts);
492  Gpu::copyAsync(Gpu::deviceToHost, p, p+numpts, hv.data());
493  Gpu::streamSynchronize();
494  ParallelDescriptor::ReduceIntMax(hv.data(),static_cast<int>(numpts));
495  Gpu::copyAsync(Gpu::hostToDevice, hv.data(), hv.data()+numpts, p);
496  }
497 
498  // Sum over component 0
499  Long num_cloudy = qc_2d.template sum<RunOn::Device>(0);
500 
501 #else
502  //
503  // We need this if we allow domain decomposition in the vertical
504  // but for now we leave it commented out
505  //
506  Long num_cloudy = Reduce::Sum<Long>(numpts,
507  [=] AMREX_GPU_DEVICE (Long i) -> Long {
508  if (p[i] == 1) {
509  return 1;
510  } else {
511  return 0;
512  }
513  });
514  ParallelDescriptor::ReduceLongSum(num_cloudy);
515 #endif
516 
517  Real num_total = Real(qc_2d.box().d_numPts());
518 
519  Real cloud_frac = num_cloudy / num_total;
520 
521  return cloud_frac;
522 }
RhoQ2_comp
#define RhoQ2_comp
Definition: ERF_IndexDefines.H:43
MicVar_SatAdj::qc
@ qc
Definition: ERF_SatAdj.H:40
WSM6Ind::p
@ p
Definition: ERF_WSM6.H:189

Referenced by PackAtmosphericStates().

Here is the caller graph for this function:

◆ compute_divergence()

void ERF::compute_divergence ( int  lev,
amrex::MultiFab &  rhs,
amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM >  rho0_u_const,
amrex::Geometry const &  geom_at_lev 
)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level zero

11 {
12  BL_PROFILE("ERF::compute_divergence()");
13 
14  auto dxInv = geom_at_lev.InvCellSizeArray();
15 
16  // ****************************************************************************
17  // Compute divergence which will form RHS
18  // Note that we replace "rho0w" with the contravariant momentum, Omega
19  // ****************************************************************************
20  if (solverChoice.terrain_type == TerrainType::EB)
21  {
22  bool already_on_centroids = true;
23  EB_computeDivergence(rhs, rho0_u_const, geom_at_lev, already_on_centroids);
24  }
25  else if (SolverChoice::mesh_type == MeshType::ConstantDz)
26  {
27  computeDivergence(rhs, rho0_u_const, geom_at_lev);
28  }
29  else
30  {
31  for ( MFIter mfi(rhs,TilingIfNotGPU()); mfi.isValid(); ++mfi)
32  {
33  Box bx = mfi.tilebox();
34  const Array4<Real const>& rho0u_arr = rho0_u_const[0]->const_array(mfi);
35  const Array4<Real const>& rho0v_arr = rho0_u_const[1]->const_array(mfi);
36  const Array4<Real const>& rho0w_arr = rho0_u_const[2]->const_array(mfi);
37  const Array4<Real >& rhs_arr = rhs.array(mfi);
38 
39  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
40  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
41  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
42  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
43 
44  if (SolverChoice::mesh_type == MeshType::StretchedDz)
45  {
46  Real* stretched_dz_d_ptr = stretched_dz_d[lev].data();
47  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
48  {
49  Real inv_dz = one/stretched_dz_d_ptr[k];
50  Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
51  rhs_arr(i,j,k) = ( (rho0u_arr(i+1,j ,k )/mf_uy(i+1,j,0) - rho0u_arr(i,j,k)/mf_uy(i,j,0)) * dxInv[0]
52  +(rho0v_arr(i ,j+1,k )/mf_vx(i,j+1,0) - rho0v_arr(i,j,k)/mf_vx(i,j,0)) * dxInv[1]
53  +(rho0w_arr(i ,j ,k+1)/mfsq - rho0w_arr(i,j,k)/mfsq ) * inv_dz ) * mfsq;
54  });
55  }
56  else
57  {
58  //
59  // Note we compute the divergence using "rho0w" == Omega
60  //
61  const Array4<Real const>& ax_arr = ax[lev]->const_array(mfi);
62  const Array4<Real const>& ay_arr = ay[lev]->const_array(mfi);
63  const Array4<Real const>& dJ_arr = detJ_cc[lev]->const_array(mfi);
64 
65  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
66  {
67  Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
68  rhs_arr(i,j,k) = ( ( ax_arr(i+1,j,k)*rho0u_arr(i+1,j,k)/mf_uy(i+1,j,0)
69  -ax_arr(i ,j,k)*rho0u_arr(i ,j,k)/mf_uy(i ,j,0) ) * dxInv[0]
70  + ( ay_arr(i,j+1,k)*rho0v_arr(i,j+1,k)/mf_vx(i,j+1,0)
71  -ay_arr(i,j ,k)*rho0v_arr(i,j ,k)/mf_vx(i,j ,0) ) * dxInv[1]
72  +( rho0w_arr(i,j,k+1)/mfsq
73  - rho0w_arr(i,j,k )/mfsq ) * dxInv[2] ) * mfsq / dJ_arr(i,j,k);
74  });
75  }
76  } // mfi
77  }
78 }
ERF::ax
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
Definition: ERF.H:1054
ERF::ay
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
Definition: ERF.H:1055
Here is the call graph for this function:

◆ compute_max_pressure_gradient_diagnostic()

void ERF::compute_max_pressure_gradient_diagnostic ( int  lev)
13 {
14  // We don't require HSE when anelastic because the pressure gradient
15  // is computed from the Poisson solve
16  if (solverChoice.anelastic[lev]) return;
17 
18  auto& lev_new = vars_new[lev];
19 
20  int ng = (solverChoice.terrain_type == TerrainType::EB) ? 3 : 1;
21 
22  const Real grav = solverChoice.gravity;
23 
24  Vector<MultiFab> gradp_temp; gradp_temp.resize(AMREX_SPACEDIM);
25  gradp_temp[0].define(vars_new[lev][Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, 0);
26  gradp_temp[0].setVal(0.);
27  gradp_temp[1].define(vars_new[lev][Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, 0);
28  gradp_temp[1].setVal(0.);
29  gradp_temp[2].define(vars_new[lev][Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, 0);
30  gradp_temp[2].setVal(0.);
31 
32  int comp = 0;
33 
34  // Use this region to take max/min of gpx without including xlo,xhi if using real_bcs
35  Box xface_domain = surroundingNodes(geom[lev].Domain(), 0);
36  int ilo = xface_domain.smallEnd(0);
37  int ihi = xface_domain.bigEnd(0);
38  if (solverChoice.use_real_bcs) {
39  xface_domain.growLo(0,-1);
40  xface_domain.growHi(0,-1);
41  }
42 
43  // Use this region to take max/min of gpy without including ylo,yhi if using real_bcs
44  Box yface_domain = surroundingNodes(geom[lev].Domain(), 1);
45  int jlo = yface_domain.smallEnd(1);
46  int jhi = yface_domain.bigEnd(1);
47  if (solverChoice.use_real_bcs) {
48  yface_domain.growLo(1,-1);
49  yface_domain.growHi(1,-1);
50  }
51 
52 
53  // Use this region to take max/min of gpz without including top and bottom faces
54  Box zface_domain = surroundingNodes(geom[lev].Domain(), 2);
55  int klo = zface_domain.smallEnd(2);
56  int khi = zface_domain.bigEnd(2);
57 
58  zface_domain.growLo(2,-1);
59  zface_domain.growHi(2,-1);
60 
61  // *******************************************************************************
62  // First check that base state satisfies EOS
63  // *******************************************************************************
64 
65  Print() << " " << std::endl;
66 
67  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
68  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
69  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp , 1);
70  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
71 
72  MultiFab dp(p_hse.boxArray(), p_hse.DistributionMap(), 1, 0);
73 
74  // Initialize to zero in case of EB covered cells
75  dp.setVal(0.);
76 
77  for (MFIter mfi(dp); mfi.isValid(); ++mfi) {
78  Box bx = mfi.validbox();
79  auto const rhse_arr = r_hse.const_array(mfi);
80  auto const phse_arr = p_hse.const_array(mfi);
81  auto const qvhse_arr = qv_hse.const_array(mfi);
82  auto const thhse_arr = th_hse.const_array(mfi);
83  auto dpeos_arr = dp.array(mfi);
84 
85  if (solverChoice.terrain_type != TerrainType::EB) {
86  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
87  Real rhotheta = rhse_arr(i,j,k) * thhse_arr(i,j,k);
88  dpeos_arr(i,j,k) = std::abs(getPgivenRTh(rhotheta, qvhse_arr(i,j,k)) - phse_arr(i,j,k));
89  });
90  } else {
91  Array4<const Real> volfrac = (get_eb(lev).get_const_factory())->getVolFrac().const_array(mfi);
92  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
93  if (volfrac(i,j,k) > zero) {
94  Real rhotheta = rhse_arr(i,j,k) * thhse_arr(i,j,k);
95  dpeos_arr(i,j,k) = std::abs(getPgivenRTh(rhotheta, qvhse_arr(i,j,k)) - phse_arr(i,j,k));
96  }
97  });
98  }
99  }
100  Real max_diff = dp.max(0);
101  if (max_diff > 1.e-8) {
102  IntVect max_loc = dp.maxIndex(0);
103  Print() << "Max value of |p_hse - p_eos| is " << max_diff << std::endl;
104  Print() << " with max in cell " << max_loc << std::endl;
105  Abort("Base state violates EOS ");
106  } else {
107  Print() << "Max value of |p_hse - p_eos| is less than 1e-8" << std::endl;
108  }
109 
110  // *******************************************************************************
111  // Now compute pressure gradients for base state pressure
112  // *******************************************************************************
113 
114  compute_gradp(p_hse, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
115  get_eb(lev), gradp_temp, solverChoice);
116 
117  Real min_gpx = gradp_temp[0].min(xface_domain,comp);
118  Real max_gpx = gradp_temp[0].max(xface_domain,comp);
119  if (max_gpx != zero || min_gpx != zero) {
120  Print() << "Min/max value of dp0/dx are " << min_gpx << " " << max_gpx << std::endl;
121  IntVect min_loc = gradp_temp[0].minIndex(comp);
122  IntVect max_loc = gradp_temp[0].maxIndex(comp);
123  if (min_loc[0] != ilo && min_loc[0] != ihi) amrex::Print() << " with min at face " << min_loc;
124  if (max_loc[0] != ilo && max_loc[0] != ihi) amrex::Print() << " with max at face " << max_loc;
125  Print() << std::endl;
126  } else {
127  Print() << "Min/max value of dp0/dx are zero " << std::endl;
128  }
129 
130  Real min_gpy = gradp_temp[1].min(yface_domain,comp);
131  Real max_gpy = gradp_temp[1].max(yface_domain,comp);
132  if (max_gpy != zero || min_gpy != zero) {
133  Print() << "Min/max value of dp0/dy are " << min_gpy << " " << max_gpy << std::endl;
134  IntVect min_loc = gradp_temp[1].minIndex(comp);
135  IntVect max_loc = gradp_temp[1].maxIndex(comp);
136  if (min_loc[1] != jlo && min_loc[1] != jhi) amrex::Print() << " with min at face " << min_loc;
137  if (max_loc[1] != jlo && max_loc[1] != jhi) amrex::Print() << " with max at face " << max_loc;
138  Print() << std::endl;
139  } else {
140  Print() << "Min/max value of dp0/dy are zero " << std::endl;
141  }
142 
143  if (solverChoice.terrain_type != TerrainType::EB) {
144  for (MFIter mfi(gradp_temp[2]); mfi.isValid(); ++mfi) {
145  Box bx = mfi.validbox(); bx.growHi(2,-1);
146  if (bx.smallEnd(2) == 0) bx.growLo(2,-1);
147  auto gpz_arr = gradp_temp[2].array(mfi);
148  auto const rhse_arr = r_hse.const_array(mfi);
149  auto const qvhse_arr = qv_hse.const_array(mfi);
150  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
151  gpz_arr(i,j,k) += grav * myhalf * ( rhse_arr(i,j,k ) * (one + qvhse_arr(i,j,k ))
152  +rhse_arr(i,j,k-1) * (one + qvhse_arr(i,j,k-1)) );
153  });
154  }
155  // EB case: check HSE only for uncovered cells
156  } else {
157  for (MFIter mfi(gradp_temp[2]); mfi.isValid(); ++mfi) {
158  Box bx = mfi.validbox(); bx.growHi(2,-1);
159  if (bx.smallEnd(2) == 0) bx.growLo(2,-1);
160  auto gpz_arr = gradp_temp[2].array(mfi);
161  auto const rhse_arr = r_hse.const_array(mfi);
162  auto const qvhse_arr = qv_hse.const_array(mfi);
163  Array4<const Real> w_volfrac = (get_eb(lev).get_w_const_factory())->getVolFrac().const_array(mfi);
164 
165  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
166  if (w_volfrac(i,j,k) > zero) {
167  gpz_arr(i,j,k) += grav * myhalf * ( rhse_arr(i,j,k ) * (one + qvhse_arr(i,j,k ))
168  +rhse_arr(i,j,k-1) * (one + qvhse_arr(i,j,k-1)) );
169  }
170  });
171  }
172  }
173 
174  Real tol;
175 #ifdef AMREX_USE_FLOAT
176  tol = Real(1.e-4);
177 #else
178  if (solverChoice.terrain_type == TerrainType::EB) {
179  tol = 1.e-4;
180  } else {
181  tol = 1.e-8;
182  }
183 #endif
184 
185  Real min_gpz = gradp_temp[2].min(zface_domain,comp);
186  Real max_gpz = gradp_temp[2].max(zface_domain,comp);
187 
188  if (std::abs(max_gpz) > tol || std::abs(min_gpz) > tol) {
189  IntVect min_loc = gradp_temp[2].minIndex(comp);
190  IntVect max_loc = gradp_temp[2].maxIndex(comp);
191  Print() << "Min/max value of dp0/dz + rho0*|g| are " << min_gpz << " " << max_gpz;
192  if (min_loc[2] != klo && min_loc[2] != khi) amrex::Print() << " with min at face " << min_loc;
193  if (max_loc[2] != klo && max_loc[2] != khi) amrex::Print() << " with max at face " << max_loc;
194  amrex::Abort("Base state is too far out of HSE");
195  } else {
196  Print() << "Min/max value of dp0/dz + rho0*|g| are less than " << tol << std::endl;
197  }
198  Print() << " " << std::endl;
199 
200  if (!solverChoice.anelastic[lev]) {
201 
202  // *******************************************************************************
203  // Now compute for full (moist) pressure
204  // *******************************************************************************
205 
206  MultiFab p(p_hse.boxArray(), p_hse.DistributionMap(), 1, ng);
207  MultiFab rho(lev_new[Vars::cons], make_alias, Rho_comp , 1);
208 
209  if (solverChoice.moisture_type != MoistureType::None) {
210 
211  for (MFIter mfi(rho); mfi.isValid(); ++mfi)
212  {
213  Box gbx = mfi.tilebox();
214  gbx.grow(IntVect(ng,ng,ng));
215  if (gbx.smallEnd(2) < 0) gbx.setSmall(2,0);
216 
217  const Array4<const Real>& cell_data = lev_new[Vars::cons].array(mfi);
218  const Array4<const Real>& r_arr = rho.array(mfi);
219  const Array4< Real>& pp_arr = p.array(mfi);
220  ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
221  {
222  Real qv_for_p = cell_data(i,j,k,RhoQ1_comp)/r_arr(i,j,k);
223  pp_arr(i,j,k) = getPgivenRTh(cell_data(i,j,k,RhoTheta_comp),qv_for_p);
224  });
225  }
226  compute_gradp(p, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
227  get_eb(lev), gradp_temp, solverChoice);
228 
229  min_gpx = gradp_temp[0].min(xface_domain,comp);
230  max_gpx = gradp_temp[0].max(xface_domain,comp);
231  if (max_gpx != zero || min_gpx != zero) {
232  Print() << "Min/Max value of x-gradient of full (moist) pressure are " << min_gpx << " " << max_gpx;
233  IntVect min_loc = gradp_temp[0].minIndex(comp);
234  IntVect max_loc = gradp_temp[0].maxIndex(comp);
235  if (min_loc[0] != ilo && min_loc[0] != ihi) amrex::Print() << " with min at face " << min_loc;
236  if (max_loc[0] != ilo && max_loc[0] != ihi) amrex::Print() << " with max at face " << max_loc;
237  Print() << std::endl;
238  } else {
239  Print() << "Min/max value of x-gradient of full (moist) pressure are zero " << std::endl;
240  }
241 
242  min_gpy = gradp_temp[1].min(yface_domain,comp);
243  max_gpy = gradp_temp[1].max(yface_domain,comp);
244  if (max_gpy != zero || min_gpy != zero) {
245  Print() << "Min/Max value of y-gradient of full (moist) pressure are " << min_gpy << " " << max_gpy;
246  IntVect min_loc = gradp_temp[1].minIndex(comp);
247  IntVect max_loc = gradp_temp[1].maxIndex(comp);
248  if (min_loc[1] != jlo && min_loc[1] != jhi) amrex::Print() << " with min at face " << min_loc;
249  if (max_loc[1] != jlo && max_loc[1] != jhi) amrex::Print() << " with max at face " << max_loc;
250  Print() << std::endl;
251  } else {
252  Print() << "Min/max value of y-gradient of full (moist) pressure are zero " << std::endl;
253  }
254 
255  MultiFab qt(rho.boxArray(), rho.DistributionMap(), 1, 1);
256  int n_qstate_into_total = micro->Get_Qstate_Moist_Size() - micro->Get_Qstate_Moist_NumConc_Size();
257  make_qt(lev_new[Vars::cons], qt, n_qstate_into_total);
258 
259  for (MFIter mfi(gradp_temp[2]); mfi.isValid(); ++mfi)
260  {
261  Box bx = mfi.validbox(); bx.growHi(2,-1);
262  if (bx.smallEnd(2) == 0) bx.growLo(2,-1);
263  auto gpz_arr = gradp_temp[2].array(mfi);
264  auto const r_arr = rho.const_array(mfi);
265  auto const qt_arr = qt.const_array(mfi);
266 
267  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
268  gpz_arr(i,j,k) += grav * myhalf * (r_arr(i,j,k )*(one+qt_arr(i,j,k )) +
269  r_arr(i,j,k-1)*(one+qt_arr(i,j,k-1)) );
270  });
271  }
272 
273  min_gpz = gradp_temp[2].min(zface_domain,comp);
274  max_gpz = gradp_temp[2].max(zface_domain,comp);
275  if (max_gpz != zero || min_gpz != zero) {
276  IntVect min_loc = gradp_temp[2].minIndex(comp);
277  IntVect max_loc = gradp_temp[2].maxIndex(comp);
278  Print() << "Min/max value of moist dp/dz + rho_m*|g| are " << min_gpz << " " << max_gpz;
279  if (min_loc[2] != klo && min_loc[2] != khi) amrex::Print() << " with min at face " << min_loc;
280  if (max_loc[2] != klo && max_loc[2] != khi) amrex::Print() << " with max at face " << max_loc;
281  Print() << std::endl;
282  } else {
283  Print() << "Min/max value of moist dp/dz + rho_m*|g| are zero " << std::endl;
284  }
285  Print() << " " << std::endl;
286  } // if moist
287  } // if !anelastic
288 }
myhalf
constexpr amrex::Real myhalf
Definition: ERF_Constants.H:13
getPgivenRTh
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getPgivenRTh(const amrex::Real rhotheta, const amrex::Real qv=amrex::Real(0))
Definition: ERF_EOS.H:81
khi
const int khi
Definition: ERF_InitCustomPert_Bubble.H:21
compute_gradp
void compute_gradp(const MultiFab &p, const Geometry &geom, const MultiFab &z_phys_nd, const MultiFab &z_phys_cc, Vector< std::unique_ptr< MultiFab >> &mapfac, const eb_ &ebfact, Vector< MultiFab > &gradp, const SolverChoice &solverChoice)
Definition: ERF_MakeGradP.cpp:113
make_qt
void make_qt(const MultiFab &cons_state, MultiFab &qt, int n_qstate_into_total)
Definition: ERF_Utils.cpp:37
eb_::get_w_const_factory
eb_aux_ const * get_w_const_factory() const noexcept
Definition: ERF_EB.H:52
BaseState::qv0_comp
@ qv0_comp
Definition: ERF_IndexDefines.H:77
SolverChoice::gravity
amrex::Real gravity
Definition: ERF_DataStruct.H:1327
SolverChoice::use_real_bcs
static bool use_real_bcs
Definition: ERF_DataStruct.H:1236
Here is the call graph for this function:

◆ compute_native_shoc_tendencies()

void ERF::compute_native_shoc_tendencies ( int  lev,
amrex::MultiFab *  cons,
amrex::MultiFab *  xvel,
amrex::MultiFab *  yvel,
amrex::MultiFab *  zvel,
amrex::Real *  w_subsid,
amrex::MultiFab *  tau13,
amrex::MultiFab *  tau23,
amrex::MultiFab *  hfx3,
amrex::MultiFab *  qfx3,
amrex::MultiFab *  eddyDiffs,
amrex::MultiFab *  z_phys_nd,
const double &  dt_advance 
)
19 {
20  AMREX_ALWAYS_ASSERT(native_shoc_driver[lev]);
21  native_shoc_driver[lev]->advance(*cons, *xvel, *yvel, *zvel,
22  tau13, tau23, hfx3, qfx3, eddyDiffs,
23  *z_phys_nd_in, Geom(lev), dt_advance);
24 }
Here is the call graph for this function:

◆ ComputeAndWriteEnsemblePerturbations()

void ERF::ComputeAndWriteEnsemblePerturbations ( )

◆ ComputeDt()

void ERF::ComputeDt ( int  step = -1,
double  cur_time_d = 0.0 
)
private

Function that calls estTimeStep for each level

12 {
13  Vector<double> dt_tmp(finest_level+1);
14 
15  for (int lev = 0; lev <= finest_level; ++lev)
16  {
17  dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
18  }
19 
20  ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
21 
22  double dt_0 = dt_tmp[0];
23  int n_factor = 1;
24  for (int lev = 0; lev <= finest_level; ++lev) {
25  dt_tmp[lev] = amrex::min(dt_tmp[lev], static_cast<double>(change_max*dt[lev]));
26  n_factor *= nsubsteps[lev];
27  dt_0 = std::min(dt_0, static_cast<double>(n_factor*dt_tmp[lev]));
28 
29  }
30  // Limit level 0 time step if requested
31  if (step == 0) {
32  dt_0 *= init_shrink;
33  if (verbose && init_shrink != one) {
34  Print() << "Timestep 0: shrink level 0 initial dt by " << init_shrink << std::endl;
35  }
36  }
37  //
38  // Limit dt by the value of stop_time.
39  // Recall that stop_time is total time, but t_new is elapsed time,
40  // so we must add start_time to t_new
41  //
42  const double eps = 1.e-3*dt_0;
43  if (cur_time_d + dt_0 > (stop_time - start_time) - eps) {
44  dt_0 = (stop_time - start_time) - cur_time_d;
45  }
46 
47  dt[0] = dt_0;
48  for (int lev = 1; lev <= finest_level; ++lev) {
49  dt[lev] = dt[lev-1] / nsubsteps[lev];
50  }
51 }
ERF::stop_time
static double stop_time
Definition: ERF.H:1161
ERF::nsubsteps
amrex::Vector< int > nsubsteps
Definition: ERF.H:916
ERF::estTimeStep
double estTimeStep(int lev, long &dt_fast_ratio) const
Definition: ERF_ComputeTimestep.cpp:60
ERF::init_shrink
static amrex::Real init_shrink
Definition: ERF.H:1176
ERF::change_max
static amrex::Real change_max
Definition: ERF.H:1177

Referenced by EvolveOneStep().

Here is the caller graph for this function:

◆ ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells ( const SolverChoice &  sc)
inlinestaticprivate
1485  {
1486  int ngrow = 0;
1487 
1488  if (sc.use_num_diff)
1489  {
1490  ngrow = 3;
1491  } else {
1492  if (
1493  (sc.advChoice.dycore_horiz_adv_type == AdvType::Centered_6th)
1494  || (sc.advChoice.dycore_vert_adv_type == AdvType::Centered_6th)
1495  || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
1496  || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Centered_6th)
1497  || (sc.advChoice.dryscal_horiz_adv_type == AdvType::Centered_6th)
1498  || (sc.advChoice.dryscal_vert_adv_type == AdvType::Centered_6th) )
1499  { ngrow = 3; }
1500  else if (
1501  (sc.advChoice.dycore_horiz_adv_type == AdvType::Upwind_5th)
1502  || (sc.advChoice.dycore_vert_adv_type == AdvType::Upwind_5th)
1503  || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
1504  || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Upwind_5th)
1505  || (sc.advChoice.dryscal_horiz_adv_type == AdvType::Upwind_5th)
1506  || (sc.advChoice.dryscal_vert_adv_type == AdvType::Upwind_5th) )
1507  { ngrow = 3; }
1508  else if (
1509  (sc.advChoice.dryscal_horiz_adv_type == AdvType::Weno_5)
1510  || (sc.advChoice.dryscal_vert_adv_type == AdvType::Weno_5)
1511  || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_5)
1512  || (sc.advChoice.moistscal_vert_adv_type == AdvType::Weno_5)
1513  || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_5Z)
1514  || (sc.advChoice.moistscal_vert_adv_type == AdvType::Weno_5Z)
1515  || (sc.advChoice.dryscal_horiz_adv_type == AdvType::Weno_5Z)
1516  || (sc.advChoice.dryscal_vert_adv_type == AdvType::Weno_5Z) )
1517  { ngrow = 3; }
1518  else if (
1519  (sc.advChoice.dryscal_horiz_adv_type == AdvType::Weno_7)
1520  || (sc.advChoice.dryscal_vert_adv_type == AdvType::Weno_7)
1521  || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_7)
1522  || (sc.advChoice.moistscal_vert_adv_type == AdvType::Weno_7)
1523  || (sc.advChoice.moistscal_horiz_adv_type == AdvType::Weno_7Z)
1524  || (sc.advChoice.moistscal_vert_adv_type == AdvType::Weno_7Z)
1525  || (sc.advChoice.dryscal_horiz_adv_type == AdvType::Weno_7Z)
1526  || (sc.advChoice.dryscal_vert_adv_type == AdvType::Weno_7Z) )
1527  { ngrow = 4; }
1528  else
1529  {
1530  if (sc.terrain_type == TerrainType::EB){
1531  ngrow = 4;
1532  } else {
1533  ngrow = 2;
1534  }
1535  }
1536  }
1537 
1538  return ngrow;
1539  }
AdvType::Weno_7
@ Weno_7
AdvType::Weno_7Z
@ Weno_7Z
AdvType::Upwind_5th
@ Upwind_5th
AdvType::Centered_6th
@ Centered_6th
AdvType::Weno_5Z
@ Weno_5Z
AdvType::Weno_5
@ Weno_5
AdvChoice::moistscal_horiz_adv_type
AdvType moistscal_horiz_adv_type
Definition: ERF_AdvStruct.H:423
AdvChoice::dycore_vert_adv_type
AdvType dycore_vert_adv_type
Definition: ERF_AdvStruct.H:420
AdvChoice::moistscal_vert_adv_type
AdvType moistscal_vert_adv_type
Definition: ERF_AdvStruct.H:424
AdvChoice::dryscal_horiz_adv_type
AdvType dryscal_horiz_adv_type
Definition: ERF_AdvStruct.H:421
AdvChoice::dycore_horiz_adv_type
AdvType dycore_horiz_adv_type
Definition: ERF_AdvStruct.H:419
AdvChoice::dryscal_vert_adv_type
AdvType dryscal_vert_adv_type
Definition: ERF_AdvStruct.H:422
SolverChoice::advChoice
AdvChoice advChoice
Definition: ERF_DataStruct.H:1247

◆ ComputeGlobalMinLocation()

void ERF::ComputeGlobalMinLocation ( const SolverChoice &  sc,
const amrex::Geometry &  geom,
const amrex::Vector< amrex::MultiFab > &  S_data,
amrex::Real *  d_val_min_ptr,
int *  d_i_min_ptr,
int *  d_j_min_ptr,
amrex::Real &  global_val_min,
int &  global_i_min,
int &  global_j_min 
)
45 {
46  Real h_val_min;
47  int h_i_min, h_j_min;
48 
49  Gpu::copy(Gpu::deviceToHost, d_val_min_ptr, d_val_min_ptr + 1, &h_val_min);
50  Gpu::copy(Gpu::deviceToHost, d_i_min_ptr, d_i_min_ptr + 1, &h_i_min);
51  Gpu::copy(Gpu::deviceToHost, d_j_min_ptr, d_j_min_ptr + 1, &h_j_min);
52  Gpu::synchronize();
53 
54  Real local_val_min = h_val_min;
55  int local_i_min = h_i_min;
56  int local_j_min = h_j_min;
57 
58  int rank = ParallelDescriptor::MyProc();
59 
60  in.value = local_val_min;
61  in.rank = rank;
62 
63  #ifdef AMREX_USE_MPI
64  MPI_Allreduce(&in, &out, 1, MPI_DOUBLE_INT, MPI_MINLOC, MPI_COMM_WORLD);
65  #else
66  out = in;
67  #endif
68 
69  global_val_min = out.value;
70  int owner_rank = out.rank;
71 
72  // Broadcast the indices from the rank that owns the minimum
73  global_i_min = local_i_min;
74  global_j_min = local_j_min;
75 
76  ParallelDescriptor::Bcast(&global_i_min, 1, owner_rank);
77  ParallelDescriptor::Bcast(&global_j_min, 1, owner_rank);
78 
79  if (rank == 0) {
80  Print() << "Global minimum distance to hurricane eye (k=0): "
81  << global_val_min << " at (i,j) = ("
82  << global_i_min << ", " << global_j_min << ")\n";
83  }
84 
85  Gpu::DeviceScalar<Real> d_eye_lat(zero), d_eye_lon(zero);
86 
87  Real* d_eye_lat_ptr = d_eye_lat.dataPtr();
88  Real* d_eye_lon_ptr = d_eye_lon.dataPtr();
89 
90  int levc = finest_level;
91  // On owner_rank, compute eye_lat and eye_lon
92  if (sc.init_type == InitType::WRFInput and rank == owner_rank) {
93  for (MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
94  const Box& box = mfi.validbox();
95  FArrayBox& fab_lat = (*(lat_m[levc]))[mfi];
96  FArrayBox& fab_lon = (*(lon_m[levc]))[mfi];
97  const Array4<Real>& lat_arr = fab_lat.array();
98  const Array4<Real>& lon_arr = fab_lon.array();
99 
100  if (box.smallEnd()[2] == 0) {
101  Box bx2d = makeSlab(box,2,0);
102  ParallelFor(bx2d, [=] AMREX_GPU_DEVICE(int i, int j, int ) {
103  if (i == global_i_min && j == global_j_min) {
104  *d_eye_lat_ptr = lat_arr(i,j,0);
105  *d_eye_lon_ptr = lon_arr(i,j,0);
106  }
107  });
108  }
109  }
110  }
111 
112  if (sc.init_type == InitType::HindCast and rank == owner_rank) {
113  // On owner_rank, compute eye_lat and eye_lon
114  if (rank == owner_rank) {
115  for (amrex::MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
116  const amrex::Box& box = mfi.validbox();
117  const auto& mf_latlon = forecast_state_interp[levc][4];
118  const auto latlon_arr = mf_latlon.array(mfi);
119  amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
120  if (i == global_i_min && j == global_j_min && k == 0) {
121  *d_eye_lat_ptr = latlon_arr(i,j,k,0);
122  *d_eye_lon_ptr = latlon_arr(i,j,k,1);
123  }
124  });
125  }
126  }
127  }
128 
129  Real eye_lat = d_eye_lat.dataValue();
130  Real eye_lon = d_eye_lon.dataValue();
131 
132  // Synchronize to ensure the owner has computed values
133  Gpu::synchronize();
134 
135  ParallelDescriptor::Bcast(&eye_lat, 1, owner_rank);
136  ParallelDescriptor::Bcast(&eye_lon, 1, owner_rank);
137 
138  const auto dx = lev_geom.CellSizeArray();
139  const auto prob_lo = lev_geom.ProbLoArray();
140 
141  Real eye_x = prob_lo[0] + (global_i_min+myhalf)*dx[0];
142  Real eye_y = prob_lo[1] + (global_j_min+myhalf)*dx[1];
143 
144  hurricane_eye_track_xy.push_back({eye_x, eye_y});
145  hurricane_eye_track_latlon.push_back({eye_lon, eye_lat});
146 }
rank
int rank
Definition: ERF_HurricaneDiagnostics.cpp:31
prob_lo
const amrex::Real * prob_lo
Definition: ERF_InitCustomPert_DataAssimilation_ISV.H:16
ERF::hurricane_eye_track_xy
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_eye_track_xy
Definition: ERF.H:178
ERF::forecast_state_interp
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_interp
Definition: ERF.H:186
ERF::hurricane_eye_track_latlon
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_eye_track_latlon
Definition: ERF.H:179
SolverChoice::init_type
static InitType init_type
Definition: ERF_DataStruct.H:1224
Here is the call graph for this function:

◆ ConfigureDriverAtmosToOceanCoupling()

void ERF::ConfigureDriverAtmosToOceanCoupling ( bool  use_coupling_driver,
bool  use_two_way_coupling,
bool  use_state_contract 
)
121 {
122  m_driver_has_atm2ocn_coupling = use_coupling_driver;
123  m_driver_uses_two_way_coupling = use_two_way_coupling;
124  m_driver_atm2ocn_uses_state_contract = use_state_contract;
125 }
ERF::m_driver_has_atm2ocn_coupling
bool m_driver_has_atm2ocn_coupling
Definition: ERF.H:704
ERF::m_driver_uses_two_way_coupling
bool m_driver_uses_two_way_coupling
Definition: ERF.H:705
ERF::m_driver_atm2ocn_uses_state_contract
bool m_driver_atm2ocn_uses_state_contract
Definition: ERF.H:706

◆ Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int  lev)
private
2714 {
2715  auto& fine_new = vars_new[lev];
2716  auto& crse_new = vars_new[lev-1];
2717  auto& ba_fine = fine_new[Vars::cons].boxArray();
2718  auto& ba_crse = crse_new[Vars::cons].boxArray();
2719  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2720  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2721 
2722  int ncomp = vars_new[lev][Vars::cons].nComp();
2723 
2724  FPr_c.emplace_back(ba_fine, dm_fine, geom[lev] ,
2725  ba_crse, dm_crse, geom[lev-1],
2726  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2727  FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2728  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2729  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2730  FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2731  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2732  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2733  FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2734  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2735  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2736 }
ERF::cf_set_width
int cf_set_width
Definition: ERF.H:1012

◆ create_background_state_for_ensemble()

void ERF::create_background_state_for_ensemble ( int  lev,
amrex::MultiFab &  mf_cc_pert,
amrex::MultiFab &  cons_pert,
amrex::MultiFab &  xvel_pert,
amrex::MultiFab &  yvel_pert,
amrex::MultiFab &  zvel_pert 
)
522 {
523 
524  ignore_unused(lev);
525  int nx_crse, ny_crse, nz_crse, ng_crse, ncomp_crse;
526  Vector<Vector<Real>> data_crse;
527  std::array<Real,3> problo_ext, probhi_ext;
528 
529  Vector<Real> data_rho, data_theta, data_xvel, data_yvel, data_zvel;
530 
531  ReadCustomDataFile(solverChoice.coarse_bckgnd_data_file,
532  nx_crse, ny_crse, nz_crse, ng_crse, ncomp_crse,
533  problo_ext, probhi_ext,
534  data_rho, data_theta, data_xvel, data_yvel, data_zvel);
535 
536  Geometry& geom_fine = geom[0];
537  // Create a cell-centered multifab on the fine mesh - ie. something with the same boxarray,
538  // distributed mapping, nGrow, but with 5 components
539  MultiFab mf_cc_fine;
540  const MultiFab& src = vars_new[0][0];
541  int ncomp = 5;
542  mf_cc_fine.define(src.boxArray(), src.DistributionMap(),
543  ncomp, src.nGrow());
544 
545  InterpolateToFineMF(data_rho, data_theta, data_xvel, data_yvel, data_zvel,
546  nx_crse, ny_crse, nz_crse,
547  problo_ext, probhi_ext,
548  mf_cc_fine,
549  geom_fine);
550 
551  ApplyNeumannBCs(geom_fine, mf_cc_fine);
552 
553  Vector<std::string> varnames = {"density","theta", "x_velocity","y_velocity","z_velocity"};
554 
555  // Add pertubrations stored in the "pert" variables in the function arguments
556  // (multiplied by the corresponding amplitude)
557  AddPertToBckgnd(mf_cc_fine, mf_cc_pert, solverChoice.ens_pert_amplitude);
558  ApplyNeumannBCs(geom_fine, mf_cc_fine);
559  //WriteSingleLevelPlotfile("1_plt_final", mf_cc_fine, varnames, geom_fine, zero, 0);
560 
561  MakeFinalMultiFabs(mf_cc_fine, cons_pert, xvel_pert, yvel_pert, zvel_pert);
562 }
InterpolateToFineMF
void InterpolateToFineMF(const Vector< Real > &data_rho, const Vector< Real > &data_theta, const Vector< Real > &data_xvel, const Vector< Real > &data_yvel, const Vector< Real > &data_zvel, int nx, int ny, int nz, const std::array< Real, 3 > &problo, const std::array< Real, 3 > &probhi, MultiFab &mf_fine, const Geometry &geom_fine)
Definition: ERF_InitForEnsemble.cpp:325
ReadCustomDataFile
void ReadCustomDataFile(const std::string &filename_custom, int &nx, int &ny, int &nz, int &ng, int &ncomp, std::array< Real, 3 > &problo_ext, std::array< Real, 3 > &probhi_ext, Vector< Real > &data_rho, Vector< Real > &data_theta, Vector< Real > &data_xvel, Vector< Real > &data_yvel, Vector< Real > &data_zvel)
Definition: ERF_InitForEnsemble.cpp:208
AddPertToBckgnd
void AddPertToBckgnd(MultiFab &mf_cc_fine, const MultiFab &mf_cc_pert, const Real &ens_pert_amplitude)
Definition: ERF_InitForEnsemble.cpp:490
ApplyNeumannBCs
void ApplyNeumannBCs(const Geometry &geom, MultiFab &mf_cc)
Definition: ERF_InitForEnsemble.cpp:162
MakeFinalMultiFabs
void MakeFinalMultiFabs(const MultiFab &mf_cc_fine, MultiFab &cons_pert, MultiFab &xvel_pert, MultiFab &yvel_pert, MultiFab &zvel_pert)
Definition: ERF_InitForEnsemble.cpp:426
SolverChoice::ens_pert_amplitude
amrex::Real ens_pert_amplitude
Definition: ERF_DataStruct.H:1484
SolverChoice::coarse_bckgnd_data_file
std::string coarse_bckgnd_data_file
Definition: ERF_DataStruct.H:1485
Here is the call graph for this function:

◆ create_random_perturbations()

void ERF::create_random_perturbations ( const int  lev,
amrex::MultiFab &  mf_cc_pert 
)
16 {
17  const MultiFab& src = vars_new[lev][Vars::cons];
18 
19  int ncomp = 5;
20  mf_cc_pert.define(src.boxArray(), src.DistributionMap(),
21  ncomp, src.nGrow());
22 
23  // Loop over cell-centered boxes
24  for (MFIter mfi(mf_cc_pert, TilingIfNotGPU()); mfi.isValid(); ++mfi)
25  {
26  const Box& bx = mfi.tilebox();
27 
28  auto const& pert_arr = mf_cc_pert.array(mfi);
29 
30  // Loop over all 5 components
31  amrex::ParallelForRNG(bx, ncomp,
32  [=] AMREX_GPU_DEVICE (int i, int j, int k, int n,
33  const amrex::RandomEngine& engine) noexcept
34  {
35  pert_arr(i,j,k,n) = amrex::Random(engine);
36  });
37  }
38 }
ParallelForRNG
ParallelForRNG(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k, const amrex::RandomEngine &engine) noexcept { const Real x=prob_lo_x+(i+myhalf) *dx;const Real y=prob_lo_y+(j+myhalf) *dy;const Real z=z_cc(i, j, k);const Real r=std::sqrt((x-xc) *(x-xc)+(y-yc) *(y-yc)+(z-zc) *(z-zc));if((z<=pert_ref_height) &&(T_0_Pert_Mag !=amrex::Real(0))) { Real rand_double=amrex::Random(engine);state_pert(i, j, k, RhoTheta_comp)=(rand_double *amrex::Real(2) - amrex::Real(1)) *T_0_Pert_Mag;if(!pert_rhotheta) { state_pert(i, j, k, RhoTheta_comp) *=r_hse(i, j, k);} } state_pert(i, j, k, RhoScalar_comp)=A_0 *std::exp(-amrex::Real(10.) *r *r);if(state_pert.nComp() > RhoKE_comp) { if(rhoKE_0 > 0) { state_pert(i, j, k, RhoKE_comp)=rhoKE_0;} else { state_pert(i, j, k, RhoKE_comp)=r_hse(i, j, k) *KE_0;} if(KE_decay_height > 0) { state_pert(i, j, k, RhoKE_comp) *=amrex::max(std::pow(1 - amrex::min(z/KE_decay_height, amrex::Real(1)), KE_decay_order), Real(1e-12));} } })
Here is the call graph for this function:

◆ DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int  i)
inlineprivate
1550  {
1551  return *datalog[i];
1552  }
ERF::datalog
amrex::Vector< std::unique_ptr< std::fstream > > datalog
Definition: ERF.H:1729

◆ DataLogName()

std::string ERF::DataLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith datalog file.

1745 { return datalogname[i]; }
ERF::datalogname
amrex::Vector< std::string > datalogname
Definition: ERF.H:1732

◆ Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int  lev)
private
2740 {
2741  auto& fine_new = vars_new[lev];
2742  auto& crse_new = vars_new[lev-1];
2743  auto& ba_fine = fine_new[Vars::cons].boxArray();
2744  auto& ba_crse = crse_new[Vars::cons].boxArray();
2745  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2746  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2747 
2748  int ncomp = fine_new[Vars::cons].nComp();
2749 
2750  FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev] ,
2751  ba_crse, dm_crse, geom[lev-1],
2752  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2753  FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2754  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2755  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2756  FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2757  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2758  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2759  FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2760  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2761  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2762 }

◆ DerDataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DerDataLog ( int  i)
inlineprivate
1557  {
1558  return *der_datalog[i];
1559  }
ERF::der_datalog
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
Definition: ERF.H:1730

◆ DerDataLogName()

std::string ERF::DerDataLogName ( int  i) const
inlineprivatenoexcept
1746 { return der_datalogname[i]; }
ERF::der_datalogname
amrex::Vector< std::string > der_datalogname
Definition: ERF.H:1733

◆ derive_diag_profiles()

void ERF::derive_diag_profiles ( double  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
205 {
206  // We assume that this is always called at level 0
207  int lev = 0;
208 
209  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
210  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
211  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, uw, vv, vw, ww, uth, vth, wth,
212  // 0 1 2 3 4 5 6 7 8 9 10 11 12 13
213  // thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr,
214  // 14 15 16 17 18 19 20 21 22 23 24 25 26 27
215  // qi, qs, qg, wthv
216  // 28 29 30 31
217  MultiFab mf_out(grids[lev], dmap[lev], 32, 0);
218 
219  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
220 
221  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
222  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
223  MultiFab w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
224 
225  average_face_to_cellcenter(mf_vels,0,
226  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
227 
228  int zdir = 2;
229  auto domain = geom[0].Domain();
230 
231  // Sum in the horizontal plane
232  h_avg_u = sumToLine(mf_vels ,0,1,domain,zdir);
233  h_avg_v = sumToLine(mf_vels ,1,1,domain,zdir);
234  h_avg_w = sumToLine(mf_vels ,2,1,domain,zdir);
235 
236  int hu_size = h_avg_u.size();
237 
238  // Divide by the total number of cells we are averaging over
239  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
240  for (int k = 0; k < hu_size; ++k) {
241  h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
242  }
243 
244  Gpu::DeviceVector<Real> d_avg_u(hu_size, zero);
245  Gpu::DeviceVector<Real> d_avg_v(hu_size, zero);
246  Gpu::DeviceVector<Real> d_avg_w(hu_size, zero);
247 
248 #if 0
249  auto* avg_u_ptr = d_avg_u.data();
250  auto* avg_v_ptr = d_avg_v.data();
251  auto* avg_w_ptr = d_avg_w.data();
252 #endif
253 
254  Gpu::copy(Gpu::hostToDevice, h_avg_u.begin(), h_avg_u.end(), d_avg_u.begin());
255  Gpu::copy(Gpu::hostToDevice, h_avg_v.begin(), h_avg_v.end(), d_avg_v.begin());
256  Gpu::copy(Gpu::hostToDevice, h_avg_w.begin(), h_avg_w.end(), d_avg_w.begin());
257 
258  int nvars = vars_new[lev][Vars::cons].nComp();
259  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
260 
261  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
262 
263  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
264  const MultiFab* eta_src = nullptr;
265  const bool have_native_shoc_diagnostics =
266  solverChoice.turbChoice[lev].uses_native_shoc() &&
267  native_shoc_driver[lev] &&
268  native_shoc_driver[lev]->has_native_diagnostics();
269  if (l_use_kturb) {
270  if (have_native_shoc_diagnostics) {
271  eta_src = &native_shoc_driver[lev]->native_diagnostics();
272  } else
273  {
274  eta_src = eddyDiffs_lev[lev].get();
275  }
276  }
277 
278  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
279  {
280  const Box& bx = mfi.tilebox();
281  const Array4<Real>& fab_arr = mf_out.array(mfi);
282  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
283  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
284  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
285  const Array4<Real>& cons_arr = mf_cons.array(mfi);
286  const Array4<Real>& p0_arr = p_hse.array(mfi);
287  const Array4<const Real>& eta_arr = (eta_src) ? eta_src->const_array(mfi) :
288  Array4<const Real>{};
289 
290  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
291  {
292  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
293  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
294  fab_arr(i, j, k, 1) = theta;
295  Real ksgs = zero;
296  if (l_use_KE) {
297  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
298  }
299  fab_arr(i, j, k, 2) = ksgs;
300 #if 1
301  if (l_use_kturb) {
302  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
303  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
304  } else {
305  fab_arr(i, j, k, 3) = zero;
306  fab_arr(i, j, k, 4) = zero;
307  }
308 #else
309  // Here we hijack the "Kturb" variable name to print out the resolved kinetic energy
310  Real upert = u_cc_arr(i,j,k) - avg_u_ptr[k];
311  Real vpert = v_cc_arr(i,j,k) - avg_v_ptr[k];
312  Real wpert = w_cc_arr(i,j,k) - avg_w_ptr[k];
313  fab_arr(i, j, k, 3) = myhalf * (upert*upert + vpert*vpert + wpert*wpert);
314 #endif
315  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
316  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
317  fab_arr(i, j, k, 7) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k); // u*w
318  fab_arr(i, j, k, 8) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
319  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k); // v*w
320  fab_arr(i, j, k,10) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k); // w*w
321  fab_arr(i, j, k,11) = u_cc_arr(i,j,k) * theta; // u*th
322  fab_arr(i, j, k,12) = v_cc_arr(i,j,k) * theta; // v*th
323  fab_arr(i, j, k,13) = w_cc_arr(i,j,k) * theta; // w*th
324  fab_arr(i, j, k,14) = theta * theta; // th*th
325 
326  // if the number of fields is changed above, then be sure to update
327  // the following def!
328  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,8) + fab_arr(i,j,k,10);
329  fab_arr(i, j, k,15) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
330  fab_arr(i, j, k,16) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
331  fab_arr(i, j, k,17) = uiui * w_cc_arr(i,j,k); // (ui*ui)*w
332 
333  if (!use_moisture) {
334  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
335  p -= p0_arr(i,j,k);
336  fab_arr(i, j, k,18) = p; // p
337  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
338  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
339  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
340  fab_arr(i, j, k,22) = zero; // qv
341  fab_arr(i, j, k,23) = zero; // qc
342  fab_arr(i, j, k,24) = zero; // qr
343  fab_arr(i, j, k,25) = zero; // w*qv
344  fab_arr(i, j, k,26) = zero; // w*qc
345  fab_arr(i, j, k,27) = zero; // w*qr
346  fab_arr(i, j, k,28) = zero; // qi
347  fab_arr(i, j, k,29) = zero; // qs
348  fab_arr(i, j, k,30) = zero; // qg
349  fab_arr(i, j, k,31) = zero; // w*thv
350  }
351  });
352  } // mfi
353 
354  if (use_moisture)
355  {
356  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
357 
358  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
359  {
360  const Box& bx = mfi.tilebox();
361  const Array4<Real>& fab_arr = mf_out.array(mfi);
362  const Array4<Real>& cons_arr = mf_cons.array(mfi);
363  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
364  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
365  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
366  const Array4<Real>& p0_arr = p_hse.array(mfi);
367 
368  int rhoqr_comp = solverChoice.moisture_indices.qr;
369 
370  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
371  {
372  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
373  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
374  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
375  zero;
376  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
377 
378  p -= p0_arr(i,j,k);
379  fab_arr(i, j, k,18) = p; // p
380  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
381  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
382  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
383  fab_arr(i, j, k,22) = qv; // qv
384  fab_arr(i, j, k,23) = qc; // qc
385  fab_arr(i, j, k,24) = qr; // qr
386  fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * qv; // w*qv
387  fab_arr(i, j, k,26) = w_cc_arr(i,j,k) * qc; // w*qc
388  fab_arr(i, j, k,27) = w_cc_arr(i,j,k) * qr; // w*qr
389  if (n_qstate_moist > 3) {
390  fab_arr(i, j, k,28) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
391  fab_arr(i, j, k,29) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
392  fab_arr(i, j, k,30) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
393  } else {
394  fab_arr(i, j, k,28) = zero; // qi
395  fab_arr(i, j, k,29) = zero; // qs
396  fab_arr(i, j, k,30) = zero; // qg
397  }
398  Real ql = qc + qr;
399  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
400  Real thv = theta * (one + epsv*qv - ql);
401  fab_arr(i, j, k,31) = w_cc_arr(i,j,k) * thv; // w*thv
402  });
403  } // mfi
404  } // use_moisture
405 
406  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
407  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
408  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
409  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
410  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
411  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
412  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
413  h_avg_uw = sumToLine(mf_out, 7,1,domain,zdir);
414  h_avg_vv = sumToLine(mf_out, 8,1,domain,zdir);
415  h_avg_vw = sumToLine(mf_out, 9,1,domain,zdir);
416  h_avg_ww = sumToLine(mf_out,10,1,domain,zdir);
417  h_avg_uth = sumToLine(mf_out,11,1,domain,zdir);
418  h_avg_vth = sumToLine(mf_out,12,1,domain,zdir);
419  h_avg_wth = sumToLine(mf_out,13,1,domain,zdir);
420  h_avg_thth = sumToLine(mf_out,14,1,domain,zdir);
421  h_avg_uiuiu = sumToLine(mf_out,15,1,domain,zdir);
422  h_avg_uiuiv = sumToLine(mf_out,16,1,domain,zdir);
423  h_avg_uiuiw = sumToLine(mf_out,17,1,domain,zdir);
424  h_avg_p = sumToLine(mf_out,18,1,domain,zdir);
425  h_avg_pu = sumToLine(mf_out,19,1,domain,zdir);
426  h_avg_pv = sumToLine(mf_out,20,1,domain,zdir);
427  h_avg_pw = sumToLine(mf_out,21,1,domain,zdir);
428  h_avg_qv = sumToLine(mf_out,22,1,domain,zdir);
429  h_avg_qc = sumToLine(mf_out,23,1,domain,zdir);
430  h_avg_qr = sumToLine(mf_out,24,1,domain,zdir);
431  h_avg_wqv = sumToLine(mf_out,25,1,domain,zdir);
432  h_avg_wqc = sumToLine(mf_out,26,1,domain,zdir);
433  h_avg_wqr = sumToLine(mf_out,27,1,domain,zdir);
434  h_avg_qi = sumToLine(mf_out,28,1,domain,zdir);
435  h_avg_qs = sumToLine(mf_out,29,1,domain,zdir);
436  h_avg_qg = sumToLine(mf_out,30,1,domain,zdir);
437  h_avg_wthv = sumToLine(mf_out,31,1,domain,zdir);
438 
439  // Divide by the total number of cells we are averaging over
440  int h_avg_u_size = static_cast<int>(h_avg_u.size());
441  for (int k = 0; k < h_avg_u_size; ++k) {
442  h_avg_rho[k] /= area_z;
443  h_avg_ksgs[k] /= area_z;
444  h_avg_Kmv[k] /= area_z;
445  h_avg_Khv[k] /= area_z;
446  h_avg_th[k] /= area_z;
447  h_avg_thth[k] /= area_z;
448  h_avg_uu[k] /= area_z;
449  h_avg_uv[k] /= area_z;
450  h_avg_uw[k] /= area_z;
451  h_avg_vv[k] /= area_z;
452  h_avg_vw[k] /= area_z;
453  h_avg_ww[k] /= area_z;
454  h_avg_uth[k] /= area_z;
455  h_avg_vth[k] /= area_z;
456  h_avg_wth[k] /= area_z;
457  h_avg_uiuiu[k] /= area_z;
458  h_avg_uiuiv[k] /= area_z;
459  h_avg_uiuiw[k] /= area_z;
460  h_avg_p[k] /= area_z;
461  h_avg_pu[k] /= area_z;
462  h_avg_pv[k] /= area_z;
463  h_avg_pw[k] /= area_z;
464  h_avg_qv[k] /= area_z;
465  h_avg_qc[k] /= area_z;
466  h_avg_qr[k] /= area_z;
467  h_avg_wqv[k] /= area_z;
468  h_avg_wqc[k] /= area_z;
469  h_avg_wqr[k] /= area_z;
470  h_avg_qi[k] /= area_z;
471  h_avg_qs[k] /= area_z;
472  h_avg_qg[k] /= area_z;
473  h_avg_wthv[k] /= area_z;
474  }
475 
476 #if 0
477  // Here we print the integrated total kinetic energy as computed in the 1D profile above
478  Real sum = zero;
479  Real dz = geom[0].ProbHi(2) / static_cast<Real>(h_avg_u_size);
480  for (int k = 0; k < h_avg_u_size; ++k) {
481  sum += h_avg_kturb[k] * h_avg_rho[k] * dz;
482  }
483  amrex::Print() << "ITKE " << time << " " << sum << " using " << h_avg_u_size << " " << dz << std::endl;
484 #endif
485 }
epsv
constexpr amrex::Real epsv
Definition: ERF_Constants.H:53
RhoQ3_comp
#define RhoQ3_comp
Definition: ERF_IndexDefines.H:44
RhoQ6_comp
#define RhoQ6_comp
Definition: ERF_IndexDefines.H:47
RhoQ5_comp
#define RhoQ5_comp
Definition: ERF_IndexDefines.H:46
RhoKE_comp
#define RhoKE_comp
Definition: ERF_IndexDefines.H:38
use_moisture
const bool use_moisture
Definition: ERF_InitCustomPert_Bomex.H:14
EddyDiff::Theta_v
@ Theta_v
Definition: ERF_IndexDefines.H:211
EddyDiff::Mom_v
@ Mom_v
Definition: ERF_IndexDefines.H:210
LsmVar_MM5::theta
@ theta
Definition: ERF_MM5.H:20
WSM6SedCellScratch::qr
@ qr
Definition: ERF_AdvanceWSM6.cpp:112
Here is the call graph for this function:

◆ derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag ( double  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
311 {
312  // We assume that this is always called at level 0
313  int lev = 0;
314 
315  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
316  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
317  // Note: "uiui" == u_i*u_i = u*u + v*v + w*w
318  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, vv, uth, vth,
319  // indices: 0 1 2 3 4 5 6 7 8 9
320  // thth, uiuiu, uiuiv, p, pu, pv, qv, qc, qr, qi, qs, qg
321  // 10 11 12 13 14 15 16 17 18 19 20 21
322  MultiFab mf_out(grids[lev], dmap[lev], 22, 0);
323 
324  // This will hold uw, vw, ww, wth, uiuiw, pw, wqv, wqc, wqr, wthv
325  // indices: 0 1 2 3 4 5 6 7 8 9
326  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 10, 0);
327 
328  // This is only used to average u and v; w is not averaged to cell centers
329  MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
330 
331  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
332  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
333  MultiFab w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers (staggered)
334 
335  int zdir = 2;
336  auto domain = geom[0].Domain();
337  Box stag_domain = domain;
338  stag_domain.convert(IntVect(0,0,1));
339 
340  int nvars = vars_new[lev][Vars::cons].nComp();
341  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
342 
343  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
344 
345  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
346  const MultiFab* eta_src = nullptr;
347  const bool have_native_shoc_diagnostics =
348  solverChoice.turbChoice[lev].uses_native_shoc() &&
349  native_shoc_driver[lev] &&
350  native_shoc_driver[lev]->has_native_diagnostics();
351  if (l_use_kturb) {
352  if (have_native_shoc_diagnostics) {
353  eta_src = &native_shoc_driver[lev]->native_diagnostics();
354  } else
355  {
356  eta_src = eddyDiffs_lev[lev].get();
357  }
358  }
359 
360  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
361  {
362  const Box& bx = mfi.tilebox();
363  const Array4<Real>& fab_arr = mf_out.array(mfi);
364  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
365  const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
366  const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
367  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
368  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
369  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
370  const Array4<Real>& cons_arr = mf_cons.array(mfi);
371  const Array4<Real>& p0_arr = p_hse.array(mfi);
372  const Array4<const Real>& eta_arr = (eta_src) ? eta_src->const_array(mfi) :
373  Array4<const Real>{};
374 
375  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
376  {
377  u_cc_arr(i,j,k) = myhalf * (u_arr(i,j,k) + u_arr(i+1,j ,k));
378  v_cc_arr(i,j,k) = myhalf * (v_arr(i,j,k) + v_arr(i ,j+1,k));
379 
380  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
381  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
382  fab_arr(i, j, k, 1) = theta;
383  Real ksgs = zero;
384  if (l_use_KE) {
385  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
386  }
387  fab_arr(i, j, k, 2) = ksgs;
388  if (l_use_kturb) {
389  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
390  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
391  } else {
392  fab_arr(i, j, k, 3) = zero;
393  fab_arr(i, j, k, 4) = zero;
394  }
395  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
396  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
397  fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
398  fab_arr(i, j, k, 8) = u_cc_arr(i,j,k) * theta; // u*th
399  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * theta; // v*th
400  fab_arr(i, j, k,10) = theta * theta; // th*th
401 
402  Real wcc = myhalf * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
403 
404  // if the number of fields is changed above, then be sure to update
405  // the following def!
406  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,7) + wcc*wcc;
407  fab_arr(i, j, k,11) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
408  fab_arr(i, j, k,12) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
409 
410  if (!use_moisture) {
411  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
412  p -= p0_arr(i,j,k);
413  fab_arr(i, j, k,13) = p; // p
414  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
415  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
416  fab_arr(i, j, k,16) = zero; // qv
417  fab_arr(i, j, k,17) = zero; // qc
418  fab_arr(i, j, k,18) = zero; // qr
419  fab_arr(i, j, k,19) = zero; // qi
420  fab_arr(i, j, k,20) = zero; // qs
421  fab_arr(i, j, k,21) = zero; // qg
422  }
423  });
424 
425  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
426  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
427  {
428  // average to z faces (first to cell centers, then in z)
429  Real uface = fourth * ( u_arr(i ,j,k) + u_arr(i ,j,k-1)
430  + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
431  Real vface = fourth * ( v_arr(i,j ,k) + v_arr(i,j ,k-1)
432  + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
433  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
434  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
435  Real thface = myhalf*(theta0 + theta1);
436  fab_arr_stag(i,j,k,0) = uface * w_fc_arr(i,j,k); // u*w
437  fab_arr_stag(i,j,k,1) = vface * w_fc_arr(i,j,k); // v*w
438  fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
439  fab_arr_stag(i,j,k,3) = thface * w_fc_arr(i,j,k); // th*w
440  Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
441  fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
442  if (!use_moisture) {
443  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp)) - p0_arr(i,j,k );
444  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
445  Real pface = myhalf * (p0 + p1);
446  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
447  fab_arr_stag(i,j,k,6) = zero; // w*qv
448  fab_arr_stag(i,j,k,7) = zero; // w*qc
449  fab_arr_stag(i,j,k,8) = zero; // w*qr
450  fab_arr_stag(i,j,k,9) = zero; // w*thv
451  }
452  });
453 
454  } // mfi
455 
456  if (use_moisture)
457  {
458  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
459 
460  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
461  {
462  const Box& bx = mfi.tilebox();
463  const Array4<Real>& fab_arr = mf_out.array(mfi);
464  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
465  const Array4<Real>& cons_arr = mf_cons.array(mfi);
466  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
467  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
468  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
469  const Array4<Real>& p0_arr = p_hse.array(mfi);
470 
471  int rhoqr_comp = solverChoice.moisture_indices.qr;
472 
473  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
474  {
475  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
476  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
477  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
478  zero;
479  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
480 
481  p -= p0_arr(i,j,k);
482  fab_arr(i, j, k,13) = p; // p
483  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
484  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
485  fab_arr(i, j, k,16) = qv; // qv
486  fab_arr(i, j, k,17) = qc; // qc
487  fab_arr(i, j, k,18) = qr; // qr
488  if (n_qstate_moist > 3) { // SAM model
489  fab_arr(i, j, k,19) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
490  fab_arr(i, j, k,20) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
491  fab_arr(i, j, k,21) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
492  } else {
493  fab_arr(i, j, k,19) = zero; // qi
494  fab_arr(i, j, k,20) = zero; // qs
495  fab_arr(i, j, k,21) = zero; // qg
496  }
497  });
498 
499  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
500  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
501  {
502  Real qv0 = cons_arr(i,j,k ,RhoQ1_comp) / cons_arr(i,j,k ,Rho_comp);
503  Real qv1 = cons_arr(i,j,k-1,RhoQ1_comp) / cons_arr(i,j,k-1,Rho_comp);
504  Real qc0 = cons_arr(i,j,k ,RhoQ2_comp) / cons_arr(i,j,k ,Rho_comp);
505  Real qc1 = cons_arr(i,j,k-1,RhoQ2_comp) / cons_arr(i,j,k-1,Rho_comp);
506  Real qr0 = (rhoqr_comp > -1) ? cons_arr(i,j,k ,RhoQ3_comp) / cons_arr(i,j,k ,Rho_comp) :
507  zero;
508  Real qr1 = (rhoqr_comp > -1) ? cons_arr(i,j,k-1,RhoQ3_comp) / cons_arr(i,j,k-1,Rho_comp) :
509  zero;
510  Real qvface = myhalf * (qv0 + qv1);
511  Real qcface = myhalf * (qc0 + qc1);
512  Real qrface = myhalf * (qr0 + qr1);
513 
514  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp), qv0) - p0_arr(i,j,k );
515  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv1) - p0_arr(i,j,k-1);
516  Real pface = myhalf * (p0 + p1);
517 
518  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
519  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
520  Real thface = myhalf*(theta0 + theta1);
521  Real ql = qcface + qrface;
522  Real thv = thface * (one + epsv*qvface - ql);
523 
524  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
525  fab_arr_stag(i,j,k,6) = qvface * w_fc_arr(i,j,k); // w*qv
526  fab_arr_stag(i,j,k,7) = qcface * w_fc_arr(i,j,k); // w*qc
527  fab_arr_stag(i,j,k,8) = qrface * w_fc_arr(i,j,k); // w*qr
528  fab_arr_stag(i,j,k,9) = thv * w_fc_arr(i,j,k); // w*thv
529  });
530  } // mfi
531  } // use_moisture
532 
533  // Sum in the horizontal plane
534  h_avg_u = sumToLine(u_cc,0,1, domain,zdir);
535  h_avg_v = sumToLine(v_cc,0,1, domain,zdir);
536  h_avg_w = sumToLine(w_fc,0,1,stag_domain,zdir);
537 
538  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
539  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
540  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
541  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
542  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
543  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
544  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
545  h_avg_vv = sumToLine(mf_out, 7,1,domain,zdir);
546  h_avg_uth = sumToLine(mf_out, 8,1,domain,zdir);
547  h_avg_vth = sumToLine(mf_out, 9,1,domain,zdir);
548  h_avg_thth = sumToLine(mf_out,10,1,domain,zdir);
549  h_avg_uiuiu = sumToLine(mf_out,11,1,domain,zdir);
550  h_avg_uiuiv = sumToLine(mf_out,12,1,domain,zdir);
551  h_avg_p = sumToLine(mf_out,13,1,domain,zdir);
552  h_avg_pu = sumToLine(mf_out,14,1,domain,zdir);
553  h_avg_pv = sumToLine(mf_out,15,1,domain,zdir);
554  h_avg_qv = sumToLine(mf_out,16,1,domain,zdir);
555  h_avg_qc = sumToLine(mf_out,17,1,domain,zdir);
556  h_avg_qr = sumToLine(mf_out,18,1,domain,zdir);
557  h_avg_qi = sumToLine(mf_out,19,1,domain,zdir);
558  h_avg_qs = sumToLine(mf_out,20,1,domain,zdir);
559  h_avg_qg = sumToLine(mf_out,21,1,domain,zdir);
560 
561  h_avg_uw = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
562  h_avg_vw = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
563  h_avg_ww = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
564  h_avg_wth = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
565  h_avg_uiuiw = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
566  h_avg_pw = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
567  h_avg_wqv = sumToLine(mf_out_stag,6,1,stag_domain,zdir);
568  h_avg_wqc = sumToLine(mf_out_stag,7,1,stag_domain,zdir);
569  h_avg_wqr = sumToLine(mf_out_stag,8,1,stag_domain,zdir);
570  h_avg_wthv = sumToLine(mf_out_stag,9,1,stag_domain,zdir);
571 
572  // Divide by the total number of cells we are averaging over
573  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
574  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
575  for (int k = 0; k < unstag_size; ++k) {
576  h_avg_u[k] /= area_z;
577  h_avg_v[k] /= area_z;
578  h_avg_rho[k] /= area_z;
579  h_avg_ksgs[k] /= area_z;
580  h_avg_Kmv[k] /= area_z;
581  h_avg_Khv[k] /= area_z;
582  h_avg_th[k] /= area_z;
583  h_avg_thth[k] /= area_z;
584  h_avg_uu[k] /= area_z;
585  h_avg_uv[k] /= area_z;
586  h_avg_vv[k] /= area_z;
587  h_avg_uth[k] /= area_z;
588  h_avg_vth[k] /= area_z;
589  h_avg_uiuiu[k] /= area_z;
590  h_avg_uiuiv[k] /= area_z;
591  h_avg_p[k] /= area_z;
592  h_avg_pu[k] /= area_z;
593  h_avg_pv[k] /= area_z;
594  h_avg_qv[k] /= area_z;
595  h_avg_qc[k] /= area_z;
596  h_avg_qr[k] /= area_z;
597  h_avg_qi[k] /= area_z;
598  h_avg_qs[k] /= area_z;
599  h_avg_qg[k] /= area_z;
600  }
601 
602  for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
603  h_avg_w[k] /= area_z;
604  h_avg_uw[k] /= area_z;
605  h_avg_vw[k] /= area_z;
606  h_avg_ww[k] /= area_z;
607  h_avg_wth[k] /= area_z;
608  h_avg_uiuiw[k] /= area_z;
609  h_avg_pw[k] /= area_z;
610  h_avg_wqv[k] /= area_z;
611  h_avg_wqc[k] /= area_z;
612  h_avg_wqr[k] /= area_z;
613  h_avg_wthv[k] /= area_z;
614  }
615 }
fourth
constexpr amrex::Real fourth
Definition: ERF_Constants.H:14
zbx
const Box zbx
Definition: ERF_SetupDiff.H:9
mp_wsm6::qc0
real(kind=kind_phys), save qc0
Definition: ERF_module_mp_wsm6.F90:46
Here is the call graph for this function:

◆ derive_stress_profiles()

void ERF::derive_stress_profiles ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
493 {
494  int lev = 0;
495 
496  // This will hold the stress tensor components
497  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
498 
499  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
500 
501  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
502 
503  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
504  {
505  const Box& bx = mfi.tilebox();
506  const Array4<Real>& fab_arr = mf_out.array(mfi);
507 
508  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
509 
510  // NOTE: These are from the last RK stage...
511  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
512  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
513  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
514  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
515  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
516  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
517 
518  // These should be re-calculated during ERF_slow_rhs_post
519  // -- just vertical SFS kinematic heat flux for now
520  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
521  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
522  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
523  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
524  Array4<const Real>{};
525  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
526  Array4<const Real>{};
527  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
528 
529  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
530  {
531  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
532  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
533  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
534  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
535  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
536  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
537  fab_arr(i, j, k, 2) = ( tau13_arr(i,j,k ) + tau13_arr(i+1,j,k )
538  + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) )
539  / ( rho_arr(i,j,k ) + rho_arr(i+1,j,k )
540  + rho_arr(i,j,k+1) + rho_arr(i+1,j,k+1) );
541  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
542  fab_arr(i, j, k, 4) = ( tau23_arr(i,j,k ) + tau23_arr(i,j+1,k )
543  + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) )
544  / ( rho_arr(i,j,k ) + rho_arr(i,j+1,k )
545  + rho_arr(i,j,k+1) + rho_arr(i,j+1,k+1) );
546  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
547  fab_arr(i, j, k, 6) = myhalf * ( hfx3_arr(i,j,k) + hfx3_arr(i,j,k+1) ) / rho_arr(i,j,k);
548  fab_arr(i, j, k, 7) = (l_use_moist) ? myhalf * ( q1fx3_arr(i,j,k) + q1fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : zero;
549  fab_arr(i, j, k, 8) = (l_use_moist) ? myhalf * ( q2fx3_arr(i,j,k) + q2fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : zero;
550  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
551  });
552  }
553 
554  int zdir = 2;
555  auto domain = geom[0].Domain();
556 
557  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
558  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
559  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
560  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
561  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
562  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
563  h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
564  h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
565  h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
566  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
567 
568  int ht_size = h_avg_tau11.size();
569 
570  // Divide by the total number of cells we are averaging over
571  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
572  for (int k = 0; k < ht_size; ++k) {
573  h_avg_tau11[k] /= area_z;
574  h_avg_tau12[k] /= area_z;
575  h_avg_tau13[k] /= area_z;
576  h_avg_tau22[k] /= area_z;
577  h_avg_tau23[k] /= area_z;
578  h_avg_tau33[k] /= area_z;
579  h_avg_hfx3[k] /= area_z;
580  h_avg_q1fx3[k] /= area_z;
581  h_avg_q2fx3[k] /= area_z;
582  h_avg_diss[k] /= area_z;
583  }
584 }
rho_arr
auto rho_arr
Definition: ERF_UpdateWSubsidence_SineMassFlux.H:3
Here is the call graph for this function:

◆ derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
623 {
624  int lev = 0;
625 
626  // This will hold the stress tensor components
627  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
628 
629  // This will hold Tau13 and Tau23
630  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 5, 0);
631 
632  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
633 
634  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
635 
636  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
637  {
638  const Box& bx = mfi.tilebox();
639  const Array4<Real>& fab_arr = mf_out.array(mfi);
640  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
641 
642  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
643 
644  // NOTE: These are from the last RK stage...
645  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
646  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
647  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
648  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
649  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
650  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
651 
652  // These should be re-calculated during ERF_slow_rhs_post
653  // -- just vertical SFS kinematic heat flux for now
654  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
655  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
656  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
657  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
658  Array4<const Real>{};
659  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
660  Array4<const Real>{};
661  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
662 
663  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
664  {
665  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
666  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
667  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
668  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
669  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
670  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
671  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
672  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
673  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
674  });
675 
676  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
677  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
678  {
679  Real rho_face = myhalf * (rho_arr(i,j,k-1) + rho_arr(i,j,k));
680  // average from edge to face center
681  fab_arr_stag(i,j,k,0) = myhalf*(tau13_arr(i,j,k) + tau13_arr(i+1,j ,k)) / rho_face;
682  fab_arr_stag(i,j,k,1) = myhalf*(tau23_arr(i,j,k) + tau23_arr(i ,j+1,k)) / rho_face;
683 
684  fab_arr_stag(i,j,k,2) = hfx3_arr(i,j,k) / rho_face;
685  fab_arr_stag(i,j,k,3) = (l_use_moist) ? q1fx3_arr(i,j,k) / rho_face : zero;
686  fab_arr_stag(i,j,k,4) = (l_use_moist) ? q2fx3_arr(i,j,k) / rho_face : zero;
687  });
688  }
689 
690  int zdir = 2;
691  auto domain = geom[0].Domain();
692  Box stag_domain = domain;
693  stag_domain.convert(IntVect(0,0,1));
694 
695  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
696  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
697 // h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
698  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
699 // h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
700  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
701 // h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
702 // h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
703 // h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
704  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
705 
706  h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
707  h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
708  h_avg_hfx3 = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
709  h_avg_q1fx3 = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
710  h_avg_q2fx3 = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
711 
712  int ht_size = h_avg_tau11.size(); // _un_staggered
713 
714  // Divide by the total number of cells we are averaging over
715  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
716  for (int k = 0; k < ht_size; ++k) {
717  h_avg_tau11[k] /= area_z;
718  h_avg_tau12[k] /= area_z;
719  h_avg_tau13[k] /= area_z;
720  h_avg_tau22[k] /= area_z;
721  h_avg_tau23[k] /= area_z;
722  h_avg_tau33[k] /= area_z;
723  h_avg_hfx3[k] /= area_z;
724  h_avg_q1fx3[k] /= area_z;
725  h_avg_q2fx3[k] /= area_z;
726  h_avg_diss[k] /= area_z;
727  }
728  // staggered heights
729  h_avg_tau13[ht_size] /= area_z;
730  h_avg_tau23[ht_size] /= area_z;
731  h_avg_hfx3[ht_size] /= area_z;
732  h_avg_q1fx3[ht_size] /= area_z;
733  h_avg_q2fx3[ht_size] /= area_z;
734 }
Here is the call graph for this function:

◆ derive_upwp()

void ERF::derive_upwp ( amrex::Vector< amrex::Real > &  h_havg)

◆ EBFactory()

amrex::EBFArrayBoxFactory const& ERF::EBFactory ( int  lev) const
inlineprivatenoexcept
1763  {
1764  return *(eb[lev]->get_const_factory());
1765  }
ERF::eb
amrex::Vector< std::unique_ptr< eb_ > > eb
Definition: ERF.H:1755

◆ erf_enforce_hse()

void ERF::erf_enforce_hse ( int  lev,
amrex::MultiFab &  dens,
amrex::MultiFab &  pres,
amrex::MultiFab &  pi,
amrex::MultiFab &  th,
amrex::MultiFab &  qv,
std::unique_ptr< amrex::MultiFab > &  z_cc 
)

Enforces hydrostatic equilibrium when using terrain.

Parameters
[in]levInteger specifying the current level
[out]densMultiFab storing base state density
[out]presMultiFab storing base state pressure
[out]piMultiFab storing base state Exner function
[in]z_ccPointer to MultiFab storing cell centered z-coordinates
213 {
214  Real l_gravity = solverChoice.gravity;
215  bool l_use_terrain = (solverChoice.mesh_type != MeshType::ConstantDz);
216 
217  const auto geomdata = geom[lev].data();
218  const Real dz = geomdata.CellSize(2);
219 
220  for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
221  {
222  // Create a flat box with same horizontal extent but only one cell in vertical
223  const Box& tbz = mfi.nodaltilebox(2);
224  int klo = tbz.smallEnd(2);
225  int khi = tbz.bigEnd(2);
226 
227  // Note we only grow by 1 because that is how big z_cc is.
228  Box b2d = tbz; // Copy constructor
229  b2d.grow(0,1);
230  b2d.grow(1,1);
231  b2d.setRange(2,0);
232 
233  // Intersect this box with the domain
234  Box zdomain = convert(geom[lev].Domain(),tbz.ixType());
235  b2d &= zdomain;
236 
237  // We integrate to the first cell (and below) by using rho in this cell
238  // If gravity == 0 this is constant pressure
239  // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
240  // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
241  // Note ng_pres_hse = 1
242 
243  // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
244  // Note that gravity is positive
245 
246  Array4<Real> rho_arr = dens.array(mfi);
247  Array4<Real> pres_arr = pres.array(mfi);
248  Array4<Real> pi_arr = pi.array(mfi);
249  Array4<Real> th_arr = theta.array(mfi);
250  Array4<Real> zcc_arr;
251  if (l_use_terrain) {
252  zcc_arr = z_cc->array(mfi);
253  }
254 
255  const Real rdOcp = solverChoice.rdOcp;
256 
257  ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
258  {
259  // Set value at surface from Newton iteration for rho
260  if (klo == 0)
261  {
262  // Physical height of the terrain at cell center
263  Real hz;
264  if (l_use_terrain) {
265  hz = zcc_arr(i,j,klo);
266  } else {
267  hz = myhalf*dz;
268  }
269 
270  pres_arr(i,j,klo) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
271  pi_arr(i,j,klo) = getExnergivenP(pres_arr(i,j,klo), rdOcp);
272  th_arr(i,j,klo) = getRhoThetagivenP(pres_arr(i,j,klo)) / rho_arr(i,j,klo);
273 
274  //
275  // Set ghost cell with dz and rho at boundary
276  // (We will set the rest of the ghost cells in the boundary condition routine)
277  //
278  pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
279  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
280  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
281 
282  } else {
283 
284  // If level > 0 and klo > 0, we need to use the value of pres_arr(i,j,klo-1) which was
285  // filled from FillPatch-ing it.
286  Real dz_loc;
287  if (l_use_terrain) {
288  dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
289  } else {
290  dz_loc = dz;
291  }
292 
293  Real dens_interp = myhalf*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
294  pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
295 
296  pi_arr(i,j,klo ) = getExnergivenP(pres_arr(i,j,klo ), rdOcp);
297  th_arr(i,j,klo ) = getRhoThetagivenP(pres_arr(i,j,klo )) / rho_arr(i,j,klo );
298 
299  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
300  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
301  }
302 
303  Real dens_interp;
304  if (l_use_terrain) {
305  for (int k = klo+1; k <= khi; k++) {
306  Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
307  dens_interp = myhalf*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
308  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
309  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
310  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
311  }
312  } else {
313  for (int k = klo+1; k <= khi; k++) {
314  dens_interp = myhalf*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
315  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
316  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
317  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
318  }
319  }
320  });
321 
322  } // mfi
323 
324  dens.FillBoundary(geom[lev].periodicity());
325  pres.FillBoundary(geom[lev].periodicity());
326  pi.FillBoundary(geom[lev].periodicity());
327  theta.FillBoundary(geom[lev].periodicity());
328  qv.FillBoundary(geom[lev].periodicity());
329 }
p_0
constexpr amrex::Real p_0
Definition: ERF_Constants.H:61
getRhoThetagivenP
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getRhoThetagivenP(const amrex::Real p, const amrex::Real qv=amrex::Real(0))
Definition: ERF_EOS.H:172
getExnergivenP
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getExnergivenP(const amrex::Real P, const amrex::Real rdOcp)
Definition: ERF_EOS.H:141
rdOcp
const Real rdOcp
Definition: ERF_InitCustomPert_Bomex.H:16
MicVar_Kess::pres
@ pres
Definition: ERF_Kessler.H:27
module_mp_morr_two_moment::pi
real(c_double), parameter, private pi
Definition: ERF_module_mp_morr_two_moment.F90:100
mp_wsm6::dens
real(kind=kind_phys), parameter, private dens
Definition: ERF_module_mp_wsm6.F90:39
SolverChoice::rdOcp
amrex::Real rdOcp
Definition: ERF_DataStruct.H:1329
Here is the call graph for this function:

◆ ERF_shared()

void ERF::ERF_shared ( )
62 {
63  if (ParallelDescriptor::IOProcessor()) {
64  const char* erf_hash = buildInfoGetGitHash(1);
65  const char* amrex_hash = buildInfoGetGitHash(2);
66  const char* buildgithash = buildInfoGetBuildGitHash();
67  const char* buildgitname = buildInfoGetBuildGitName();
68 
69  if (strlen(erf_hash) > 0) {
70  Print() << "\n"
71  << "ERF git hash: " << erf_hash << "\n";
72  }
73  if (strlen(amrex_hash) > 0) {
74  Print() << "AMReX git hash: " << amrex_hash << "\n";
75  }
76  if (strlen(buildgithash) > 0) {
77  Print() << buildgitname << " git hash: " << buildgithash << "\n";
78  }
79 
80  Print() << "\n";
81  }
82 
83  int nlevs_max = max_level + 1;
84 
85 #ifdef ERF_USE_WINDFARM
86  Nturb.resize(nlevs_max);
87  vars_windfarm.resize(nlevs_max);
88  SMark.resize(nlevs_max);
89 #endif
90 
91  qheating_rates.resize(nlevs_max);
92  rad_fluxes.resize(nlevs_max);
93 
94  // NOTE: size lsm before readparams (chooses the model at all levels)
95  lsm.ReSize(nlevs_max);
96  lsm_data.resize(nlevs_max);
97  lsm_flux.resize(nlevs_max);
98 
99  rhotheta_src.resize(nlevs_max);
100  rhoqt_src.resize(nlevs_max);
101 
102  // NOTE: size canopy model before readparams (if file exists, we construct)
103  m_forest_drag.resize(nlevs_max);
104  for (int lev = 0; lev <= max_level; ++lev) { m_forest_drag[lev] = nullptr;}
105 
106  ReadParameters();
107  // Create one invocation identity after inputs are available and before
108  // InitData can read restart metadata or write an output on restart.
109  execution_provenance = erf_provenance::initialize_execution_provenance();
110  initializeMicrophysics(nlevs_max);
111 
112 #ifdef ERF_USE_WINDFARM
113  initializeWindFarm(nlevs_max);
114 #endif
115 
116 #ifdef ERF_USE_EAMXX_SHOC
117  eamxx_shoc_interface.resize(nlevs_max);
118  for (int lev = 0; lev <= max_level; ++lev) {
119  if (solverChoice.turbChoice[lev].uses_eamxx_shoc()) {
120  eamxx_shoc_interface[lev] = std::make_unique<SHOCInterface>(lev, solverChoice);
121  }
122  }
123 #endif
124 
125  native_shoc_driver.resize(nlevs_max);
126  for (int lev = 0; lev <= max_level; ++lev) {
127  if (solverChoice.turbChoice[lev].uses_native_shoc()) {
128  native_shoc_driver[lev] = std::make_unique<ShocDriver>(lev, solverChoice);
129  }
130  }
131 
132  rad.resize(nlevs_max);
133  for (int lev = 0; lev <= max_level; ++lev) {
134  if (solverChoice.rad_type == RadiationType::RRTMGP) {
135 #ifdef ERF_USE_RRTMGP
136  rad[lev] = std::make_unique<Radiation>(lev, solverChoice);
137  // pass radiation datalog frequency to model - RRTMGP needs to know when to save data for profiles
138  rad[lev]->setDataLogFrequency(rad_datalog_int);
139 #endif
140  } else if (solverChoice.rad_type != RadiationType::None) {
141  Abort("Don't know this radiation model!");
142  }
143  }
144  const std::string& pv3d_1 = "plot_vars_1" ; setPlotVariables(pv3d_1,plot3d_var_names_1);
145  const std::string& pv3d_2 = "plot_vars_2" ; setPlotVariables(pv3d_2,plot3d_var_names_2);
146 
147  // This is only used when we have mesh_type == MeshType::StretchedDz
148  stretched_dz_h.resize(nlevs_max);
149  stretched_dz_d.resize(nlevs_max);
150 
151  // Initialize staggered vertical levels for grid stretching or terrain, and
152  // to simplify Rayleigh damping layer calculations.
153  zlevels_stag.resize(max_level+1);
154  init_zlevels(zlevels_stag,
155  stretched_dz_h,
156  stretched_dz_d,
157  geom,
158  refRatio(),
159  solverChoice.grid_stretching_ratio,
160  solverChoice.zsurf,
161  solverChoice.dz0);
162 
163  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
164  SolverChoice::mesh_type == MeshType::VariableDz) {
165  int nz = geom[0].Domain().length(2) + 1; // staggered
166  if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > Real(1.0e-4)) {
167  Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
168  << " does not match highest requested z level " << zlevels_stag[0][nz-1]
169  << std::endl;
170  }
171  if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > Real(1.0e-4)) {
172  Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
173  << " does not match lowest requested level " << zlevels_stag[0][0]
174  << std::endl;
175  }
176 
177  // Redefine the problem domain here?
178  }
179 
180  // Get lo/hi indices for massflux calc
181  if ((solverChoice.const_massflux_u != 0) || (solverChoice.const_massflux_v != 0)) {
182  if (solverChoice.mesh_type == MeshType::ConstantDz) {
183  const bool zlo_unset = (solverChoice.const_massflux_layer_lo == amrex::Real(-bogus_large_value));
184  const bool zhi_unset = (solverChoice.const_massflux_layer_hi == amrex::Real( bogus_large_value));
185  const Real massflux_zlo = solverChoice.const_massflux_layer_lo - geom[0].ProbLo(2);
186  const Real massflux_zhi = solverChoice.const_massflux_layer_hi - geom[0].ProbLo(2);
187  const Real dz = geom[0].CellSize(2);
188  if (zlo_unset) {
189  solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
190  } else {
191  solverChoice.massflux_klo = static_cast<int>(std::ceil(massflux_zlo / dz - myhalf));
192  }
193  if (zhi_unset) {
194  solverChoice.massflux_khi = geom[0].Domain().bigEnd(2);
195  } else {
196  solverChoice.massflux_khi = static_cast<int>(std::floor(massflux_zhi / dz - myhalf));
197  }
198  } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
199  const Real massflux_zlo = solverChoice.const_massflux_layer_lo;
200  const Real massflux_zhi = solverChoice.const_massflux_layer_hi;
201  solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
202  solverChoice.massflux_khi = geom[0].Domain().bigEnd(2) + 1;
203  for (int k=0; k <= geom[0].Domain().bigEnd(2)+1; ++k) {
204  if (zlevels_stag[0][k] <= massflux_zlo) solverChoice.massflux_klo = k;
205  if (zlevels_stag[0][k] <= massflux_zhi) solverChoice.massflux_khi = k;
206  }
207  } else { // solverChoice.mesh_type == MeshType::VariableDz
208  Error("Const massflux with variable dz not supported -- planar averages are on k rather than constant-z planes");
209  }
210 
211  Print() << "Constant mass flux based on k in ["
212  << solverChoice.massflux_klo << ", " << solverChoice.massflux_khi << "]" << std::endl;
213  }
214 
215 #ifdef ERF_REMORA_FORCE_PROBINIT_LINK
216  extern void erf_probinit_link_anchor_func () noexcept;
217  erf_probinit_link_anchor_func();
218 #endif
219  prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
220 
221  // Geometry on all levels has been defined already.
222 
223  // No valid BoxArray and DistributionMapping have been defined.
224  // But the arrays for them have been resized.
225 
226  t_new.resize(nlevs_max, zero);
227  t_old.resize(nlevs_max, -bogus_large_value);
228  dt.resize(nlevs_max, std::min(static_cast<double>(bogus_large_value),dt_max_initial));
229  dt_mri_ratio.resize(nlevs_max, 1);
230 
231  vars_new.resize(nlevs_max);
232  vars_old.resize(nlevs_max);
233  gradp.resize(nlevs_max);
234 
235  // We resize this regardless in order to pass it without error
236  pp_inc.resize(nlevs_max);
237 
238  // Used in the fast substepping only
239  lagged_delta_rt.resize(nlevs_max);
240  avg_xmom.resize(nlevs_max);
241  avg_ymom.resize(nlevs_max);
242  avg_zmom.resize(nlevs_max);
243 
244  rU_new.resize(nlevs_max);
245  rV_new.resize(nlevs_max);
246  rW_new.resize(nlevs_max);
247 
248  rU_old.resize(nlevs_max);
249  rV_old.resize(nlevs_max);
250  rW_old.resize(nlevs_max);
251 
252  // xmom_crse_rhs.resize(nlevs_max);
253  // ymom_crse_rhs.resize(nlevs_max);
254  zmom_crse_rhs.resize(nlevs_max);
255 
256  for (int lev = 0; lev < nlevs_max; ++lev) {
257  vars_new[lev].resize(Vars::NumTypes);
258  vars_old[lev].resize(Vars::NumTypes);
259  gradp[lev].resize(AMREX_SPACEDIM);
260  }
261 
262  // Time integrator
263  mri_integrator_mem.resize(nlevs_max);
264 
265  // Physical boundary conditions
266  physbcs_cons.resize(nlevs_max);
267  physbcs_u.resize(nlevs_max);
268  physbcs_v.resize(nlevs_max);
269  physbcs_w.resize(nlevs_max);
270  physbcs_base.resize(nlevs_max);
271 
272  // Planes to hold Dirichlet values at boundaries
273  xvel_bc_data.resize(nlevs_max);
274  yvel_bc_data.resize(nlevs_max);
275  zvel_bc_data.resize(nlevs_max);
276  th_bc_data.resize(nlevs_max);
277 
278  advflux_reg.resize(nlevs_max);
279 
280  // Stresses
281  Tau.resize(nlevs_max);
282  Tau_corr.resize(nlevs_max);
283  SFS_hfx1_lev.resize(nlevs_max); SFS_hfx2_lev.resize(nlevs_max); SFS_hfx3_lev.resize(nlevs_max);
284  SFS_diss_lev.resize(nlevs_max);
285  SFS_q1fx1_lev.resize(nlevs_max); SFS_q1fx2_lev.resize(nlevs_max); SFS_q1fx3_lev.resize(nlevs_max);
286  SFS_q2fx3_lev.resize(nlevs_max);
287  eddyDiffs_lev.resize(nlevs_max);
288  SmnSmn_lev.resize(nlevs_max);
289  Tau_EB.resize(nlevs_max);
290  hfx3_EB.resize(nlevs_max);
291 
292  // Sea surface temps
293  sst_lev.resize(nlevs_max);
294  tsk_lev.resize(nlevs_max);
295  lmask_lev.resize(nlevs_max);
296 
297  // Land and soil grid type and urban fractions
298  land_type_lev.resize(nlevs_max);
299  soil_type_lev.resize(nlevs_max);
300  urb_frac_lev.resize(nlevs_max);
301 
302  // Metric terms
303  z_phys_nd.resize(nlevs_max);
304  z_phys_cc.resize(nlevs_max);
305  detJ_cc.resize(nlevs_max);
306  ax.resize(nlevs_max);
307  ay.resize(nlevs_max);
308  az.resize(nlevs_max);
309 
310  z_phys_nd_new.resize(nlevs_max);
311  detJ_cc_new.resize(nlevs_max);
312 
313  z_phys_nd_src.resize(nlevs_max);
314  z_phys_cc_src.resize(nlevs_max);
315  detJ_cc_src.resize(nlevs_max);
316  ax_src.resize(nlevs_max);
317  ay_src.resize(nlevs_max);
318  az_src.resize(nlevs_max);
319 
320  z_t_rk.resize(nlevs_max);
321 
322  terrain_blanking.resize(nlevs_max);
323 
324  // Wall distance
325  walldist.resize(nlevs_max);
326 
327  // BoxArrays to make MultiFabs needed to convert WRFBdy data
328  ba1d.resize(nlevs_max);
329  ba2d.resize(nlevs_max);
330 
331  // MultiFabs needed to convert WRFBdy data
332  mf_PSFC.resize(nlevs_max);
333 
334  // Map factors
335  mapfac.resize(nlevs_max);
336 
337  // Fine mask
338  fine_mask.resize(nlevs_max);
339 
340  // Thin immersed body
341  xflux_imask.resize(nlevs_max);
342  yflux_imask.resize(nlevs_max);
343  zflux_imask.resize(nlevs_max);
344  //overset_imask.resize(nlevs_max);
345  thin_xforce.resize(nlevs_max);
346  thin_yforce.resize(nlevs_max);
347  thin_zforce.resize(nlevs_max);
348 
349  // Base state
350  base_state.resize(nlevs_max);
351  base_state_new.resize(nlevs_max);
352 
353  // Wave coupling data
354  Hwave.resize(nlevs_max);
355  Lwave.resize(nlevs_max);
356  for (int lev = 0; lev < max_level; ++lev)
357  {
358  Hwave[lev] = nullptr;
359  Lwave[lev] = nullptr;
360  }
361  Hwave_onegrid.resize(nlevs_max);
362  Lwave_onegrid.resize(nlevs_max);
363  for (int lev = 0; lev < max_level; ++lev)
364  {
365  Hwave_onegrid[lev] = nullptr;
366  Lwave_onegrid[lev] = nullptr;
367  }
368 
369  // Theta prim for MOST
370  Theta_prim.resize(nlevs_max);
371 
372  // Qv prim for MOST
373  Qv_prim.resize(nlevs_max);
374 
375  // Qr prim for MOST
376  Qr_prim.resize(nlevs_max);
377 
378  // Time averaged velocity field
379  vel_t_avg.resize(nlevs_max);
380  t_avg_cnt.resize(nlevs_max);
381 
382  // Size lat long arrays and default to null pointers
383  lat_m.resize(nlevs_max);
384  lon_m.resize(nlevs_max);
385  for (int lev = 0; lev < max_level; ++lev) {
386  lat_m[lev] = nullptr;
387  lon_m[lev] = nullptr;
388  }
389 
390  // Variable coriolis
391  sinPhi_m.resize(nlevs_max);
392  cosPhi_m.resize(nlevs_max);
393  for (int lev = 0; lev < max_level; ++lev) {
394  sinPhi_m[lev] = nullptr;
395  cosPhi_m[lev] = nullptr;
396  }
397 
398  // Rayleigh damping
399  h_rayleigh_ptrs.resize(nlevs_max);
400  d_rayleigh_ptrs.resize(nlevs_max);
401  h_sinesq_ptrs.resize(nlevs_max);
402  d_sinesq_ptrs.resize(nlevs_max);
403  h_sinesq_stag_ptrs.resize(nlevs_max);
404  d_sinesq_stag_ptrs.resize(nlevs_max);
405 
406  // Initialize tagging criteria for mesh refinement
407  refinement_criteria_setup();
408 
409  for (int lev = 0; lev < max_level; ++lev)
410  {
411  Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
412  ref_ratio[lev][0] << " " << ref_ratio[lev][1] << " " << ref_ratio[lev][2] << std::endl;
413  }
414 
415  // We will create each of these in MakeNewLevelFromScratch
416  eb.resize(max_level+1);
417  for (int lev = 0; lev < max_level + 1; lev++){
418  eb[lev] = std::make_unique<eb_>();
419  }
420 
421  //
422  // Construct the EB data structures and store in a separate class
423  //
424  // This is needed before initializing level MultiFabs
425  if ( solverChoice.terrain_type == TerrainType::EB ||
426  solverChoice.terrain_type == TerrainType::ImmersedForcing)
427  {
428  std::string geometry ="terrain";
429  ParmParse pp_eb2("eb2");
430  pp_eb2.queryAdd("geometry", geometry);
431 
432  constexpr int ngrow_for_eb = 4; // This is the default in amrex but we need to explicitly pass it here since
433  // we want to also pass the build_coarse_level_by_coarsening argument
434  const bool build_eb_for_multigrid = (solverChoice.terrain_type == TerrainType::EB &&
435  ((solverChoice.project_initial_velocity[0] == 1) ||
436  solverChoice.anelastic[0] == 1));
437  // Note this just needs to be an integer > number of V-cycles one might use
438  const int max_coarsening_level = (build_eb_for_multigrid) ? 100 : 0;
439  const bool build_coarse_level_by_coarsening(false);
440 
441  // Define GeometryShop using the implicit function
442  if (geometry == "terrain") {
443  Box terrain_bx(surroundingNodes(geom[max_level].Domain())); terrain_bx.grow(3);
444  FArrayBox terrain_fab(makeSlab(terrain_bx,2,0),1);
445  double dummy_time = 0.0;
446  prob->init_terrain_surface(geom[max_level], terrain_fab, dummy_time);
447  TerrainIF implicit_fun(terrain_fab, geom[max_level], stretched_dz_d[max_level]);
448  auto gshop = EB2::makeShop(implicit_fun);
449  if (build_eb_for_multigrid) {
450  EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
451  ngrow_for_eb, build_coarse_level_by_coarsening);
452  } else {
453  EB2::Build(gshop, this->Geom(), ngrow_for_eb);
454  }
455  } else if (geometry == "plane") {
456  RealArray plane_point{zero, zero, zero};
457  RealArray plane_normal{zero, zero, -one}; // pointing into the solid region
458  pp_eb2.query("plane_point", plane_point);
459  pp_eb2.query("plane_normal", plane_normal);
460  EB2::PlaneIF implicit_fun(plane_point, plane_normal, true);
461  auto gshop = EB2::makeShop(implicit_fun);
462  if (build_eb_for_multigrid) {
463  EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
464  ngrow_for_eb, build_coarse_level_by_coarsening);
465  } else {
466  EB2::Build(gshop, this->Geom(), ngrow_for_eb);
467  }
468  } else if (geometry == "box") {
469  RealArray box_lo{zero, zero, zero};
470  RealArray box_hi{zero, zero, zero};
471  pp_eb2.query("box_lo", box_lo);
472  pp_eb2.query("box_hi", box_hi);
473  EB2::BoxIF implicit_fun(box_lo, box_hi, false);
474  auto gshop = EB2::makeShop(implicit_fun);
475  if (build_eb_for_multigrid) {
476  EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
477  ngrow_for_eb, build_coarse_level_by_coarsening);
478  } else {
479  EB2::Build(gshop, this->Geom(), ngrow_for_eb);
480  }
481  } else if (geometry == "sphere") {
482  auto ProbLoArr = geom[max_level].ProbLoArray();
483  auto ProbHiArr = geom[max_level].ProbHiArray();
484  const Real xcen = myhalf * (ProbLoArr[0] + ProbHiArr[0]);
485  const Real ycen = myhalf * (ProbLoArr[1] + ProbHiArr[1]);
486  RealArray sphere_center = {xcen, ycen, zero};
487  EB2::SphereIF implicit_fun(myhalf, sphere_center, false);
488  auto gshop = EB2::makeShop(implicit_fun);
489  if (build_eb_for_multigrid) {
490  EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level,
491  ngrow_for_eb, build_coarse_level_by_coarsening);
492  } else {
493  EB2::Build(gshop, this->Geom(), ngrow_for_eb);
494  }
495  }
496  }
497 
498  if ( solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
499  constexpr int ngrow_for_eb = 4;
500  Box buildings_bx(surroundingNodes(geom[max_level].Domain())); buildings_bx.grow(3);
501  FArrayBox buildings_fab(makeSlab(buildings_bx,2,0),1);
502  double dummy_time = 0.0;
503  prob->init_buildings_surface(geom[max_level], buildings_fab, dummy_time);
504  TerrainIF implicit_fun(buildings_fab, geom[max_level], stretched_dz_d[max_level]);
505  auto gshop = EB2::makeShop(implicit_fun);
506  EB2::Build(gshop, this->Geom(), ngrow_for_eb);
507  }
508 
509  forecast_state_1.resize(nlevs_max);
510  forecast_state_2.resize(nlevs_max);
511  forecast_state_interp.resize(nlevs_max);
512 
513  surface_state_1.resize(nlevs_max);
514  surface_state_2.resize(nlevs_max);
515  surface_state_interp.resize(nlevs_max);
516 }
init_zlevels
void init_zlevels(Vector< Vector< Real >> &zlevels_stag, Vector< Vector< Real >> &stretched_dz_h, Vector< Gpu::DeviceVector< Real >> &stretched_dz_d, Vector< Geometry > const &geom, Vector< IntVect > const &ref_ratio, const Real grid_stretching_ratio, const Real zsurf, const Real dz0)
Definition: ERF_InitZLevels.cpp:11
amrex_probinit
std::unique_ptr< ProblemBase > amrex_probinit(const amrex_real *problo, const amrex_real *probhi) AMREX_ATTRIBUTE_WEAK
ERF::t_new
amrex::Vector< double > t_new
Definition: ERF.H:919
ERF::Hwave_onegrid
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
Definition: ERF.H:1091
ERF::thin_yforce
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
Definition: ERF.H:1127
ERF::setPlotVariables
void setPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:14
ERF::gradp
amrex::Vector< amrex::Vector< amrex::MultiFab > > gradp
Definition: ERF.H:931
ERF::ReadParameters
void ReadParameters()
Definition: ERF.cpp:1945
ERF::mf_PSFC
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_PSFC
Definition: ERF.H:1394
ERF::z_phys_nd_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
Definition: ERF.H:1058
ERF::base_state_new
amrex::Vector< amrex::MultiFab > base_state_new
Definition: ERF.H:1086
ERF::az
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
Definition: ERF.H:1056
ERF::terrain_blanking
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
Definition: ERF.H:1071
ERF::z_phys_nd_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
Definition: ERF.H:1065
ERF::thin_zforce
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
Definition: ERF.H:1128
ERF::plot3d_var_names_2
amrex::Vector< std::string > plot3d_var_names_2
Definition: ERF.H:1229
ERF::thin_xforce
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
Definition: ERF.H:1126
ERF::th_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
Definition: ERF.H:886
ERF::surface_state_1
amrex::Vector< amrex::MultiFab > surface_state_1
Definition: ERF.H:187
ERF::z_t_rk
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
Definition: ERF.H:1068
ERF::Lwave_onegrid
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
Definition: ERF.H:1092
ERF::h_sinesq_ptrs
amrex::Vector< amrex::Vector< amrex::Real > > h_sinesq_ptrs
Definition: ERF.H:1443
ERF::execution_provenance
erf_provenance::ExecutionProvenance execution_provenance
Definition: ERF.H:1171
ERF::fine_mask
amrex::Vector< std::unique_ptr< amrex::MultiFab > > fine_mask
Definition: ERF.H:1080
ERF::m_forest_drag
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
Definition: ERF.H:1471
ERF::ba1d
amrex::Vector< amrex::BoxArray > ba1d
Definition: ERF.H:1385
ERF::xvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
Definition: ERF.H:883
ERF::rad_datalog_int
int rad_datalog_int
Definition: ERF.H:1008
ERF::surface_state_2
amrex::Vector< amrex::MultiFab > surface_state_2
Definition: ERF.H:188
ERF::detJ_cc_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
Definition: ERF.H:1060
ERF::ay_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
Definition: ERF.H:1062
ERF::yflux_imask
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
Definition: ERF.H:1121
ERF::lsm_flux
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
Definition: ERF.H:992
ERF::plot3d_var_names_1
amrex::Vector< std::string > plot3d_var_names_1
Definition: ERF.H:1228
ERF::refinement_criteria_setup
void refinement_criteria_setup()
Definition: ERF_Tagging.cpp:472
ERF::Tau_corr
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau_corr
Definition: ERF.H:1020
ERF::ax_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
Definition: ERF.H:1061
ERF::urb_frac_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > urb_frac_lev
Definition: ERF.H:1032
ERF::z_phys_cc_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc_src
Definition: ERF.H:1059
ERF::forecast_state_2
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_2
Definition: ERF.H:185
ERF::dt_max_initial
static double dt_max_initial
Definition: ERF.H:1178
ERF::soil_type_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > soil_type_lev
Definition: ERF.H:1031
ERF::zlevels_stag
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
Definition: ERF.H:1047
ERF::lsm_data
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
Definition: ERF.H:990
ERF::stretched_dz_h
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
Definition: ERF.H:1082
ERF::az_src
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
Definition: ERF.H:1063
ERF::Lwave
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
Definition: ERF.H:1090
ERF::land_type_lev
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > land_type_lev
Definition: ERF.H:1030
ERF::forecast_state_1
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_1
Definition: ERF.H:184
ERF::zflux_imask
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
Definition: ERF.H:1122
ERF::zvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
Definition: ERF.H:885
ERF::h_sinesq_stag_ptrs
amrex::Vector< amrex::Vector< amrex::Real > > h_sinesq_stag_ptrs
Definition: ERF.H:1444
ERF::detJ_cc_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
Definition: ERF.H:1066
ERF::yvel_bc_data
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
Definition: ERF.H:884
ERF::surface_state_interp
amrex::Vector< amrex::MultiFab > surface_state_interp
Definition: ERF.H:189
ERF::h_rayleigh_ptrs
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
Definition: ERF.H:1439
ERF::Hwave
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
Definition: ERF.H:1089
ERF::Tau_EB
amrex::Vector< amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > > Tau_EB
Definition: ERF.H:1042
ERF::xflux_imask
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
Definition: ERF.H:1120
ERF::initializeMicrophysics
void initializeMicrophysics(const int &)
Definition: ERF.cpp:1713
LandSurface::ReSize
void ReSize(const int &nlev)
Definition: ERF_LandSurface.H:27
TerrainIF
Definition: ERF_EBIFTerrain.H:14
amrex::buildInfoGetGitHash
const char * buildInfoGetGitHash(int i)
erf_provenance::initialize_execution_provenance
ExecutionProvenance initialize_execution_provenance()
Definition: ERF_Provenance.cpp:559
SolverChoice::dz0
amrex::Real dz0
Definition: ERF_DataStruct.H:1334
SolverChoice::const_massflux_layer_lo
amrex::Real const_massflux_layer_lo
Definition: ERF_DataStruct.H:1465
SolverChoice::const_massflux_v
amrex::Real const_massflux_v
Definition: ERF_DataStruct.H:1463
SolverChoice::massflux_klo
int massflux_klo
Definition: ERF_DataStruct.H:1467
SolverChoice::grid_stretching_ratio
amrex::Real grid_stretching_ratio
Definition: ERF_DataStruct.H:1332
SolverChoice::const_massflux_u
amrex::Real const_massflux_u
Definition: ERF_DataStruct.H:1462
SolverChoice::zsurf
amrex::Real zsurf
Definition: ERF_DataStruct.H:1333
SolverChoice::buildings_type
static BuildingsType buildings_type
Definition: ERF_DataStruct.H:1233
SolverChoice::const_massflux_layer_hi
amrex::Real const_massflux_layer_hi
Definition: ERF_DataStruct.H:1466
SolverChoice::massflux_khi
int massflux_khi
Definition: ERF_DataStruct.H:1468
Here is the call graph for this function:

◆ ErrorEst()

void ERF::ErrorEst ( int  lev,
amrex::TagBoxArray &  tags,
amrex::Real  time,
int  ngrow 
)
override

Function to tag cells for refinement – this overrides the pure virtual function in AmrCore

Parameters
[in]levclevel of refinement at which we tag cells (0 is coarsest level)
[out]tagsarray of tagged cells
[in]timecurrent time
[in]ngrownumber of ghost cells (not used here)
26 {
27  const int clearval = TagBox::CLEAR;
28  const int tagval = TagBox::SET;
29 
30 #ifdef ERF_USE_NETCDF
31  if ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) {
32  int ratio;
33  Box subdomain;
34 
35  // This is the number of boxes that may have already been defined in the refinement_criteria_setup routine.
36  // If nb == 0 then no boxes have been specified in the inputs file, and we will use the boxes given in wrfinput_d*
37  // If nb > 0 then boxes have been specified in the inputs file, and we will use the specified boxes as long
38  // as we can ensure that they are contained inside the boxes given in wrfinput_d*
39  int nb_prespecified = num_boxes_at_level[levc+1];
40 
41  if (!nc_init_file[levc+1].empty())
42  {
43  double levc_start_time = read_start_time_from_wrfinput(levc , nc_init_file[levc ][0]);
44  if (solverChoice.init_type == InitType::WRFInput) {
45  amrex::Print() << " WRFInput time at level " << levc << " is " << levc_start_time << std::endl;
46  } else if (solverChoice.init_type == InitType::Metgrid) {
47  amrex::Print() << " met_em time at level " << levc << " is " << levc_start_time << std::endl;
48  }
49 
50  for (int isub = 0; isub < nc_init_file[levc+1].size(); isub++) {
51  if (!have_read_nc_init_file[levc+1][isub])
52  {
53  double levf_start_time = read_start_time_from_wrfinput(levc+1, nc_init_file[levc+1][isub]);
54  if (solverChoice.init_type == InitType::WRFInput) {
55  amrex::Print() << " WRFInput start_time at level " << levc+1 << " is " << levf_start_time << std::endl;
56  } else if (solverChoice.init_type == InitType::Metgrid) {
57  amrex::Print() << " met_em start time at level " << levc+1 << " is " << levf_start_time << std::endl;
58  }
59 
60  // We assume there is only one subdomain at levc; otherwise we don't know
61  // which one is the parent of the fine region we are trying to create
62  AMREX_ALWAYS_ASSERT(subdomains[levc].size() == 1);
63 
64  if ((solverChoice.init_type == InitType::WRFInput) && ((ref_ratio[levc][2]) != 1)) {
65  amrex::Abort("The ref_ratio specified in the inputs file must have 1 in the z direction; please use ref_ratio_vect rather than ref_ratio");
66  }
67 
68  if ( levf_start_time <= (levc_start_time + t_new[levc]) ) {
69  if (solverChoice.init_type == InitType::WRFInput) {
70  amrex::Print() << " WRFInput file to read: " << nc_init_file[levc+1][isub] << std::endl;
71  subdomain = read_subdomain_from_wrfinput(levc, nc_init_file[levc+1][isub], ratio);
72  amrex::Print() << " WRFInput subdomain " << isub << " at level " << levc+1 << " is " << subdomain << std::endl;
73  } else if (solverChoice.init_type == InitType::Metgrid) {
74  amrex::Print() << "met_em file to read: " << nc_init_file[levc+1][0] << std::endl;
75  const Box& domain = geom[levc].Domain();
76  int klo = domain.smallEnd(2);
77  int khi = domain.bigEnd(2);
78  subdomain = read_subdomain_from_metgrid(levc, nc_init_file[levc+1][0], ratio, klo, khi);
79  amrex::Print() << " met_em subdomain at level " << levc+1 << " is " << subdomain << std::endl;
80  }
81 
82  if ( (ratio != ref_ratio[levc][0]) || (ratio != ref_ratio[levc][1]) ) {
83  amrex::Print() << "File " << nc_init_file[levc+1][0] << " has refinement ratio = " << ratio << std::endl;
84  amrex::Print() << "The inputs file has refinement ratio = " << ref_ratio[levc] << std::endl;
85  amrex::Abort("These must be the same -- please edit your inputs file and try again.");
86  }
87 
88  subdomain.coarsen(ref_ratio[levc]);
89 
90  // Recall we asserted that there is only one box at level levc
91  Box coarser_level(subdomains[levc][0].minimalBox());
92  subdomain.shift(coarser_level.smallEnd());
93 
94  if (verbose > 0) {
95  amrex::Print() << " Crse version of subdomain available for tagging is" << subdomain << std::endl;
96  }
97 
98  Box new_fine(subdomain);
99  if (solverChoice.init_type == InitType::WRFInput) {
100  new_fine.refine(IntVect(ratio,ratio,1));
101  } else if (solverChoice.init_type == InitType::Metgrid) {
102  new_fine.refine(ref_ratio[levc]);
103  }
104  if (nb_prespecified == 0) {
105  num_boxes_at_level[levc+1] += 1;
106  boxes_at_level[levc+1].push_back(new_fine);
107  } else {
108  if (!new_fine.contains(boxes_at_level[levc+1][isub])) {
109  amrex::Print() << "\n";
110  amrex::Print() << "Box available in wrfinputs file " << new_fine << std::endl;
111  amrex::Print() << "Box requested for refinement in inputs file " << boxes_at_level[levc+1][isub] << std::endl;
112  amrex::Abort("Specified boxes must be contained within boxes specified in wrfinput at this level");
113  }
114  }
115 
116  Box coarsened_bx(boxes_at_level[levc+1][isub]); coarsened_bx.coarsen(ref_ratio[levc]);
117 
118  for (MFIter mfi(tags); mfi.isValid(); ++mfi)
119  {
120  auto tag_arr = tags.array(mfi); // Get device-accessible array
121 
122  Box bx = mfi.validbox() & coarsened_bx;
123 
124  if (!bx.isEmpty()) {
125  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
126  tag_arr(i,j,k) = TagBox::SET;
127  });
128  }
129  }
130  } // time is right
131  } else {
132  // Re-tag this region
133  for (MFIter mfi(tags); mfi.isValid(); ++mfi)
134  {
135  auto tag_arr = tags.array(mfi); // Get device-accessible array
136 
137  Box existing_bx_coarsened(boxes_at_level[levc+1][isub]);
138  existing_bx_coarsened.coarsen(ref_ratio[levc]);
139 
140  Box bx = mfi.validbox(); bx &= existing_bx_coarsened;
141 
142  if (!bx.isEmpty()) {
143  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
144  tag_arr(i,j,k) = TagBox::SET;
145  });
146  }
147  }
148  } // has file been read?
149  } // isub
150  return;
151  } // file not empty
152  }
153 #endif
154 
155  //
156  // Make sure the ghost cells of the level we are tagging at are filled
157  // in case we take differences that require them
158  // NOTE: We are Fillpatching only the cell-centered variables here
159  //
160  MultiFab& S_new = vars_new[levc][Vars::cons];
161  MultiFab& U_new = vars_new[levc][Vars::xvel];
162  MultiFab& V_new = vars_new[levc][Vars::yvel];
163  MultiFab& W_new = vars_new[levc][Vars::zvel];
164  //
165  if (levc == 0) {
166  FillPatchCrseLevel(levc, time, {&S_new, &U_new, &V_new, &W_new});
167  } else {
168  FillPatchFineLevel(levc, time, {&S_new, &U_new, &V_new, &W_new},
169  {&S_new, &rU_new[levc], &rV_new[levc], &rW_new[levc]},
170  base_state[levc], base_state[levc],
171  false, true);
172  }
173 
174  for (int t=0; t < ref_tags.size(); ++t)
175  {
176  //
177  // This mf must have ghost cells because we may take differences between adjacent values
178  //
179  std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 1);
180  mf->setVal(0.0);
181 
182  RealBox real_box = ref_tags[t].GetInfo().m_realbox;
183  if (real_box.ok()) {
184  ParmParse pp(pp_prefix); int lev_for_box; Vector<std::string> refinement_indicators;
185  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
186  std::string ref_prefix = pp_prefix + "." + refinement_indicators[t];
187  update_box_for_refinement(ref_prefix, lev_for_box, real_box, time);
188  ref_tags[t].GetInfo().SetRealBox(real_box);
189  }
190 
191  // This allows dynamic refinement based on the value of the density
192  if (ref_tags[t].Field() == "density")
193  {
194  MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,1);
195 
196  // Refinement based on a moisture mixing ratio (qv, qc, qi, qr, qs, qg).
197  // The map from name to RhoQ component depends on the moisture model and
198  // is held in solverChoice.moisture_indices.
199  } else if ( (ref_tags[t].Field() == "qv") ||
200  (ref_tags[t].Field() == "qc") ||
201  (ref_tags[t].Field() == "qi") ||
202  (ref_tags[t].Field() == "qr") ||
203  (ref_tags[t].Field() == "qs") ||
204  (ref_tags[t].Field() == "qg") )
205  {
206  const auto& mi = solverChoice.moisture_indices;
207  int qcomp = -1;
208  if (ref_tags[t].Field() == "qv") { qcomp = mi.qv; }
209  else if (ref_tags[t].Field() == "qc") { qcomp = mi.qc; }
210  else if (ref_tags[t].Field() == "qi") { qcomp = mi.qi; }
211  else if (ref_tags[t].Field() == "qr") { qcomp = mi.qr; }
212  else if (ref_tags[t].Field() == "qs") { qcomp = mi.qs; }
213  else if (ref_tags[t].Field() == "qg") { qcomp = mi.qg; }
214  AMREX_ALWAYS_ASSERT(qcomp >= 0);
215  MultiFab::Copy( *mf, vars_new[levc][Vars::cons], qcomp, 0, 1, 1);
216  MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
217 
218  // qt = total condensed water mixing ratio (qc + qi + qr + qs + qg).
219  // Excludes qv by convention here.
220  } else if (ref_tags[t].Field() == "qt") {
221  const auto& mi = solverChoice.moisture_indices;
222  const int idx_qc = mi.qc, idx_qi = mi.qi, idx_qr = mi.qr,
223  idx_qs = mi.qs, idx_qg = mi.qg;
224  AMREX_ALWAYS_ASSERT(idx_qc >= 0 || idx_qi >= 0 || idx_qr >= 0 ||
225  idx_qs >= 0 || idx_qg >= 0);
226  mf->setVal(0.0);
227  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
228  {
229  const Box& bx = mfi.growntilebox();
230  auto qt_arr = mf->array(mfi);
231  auto cons_arr = vars_new[levc][Vars::cons].const_array(mfi);
232  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
233  const Real rho_inv = Real(1.0) / cons_arr(i,j,k,Rho_comp);
234  Real q = Real(0.0);
235  if (idx_qc >= 0) { q += cons_arr(i,j,k,idx_qc) * rho_inv; }
236  if (idx_qi >= 0) { q += cons_arr(i,j,k,idx_qi) * rho_inv; }
237  if (idx_qr >= 0) { q += cons_arr(i,j,k,idx_qr) * rho_inv; }
238  if (idx_qs >= 0) { q += cons_arr(i,j,k,idx_qs) * rho_inv; }
239  if (idx_qg >= 0) { q += cons_arr(i,j,k,idx_qg) * rho_inv; }
240  qt_arr(i,j,k) = q;
241  });
242  }
243 
244  // This allows dynamic refinement based on the value of the z-component of vorticity
245  } else if (ref_tags[t].Field() == "vorticity" ) {
246  MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
247  average_face_to_cellcenter(mf_cc_vel,0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new}, 1);
248 
249  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
250  {
251  const Box& bx = mfi.tilebox();
252  auto& dfab = (*mf)[mfi];
253  auto& sfab = mf_cc_vel[mfi];
254  auto& zfab = (*z_phys_cc[levc])[mfi];
255  derived::erf_dervortz(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
256  }
257 
258  // This allows dynamic refinement based on the value of the scalar/theta
259  } else if ( (ref_tags[t].Field() == "scalar" ) ||
260  (ref_tags[t].Field() == "theta" ) )
261  {
262  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
263  {
264  const Box& bx = mfi.tilebox();
265  auto& dfab = (*mf)[mfi];
266  auto& sfab = vars_new[levc][Vars::cons][mfi];
267  auto& zfab = (*z_phys_cc[levc])[mfi];
268  if (ref_tags[t].Field() == "scalar") {
269  derived::erf_derscalar(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
270  } else if (ref_tags[t].Field() == "theta") {
271  derived::erf_dertheta(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
272  }
273  } // mfi
274  // This allows dynamic refinement based on the value of updraft helicity
275  } else if (ref_tags[t].Field() == "helicity")
276  {
277  MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
278  average_face_to_cellcenter(mf_cc_vel,0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new}, 1);
279 
280  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
281  {
282  const Box& bx = mfi.tilebox();
283  auto& dfab = (*mf)[mfi];
284  auto& sfab = mf_cc_vel[mfi];
285  auto& zfab = (*z_phys_cc[levc])[mfi];
286 
287  derived::erf_derhelicity(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
288  }
289  } else if (ref_tags[t].Field() == "max_reflectivity")
290  {
291  if (solverChoice.moisture_type == MoistureType::Morrison ||
292  solverChoice.moisture_type == MoistureType::SAM) {
293  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
294  {
295  const Box& bx = mfi.tilebox();
296  auto& dfab = (*mf)[mfi];
297  auto& sfab = vars_new[levc][Vars::cons][mfi];
298  auto& zfab = (*z_phys_cc[levc])[mfi];
299 
300  derived::erf_dermaxreflectivity(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
301  }
302  } else {
303  Abort("Max reflectivity is only available with Morrison and SAM microphysics.");
304  }
305  // This allows dynamic refinement based on the terrain blanking
306  } else if ( (SolverChoice::terrain_type == TerrainType::ImmersedForcing) &&
307  (ref_tags[t].Field() == "terrain_blanking") )
308  {
309  MultiFab::Copy(*mf,*terrain_blanking[levc],0,0,1,1);
310  }
311  else if (ref_tags[t].Field() == "velmag")
312  {
313  ParmParse pp(pp_prefix);
314  Vector<std::string> refinement_indicators;
315  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
316  Real velmag_threshold;
317  bool is_hurricane_tracker = false;
318  for (int i=0; i<refinement_indicators.size(); ++i)
319  {
320  if (refinement_indicators[i]=="hurricane_tracker") {
321  is_hurricane_tracker = true;
322  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
323  ParmParse ppr(ref_prefix);
324  ppr.get("value_greater", velmag_threshold);
325  break;
326  }
327  }
328 
329  Vector<MultiFab> mf_cc_vel(1);
330  mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
331  average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
332 
333  if (is_hurricane_tracker) {
334  HurricaneTracker(levc, time, mf_cc_vel[0], velmag_threshold, &tags);
335  } else {
336  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
337  {
338  const Box& bx = mfi.tilebox();
339  auto& dfab = (*mf)[mfi];
340  auto& sfab = mf_cc_vel[0][mfi];
341  auto& zfab = (*z_phys_cc[levc])[mfi];
342  derived::erf_dermagvel(bx, dfab, 0, 1, sfab, zfab, Geom(levc), time, nullptr, levc);
343  }
344  }
345 
346 #ifdef ERF_USE_PARTICLES
347  } else {
348  //
349  // Particle-derived refinement. Two forms of `field_name` are
350  // supported:
351  // <species>_count : particle count per cell
352  // <species>_<mesh_var> : Eulerian mesh variable provided
353  // by the species' `meshPlotVarNames()`
354  // (e.g. `super_droplets_moisture_mass_density`,
355  // `tracer_particles_mass_density`).
356  //
357  // In both cases the field is deposited at every level in
358  // [levc, finest_level] and averaged down level-by-level so that
359  // a signal present only on a finer level still triggers
360  // refinement at the coarser level being tagged. Without this,
361  // particles localised at level 1 would not register at level 0
362  // and the fine grid would disappear at the next regrid.
363  //
364  const auto& particles_namelist( particleData.getNames() );
365  mf->setVal(0.0);
366  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
367  {
368  auto* pc = particleData[particles_namelist[i]];
369  const std::string& sp_name = particles_namelist[i];
370  const std::string& field = ref_tags[t].Field();
371 
372  const std::string count_str = sp_name + "_count";
373  const std::string prefix = sp_name + "_";
374  std::string mesh_var;
375  if (field != count_str
376  && field.size() > prefix.size()
377  && field.compare(0, prefix.size(), prefix) == 0)
378  {
379  const std::string suffix = field.substr(prefix.size());
380  for (const auto& v : pc->meshPlotVarNames()) {
381  if (v == suffix) { mesh_var = v; break; }
382  }
383  }
384  if (field != count_str && mesh_var.empty()) { continue; }
385 
386  pc->resizeData();
387  const int pc_nlevs = static_cast<int>(pc->GetParticles().size());
388 
389  // Deposit at each level into per-level MultiFabs.
390  Vector<MultiFab> per_lev(finest_level+1);
391  for (int lev = levc; lev <= finest_level; lev++) {
392  per_lev[lev].define(grids[lev], dmap[lev], 1, 0);
393  per_lev[lev].setVal(0);
394  if (field == count_str) {
395  if (lev < pc_nlevs) {
396  pc->IncrementWithTotal(per_lev[lev], lev);
397  }
398  } else {
399  pc->computeMeshVar(mesh_var, per_lev[lev],
400  *z_phys_nd[lev], lev);
401  }
402  }
403 
404  // Average down level-by-level from finest to levc. This
405  // avoids multi-level coarsening (e.g. L2->L0 with ratio
406  // (4,1,4)) which can fail when fine-level boxes are not
407  // aligned to the composite refinement ratio.
408  for (int lev = finest_level; lev > levc; lev--) {
409  MultiFab temp_crse(grids[lev-1], dmap[lev-1], 1, 0);
410  temp_crse.setVal(0);
411  average_down(per_lev[lev], temp_crse,
412  0, 1, ref_ratio[lev-1]);
413  MultiFab::Add(per_lev[lev-1], temp_crse, 0, 0, 1, 0);
414  }
415 
416  MultiFab::Copy(*mf, per_lev[levc], 0, 0, 1, 0);
417  }
418 #endif
419  }
420 
421  ref_tags[t](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
422  } // loop over t
423 
424  // ********************************************************************************************
425  // Refinement based on 2d distance from the "eye" which is defined here as the (x,y) location of
426  // the integrated qv
427  // ********************************************************************************************
428  ParmParse pp(pp_prefix);
429  Vector<std::string> refinement_indicators;
430  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
431  for (int i=0; i<refinement_indicators.size(); ++i)
432  {
433  if ( (refinement_indicators[i]=="storm_tracker") && (solverChoice.moisture_type != MoistureType::None) )
434  {
435  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
436  ParmParse ppr(ref_prefix);
437 
438  double ref_start_time = -1.0;
439  ppr.query("start_time",ref_start_time);
440 
441  if (time >= ref_start_time) {
442 
443  Real max_radius = -one;
444  ppr.get("max_radius", max_radius);
445 
446  // Create the volume-weighted sum of (rho qv) in each column
447  MultiFab mf_qv_int(ba2d[levc], dmap[levc], 1, 0); mf_qv_int.setVal(0.);
448 
449  // Define the 2D MultiFab holding the column-integrated (rho qv)
450  volWgtColumnSum(levc, S_new, RhoQ1_comp, mf_qv_int, *detJ_cc[levc]);
451 
452  // Find the max value in the domain
453  IntVect eye = mf_qv_int.maxIndex(0);
454 
455  const auto dx = geom[levc].CellSizeArray();
456  const auto prob_lo = geom[levc].ProbLoArray();
457 
458  Real eye_x = prob_lo[0] + (eye[0] + myhalf) * dx[0];
459  Real eye_y = prob_lo[1] + (eye[1] + myhalf) * dx[1];
460 
461  tag_on_distance_from_eye(geom[levc], &tags, eye_x, eye_y, max_radius);
462  }
463  }
464  }
465 }
tag_on_distance_from_eye
void tag_on_distance_from_eye(const Geometry &cgeom, TagBoxArray *tags, const Real eye_x, const Real eye_y, const Real rad_tag)
Definition: ERF_RefineHurricane.cpp:82
ERF::boxes_at_level
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
Definition: ERF.H:913
ERF::HurricaneTracker
void HurricaneTracker(int lev, double time, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::TagBoxArray *tags=nullptr)
Definition: ERF_RefineHurricane.cpp:109
ERF::volWgtColumnSum
void volWgtColumnSum(int lev, const amrex::MultiFab &mf, int comp, amrex::MultiFab &mf_2d, const amrex::MultiFab &dJ)
Definition: ERF_VolWgtSum.cpp:82
ERF::update_box_for_refinement
void update_box_for_refinement(std::string &ref_prefix, int &lev_for_box, amrex::RealBox &real_box, const double time)
Definition: ERF_RefineBox.cpp:336
ERF::nc_init_file
static amrex::Vector< amrex::Vector< std::string > > nc_init_file
Definition: ERF.H:1353
ERF::subdomains
amrex::Vector< amrex::Vector< amrex::BoxArray > > subdomains
Definition: ERF.H:1478
ERF::have_read_nc_init_file
static amrex::Vector< amrex::Vector< int > > have_read_nc_init_file
Definition: ERF.H:1354
ERF::ref_tags
static amrex::Vector< amrex::AMRErrorTag > ref_tags
Definition: ERF.H:1476
ERF::num_boxes_at_level
amrex::Vector< int > num_boxes_at_level
Definition: ERF.H:911
WSM6Ind::q
@ q
Definition: ERF_WSM6.H:182
WSM6Ind::t
@ t
Definition: ERF_WSM6.H:181
derived::erf_derhelicity
void erf_derhelicity(const Box &bx, FArrayBox &derfab, int dcomp, int, const FArrayBox &datfab, const FArrayBox &zcc_fab, const Geometry &geomdata, Real, const int *, const int)
Definition: ERF_Derive.cpp:647
derived::erf_dermagvel
void erf_dermagvel(const Box &bx, FArrayBox &derfab, int dcomp, int ncomp, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:480
derived::erf_dervortz
void erf_dervortz(const Box &bx, FArrayBox &derfab, int dcomp, int ncomp, const FArrayBox &datfab, const FArrayBox &, const Geometry &geomdata, Real, const int *, const int)
Definition: ERF_Derive.cpp:415
derived::erf_derscalar
void erf_derscalar(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:320
derived::erf_dertheta
void erf_dertheta(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:298
derived::erf_dermaxreflectivity
void erf_dermaxreflectivity(const Box &bx, FArrayBox &derfab, int dcomp, int, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:566
module_mp_morr_two_moment::isub
integer, private isub
Definition: ERF_module_mp_morr_two_moment.F90:164
MoistureComponentIndices::qv
int qv
Definition: ERF_DataStruct.H:107
MoistureComponentIndices::qc
int qc
Definition: ERF_DataStruct.H:108
Here is the call graph for this function:

◆ estTimeStep()

double ERF::estTimeStep ( int  level,
long &  dt_fast_ratio 
) const

Function that calls estTimeStep for each level

Parameters
[in]levellevel of refinement (coarsest level i 0)
[out]dt_fast_ratioratio of slow to fast time step
61 {
62  BL_PROFILE("ERF::estTimeStep()");
63 
64  double estdt_comp = bogus_large_value;
65  double estdt_lowM = bogus_large_value;
66 
67  // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation
68  const int nxc = geom[0].Domain().length(0);
69  const int nyc = geom[0].Domain().length(1);
70 
71  auto const dxinv = geom[level].InvCellSizeArray();
72  auto const dzinv = one / dz_min[level];
73 
74  MultiFab const& S_new = vars_new[level][Vars::cons];
75 
76  MultiFab ccvel(grids[level],dmap[level],3,0);
77 
78  average_face_to_cellcenter(ccvel,0,
79  Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
80  &vars_new[level][Vars::yvel],
81  &vars_new[level][Vars::zvel]});
82 
83  bool l_substepping = (solverChoice.substepping_type[level] == SubsteppingType::Implicit);
84  int l_anelastic = solverChoice.anelastic[level];
85 
86  bool l_comp_substepping_diag = (verbose && l_substepping && !l_anelastic && solverChoice.substepping_diag);
87 
88  Real estdt_comp_inv;
89  Real estdt_vert_comp_inv;
90  Real estdt_vert_lowM_inv;
91 
92  if (l_substepping && (nxc==1) && (nyc==1)) {
93  // SCM -- should not depend on dx or dy; force minimum number of substeps
94  estdt_comp_inv = std::numeric_limits<Real>::min();
95  }
96  else if (solverChoice.terrain_type == TerrainType::EB)
97  {
98  const eb_& eb_lev = get_eb(level);
99  const MultiFab& detJ = (eb_lev.get_const_factory())->getVolFrac();
100 
101  estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
102  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
103  Array4<Real const> const& s,
104  Array4<Real const> const& u,
105  Array4<Real const> const& vf) -> Real
106  {
107  Real new_comp_dt = -bogus_large_value;
108  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
109  {
110  if (vf(i,j,k) > zero)
111  {
112  const Real rho = s(i, j, k, Rho_comp);
113  const Real rhotheta = s(i, j, k, RhoTheta_comp);
114 
115  // NOTE: even when moisture is present,
116  // we only use the partial pressure of the dry air
117  // to compute the soundspeed
118  Real pressure = getPgivenRTh(rhotheta);
119  Real c = std::sqrt(Gamma * pressure / rho);
120 
121  // If we are doing implicit acoustic substepping, then the z-direction does not contribute
122  // to the computation of the time step
123  if (l_substepping) {
124  if ((nxc > 1) && (nyc==1)) {
125  // 2-D in x-z
126  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
127  } else if ((nyc > 1) && (nxc==1)) {
128  // 2-D in y-z
129  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
130  } else {
131  // 3-D
132  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
133  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
134  }
135 
136  // If we are not doing implicit acoustic substepping, then the z-direction contributes
137  // to the computation of the time step
138  } else {
139  if (nxc > 1 && nyc > 1) {
140  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
141  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
142  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
143  } else if (nxc > 1) {
144  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
145  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
146  } else if (nyc > 1) {
147  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
148  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
149  } else {
150  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
151  }
152 
153  }
154  }
155  });
156  return new_comp_dt;
157  });
158 
159  } else {
160  const MultiFab& detJ = *detJ_cc[level];
161  estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
162  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
163  Array4<Real const> const& s,
164  Array4<Real const> const& u,
165  Array4<Real const> const& dJ) -> Real
166  {
167  Real new_comp_dt = -bogus_large_value;
168  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
169  {
170  {
171  const Real rho = s(i, j, k, Rho_comp);
172  const Real rhotheta = s(i, j, k, RhoTheta_comp);
173 
174  Real idz_loc = dxinv[2] / dJ(i,j,k);
175 
176  // NOTE: even when moisture is present,
177  // we only use the partial pressure of the dry air
178  // to compute the soundspeed
179  Real pressure = getPgivenRTh(rhotheta);
180  Real c = std::sqrt(Gamma * pressure / rho);
181 
182  // If we are doing implicit acoustic substepping, then the z-direction is not constrained
183  // by the speed of sound for the computation of the time step
184  if (l_substepping) {
185  if ((nxc > 1) && (nyc==1)) {
186  // 2-D in x-z
187  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
188  } else if ((nyc > 1) && (nxc==1)) {
189  // 2-D in y-z
190  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
191  } else {
192  // 3-D
193  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
194  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
195  ((amrex::Math::abs(u(i,j,k,2)) )*idz_loc ),new_comp_dt);
196  }
197 
198  // If we are not doing implicit acoustic substepping, then the z-direction contributes
199  // to the computation of the time step
200  } else {
201  if (nxc > 1 && nyc > 1) {
202  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
203  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
204  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
205  } else if (nxc > 1) {
206  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
207  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
208  } else if (nyc > 1) {
209  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
210  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
211  } else {
212  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
213  }
214 
215  }
216  }
217  });
218  return new_comp_dt;
219  });
220  } // not EB
221 
222  ParallelDescriptor::ReduceRealMax(estdt_comp_inv);
223  // Globally empty level -> ReduceMax = lowest(); treat level as non-constraining.
224  estdt_comp = (estdt_comp_inv > Real(0.0)) ? (cfl / estdt_comp_inv) : bogus_large_value;
225 
226  Real estdt_lowM_inv = ReduceMax(ccvel, 0,
227  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
228  Array4<Real const> const& u) -> Real
229  {
230  Real new_lm_dt = -bogus_large_value;
231  Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
232  {
233  new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
234  ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
235  ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
236  });
237  return new_lm_dt;
238  });
239 
240  ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
241  if (estdt_lowM_inv > 0.0_rt)
242  estdt_lowM = cfl / estdt_lowM_inv;
243 
244  // Additional vertical diagnostics
245  if (l_comp_substepping_diag) {
246  estdt_vert_comp_inv = ReduceMax(S_new, ccvel, 0,
247  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
248  Array4<Real const> const& s,
249  Array4<Real const> const& u) -> Real
250  {
251  Real new_comp_dt = -bogus_large_value;
252  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
253  {
254  {
255  const Real rho = s(i, j, k, Rho_comp);
256  const Real rhotheta = s(i, j, k, RhoTheta_comp);
257 
258  // NOTE: even when moisture is present,
259  // we only use the partial pressure of the dry air
260  // to compute the soundspeed
261  Real pressure = getPgivenRTh(rhotheta);
262  Real c = std::sqrt(Gamma * pressure / rho);
263 
264  // Look at z-direction only
265  new_comp_dt = amrex::max((amrex::Math::abs(u(i,j,k,2)) + c) * dzinv, new_comp_dt);
266  }
267  });
268  return new_comp_dt;
269  });
270 
271  estdt_vert_lowM_inv = ReduceMax(ccvel, 0,
272  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
273  Array4<Real const> const& u) -> Real
274  {
275  Real new_lowM_dt = -bogus_large_value;
276  amrex::Loop(b, [=,&new_lowM_dt] (int i, int j, int k) noexcept
277  {
278  new_lowM_dt = amrex::max((amrex::Math::abs(u(i,j,k,2))) * dzinv, new_lowM_dt);
279  });
280  return new_lowM_dt;
281  });
282 
283  ParallelDescriptor::ReduceRealMax(estdt_vert_comp_inv);
284  ParallelDescriptor::ReduceRealMax(estdt_vert_lowM_inv);
285  }
286 
287  if (verbose) {
288  if (fixed_dt[level] <= zero) {
289  Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " <<
290  one/dxinv[0] << " " << one/dxinv[1] << " " << dz_min[level] << std::endl;
291  Print() << "Compressible dt at level " << level << ": " << estdt_comp << std::endl;
292  if (estdt_lowM_inv > 0.0_rt) {
293  Print() << "Anelastic dt at level " << level << ": " << estdt_lowM << std::endl;
294  } else {
295  Print() << "Anelastic dt at level " << level << ": undefined " << std::endl;
296  }
297  }
298 
299  if (fixed_dt[level] > zero) {
300  Print() << "Based on cfl of one " << std::endl;
301  Print() << "Compressible dt at level " << level << " would be: " << estdt_comp/cfl << std::endl;
302  if (estdt_lowM_inv > 0.0_rt) {
303  Print() << "Anelastic dt at level " << level << " would be: " << estdt_lowM/cfl << std::endl;
304  } else {
305  Print() << "Anelastic dt at level " << level << " would be undefined " << std::endl;
306  }
307  Print() << "Fixed dt at level " << level << " is: " << fixed_dt[level] << std::endl;
308  if (fixed_fast_dt[level] > zero) {
309  Print() << "Fixed fast dt at level " << level << " is: " << fixed_fast_dt[level] << std::endl;
310  }
311  }
312  }
313 
314  if (solverChoice.substepping_type[level] != SubsteppingType::None) {
315  if (fixed_dt[level] > zero && fixed_fast_dt[level] > zero) {
316  dt_fast_ratio = static_cast<long>( fixed_dt[level] / fixed_fast_dt[level] );
317  if (dt_fast_ratio < 1) {
318  Abort("Invalid fixed_fast_dt: must be <= fixed_dt so mri_dt_ratio >= 1");
319  }
320  } else if (fixed_dt[level] > zero) {
321  // Max CFL_c = one for substeps by default, but we enforce a min of 4 substeps
322  auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
323  dt_fast_ratio = static_cast<long>( std::max(fixed_dt[level]/static_cast<double>(dt_sub_max), 4.0) );
324  } else {
325  // auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
326  // dt_fast_ratio = static_cast<long>( std::max(estdt_comp/dt_sub_max,Real(4.)) );
327  dt_fast_ratio = static_cast<long>( std::max(cfl / sub_cfl, Real(4.)) );
328  }
329 
330  // Force time step ratio to be an even value
331  if (solverChoice.force_stage1_single_substep) {
332  if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
333  } else {
334  if ( dt_fast_ratio%6 != 0) {
335  Print() << "mri_dt_ratio = " << dt_fast_ratio
336  << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
337  dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/Real(6.0)) * 6);
338  }
339  }
340 
341  if (verbose) {
342  Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
343  }
344  } // if substepping
345 
346  // Print out some extra diagnostics -- dt calcs are repeated so as to not
347  // disrupt the overall code flow...
348  if (l_comp_substepping_diag) {
349  double dt_diag = (fixed_dt[level] > zero) ? fixed_dt[level] : static_cast<double>(estdt_comp);
350  int ns = (fixed_mri_dt_ratio > zero) ? fixed_mri_dt_ratio : dt_fast_ratio;
351 
352  // horizontal acoustic CFL must be < 1 (fully explicit)
353  // vertical acoustic CFL may be > 1
354  Print() << "effective horiz,vert acoustic CFL with " << ns << " substeps : "
355  << (dt_diag / ns) * estdt_comp_inv << " "
356  << (dt_diag / ns) * estdt_vert_comp_inv << std::endl;
357 
358  // vertical advective CFL should be < 1, otherwise w-damping may be needed
359  Print() << "effective vert advective CFL : "
360  << dt_diag * estdt_vert_lowM_inv << std::endl;
361  }
362 
363  if (fixed_dt[level] > zero) {
364  return fixed_dt[level];
365  } else {
366  // Anelastic (substepping is not allowed)
367  if (l_anelastic) {
368 
369  // Make sure that timestep is less than the dt_max
370  estdt_lowM = std::min(estdt_lowM, dt_max);
371 
372  // On the first timestep enforce dt_max_initial
373  if (istep[level] == 0) {
374  return std::min(dt_max_initial, estdt_lowM);
375  } else {
376  return estdt_lowM;
377  }
378 
379 
380  // Compressible with or without substepping
381  } else {
382  return estdt_comp;
383  }
384  }
385 }
Gamma
constexpr amrex::Real Gamma
Definition: ERF_Constants.H:62
ERF::dz_min
amrex::Vector< amrex::Real > dz_min
Definition: ERF.H:1480
ERF::fixed_fast_dt
amrex::Vector< double > fixed_fast_dt
Definition: ERF.H:1183
ERF::cfl
static amrex::Real cfl
Definition: ERF.H:1174
ERF::dt_max
static double dt_max
Definition: ERF.H:1179
ERF::fixed_dt
amrex::Vector< double > fixed_dt
Definition: ERF.H:1182
ERF::sub_cfl
static amrex::Real sub_cfl
Definition: ERF.H:1175
eb_
Definition: ERF_EB.H:13
MicVar_Morr::ns
@ ns
Definition: ERF_Morrison.H:48
SolverChoice::force_stage1_single_substep
int force_stage1_single_substep
Definition: ERF_DataStruct.H:1254
SolverChoice::substepping_type
amrex::Vector< SubsteppingType > substepping_type
Definition: ERF_DataStruct.H:1256
SolverChoice::substepping_diag
bool substepping_diag
Definition: ERF_DataStruct.H:1264
Here is the call graph for this function:

◆ Evolve()

void ERF::Evolve ( )
137 {
138  BL_PROFILE_VAR("ERF::Evolve()", evolve);
139 
140  //
141  // cur_time = t_new is elapsed time, not total time
142  // stop_time is total time
143  // Tracked in double to avoid float32 drift over many timesteps in single-precision builds.
144  double cur_time = static_cast<double>(t_new[0]);
145 
146  // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
147  // for finer levels (with or without subcycling)
148  for (int step = istep[0]; (step < max_step) && (start_time+cur_time < stop_time); ++step)
149  {
150  if (use_datetime) {
151  Print() << "\n" << getTimestamp(start_time+cur_time, datetime_format)
152  << " (" << cur_time << " s elapsed)" << std::endl;
153  }
154  Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
155 
156  ComputeDt(step, cur_time);
157 
158  // Make sure we have read enough of the boundary plane data to make it through this timestep
159  if (input_bndry_planes)
160  {
161  m_r2d->read_input_files(cur_time+start_time,dt[0],m_bc_extdir_vals);
162  }
163 
164 #ifdef ERF_USE_PARTICLES
165  // We call this every time step with the knowledge that the particles may be
166  // initialized at a later time than the simulation start time.
167  // The ParticleContainer carries a "start time" so the initialization will happen
168  // only when a) time > start_time, and b) particles have not yet been initialized
169  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,cur_time);
170 #endif
171 
172  if(solverChoice.init_type == InitType::HindCast and
173  solverChoice.hindcast_lateral_forcing) {
174  for(int lev=0;lev<finest_level+1;lev++){
175  WeatherDataInterpolation(lev,cur_time,z_phys_nd,false);
176  }
177  }
178 
179  if(solverChoice.init_type == InitType::HindCast and
180  solverChoice.hindcast_surface_bcs) {
181  for(int lev=0;lev<finest_level+1;lev++){
182  SurfaceDataInterpolation(lev,cur_time,z_phys_nd,false);
183  }
184  }
185 
186  auto dEvolveTime0 = amrex::second();
187 
188  int iteration = 1;
189  timeStep(0, cur_time, iteration);
190 
191  cur_time += static_cast<double>(dt[0]);
192  t_new[0] = cur_time;
193 
194  Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
195  << " DT = " << dt[0] << std::endl;
196 
197  if (check_for_nans > 0) {
198  amrex::Print() << "Testing new state and vels for NaNs at end of timestep" << std::endl;
199  for (int lev = 0; lev <= finest_level; ++lev) {
200  check_state_for_nans(vars_new[lev][IntVars::cons]);
201  check_vels_for_nans(vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],vars_new[lev][Vars::zvel]);
202  }
203  }
204 
205  if (verbose > 0)
206  {
207  auto dEvolveTime = amrex::second() - dEvolveTime0;
208  ParallelDescriptor::ReduceRealMax(dEvolveTime,ParallelDescriptor::IOProcessorNumber());
209  amrex::Print() << "Timestep time = " << dEvolveTime << " seconds." << '\n';
210  }
211 
212  post_timestep(step, cur_time, dt[0]);
213 
214  WriteAtIntermediateTime(step, cur_time);
215 
216 #ifdef AMREX_MEM_PROFILING
217  {
218  std::ostringstream ss;
219  ss << "[STEP " << step+1 << "]";
220  MemProfiler::report(ss.str());
221  }
222 #endif
223 
224  if (start_time+cur_time >= stop_time - 1.e-6*dt[0]) break;
225  }
226 
227  WriteAtFinalTime();
228 
229  BL_PROFILE_VAR_STOP(evolve);
230 }
getTimestamp
AMREX_FORCE_INLINE std::string getTimestamp(const double epoch_real, const std::string &datetime_format, bool add_long_frac=true)
Definition: ERF_EpochTime.H:103
ERF::max_step
int max_step
Definition: ERF.H:1157
ERF::WriteAtIntermediateTime
void WriteAtIntermediateTime(int nstep, double time)
Definition: ERF.cpp:233
ERF::post_timestep
void post_timestep(int nstep, double time, double dt_lev)
Definition: ERF.cpp:311
ERF::SurfaceDataInterpolation
void SurfaceDataInterpolation(const int nlevs, const double time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)
Definition: ERF_SurfaceDataInterpolation.cpp:142
ERF::m_bc_extdir_vals
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_extdir_vals
Definition: ERF.H:1108
ERF::WeatherDataInterpolation
void WeatherDataInterpolation(const int nlevs, const double time, amrex::Vector< std::unique_ptr< amrex::MultiFab >> &z_phys_nd, bool regrid_forces_file_read)
Definition: ERF_WeatherDataInterpolation.cpp:347
ERF::input_bndry_planes
static int input_bndry_planes
Definition: ERF.H:1411
ERF::datetime_format
const std::string datetime_format
Definition: ERF.H:1164
ERF::use_datetime
bool use_datetime
Definition: ERF.H:1163
ERF::ComputeDt
void ComputeDt(int step=-1, double cur_time_d=0.0)
Definition: ERF_ComputeTimestep.cpp:11
ERF::m_r2d
std::unique_ptr< ReadBndryPlanes > m_r2d
Definition: ERF.H:1469
ERF::timeStep
void timeStep(int lev, double time, int iteration)
Definition: ERF_TimeStep.cpp:18
ERF::WriteAtFinalTime
void WriteAtFinalTime()
Definition: ERF.cpp:276
SolverChoice::hindcast_lateral_forcing
bool hindcast_lateral_forcing
Definition: ERF_DataStruct.H:1472
SolverChoice::hindcast_surface_bcs
bool hindcast_surface_bcs
Definition: ERF_DataStruct.H:1473

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ EvolveOneStep()

double ERF::EvolveOneStep ( double  time,
double  dt_request 
)
93 {
94  double cur_time = t_new[0];
95  const int step = istep[0];
96 
97  if (start_time + cur_time >= stop_time) {
98  return 0.0;
99  }
100 
101  ComputeDt(step);
102 
103  int iteration = 1;
104  timeStep(0, cur_time, iteration);
105  cur_time += dt[0];
106 
107  post_timestep(step, cur_time, dt[0]);
108 
109  // ****************************************************************************************
110  // Write plotfiles at intermediate times
111  // ****************************************************************************************
112  WriteAtIntermediateTime(step, cur_time);
113 
114  return dt[0];
115 }
Here is the call graph for this function:

◆ fill_from_bndryregs()

void ERF::fill_from_bndryregs ( const amrex::Vector< amrex::MultiFab * > &  mfs,
double  time 
)
14 {
15  //
16  // We now assume that if we read in on one face, we read in on all faces
17  //
18  AMREX_ALWAYS_ASSERT(m_r2d);
19 
20  int lev = 0;
21  const Box& domain = geom[lev].Domain();
22 
23  const auto& dom_lo = lbound(domain);
24  const auto& dom_hi = ubound(domain);
25 
26  // Boundary-plane files are indexed by absolute simulation time.
27  Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time + start_time);
28 
29  const BCRec* bc_ptr = domain_bcs_type_d.data();
30 
31  // xlo: ori = 0
32  // ylo: ori = 1
33  // zlo: ori = 2
34  // xhi: ori = 3
35  // yhi: ori = 4
36  // zhi: ori = 5
37  const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
38  const auto& bdatylo = (*bndry_data[1])[lev].const_array();
39  const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
40  const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
41 
42  int bccomp;
43 
44  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
45  {
46  MultiFab& mf = *mfs[var_idx];
47  const int icomp = 0;
48  const int ncomp = mf.nComp();
49 
50  if (var_idx == Vars::xvel) {
51  bccomp = BCVars::xvel_bc;
52  } else if (var_idx == Vars::yvel) {
53  bccomp = BCVars::yvel_bc;
54  } else if (var_idx == Vars::zvel) {
55  bccomp = BCVars::zvel_bc;
56  } else if (var_idx == Vars::cons) {
57  bccomp = BCVars::cons_bc;
58  }
59 
60 #ifdef AMREX_USE_OMP
61 #pragma omp parallel if (Gpu::notInLaunchRegion())
62 #endif
63  for (MFIter mfi(mf); mfi.isValid(); ++mfi)
64  {
65  const Array4<Real>& dest_arr = mf.array(mfi);
66  Box bx = mfi.growntilebox();
67 
68  // x-faces
69  {
70  Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
71  if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
72 
73  Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
74 
75  ParallelFor(
76  bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
77  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
78  BCVars::RhoScalar_bc_comp : icomp+n;
79  if (bc_ptr[bc_comp].lo(0) == ERFBCType::ext_dir_ingested) {
80  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
81  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
82  dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
83  }
84  },
85  bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
86  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
87  BCVars::RhoScalar_bc_comp : icomp+n;
88  if (bc_ptr[bc_comp].hi(0) == ERFBCType::ext_dir_ingested) {
89  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
90  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
91  dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
92  }
93  }
94  );
95  } // x-faces
96 
97  // y-faces
98  {
99  Box bx_ylo(bx); bx_ylo.setBig (1,dom_lo.y-1);
100  if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
101 
102  Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
103 
104  ParallelFor(
105  bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
106  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
107  BCVars::RhoScalar_bc_comp : icomp+n;
108  if (bc_ptr[bc_comp].lo(1) == ERFBCType::ext_dir_ingested) {
109  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
110  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
111  dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
112  }
113  },
114  bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
115  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
116  BCVars::RhoScalar_bc_comp : icomp+n;
117  if (bc_ptr[bc_comp].hi(1) == ERFBCType::ext_dir_ingested) {
118  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
119  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
120  dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
121  }
122  }
123  );
124  } // y-faces
125  } // mf
126  } // var_idx
127 }
RhoScalar_comp
#define RhoScalar_comp
Definition: ERF_IndexDefines.H:40
NSCALARS
#define NSCALARS
Definition: ERF_IndexDefines.H:16
dom_hi
const auto & dom_hi
Definition: ERF_SetupVertDiff.H:2
dom_lo
const auto & dom_lo
Definition: ERF_SetupVertDiff.H:1
ERF::domain_bcs_type_d
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
Definition: ERF.H:1102
BCVars::RhoScalar_bc_comp
@ RhoScalar_bc_comp
Definition: ERF_IndexDefines.H:90
ERFBCType::ext_dir_ingested
@ ext_dir_ingested
Definition: ERF_IndexDefines.H:252
Here is the call graph for this function:

◆ fill_rhs()

void ERF::fill_rhs ( amrex::MultiFab &  rhs_mf,
const amrex::MultiFab &  state_mf,
double  time,
const amrex::Geometry &  geom 
)
private

◆ FillBdyCCVels()

void ERF::FillBdyCCVels ( amrex::MultiFab &  mf_cc_vel,
amrex::Geometry &  lev_geom 
)
12 {
13  // Impose bc's at domain boundaries
14  Box domain(lev_geom.Domain());
15 
16  int ihi = domain.bigEnd(0);
17  int jhi = domain.bigEnd(1);
18  int khi = domain.bigEnd(2);
19 
20  // Impose periodicity first
21  mf_cc_vel.FillBoundary(lev_geom.periodicity());
22 
23  int jper = (lev_geom.isPeriodic(1));
24  int kper = (lev_geom.isPeriodic(2));
25 
26  for (MFIter mfi(mf_cc_vel, TilingIfNotGPU()); mfi.isValid(); ++mfi)
27  {
28  const Box& bx = mfi.tilebox();
29  const Array4<Real>& vel_arr = mf_cc_vel.array(mfi);
30 
31  if (!lev_geom.isPeriodic(0)) {
32  // Low-x side
33  if (bx.smallEnd(0) <= domain.smallEnd(0)) {
34  Real multn = ( (phys_bc_type[0] == ERF_BC::slip_wall ) ||
35  (phys_bc_type[0] == ERF_BC::no_slip_wall) ||
36  (phys_bc_type[0] == ERF_BC::symmetry ) ) ? -one : one;
37  Real multt = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -one : one;
38  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
39  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
40  {
41  vel_arr(-1,j,k,0) = multn*vel_arr(0,j,k,0); // u
42  vel_arr(-1,j,k,1) = multt*vel_arr(0,j,k,1); // v
43  vel_arr(-1,j,k,2) = multt*vel_arr(0,j,k,2); // w
44  });
45  }
46 
47  // High-x side
48  if (bx.bigEnd(0) >= domain.bigEnd(0)) {
49  Real multn = ( (phys_bc_type[3] == ERF_BC::slip_wall ) ||
50  (phys_bc_type[3] == ERF_BC::no_slip_wall) ||
51  (phys_bc_type[3] == ERF_BC::symmetry ) ) ? -one : one;
52  Real multt = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -one : one;
53  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
54  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
55  {
56  vel_arr(ihi+1,j,k,0) = multn*vel_arr(ihi,j,k,0); // u
57  vel_arr(ihi+1,j,k,1) = multt*vel_arr(ihi,j,k,1); // v
58  vel_arr(ihi+1,j,k,2) = multt*vel_arr(ihi,j,k,2); // w
59  });
60  }
61  } // !periodic
62 
63  if (!lev_geom.isPeriodic(1)) {
64  // Low-y side
65  if (bx.smallEnd(1) <= domain.smallEnd(1)) {
66  Real multn = ( (phys_bc_type[1] == ERF_BC::slip_wall ) ||
67  (phys_bc_type[1] == ERF_BC::no_slip_wall) ||
68  (phys_bc_type[1] == ERF_BC::symmetry ) ) ? -one : one;
69  Real multt = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -one : one;
70  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
71  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
72  {
73  vel_arr(i,-1,k,0) = multt*vel_arr(i,0,k,0); // u
74  vel_arr(i,-1,k,1) = multn*vel_arr(i,0,k,1); // u
75  vel_arr(i,-1,k,2) = multt*vel_arr(i,0,k,2); // w
76  });
77  }
78 
79  // High-y side
80  if (bx.bigEnd(1) >= domain.bigEnd(1)) {
81  Real multn = ( (phys_bc_type[4] == ERF_BC::slip_wall ) ||
82  (phys_bc_type[4] == ERF_BC::no_slip_wall) ||
83  (phys_bc_type[4] == ERF_BC::symmetry ) ) ? -one : one;
84  Real multt = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -one : one;
85  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
86  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
87  {
88  vel_arr(i,jhi+1,k,0) = multt*vel_arr(i,jhi,k,0); // u
89  vel_arr(i,jhi+1,k,1) = multn*vel_arr(i,jhi,k,1); // v
90  vel_arr(i,jhi+1,k,2) = multt*vel_arr(i,jhi,k,2); // w
91  });
92  }
93  } // !periodic
94 
95  if (!lev_geom.isPeriodic(2)) {
96  // Low-z side
97  if (bx.smallEnd(2) <= domain.smallEnd(2)) {
98  Real multn = ( (phys_bc_type[2] == ERF_BC::slip_wall ) ||
99  (phys_bc_type[2] == ERF_BC::no_slip_wall) ||
100  (phys_bc_type[2] == ERF_BC::symmetry ) ) ? -one : one;
101  Real multt = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -one : one;
102  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
103  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
104  {
105  vel_arr(i,j,-1,0) = multt*vel_arr(i,j,0,0); // u
106  vel_arr(i,j,-1,1) = multt*vel_arr(i,j,0,1); // v
107  vel_arr(i,j,-1,2) = multn*vel_arr(i,j,0,2); // w
108  });
109  }
110 
111  // High-z side
112  if (bx.bigEnd(2) >= domain.bigEnd(2)) {
113  Real multn = ( (phys_bc_type[5] == ERF_BC::slip_wall ) ||
114  (phys_bc_type[5] == ERF_BC::no_slip_wall) ||
115  (phys_bc_type[5] == ERF_BC::symmetry ) ) ? -one : one;
116  Real multt = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -one : one;
117  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
118  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
119  {
120  vel_arr(i,j,khi+1,0) = multt*vel_arr(i,j,khi,0); // u
121  vel_arr(i,j,khi+1,1) = multt*vel_arr(i,j,khi,1); // v
122  vel_arr(i,j,khi+1,2) = multn*vel_arr(i,j,khi,2); // w
123  });
124  }
125  } // !periodic
126  } // MFIter
127 
128  // Impose periodicity again
129  mf_cc_vel.FillBoundary(lev_geom.periodicity());
130 }
ERF_BC::slip_wall
@ slip_wall
ERF_BC::symmetry
@ symmetry
ERF_BC::no_slip_wall
@ no_slip_wall
Here is the call graph for this function:

◆ FillCoarsePatch()

void ERF::FillCoarsePatch ( int  lev,
double  time 
)
private
22 {
23  BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
24  AMREX_ASSERT(lev > 0);
25 
26  //
27  //****************************************************************************************************************
28  // First fill velocities and density at the COARSE level so we can convert velocity to momenta at the COARSE level
29  //****************************************************************************************************************
30  //
31  bool cons_only = false;
32  if (lev == 1) {
33  FillPatchCrseLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
34  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
35  cons_only);
36  } else {
37  FillPatchFineLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
38  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
39  {&vars_new[lev-1][Vars::cons],
40  &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
41  base_state[lev-1], base_state[lev-1],
42  false, cons_only);
43  }
44 
45  //
46  // ************************************************
47  // Convert velocity to momentum at the COARSE level
48  // ************************************************
49  //
50  const MultiFab* c_vfrac = nullptr;
51  if (solverChoice.terrain_type == TerrainType::EB) {
52  c_vfrac = &((get_eb(lev-1).get_const_factory())->getVolFrac());
53  }
54 
55  VelocityToMomentum(vars_new[lev-1][Vars::xvel], IntVect{0},
56  vars_new[lev-1][Vars::yvel], IntVect{0},
57  vars_new[lev-1][Vars::zvel], IntVect{0},
58  vars_new[lev-1][Vars::cons],
59  rU_new[lev-1],
60  rV_new[lev-1],
61  rW_new[lev-1],
62  Geom(lev-1).Domain(),
63  domain_bcs_type, c_vfrac);
64 
65  // Fill ghost cells of coarse momentum before interpolation to fine level.
66  // VelocityToMomentum above fills only valid cells (IntVect{0} grow). On restart
67  // from a non-AMR checkpoint, init_stuff initialises rU/rV/rW_new[lev-1] with large
68  // sentinel values for ALL cells including ghost cells; the checkpoint read then
69  // overwrites only valid cells. InterpFromCoarseLevel (see comments below) ASSUMES
70  // ghost cells at lev-1 are already filled and uses them in its stencil near periodic
71  // boundaries. Without this FillBoundary, those sentinel ghost cells contaminate the
72  // fine-level interpolation, producing unphysical velocities that blow up WENO5.
73  rU_new[lev-1].FillBoundary(geom[lev-1].periodicity());
74  rV_new[lev-1].FillBoundary(geom[lev-1].periodicity());
75  rW_new[lev-1].FillBoundary(geom[lev-1].periodicity());
76 
77  //
78  // *****************************************************************
79  // Interpolate all cell-centered variables from coarse to fine level
80  // *****************************************************************
81  //
82  Interpolater* mapper_c = &cell_cons_interp;
83  Interpolater* mapper_f = &face_cons_linear_interp;
84 
85  //
86  //************************************************************************************************
87  // Interpolate cell-centered data from coarse to fine level
88  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
89  // ************************************************************************************************
90  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
91  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
92 
93  InterpFromCoarseLevel(vars_new[lev ][Vars::cons], ngvect_cons, IntVect(0,0,0),
94  vars_new[lev-1][Vars::cons], 0, 0, ncomp_cons,
95  geom[lev-1], geom[lev],
96  refRatio(lev-1), mapper_c, domain_bcs_type, BCVars::cons_bc);
97 
98  // ***************************************************************************
99  // Physical bc's for cell centered variables at domain boundary
100  // ***************************************************************************
101  (*physbcs_cons[lev])(vars_new[lev][Vars::cons],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
102  0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc,true);
103 
104  //
105  //************************************************************************************************
106  // Interpolate x-momentum from coarse to fine level
107  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
108  // ************************************************************************************************
109  //
110  InterpFromCoarseLevel(rU_new[lev], IntVect{0}, IntVect{0}, rU_new[lev-1], 0, 0, 1,
111  geom[lev-1], geom[lev],
112  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::xvel_bc);
113 
114  //
115  //************************************************************************************************
116  // Interpolate y-momentum from coarse to fine level
117  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
118  // ************************************************************************************************
119  //
120  InterpFromCoarseLevel(rV_new[lev], IntVect{0}, IntVect{0}, rV_new[lev-1], 0, 0, 1,
121  geom[lev-1], geom[lev],
122  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::yvel_bc);
123 
124  //************************************************************************************************
125  // Interpolate z-momentum from coarse to fine level
126  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
127  // ************************************************************************************************
128  InterpFromCoarseLevel(rW_new[lev], IntVect{0}, IntVect{0}, rW_new[lev-1], 0, 0, 1,
129  geom[lev-1], geom[lev],
130  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::zvel_bc);
131  //
132  // *********************************************************
133  // After interpolation of momentum, convert back to velocity
134  // *********************************************************
135  //
136  for (int which_lev = lev-1; which_lev <= lev; which_lev++)
137  {
138  c_vfrac = nullptr;
139  if (solverChoice.terrain_type == TerrainType::EB) {
140  c_vfrac = &((get_eb(which_lev).get_const_factory())->getVolFrac());
141  }
142 
143  MomentumToVelocity(vars_new[which_lev][Vars::xvel],
144  vars_new[which_lev][Vars::yvel],
145  vars_new[which_lev][Vars::zvel],
146  vars_new[which_lev][Vars::cons],
147  rU_new[which_lev],
148  rV_new[which_lev],
149  rW_new[which_lev],
150  Geom(which_lev).Domain(),
151  domain_bcs_type, c_vfrac);
152  }
153 
154  // ***************************************************************************
155  // Physical bc's at domain boundary
156  // ***************************************************************************
157  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
158 
159  (*physbcs_u[lev])(vars_new[lev][Vars::xvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
160  ngvect_vels,time,BCVars::xvel_bc,true);
161  (*physbcs_v[lev])(vars_new[lev][Vars::yvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
162  ngvect_vels,time,BCVars::yvel_bc,true);
163  (*physbcs_w[lev])(vars_new[lev][Vars::zvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
164  ngvect_vels,time,BCVars::zvel_bc,true);
165 
166  // ***************************************************************************
167  // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
168  // ***************************************************************************
169 }
ERF::FillCoarsePatch
void FillCoarsePatch(int lev, double time)
Definition: ERF_FillCoarsePatch.cpp:21
Here is the call graph for this function:

◆ FillForecastStateMultiFabs()

void ERF::FillForecastStateMultiFabs ( const int  lev,
const std::string &  filename,
const std::unique_ptr< amrex::MultiFab > &  z_phys_nd,
amrex::Vector< amrex::Vector< amrex::MultiFab >> &  forecast_state 
)
68 {
69 
70  Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
71  Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
72 
73  ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
74  xvec_h, yvec_h, zvec_h, rho_h,
75  uvel_h, vvel_h, wvel_h,
76  theta_h, qv_h, qc_h, qr_h);
77 
78  Real zmax = *std::max_element(zvec_h.begin(), zvec_h.end());
79 
80  const auto prob_lo_erf = geom[lev].ProbLoArray();
81  const auto prob_hi_erf = geom[lev].ProbHiArray();
82  const auto dx_erf = geom[lev].CellSizeArray();
83 
84  if (prob_hi_erf[2] >= zmax) {
85  Abort("ERROR: the maximum z of the domain (" + std::to_string(prob_hi_erf[2]) +
86  ") should be less than the maximum z in the forecast data (" + std::to_string(zmax) +
87  "). Change geometry.prob_hi[2] in the inputs to be less than " + std::to_string(zmax) + "."
88  );
89  }
90 
91  if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
92  amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
93  }
94  if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
95  amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
96  }
97  if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
98  amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
99  }
100  if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
101  amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
102  }
103 
104 
105  int nx = static_cast<int>(xvec_h.size());
106  int ny = static_cast<int>(yvec_h.size());
107  int nz = static_cast<int>(zvec_h.size());
108 
109  amrex::Real dxvec = (xvec_h[nx-1]-xvec_h[0])/(nx-1);
110  amrex::Real dyvec = (yvec_h[ny-1]-yvec_h[0])/(ny-1);
111 
112  amrex::Gpu::DeviceVector<Real> latvec_d(nx*ny), lonvec_d(nx*ny), zvec_d(nz);
113  amrex::Gpu::DeviceVector<Real> xvec_d(nx*ny*nz), yvec_d(nx*ny*nz);
114  amrex::Gpu::DeviceVector<Real> rho_d(nx*ny*nz), uvel_d(nx*ny*nz), vvel_d(nx*ny*nz), wvel_d(nx*ny*nz),
115  theta_d(nx*ny*nz), qv_d(nx*ny*nz), qc_d(nx*ny*nz), qr_d(nx*ny*nz);
116 
117  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, latvec_h.begin(), latvec_h.end(), latvec_d.begin());
118  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, lonvec_h.begin(), lonvec_h.end(), lonvec_d.begin());
119 
120  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, xvec_h.begin(), xvec_h.end(), xvec_d.begin());
121  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, yvec_h.begin(), yvec_h.end(), yvec_d.begin());
122  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, zvec_h.begin(), zvec_h.end(), zvec_d.begin());
123  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, rho_h.begin(), rho_h.end(), rho_d.begin());
124  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, theta_h.begin(), theta_h.end(), theta_d.begin());
125  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, uvel_h.begin(), uvel_h.end(), uvel_d.begin());
126  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, vvel_h.begin(), vvel_h.end(), vvel_d.begin());
127  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, wvel_h.begin(), wvel_h.end(), wvel_d.begin());
128  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qv_h.begin(), qv_h.end(), qv_d.begin());
129  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qc_h.begin(), qc_h.end(), qc_d.begin());
130  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, qr_h.begin(), qr_h.end(), qr_d.begin());
131 
132  amrex::Gpu::streamSynchronize();
133 
134  Real* latvec_d_ptr = latvec_d.data();
135  Real* lonvec_d_ptr = lonvec_d.data();
136  Real* xvec_d_ptr = xvec_d.data();
137  Real* yvec_d_ptr = yvec_d.data();
138  Real* zvec_d_ptr = zvec_d.data();
139  Real* rho_d_ptr = rho_d.data();
140  Real* uvel_d_ptr = uvel_d.data();
141  Real* vvel_d_ptr = vvel_d.data();
142  Real* wvel_d_ptr = wvel_d.data();
143  Real* theta_d_ptr = theta_d.data();
144  Real* qv_d_ptr = qv_d.data();
145  Real* qc_d_ptr = qc_d.data();
146  Real* qr_d_ptr = qr_d.data();
147 
148  MultiFab& erf_mf_cons = forecast_state[lev][Vars::cons];
149  MultiFab& erf_mf_xvel = forecast_state[lev][Vars::xvel];
150  MultiFab& erf_mf_yvel = forecast_state[lev][Vars::yvel];
151  MultiFab& erf_mf_zvel = forecast_state[lev][Vars::zvel];
152  MultiFab& erf_mf_latlon = forecast_state[lev][4];
153 
154  erf_mf_cons.setVal(0.0);
155  erf_mf_xvel.setVal(0.0);
156  erf_mf_yvel.setVal(0.0);
157  erf_mf_zvel.setVal(0.0);
158  erf_mf_latlon.setVal(0.0);
159 
160  // Interpolate the data on to the ERF mesh
161 
162  for (MFIter mfi(erf_mf_cons); mfi.isValid(); ++mfi) {
163  const auto z_arr = (a_z_phys_nd) ? a_z_phys_nd->const_array(mfi) :
164  Array4<const Real> {};
165  const Array4<Real> &fine_cons_arr = erf_mf_cons.array(mfi);
166  const Array4<Real> &fine_xvel_arr = erf_mf_xvel.array(mfi);
167  const Array4<Real> &fine_yvel_arr = erf_mf_yvel.array(mfi);
168  const Array4<Real> &fine_zvel_arr = erf_mf_zvel.array(mfi);
169  const Array4<Real> &fine_latlon_arr = erf_mf_latlon.array(mfi);
170 
171 
172  const Box& gbx = mfi.growntilebox(); // tilebox + ghost cells
173 
174  const Box &gtbx = mfi.tilebox(IntVect(1,0,0));
175  const Box &gtby = mfi.tilebox(IntVect(0,1,0));
176  const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
177  const auto prob_lo = geom[lev].ProbLoArray();
178  const auto dx = geom[lev].CellSizeArray();
179  //const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
180 
181  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
182  // Geometry (note we must include these here to get the data on device)
183  const Real x = prob_lo[0] + (i + myhalf) * dx[0];
184  const Real y = prob_lo[1] + (j + myhalf) * dx[1];
185  //const Real z = prob_lo[2] + (k + myhalf) * dx[2];
186  const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/two;
187 
188  // First interpolate where the weather data is available from
189  Real tmp_rho, tmp_theta, tmp_qv, tmp_qc, tmp_qr, tmp_lat, tmp_lon;
190  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
191  dxvec, dyvec,
192  nx, ny, nz,
193  x, y, z,
194  rho_d_ptr, tmp_rho);
195 
196  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
197  dxvec, dyvec,
198  nx, ny, nz,
199  x, y, z,
200  theta_d_ptr, tmp_theta);
201 
202  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
203  dxvec, dyvec,
204  nx, ny, nz,
205  x, y, z,
206  qv_d_ptr, tmp_qv);
207 
208  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
209  dxvec, dyvec,
210  nx, ny, nz,
211  x, y, z,
212  qc_d_ptr, tmp_qc);
213 
214  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
215  dxvec, dyvec,
216  nx, ny, nz,
217  x, y, z,
218  qr_d_ptr, tmp_qr);
219 
220  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
221  dxvec, dyvec,
222  nx, ny, 1,
223  x, y, zero,
224  latvec_d_ptr, tmp_lat);
225 
226  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
227  dxvec, dyvec,
228  nx, ny, 1,
229  x, y, zero,
230  lonvec_d_ptr, tmp_lon);
231 
232  fine_cons_arr(i,j,k,Rho_comp) = tmp_rho;
233  fine_latlon_arr(i,j,k,0) = tmp_lat;
234  fine_latlon_arr(i,j,k,1) = tmp_lon;
235  });
236 
237  ParallelFor(gtbx, gtby, gtbz,
238  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
239  // Physical location of the fine node
240  Real x = prob_lo_erf[0] + i * dx_erf[0];
241  Real y = prob_lo_erf[1] + (j+myhalf) * dx_erf[1];
242  //Real z = prob_lo_erf[2] + (k+myhalf) * dx_erf[2];
243  const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/two;
244 
245  Real tmp_uvel;
246  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
247  dxvec, dyvec,
248  nx, ny, nz,
249  x, y, z,
250  uvel_d_ptr, tmp_uvel);
251 
252  fine_xvel_arr(i, j, k, 0) = tmp_uvel;
253  },
254  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
255  // Physical location of the fine node
256  Real x = prob_lo_erf[0] + (i+myhalf) * dx_erf[0];
257  Real y = prob_lo_erf[1] + j * dx_erf[1];
258  //Real z = prob_lo_erf[2] + (k+myhalf) * dx_erf[2];
259  const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/two;
260 
261  Real tmp_vvel;
262  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
263  dxvec, dyvec,
264  nx, ny, nz,
265  x, y, z,
266  vvel_d_ptr, tmp_vvel);
267 
268  fine_yvel_arr(i, j, k, 0) = tmp_vvel;
269  },
270  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
271  // Physical location of the fine node
272  Real x = prob_lo_erf[0] + (i+myhalf) * dx_erf[0];
273  Real y = prob_lo_erf[1] + (j+myhalf) * dx_erf[1];
274  Real z = prob_lo_erf[2] + k * dx_erf[2];
275  //const Real z = (z_arr(i,j,k) + z_arr(i,j,k+1))/two;
276 
277  Real tmp_wvel;
278  bilinear_interpolation(xvec_d_ptr, yvec_d_ptr, zvec_d_ptr,
279  dxvec, dyvec,
280  nx, ny, nz,
281  x, y, z,
282  wvel_d_ptr, tmp_wvel);
283 
284  fine_zvel_arr(i, j, k, 0) = tmp_wvel;
285  });
286  }
287 
288  /*Vector<std::string> varnames = {
289  "rho", "uvel", "vvel", "wvel", "theta", "qv", "qc", "qr"
290  }; // Customize variable names
291 
292  Vector<std::string> varnames_cons = {
293  "rho", "rhotheta", "ke", "sc", "rhoqv", "rhoqc", "rhoqr"
294  }; // Customize variable names
295 
296  Vector<std::string> varnames_plot_mf = {
297  "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
298  }; // Customize variable names
299 
300  const double time = 0.0;
301 
302  std::string pltname = "plt_interp";
303 
304  MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
305  10, 0);
306 
307  plot_mf.setVal(0.0);
308 
309  for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
310  const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
311  const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
312  const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
313  const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
314  const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
315  const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
316 
317  const Box& bx = mfi.validbox();
318 
319  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
320  plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
321  plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
322  plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
323  plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
324  plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
325 
326  plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/two;
327  plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/two;
328  plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/two;
329 
330  plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
331  plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
332  });
333  }
334 
335 
336  WriteSingleLevelPlotfile(
337  pltname,
338  plot_mf,
339  varnames_plot_mf,
340  geom[0],
341  time,
342  0 // level
343  );*/
344 }
Coord::y
@ y
Coord::x
@ x
Coord::z
@ z
bilinear_interpolation
AMREX_FORCE_INLINE AMREX_GPU_HOST_DEVICE void bilinear_interpolation(const amrex::Real *xvec, const amrex::Real *yvec, const amrex::Real *zvec, const amrex::Real dxvec, const amrex::Real dyvec, const int nx, const int ny, const int nz, const amrex::Real x, const amrex::Real y, const amrex::Real z, const amrex::Real *varvec, amrex::Real &tmp_var)
Definition: ERF_Interpolation_Bilinear.H:42
ReadCustomBinaryIC
void ReadCustomBinaryIC(const std::string filename, amrex::Vector< amrex::Real > &latvec_h, amrex::Vector< amrex::Real > &lonvec_h, amrex::Vector< amrex::Real > &xvec_h, amrex::Vector< amrex::Real > &yvec_h, amrex::Vector< amrex::Real > &zvec_h, amrex::Vector< amrex::Real > &rho_h, amrex::Vector< amrex::Real > &uvel_h, amrex::Vector< amrex::Real > &vvel_h, amrex::Vector< amrex::Real > &wvel_h, amrex::Vector< amrex::Real > &theta_h, amrex::Vector< amrex::Real > &qv_h, amrex::Vector< amrex::Real > &qc_h, amrex::Vector< amrex::Real > &qr_h)
Definition: ERF_ReadCustomBinaryIC.H:15
rho_h
Vector< Real > rho_h(khi+1, zero)
rho_d
Gpu::DeviceVector< Real > rho_d(khi+1, zero)
Here is the call graph for this function:

◆ FillIntermediatePatch()

void ERF::FillIntermediatePatch ( int  lev,
double  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
int  ng_cons,
int  ng_vel,
bool  cons_only,
int  icomp_cons,
int  ncomp_cons 
)
private
33 {
34  BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
35  Interpolater* mapper;
36 
37  Real time = static_cast<Real>(time_d);
38 
39  PhysBCFunctNoOp null_bc;
40 
41  //
42  // ***************************************************************************
43  // The first thing we do is interpolate the momenta on the "valid" faces of
44  // the fine grids (where the interface is coarse/fine not fine/fine) -- this
45  // will not be over-written by interpolation below because the FillPatch
46  // operators see these as valid faces. But we must have these interpolated
47  // values in the fine data before we call FillPatchTwoLevels.
48  //
49  // Also -- note that we might be filling values by interpolation at physical boundaries
50  // here but that's ok because we will overwrite those values when we impose
51  // the physical bc's below
52  // ***************************************************************************
53  if (lev>0) {
54  if (cf_set_width > 0) {
55  // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
56  FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
57  }
58  if ( !cons_only && (cf_set_width >= 0) ) {
59  FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
60  FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
61  FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
62  }
63  }
64 
65  // amrex::Print() << "LEVEL " << lev << " CONS ONLY " << cons_only <<
66  // " ICOMP NCOMP " << icomp_cons << " " << ncomp_cons << " NGHOST " << ng_cons << std::endl;
67 
68  if (!cons_only) {
69  AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
70  AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
71  }
72 
73  // Enforce no penetration for thin immersed body
74  if (!cons_only) {
75  // Enforce no penetration for thin immersed body
76  if (xflux_imask[lev]) {
77  ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
78  }
79  if (yflux_imask[lev]) {
80  ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
81  }
82  if (zflux_imask[lev]) {
83  ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
84  }
85  }
86 
87  //
88  // We now start working on conserved quantities + VELOCITY
89  //
90  if (lev == 0)
91  {
92  // We don't do anything here because we will call the physbcs routines below,
93  // which calls FillBoundary and fills other domain boundary conditions
94  // Physical boundaries will be filled below
95 
96  if (!cons_only)
97  {
98  // ***************************************************************************
99  // We always come in to this call with updated momenta but we need to create updated velocity
100  // in order to impose the rest of the bc's
101  // ***************************************************************************
102  const MultiFab* c_vfrac = nullptr;
103  if (solverChoice.terrain_type == TerrainType::EB) {
104  c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
105  }
106 
107  // This only fills VALID region of velocity
108  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
109  *mfs_vel[Vars::cons],
110  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
111  Geom(lev).Domain(), domain_bcs_type, c_vfrac);
112  }
113  }
114  else
115  {
116  //
117  // We must fill a temporary then copy it back so we don't double add/subtract
118  //
119  MultiFab mf(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
120  mfs_vel[Vars::cons]->nComp() ,mfs_vel[Vars::cons]->nGrowVect());
121  //
122  // Set all components to bogus_large_value, then copy just the density from *mfs_vel[Vars::cons]
123  //
124  mf.setVal(bogus_large_value);
125  MultiFab::Copy(mf,*mfs_vel[Vars::cons],Rho_comp,Rho_comp,1,mf.nGrowVect());
126 
127  Vector<MultiFab*> fmf = {mfs_vel[Vars::cons],mfs_vel[Vars::cons]};
128  Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
129  Vector<Real> ctime = {static_cast<Real>(t_old[lev-1]), static_cast<Real>(t_new[lev-1])};
130  Vector<Real> ftime = {static_cast<Real>(time), static_cast<Real>(time)};
131 
132  if (interpolation_type == StateInterpType::Perturbational)
133  {
134  if (icomp_cons+ncomp_cons > 1)
135  {
136  // Divide (rho theta) by rho to get theta
137  MultiFab::Divide(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
138 
139  // Subtract theta_0 from theta
140  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
141 
142  if (!amrex::almostEqual(time,ctime[1])) {
143  MultiFab::Divide(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
144  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
145  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
146  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
147  }
148  if (!amrex::almostEqual(time,ctime[0])) {
149  MultiFab::Divide(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
150  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
151  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
152  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
153  }
154  }
155 
156  // Subtract rho_0 from rho before we interpolate -- note we only subtract
157  // on valid region of mf since the ghost cells will be filled below
158  if (icomp_cons == 0)
159  {
160  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
161 
162  if (!amrex::almostEqual(time,ctime[1])) {
163  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
164  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
165  }
166  if (!amrex::almostEqual(time,ctime[0])) {
167  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
168  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
169  }
170  }
171  } // interpolation_type == StateInterpType::Perturbational
172 
173  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
174  mapper = &cell_cons_interp;
175  FillPatchTwoLevels(mf, IntVect{ng_cons}, IntVect(0,0,0),
176  time, cmf, ctime, fmf, ftime,
177  icomp_cons, icomp_cons, ncomp_cons, geom[lev-1], geom[lev],
178  refRatio(lev-1), mapper, domain_bcs_type,
179  icomp_cons);
180 
181  if (interpolation_type == StateInterpType::Perturbational)
182  {
183  if (icomp_cons == 0)
184  {
185  // Restore the coarse values to what they were
186  if (!amrex::almostEqual(time,ctime[1])) {
187  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
188  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
189  }
190  if (!amrex::almostEqual(time,ctime[0])) {
191  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
192  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
193  }
194 
195  // Set values in the cells outside the domain boundary so that we can do the Add
196  // without worrying about uninitialized values outside the domain -- these
197  // will be filled in the physbcs call
198  mf.setDomainBndry(bogus_large_value,Rho_comp,1,geom[lev]);
199 
200  // Add rho_0 back to rho after we interpolate -- on all the valid + ghost region
201  MultiFab::Add(mf, base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{ng_cons});
202  }
203 
204  if (icomp_cons+ncomp_cons > 1)
205  {
206  // Add theta_0 to theta
207  if (!amrex::almostEqual(time,ctime[1])) {
208  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
209  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
210  MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
211  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
212  }
213  if (!amrex::almostEqual(time,ctime[0])) {
214  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
215  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
216  MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
217  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
218  }
219 
220  // Multiply theta by rho to get (rho theta)
221  MultiFab::Multiply(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
222 
223  // Add theta_0 to theta
224  MultiFab::Add(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
225 
226  // Add theta_0 back to theta
227  MultiFab::Add(mf,base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{ng_cons});
228 
229  // Multiply (theta) by rho to get (rho theta)
230  MultiFab::Multiply(mf,mf,Rho_comp,RhoTheta_comp,1,IntVect{ng_cons});
231  }
232  } // interpolation_type == StateInterpType::Perturbational
233 
234  // Impose physical bc's on fine data (note time and 0 are not used)
235  // Note that we do this after the FillPatch because imposing physical bc's on fine ghost
236  // cells that need to be filled from coarse requires that we have done the interpolation first
237  bool do_fb = true; bool do_terrain_adjustment = false;
238  (*physbcs_cons[lev])(mf,*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
239  icomp_cons,ncomp_cons,IntVect{ng_cons},time,BCVars::cons_bc,
240  do_fb, do_terrain_adjustment);
241 
242  // Make sure to only copy back the components we worked on
243  MultiFab::Copy(*mfs_vel[Vars::cons],mf,icomp_cons,icomp_cons,ncomp_cons,IntVect{ng_cons});
244 
245  // *****************************************************************************************
246 
247  if (!cons_only)
248  {
249  // ***************************************************************************
250  // We always come in to this call with updated momenta but we need to create updated velocity
251  // in order to impose the rest of the bc's
252  // ***************************************************************************
253  const MultiFab* c_vfrac = nullptr;
254  if (solverChoice.terrain_type == TerrainType::EB) {
255  c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
256  }
257 
258  // This only fills VALID region of velocity
259  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
260  *mfs_vel[Vars::cons],
261  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
262  Geom(lev).Domain(), domain_bcs_type, c_vfrac);
263 
264  mapper = &face_cons_linear_interp;
265 
266  //
267  // NOTE: All interpolation here happens on velocities not momenta;
268  // note we only do the interpolation and FillBoundary here,
269  // physical bc's are imposed later
270  //
271  // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
272  // unlike the FillSet calls above which filled momenta on the coarse/fine bdy
273  //
274 
275  MultiFab& mfu = *mfs_vel[Vars::xvel];
276 
277  fmf = {&mfu,&mfu};
278  cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
279 
280  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
281  FillPatchTwoLevels(mfu, IntVect{ng_vel}, IntVect(0,0,0),
282  time, cmf, ctime, fmf, ftime,
283  0, 0, 1, geom[lev-1], geom[lev],
284  refRatio(lev-1), mapper, domain_bcs_type,
285  BCVars::xvel_bc);
286 
287  // *****************************************************************************************
288 
289  MultiFab& mfv = *mfs_vel[Vars::yvel];
290 
291  fmf = {&mfv,&mfv};
292  cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
293 
294  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
295  FillPatchTwoLevels(mfv, IntVect{ng_vel}, IntVect(0,0,0),
296  time, cmf, ctime, fmf, ftime,
297  0, 0, 1, geom[lev-1], geom[lev],
298  refRatio(lev-1), mapper, domain_bcs_type,
299  BCVars::yvel_bc);
300 
301  // *****************************************************************************************
302 
303  MultiFab& mfw = *mfs_vel[Vars::zvel];
304 
305  fmf = {&mfw,&mfw};
306  cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
307 
308  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
309  FillPatchTwoLevels(mfw, IntVect{ng_vel}, IntVect(0,0,0),
310  time, cmf, ctime, fmf, ftime,
311  0, 0, 1, geom[lev-1], geom[lev],
312  refRatio(lev-1), mapper, domain_bcs_type,
313  BCVars::zvel_bc);
314  } // !cons_only
315  } // lev > 0
316 
317  // ***************************************************************************
318  // Physical bc's at domain boundary
319  // ***************************************************************************
320  IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
321  IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
322 
323  bool do_fb = true;
324 
325 #ifdef ERF_USE_NETCDF
326  if (solverChoice.use_real_bcs && (lev==0)) {
327  fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
328  do_fb = false;
329  }
330 #endif
331 
332  if (m_r2d && !solverChoice.use_real_bcs) { fill_from_bndryregs(mfs_vel,time); }
333 
334  // We call this even if use_real_bcs is true because these will fill the vertical bcs
335  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
336  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
337  if (!cons_only) {
338  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
339  ngvect_vels,time,BCVars::xvel_bc, do_fb);
340  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
341  ngvect_vels,time,BCVars::yvel_bc, do_fb);
342  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
343  ngvect_vels,time,BCVars::zvel_bc, do_fb);
344  }
345  // ***************************************************************************
346 
347  // We always come in to this call with momenta so we need to leave with momenta!
348  // We need to make sure we convert back on all ghost cells/faces because this is
349  // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
350  if (!cons_only)
351  {
352  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
353  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
354  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
355 
356  const MultiFab* c_vfrac = nullptr;
357  if (solverChoice.terrain_type == TerrainType::EB) {
358  c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
359  }
360 
361  VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
362  *mfs_vel[Vars::yvel], ngv,
363  *mfs_vel[Vars::zvel], ngw,
364  *mfs_vel[Vars::cons],
365  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
366  Geom(lev).Domain(),
367  domain_bcs_type, c_vfrac);
368  }
369 
370  // NOTE: There are not FillBoundary calls here for the following reasons:
371  // Removal of the FillBoundary (FB) calls has bee completed for the following reasons:
372  //
373  // one physbc_cons is called before VelocityToMomentum and a FB is completed in that functor.
374  // Therefore, the conserved CC vars have their inter-rank ghost cells filled and then their
375  // domain ghost cells filled from the BC operations. We should not call FB on this MF again.
376  //
377  // two physbc_u/v/w is also called before VelocityToMomentum and a FB is completed those functors.
378  // Furthermore, VelocityToMomentum operates on a growntilebox so we exit that routine with momentum
379  // filled everywhere---i.e., physbc_u/v/w fills velocity ghost cells (inter-rank and domain)
380  // and then V2M does the conversion to momenta everywhere; so there is again no need to do a FB on momenta.
381 }
ApplyMask
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:445
ERF::FillIntermediatePatch
void FillIntermediatePatch(int lev, double time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
Definition: ERF_FillIntermediatePatch.cpp:28
ERF::fill_from_bndryregs
void fill_from_bndryregs(const amrex::Vector< amrex::MultiFab * > &mfs, double time)
Definition: ERF_BoundaryConditionsBndryReg.cpp:13
IntVars::NumTypes
@ NumTypes
Definition: ERF_IndexDefines.H:197
Here is the call graph for this function:

◆ FillPatchCrseLevel()

void ERF::FillPatchCrseLevel ( int  lev,
double  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
bool  cons_only = false 
)
private
293 {
294  BL_PROFILE_VAR("ERF::FillPatchCrseLevel()",ERF_FillPatchCrseLevel);
295 
296  AMREX_ALWAYS_ASSERT(lev == 0);
297 
298  Real time = static_cast<Real>(time_d);
299 
300  IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
301  IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
302 
303  Vector<Real> ftime = {static_cast<Real>(t_old[lev]), static_cast<Real>(t_new[lev])};
304 
305  //
306  // Below we call FillPatchSingleLevel which does NOT fill ghost cells outside the domain
307  //
308 
309  Vector<MultiFab*> fmf;
310  Vector<MultiFab*> fmf_u;
311  Vector<MultiFab*> fmf_v;
312  Vector<MultiFab*> fmf_w;
313 
314  if (amrex::almostEqual(time,ftime[0])) {
315  fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
316  } else if (amrex::almostEqual(time,ftime[1])) {
317  fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
318  } else {
319  fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
320  }
321 
322  const int ncomp = mfs_vel[Vars::cons]->nComp();
323 
324  FillPatchSingleLevel(*mfs_vel[Vars::cons], ngvect_cons, time, fmf, IntVect(0,0,0), ftime,
325  0, 0, ncomp, geom[lev]);
326 
327  if (!cons_only) {
328  if (amrex::almostEqual(time,ftime[0])) {
329  fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
330  fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
331  fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
332  } else if (amrex::almostEqual(time,ftime[1])) {
333  fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
334  fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
335  fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
336  } else {
337  fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
338  fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
339  fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
340  }
341  FillPatchSingleLevel(*mfs_vel[Vars::xvel], ngvect_vels, time, fmf_u,
342  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
343 
344  FillPatchSingleLevel(*mfs_vel[Vars::yvel], ngvect_vels, time, fmf_v,
345  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
346 
347  FillPatchSingleLevel(*mfs_vel[Vars::zvel], ngvect_vels, time, fmf_w,
348  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
349  } // !cons_only
350 
351  // ***************************************************************************
352  // Physical bc's at domain boundary
353  // ***************************************************************************
354  int icomp_cons = 0;
355  int ncomp_cons = mfs_vel[Vars::cons]->nComp();
356 
357  bool do_fb = true;
358 
359 #ifdef ERF_USE_NETCDF
360  if(solverChoice.use_real_bcs && (lev==0)) {
361  fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
362  do_fb = false;
363  }
364 #endif
365 
366  if (m_r2d && !solverChoice.use_real_bcs) { fill_from_bndryregs(mfs_vel,time); }
367 
368  // We call this even if use_real_bcs is true because these will fill the vertical bcs
369  // Note that we call FillBoundary inside the physbcs call
370  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
371  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
372  if (!cons_only) {
373  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
374  ngvect_vels,time,BCVars::xvel_bc, do_fb);
375  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
376  ngvect_vels,time,BCVars::yvel_bc, do_fb);
377  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
378  ngvect_vels,time,BCVars::zvel_bc, do_fb);
379  }
380 }
Here is the call graph for this function:

◆ FillPatchFineLevel()

void ERF::FillPatchFineLevel ( int  lev,
double  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
const amrex::MultiFab &  old_base_state,
const amrex::MultiFab &  new_base_state,
bool  fillset = true,
bool  cons_only = false 
)
private
26 {
27  BL_PROFILE_VAR("ERF::FillPatchFineLevel()",ERF_FillPatchFineLevel);
28 
29  AMREX_ALWAYS_ASSERT(lev > 0);
30 
31  Real time = static_cast<Real>(time_d);
32 
33  Interpolater* mapper = nullptr;
34 
35  PhysBCFunctNoOp null_bc;
36 
37  //
38  // ***************************************************************************
39  // The first thing we do is interpolate the momenta on the "valid" faces of
40  // the fine grids (where the interface is coarse/fine not fine/fine) -- this
41  // will not be over-written below because the FillPatch operators see these as
42  // valid faces.
43  //
44  // Note that we interpolate momentum not velocity, but all the other boundary
45  // conditions are imposed on velocity, so we convert to momentum here then
46  // convert back.
47  // ***************************************************************************
48  if (fillset) {
49  if (cf_set_width > 0) {
50  FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
51  }
52  if (cf_set_width >= 0 && !cons_only) {
53 
54  const MultiFab* c_vfrac = nullptr;
55  if (solverChoice.terrain_type == TerrainType::EB) {
56  c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
57  }
58 
59  VelocityToMomentum(*mfs_vel[Vars::xvel], IntVect{0},
60  *mfs_vel[Vars::yvel], IntVect{0},
61  *mfs_vel[Vars::zvel], IntVect{0},
62  *mfs_vel[Vars::cons],
63  *mfs_mom[IntVars::xmom],
64  *mfs_mom[IntVars::ymom],
65  *mfs_mom[IntVars::zmom],
66  Geom(lev).Domain(),
67  domain_bcs_type, c_vfrac);
68 
69  FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
70  FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
71  FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
72 
73  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
74  *mfs_vel[Vars::cons],
75  *mfs_mom[IntVars::xmom],
76  *mfs_mom[IntVars::ymom],
77  *mfs_mom[IntVars::zmom],
78  Geom(lev).Domain(),
79  domain_bcs_type, c_vfrac);
80  }
81  }
82 
83  IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
84  IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
85 
86  Vector<Real> ftime = {static_cast<Real>(t_old[lev ]), static_cast<Real>(t_new[lev ])};
87  Vector<Real> ctime = {static_cast<Real>(t_old[lev-1]), static_cast<Real>(t_new[lev-1])};
88 
89  amrex::Real small_dt = Real(1.e-8) * (ftime[1] - ftime[0]);
90 
91  Vector<MultiFab*> fmf;
92  if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
93  fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
94  } else if (amrex::almostEqual(time,ftime[1])) {
95  fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
96  } else {
97  fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
98  }
99  Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
100 
101  // We must fill a temporary then copy it back so we don't double add/subtract
102  MultiFab mf_c(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
103  mfs_vel[Vars::cons]->nComp() ,mfs_vel[Vars::cons]->nGrowVect());
104 
105  mapper = &cell_cons_interp;
106 
107  if (interpolation_type == StateInterpType::Perturbational)
108  {
109  // Divide (rho theta) by rho to get theta (before we subtract rho0 from rho!)
110  if (!amrex::almostEqual(time,ctime[1])) {
111  MultiFab::Divide(vars_old[lev-1][Vars::cons],vars_old[lev-1][Vars::cons],
112  Rho_comp,RhoTheta_comp,1,ngvect_cons);
113  MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
114  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
115  MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
116  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
117  }
118  if (!amrex::almostEqual(time,ctime[0])) {
119  MultiFab::Divide(vars_new[lev-1][Vars::cons],vars_new[lev-1][Vars::cons],
120  Rho_comp,RhoTheta_comp,1,ngvect_cons);
121  MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
122  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
123  MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
124  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
125  }
126 
127  if (!amrex::almostEqual(time,ftime[1])) {
128  MultiFab::Divide(vars_old[lev ][Vars::cons],vars_old[lev ][Vars::cons],
129  Rho_comp,RhoTheta_comp,1,IntVect{0});
130  MultiFab::Subtract(vars_old[lev ][Vars::cons],old_base_state,
131  BaseState::r0_comp,Rho_comp,1,IntVect{0});
132  MultiFab::Subtract(vars_old[lev ][Vars::cons],old_base_state,
133  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
134  }
135  if (!amrex::almostEqual(time,ftime[0])) {
136  MultiFab::Divide(vars_new[lev ][Vars::cons],vars_new[lev ][Vars::cons],
137  Rho_comp,RhoTheta_comp,1,IntVect{0});
138  MultiFab::Subtract(vars_new[lev ][Vars::cons],old_base_state,
139  BaseState::r0_comp,Rho_comp,1,IntVect{0});
140  MultiFab::Subtract(vars_new[lev ][Vars::cons],old_base_state,
141  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
142  }
143  }
144 
145  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
146  FillPatchTwoLevels(mf_c, ngvect_cons, IntVect(0,0,0),
147  time, cmf, ctime, fmf, ftime,
148  0, 0, mf_c.nComp(), geom[lev-1], geom[lev],
149  refRatio(lev-1), mapper, domain_bcs_type,
150  BCVars::cons_bc);
151 
152  if (interpolation_type == StateInterpType::Perturbational)
153  {
154  // Restore the coarse values to what they were
155  if (!amrex::almostEqual(time,ctime[1])) {
156  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
157  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
158  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
159  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
160  MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
161  Rho_comp,RhoTheta_comp,1,ngvect_cons);
162  }
163  if (!amrex::almostEqual(time,ctime[0])) {
164  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
165  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
166  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
167  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
168  MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
169  Rho_comp,RhoTheta_comp,1,ngvect_cons);
170  }
171 
172  if (!amrex::almostEqual(time,ftime[1])) {
173  MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev ],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
174  MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev ],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
175  MultiFab::Multiply(vars_old[lev][Vars::cons], vars_old[lev][Vars::cons],
176  Rho_comp,RhoTheta_comp,1,ngvect_cons);
177  }
178  if (!amrex::almostEqual(time,ftime[0])) {
179  MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
180  MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
181  MultiFab::Multiply(vars_new[lev][Vars::cons], vars_new[lev][Vars::cons],
182  Rho_comp,RhoTheta_comp,1,ngvect_cons);
183  }
184 
185  // Set values in the cells outside the domain boundary so that we can do the Add
186  // without worrying about uninitialized values outside the domain -- these
187  // will be filled in the physbcs call
188  mf_c.setDomainBndry(bogus_large_value,0,2,geom[lev]); // Do both rho and (rho theta) together
189 
190  // Add rho_0 back to rho and theta_0 back to theta
191  MultiFab::Add(mf_c, new_base_state,BaseState::r0_comp,Rho_comp,1,ngvect_cons);
192  MultiFab::Add(mf_c, new_base_state,BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
193 
194  // Multiply (theta) by rho to get (rho theta)
195  MultiFab::Multiply(mf_c,mf_c,Rho_comp,RhoTheta_comp,1,ngvect_cons);
196  }
197 
198  MultiFab::Copy(*mfs_vel[Vars::cons],mf_c,0,0,mf_c.nComp(),mf_c.nGrowVect());
199 
200  // ***************************************************************************************
201 
202  if (!cons_only)
203  {
204  mapper = &face_cons_linear_interp;
205 
206  MultiFab& mf_u = *mfs_vel[Vars::xvel];
207  MultiFab& mf_v = *mfs_vel[Vars::yvel];
208  MultiFab& mf_w = *mfs_vel[Vars::zvel];
209 
210  Vector<MultiFab*> fmf_u; Vector<MultiFab*> fmf_v; Vector<MultiFab*> fmf_w;
211  Vector<MultiFab*> cmf_u; Vector<MultiFab*> cmf_v; Vector<MultiFab*> cmf_w;
212 
213  // **********************************************************************
214 
215  if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
216  fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
217  fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
218  fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
219  } else if ( amrex::almostEqual(time,ftime[1]) ) {
220  fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
221  fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
222  fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
223  } else {
224  fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
225  fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
226  fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
227  }
228  cmf_u = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
229  cmf_v = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
230  cmf_w = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
231 
232  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
233  FillPatchTwoLevels(mf_u, ngvect_vels, IntVect(0,0,0),
234  time, cmf_u, ctime, fmf_u, ftime,
235  0, 0, 1, geom[lev-1], geom[lev],
236  refRatio(lev-1), mapper, domain_bcs_type,
237  BCVars::xvel_bc);
238 
239  FillPatchTwoLevels(mf_v, ngvect_vels, IntVect(0,0,0),
240  time, cmf_v, ctime, fmf_v, ftime,
241  0, 0, 1, geom[lev-1], geom[lev],
242  refRatio(lev-1), mapper, domain_bcs_type,
243  BCVars::yvel_bc);
244 
245  // We put these here because these may be used in constructing omega outside the
246  // domain when fillpatching w
247  bool do_fb = true;
248  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
249  ngvect_vels,time,BCVars::xvel_bc, do_fb);
250  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
251  ngvect_vels,time,BCVars::yvel_bc, do_fb);
252 
253  // **********************************************************************
254 
255  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
256  FillPatchTwoLevels(mf_w, ngvect_vels, IntVect(0,0,0),
257  time, cmf_w, ctime, fmf_w, ftime,
258  0, 0, 1, geom[lev-1], geom[lev],
259  refRatio(lev-1), mapper, domain_bcs_type,
260  BCVars::zvel_bc);
261  } // !cons_only
262 
263  // ***************************************************************************
264  // Physical bc's at domain boundary
265  // ***************************************************************************
266  int icomp_cons = 0;
267  int ncomp_cons = mfs_vel[Vars::cons]->nComp();
268 
269  bool do_fb = true;
270 
271  if (m_r2d && !solverChoice.use_real_bcs) fill_from_bndryregs(mfs_vel,time);
272 
273  // We call this even if use_real_bcs is true because these will fill the vertical bcs
274  // Note that we call FillBoundary inside the physbcs call
275  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
276  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
277  if (!cons_only) {
278  // Note that we need to fill u and v in the case of terrain because we will use
279  // these in the call of WFromOmega in lateral ghost cells of the fine grid
280  // (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
281  // ngvect_vels,time,BCVars::xvel_bc, do_fb);
282  // (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
283  // ngvect_vels,time,BCVars::yvel_bc, do_fb);
284  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
285  ngvect_vels,time,BCVars::zvel_bc, do_fb);
286  }
287 }
Here is the call graph for this function:

◆ FillSurfaceStateMultiFabs()

void ERF::FillSurfaceStateMultiFabs ( const int  lev,
const std::string &  filename,
amrex::Vector< amrex::MultiFab > &  surface_state 
)
22 {
23  // Open the binary file in input mode
24  std::ifstream infile(filename, std::ios::binary);
25  if (!infile) {
26  std::cerr << "Error: Could not open file " << filename << std::endl;
27  }
28  Vector<Real> xvec_h, yvec_h, zvec_h;
29  Vector<Real> sst_h, q_star_h, t_star_h, u_star_h, ls_mask_h;
30 
31  int nx, ny, nz, ndata;
32  float value;
33 
34  // Read the four integers
35  infile.read(reinterpret_cast<char*>(&nx), sizeof(int));
36  infile.read(reinterpret_cast<char*>(&ny), sizeof(int));
37  infile.read(reinterpret_cast<char*>(&nz), sizeof(int));
38  infile.read(reinterpret_cast<char*>(&ndata), sizeof(int));
39 
40  amrex::Gpu::DeviceVector<Real> xvec_d(nx*ny*nz), yvec_d(nx*ny*nz), zvec_d(nz);
41  for(int i=0; i<nx; i++) {
42  infile.read(reinterpret_cast<char*>(&value), sizeof(float));
43  xvec_h.emplace_back(value);
44  }
45  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, xvec_h.begin(), xvec_h.end(), xvec_d.begin());
46 
47  for(int j=0; j<ny; j++) {
48  infile.read(reinterpret_cast<char*>(&value), sizeof(float));
49  yvec_h.emplace_back(value);
50  }
51  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, yvec_h.begin(), yvec_h.end(), yvec_d.begin());
52 
53  for(int k=0; k<nz; k++) {
54  infile.read(reinterpret_cast<char*>(&value), sizeof(float));
55  zvec_h.emplace_back(value);
56  }
57  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, zvec_h.begin(), zvec_h.end(), zvec_d.begin());
58 
59  // Vector to store the data
60 
61  Vector<Real>* data_h = nullptr; // Declare pointer outside the loop
62 
63  Real* xvec_d_ptr = xvec_d.data();
64  Real* yvec_d_ptr = yvec_d.data();
65 
66  Real dxvec = (xvec_h[nx-1]-xvec_h[0])/(nx-1);
67  Real dyvec = (yvec_h[ny-1]-yvec_h[0])/(ny-1);
68 
69  // Read the file
70  for(int idx=0; idx<ndata; idx++){
71  if(idx == 0){
72  data_h = &sst_h;
73  } else if (idx==1) {
74  data_h = &q_star_h;
75  } else if (idx==2) {
76  data_h = &t_star_h;
77  } else if (idx==3) {
78  data_h = &u_star_h;
79  } else if(idx==4) {
80  data_h = &ls_mask_h;
81  }
82  for(int k=0; k<nz; k++) {
83  for(int j=0; j<ny; j++) {
84  for(int i=0; i<nx; i++) {
85  infile.read(reinterpret_cast<char*>(&value), sizeof(float));
86  //if(idx == 3) {
87  //printf("theta is %0.15g, %0.15g, %0.15g %0.15g\n", xvec_h[i], yvec_h[j], zvec_h[k], value);
88  //}
89  data_h->emplace_back(value);
90  }
91  }
92  }
93  }
94 
95  infile.close();
96 
97  amrex::Gpu::DeviceVector<Real> ls_mask_d(nx*ny*nz), sst_d(nx*ny*nz);
98 
99  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, ls_mask_h.begin(), ls_mask_h.end(), ls_mask_d.begin());
100  amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, sst_h.begin(), sst_h.end(), sst_d.begin());
101 
102  Real* ls_mask_d_ptr = ls_mask_d.data();
103  Real* sst_d_ptr = sst_d.data();
104 
105  const auto prob_lo = geom[lev].ProbLo();
106  const auto dx = geom[lev].CellSize();
107 
108  for (amrex::MFIter mfi(surface_state[lev]); mfi.isValid(); ++mfi) {
109  const Box gbx = mfi.growntilebox();
110  const Array4<Real>& surf_arr = surface_state[lev].array(mfi);
111 
112  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
113 
114  if(k == 0) {
115  const Real x = prob_lo[0] + (i + myhalf) * dx[0];
116  const Real y = prob_lo[1] + (j + myhalf) * dx[1];
117 
118  // First interpolate where the weather data is available from
119  Real tmp_ls_mask, tmp_sst;
120 
121  bilinear_interpolation_2d(xvec_d_ptr, yvec_d_ptr,
122  dxvec, dyvec,
123  nx, ny,
124  x, y,
125  ls_mask_d_ptr, tmp_ls_mask);
126 
127  bilinear_interpolation_2d(xvec_d_ptr, yvec_d_ptr,
128  dxvec, dyvec,
129  nx, ny,
130  x, y,
131  sst_d_ptr, tmp_sst);
132 
133  surf_arr(i, j, 0) = std::min(tmp_ls_mask, amrex::Real(1.0));
134  surf_arr(i, j, 1) = tmp_sst;
135  }
136  });
137  }
138 
139 }
value
Real value
Definition: ERF_HurricaneDiagnostics.cpp:30
idx
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE int idx(int i, int j, int k, int nx, int ny)
Definition: ERF_InitForEnsemble.cpp:287
bilinear_interpolation_2d
AMREX_FORCE_INLINE AMREX_GPU_HOST_DEVICE void bilinear_interpolation_2d(const amrex::Real *xvec, const amrex::Real *yvec, const amrex::Real dxvec, const amrex::Real dyvec, const int nx, const int ny, amrex::Real x, amrex::Real y, const amrex::Real *varvec, amrex::Real &tmp_var)
Definition: ERF_Interpolation_Bilinear.H:156
Here is the call graph for this function:

◆ FindInitialEye()

bool ERF::FindInitialEye ( int  lev,
const amrex::MultiFab &  cc_vel,
const amrex::Real  velmag_threshold,
amrex::Real &  eye_x,
amrex::Real &  eye_y 
)
15 {
16  const auto dx = geom[levc].CellSizeArray();
17  const auto prob_lo = geom[levc].ProbLoArray();
18 
19  Gpu::DeviceVector<Real> d_coords(2, zero);
20  Gpu::DeviceVector<int> d_found(1,0);
21 
22  Real* d_coords_ptr = d_coords.data();
23  int* d_found_ptr = d_found.data();
24 
25  for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi)
26  {
27  const Box& box = mfi.validbox();
28  const Array4<const Real>& vel_arr = mf_cc_vel.const_array(mfi);
29 
30  ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k)
31  {
32  Real magnitude = std::sqrt(vel_arr(i,j,k,0) * vel_arr(i,j,k,0) +
33  vel_arr(i,j,k,1) * vel_arr(i,j,k,1) +
34  vel_arr(i,j,k,2) * vel_arr(i,j,k,2));
35 
36  magnitude *= Real(3.6);
37 
38  Real z = prob_lo[2] + (k + myhalf) * dx[2];
39 
40  // Check if magnitude exceeds threshold
41  if (z < Real(2000.) && magnitude > velmag_threshold) {
42  // Use atomic operations to set found flag and store coordinates
43  Gpu::Atomic::Add(&d_found_ptr[0], 1); // Mark as found
44 
45  Real x = prob_lo[0] + (i + myhalf) * dx[0];
46  Real y = prob_lo[1] + (j + myhalf) * dx[1];
47 
48  // Store coordinates
49  Gpu::Atomic::Add(&d_coords_ptr[0],x); // Store x index
50  Gpu::Atomic::Add(&d_coords_ptr[1],y); // Store x index
51  }
52  });
53  }
54 
55  // Synchronize to ensure all threads complete their execution
56  amrex::Gpu::streamSynchronize(); // Wait for all GPU threads to finish
57 
58  Vector<int> h_found(1,0);
59  Gpu::copy(Gpu::deviceToHost, d_found.begin(), d_found.end(), h_found.begin());
60  ParallelAllReduce::Sum(h_found.data(), h_found.size(), ParallelContext::CommunicatorAll());
61 
62  // Broadcast coordinates if found
63  if (h_found[0] > 0) {
64  Vector<Real> h_coords(2,-bogus_large_value);
65  Gpu::copy(Gpu::deviceToHost, d_coords.begin(), d_coords.end(), h_coords.begin());
66 
67  ParallelAllReduce::Sum(h_coords.data(), h_coords.size(), ParallelContext::CommunicatorAll());
68 
69  eye_x = h_coords[0]/h_found[0];
70  eye_y = h_coords[1]/h_found[0];
71 
72  } else {
73  // Random large negative numbers so we don't trigger refinement in this case
74  eye_x = -bogus_large_value;
75  eye_y = -bogus_large_value;
76  }
77 
78  return (h_found[0] > 0);
79 }
Here is the call graph for this function:

◆ get_eb()

eb_ const& ERF::get_eb ( int  lev) const
inlineprivatenoexcept
1757  {
1758  AMREX_ASSERT(lev >= 0 && lev < eb.size() && eb[lev] != nullptr);
1759  return *eb[lev];
1760  }

◆ getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int  lev)
inlineprivate
1543  {
1544  return advflux_reg[lev];
1545  }

◆ getCPUTime()

static double ERF::getCPUTime ( )
inlinestaticprivate
1635  {
1636  int numCores = amrex::ParallelDescriptor::NProcs();
1637 #ifdef _OPENMP
1638  numCores = numCores * omp_get_max_threads();
1639 #endif
1640 
1641  double T =
1642  numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
1643  previousCPUTimeUsed;
1644 
1645  return T;
1646  }
T
Real T
Definition: ERF_InitCustomPert_Bubble.H:106
ERF::startCPUTime
static double startCPUTime
Definition: ERF.H:1630
ERF::previousCPUTimeUsed
static double previousCPUTimeUsed
Definition: ERF.H:1631

◆ GetOceanToAtmosSurfaceLayout()

void ERF::GetOceanToAtmosSurfaceLayout ( amrex::BoxArray &  ba,
amrex::DistributionMapping &  dm 
)
136 {
137  auto* sst_ptr = lsm.Get_Data_Ptr(0, 0);
138  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
139  sst_ptr != nullptr,
140  "ERF::GetOceanToAtmosSurfaceLayout requires OceanSurf level-0 surface storage after InitData.");
141  ba = sst_ptr->boxArray();
142  dm = sst_ptr->DistributionMap();
143 }
Here is the call graph for this function:

◆ GotoNextLine()

void ERF::GotoNextLine ( std::istream &  is)
staticprivate

Utility to skip to next line in Header file input stream.

26 {
27  constexpr std::streamsize bl_ignore_max { 100000 };
28  is.ignore(bl_ignore_max, '\n');
29 }

◆ HurricaneEyeTracker()

void ERF::HurricaneEyeTracker ( const SolverChoice &  solverChoice)
476 {
477  static bool is_start = true;
478  int levc=finest_level;
479 
480  const MoistureType moisture_type = sc.moisture_type;
481  const Real hurricane_eye_latitude = sc.hurricane_eye_latitude;
482  const Real hurricane_eye_longitude = sc.hurricane_eye_longitude;
483 
484  if(is_start and restart_chkfile.empty()){
485  HurricaneEyeTrackerInitial(sc, geom[levc],
486  vars_new[levc],
487  hurricane_eye_latitude,
488  hurricane_eye_longitude);
489  is_start = false;
490  } else {
491  if(!restart_chkfile.empty()) {
492  ReadStormTrackerRestart();
493  }
494  HurricaneEyeTrackerNotInitial(sc, geom[levc], vars_new[levc],
495  moisture_type);
496  }
497  HurricaneTrackerCircle();
498 }
ERF::HurricaneEyeTrackerNotInitial
void HurricaneEyeTrackerNotInitial(const SolverChoice &sc, const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, MoistureType moisture_type)
Definition: ERF_HurricaneDiagnostics.cpp:250
ERF::HurricaneEyeTrackerInitial
void HurricaneEyeTrackerInitial(const SolverChoice &sc, const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, const amrex::Real &hurricane_eye_latitude, const amrex::Real &hurricane_eye_longitude)
Definition: ERF_HurricaneDiagnostics.cpp:175
ERF::restart_chkfile
std::string restart_chkfile
Definition: ERF.H:1167
ERF::ReadStormTrackerRestart
void ReadStormTrackerRestart()
Definition: ERF_HurricaneDiagnostics.cpp:315
ERF::HurricaneTrackerCircle
void HurricaneTrackerCircle()
Definition: ERF_HurricaneDiagnostics.cpp:149

◆ HurricaneEyeTrackerInitial()

void ERF::HurricaneEyeTrackerInitial ( const SolverChoice &  sc,
const amrex::Geometry &  geom,
const amrex::Vector< amrex::MultiFab > &  S_data,
const amrex::Real &  hurricane_eye_latitude,
const amrex::Real &  hurricane_eye_longitude 
)
180 {
181  int levc = finest_level;
182  Gpu::DeviceScalar<Real> d_val_min(1e10);
183  Gpu::DeviceScalar<int> d_i_min(-1), d_j_min(-1);
184 
185  Real* d_val_min_ptr = d_val_min.dataPtr();
186  int* d_i_min_ptr = d_i_min.dataPtr();
187  int* d_j_min_ptr = d_j_min.dataPtr();
188 
189  if(sc.init_type == InitType::WRFInput){
190  for (MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
191  const Box& box = mfi.validbox();
192  FArrayBox& fab_lat = (*(lat_m[levc]))[mfi];
193  FArrayBox& fab_lon = (*(lon_m[levc]))[mfi];
194  const Array4<Real>& lat_arr = fab_lat.array();
195  const Array4<Real>& lon_arr = fab_lon.array();
196 
197  ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
198  if (k==0) {
199 
200  Real dlat = lat_arr(i,j,0) - hurricane_eye_latitude;
201  Real dlon = lon_arr(i,j,0) - hurricane_eye_longitude;
202  Real dist = std::sqrt(dlat*dlat + dlon*dlon);
203  // Atomic min using device pointer from DeviceVector
204  Real old = Gpu::Atomic::Min(&d_val_min_ptr[0], dist);
205  //Gpu::Atomic::Min(&d_val_min_ptr[0], dist);
206  if (dist < old) {
207  // We are the new minimum; record indices
208  d_i_min_ptr[0] = i;
209  d_j_min_ptr[0] = j;
210  }
211  }
212  });
213  }
214  }
215 
216  if(sc.init_type == InitType::HindCast){
217  for (amrex::MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
218  const amrex::Box& box = mfi.validbox();
219  const auto& mf_latlon = forecast_state_interp[levc][4];
220  const auto latlon_arr = mf_latlon.array(mfi);
221 
222  amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
223  if (k==0) {
224 
225  amrex::Real dlat = latlon_arr(i,j,k,0) - hurricane_eye_latitude;
226  amrex::Real dlon = latlon_arr(i,j,k,1) - hurricane_eye_longitude;
227  amrex::Real dist = std::sqrt(dlat*dlat + dlon*dlon);
228  // Atomic min using device pointer from DeviceVector
229  amrex::Real old = amrex::Gpu::Atomic::Min(&d_val_min_ptr[0], dist);
230  //amrex::Gpu::Atomic::Min(&d_val_min_ptr[0], dist);
231  if (dist < old) {
232  // We are the new minimum; record indices
233  d_i_min_ptr[0] = i;
234  d_j_min_ptr[0] = j;
235  }
236  }
237  });
238  }
239  }
240 
241  Real global_val_min;
242  int global_i_min, global_j_min;
243 
244  ComputeGlobalMinLocation(sc, lev_geom, S_data,
245  d_val_min_ptr, d_i_min_ptr, d_j_min_ptr,
246  global_val_min, global_i_min, global_j_min);
247 }
ERF::ComputeGlobalMinLocation
void ComputeGlobalMinLocation(const SolverChoice &sc, const amrex::Geometry &geom, const amrex::Vector< amrex::MultiFab > &S_data, amrex::Real *d_val_min_ptr, int *d_i_min_ptr, int *d_j_min_ptr, amrex::Real &global_val_min, int &global_i_min, int &global_j_min)
Definition: ERF_HurricaneDiagnostics.cpp:36
Here is the call graph for this function:

◆ HurricaneEyeTrackerNotInitial()

void ERF::HurricaneEyeTrackerNotInitial ( const SolverChoice &  sc,
const amrex::Geometry &  geom,
const amrex::Vector< amrex::MultiFab > &  S_data,
MoistureType  moisture_type 
)
254 {
255 
256  if (hurricane_eye_track_xy.empty()) {
257  Print() << "Error: hurricane_eye_track_xy is empty!\n";
258  Abort("Attempted to access hurricane_eye_track_xy[0]");
259  }
260 
261  Real tmp_x_eye = hurricane_eye_track_xy.back()[0];
262  Real tmp_y_eye = hurricane_eye_track_xy.back()[1];
263 
264  if(ParallelDescriptor::IOProcessor()){
265  std::cout << "The value of x y are " << tmp_x_eye << " " << tmp_y_eye << std::endl;
266  }
267 
268  Gpu::DeviceScalar<Real> d_val_min(1e10);
269  Gpu::DeviceScalar<int> d_i_min(-1), d_j_min(-1);
270 
271  Real* d_val_min_ptr = d_val_min.dataPtr();
272  int* d_i_min_ptr = d_i_min.dataPtr();
273  int* d_j_min_ptr = d_j_min.dataPtr();
274 
275  bool use_moisture = (moisture_type != MoistureType::None);
276  const int ncomp = S_data[IntVars::cons].nComp();
277 
278  const auto dx = lev_geom.CellSizeArray();
279  const auto prob_lo = lev_geom.ProbLoArray();
280 
281  for (MFIter mfi(S_data[IntVars::cons]); mfi.isValid(); ++mfi) {
282  const Box& box = mfi.validbox();
283  const Array4<Real const>& S_arr = S_data[IntVars::cons].const_array(mfi);
284 
285  ParallelFor(box,[=] AMREX_GPU_DEVICE(int i, int j, int k) {
286  if(k==0) {
287  Real x = prob_lo[0] + (i+myhalf)*dx[0];
288  Real y = prob_lo[1] + (j+myhalf)*dx[1];
289  Real dist = std::sqrt((x-tmp_x_eye)*(x-tmp_x_eye) + (y-tmp_y_eye)*(y-tmp_y_eye));
290  if(dist < 200e3) {
291  Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
292  const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
293  Real pressure = getPgivenRTh(rhotheta,qv_for_p);
294  Real old = Gpu::Atomic::Min(&d_val_min_ptr[0], pressure);
295  //Gpu::Atomic::Min(&d_val_min_ptr[0], dist);
296  if (old > pressure) {
297  // We are the new minimum; record indices
298  d_i_min_ptr[0] = i;
299  d_j_min_ptr[0] = j;
300  }
301  }
302  }
303  });
304  }
305 
306  Real global_val_min;
307  int global_i_min, global_j_min;
308 
309  ComputeGlobalMinLocation(sc, lev_geom, S_data,
310  d_val_min_ptr, d_i_min_ptr, d_j_min_ptr,
311  global_val_min, global_i_min, global_j_min);
312 }
Here is the call graph for this function:

◆ HurricaneMaxVelTracker()

void ERF::HurricaneMaxVelTracker ( const amrex::Geometry &  geom,
const amrex::MultiFab &  mf_cc_vel,
const double &  time 
)
504 {
505  const int ncomp = AMREX_SPACEDIM;
506 
507  Real* d_val_max_ptr;
508  Gpu::DeviceVector<Real> d_val_max(1, -bogus_large_value);
509  d_val_max_ptr = d_val_max.data();
510 
511  const auto [x_last, y_last] = hurricane_eye_track_xy.back();
512  const auto dx = lev_geom.CellSizeArray();
513  const auto prob_lo = lev_geom.ProbLoArray();
514 
515  Real x_eye = x_last;
516  Real y_eye = y_last;
517 
518  for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi) {
519  const Box& box = mfi.validbox();
520  const auto& vel_arr = mf_cc_vel.const_array(mfi);
521 
522  ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
523  Real x = prob_lo[0] + (i+myhalf)*dx[0];
524  Real y = prob_lo[1] + (j+myhalf)*dx[1];
525  Real dist = std::sqrt((x-x_eye)*(x-x_eye) +
526  (y-y_eye)*(y-y_eye));
527  if(k==1 && dist < 200e3) {
528  Real velmag = zero;
529  for (int comp = 0; comp < ncomp; ++comp) {
530  Real vel = vel_arr(i, j, k, comp);
531  velmag += vel * vel;
532  }
533  velmag = std::sqrt(velmag)*Real(3.6); // km/hr
534  Gpu::Atomic::Max(&d_val_max_ptr[0], velmag);
535  }
536  });
537  }
538 
539  Gpu::synchronize();
540 
541  Real h_val_max_local = -bogus_large_value;
542  Gpu::copy(Gpu::deviceToHost, d_val_max.begin(), d_val_max.end(), &h_val_max_local);
543 
544  Real h_val_max_global = -bogus_large_value;
545  #ifdef AMREX_USE_MPI
546  MPI_Allreduce(&h_val_max_local, &h_val_max_global, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
547  #else
548  h_val_max_global = h_val_max_local;
549  #endif
550 
551  double time_in_hrs = time / 3600.0;
552  hurricane_maxvel_vs_time.push_back({static_cast<Real>(time_in_hrs), h_val_max_global});
553 }
ERF::hurricane_maxvel_vs_time
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_maxvel_vs_time
Definition: ERF.H:180
Here is the call graph for this function:

◆ HurricaneMinPressureTracker()

void ERF::HurricaneMinPressureTracker ( MoistureType  moisture_type,
const amrex::Geometry &  geom,
const amrex::MultiFab &  mf_cons_var,
const double &  time 
)
560 {
561 
562 
563  Real* d_val_min_ptr;
564  Gpu::DeviceVector<Real> d_val_min(1, bogus_large_value);
565  d_val_min_ptr = d_val_min.data();
566 
567  const Real x_last = hurricane_eye_track_xy.back()[0];
568  const Real y_last = hurricane_eye_track_xy.back()[1];
569  const auto dx = lev_geom.CellSizeArray();
570  const auto prob_lo = lev_geom.ProbLoArray();
571 
572  const int ncomp = mf_cons_var.nComp();
573  bool use_moisture = (moisture_type != MoistureType::None);
574 
575  for (MFIter mfi(mf_cons_var); mfi.isValid(); ++mfi) {
576  const Box& box = mfi.validbox();
577  const auto& S_arr = mf_cons_var.const_array(mfi);
578 
579  ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
580  Real x = prob_lo[0] + (i+myhalf)*dx[0];
581  Real y = prob_lo[1] + (j+myhalf)*dx[1];
582  Real dist2 = (x-x_last)*(x-x_last) +
583  (y-y_last)*(y-y_last);
584  if(k==1 && dist2 < 200e3*200e3) {
585  const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
586  const Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
587  const Real pressure = getPgivenRTh(rhotheta,qv_for_p);
588  Gpu::Atomic::Min(&d_val_min_ptr[0], pressure);
589  }
590  });
591  }
592 
593  Gpu::synchronize();
594 
595  Real h_val_min_local = bogus_large_value;
596  Gpu::copy(Gpu::deviceToHost, d_val_min.begin(), d_val_min.end(), &h_val_min_local);
597 
598  Real h_val_min_global = bogus_large_value;
599  #ifdef AMREX_USE_MPI
600  MPI_Allreduce(&h_val_min_local, &h_val_min_global, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
601  #else
602  h_val_min_global = h_val_min_local;
603  #endif
604 
605  double time_in_hrs = time / 3600.0;
606  hurricane_minpressure_vs_time.push_back({static_cast<Real>(time_in_hrs), h_val_min_global});
607 }
ERF::hurricane_minpressure_vs_time
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_minpressure_vs_time
Definition: ERF.H:181
Here is the call graph for this function:

◆ HurricaneTracker()

void ERF::HurricaneTracker ( int  lev,
double  time,
const amrex::MultiFab &  cc_vel,
const amrex::Real  velmag_threshold,
amrex::TagBoxArray *  tags = nullptr 
)
114 {
115  bool is_found;
116 
117  Real eye_x, eye_y;
118 
119  if (time==zero || hurricane_eye_track_xy.empty()) {
120  is_found = FindInitialEye(levc, mf_cc_vel, velmag_threshold, eye_x, eye_y);
121  } else {
122  is_found = true;
123  const auto& last = hurricane_eye_track_xy.back();
124  eye_x = last[0];
125  eye_y = last[1];
126  }
127 
128  if (is_found) {
129  const int exponent = max_level-1-levc;
130  Real rad_tag = std::ldexp(Real(4.e5), exponent);
131  tag_on_distance_from_eye(geom[levc], tags, eye_x, eye_y, rad_tag);
132  }
133 }
tag_on_distance_from_eye
void tag_on_distance_from_eye(const Geometry &cgeom, TagBoxArray *tags, const Real eye_x, const Real eye_y, const Real rad_tag)
Definition: ERF_RefineHurricane.cpp:82
ERF::FindInitialEye
bool FindInitialEye(int lev, const amrex::MultiFab &cc_vel, const amrex::Real velmag_threshold, amrex::Real &eye_x, amrex::Real &eye_y)
Definition: ERF_RefineHurricane.cpp:11
Here is the call graph for this function:

◆ HurricaneTrackerCircle()

void ERF::HurricaneTrackerCircle ( )
150 {
151  // Check that there is at least one eye position
152  if (hurricane_eye_track_xy.empty()) return;
153 
154  // Get the last known (x, y) position of the eye
155  const auto [x_last, y_last] = hurricane_eye_track_xy.back();
156 
157  // Define circle properties
158  const int n_points = 100; // number of points on the circle
159  const Real radius = 200e3; // radius in meters (example: 50 km)
160 
161  // Clear previous points and reserve space
162  hurricane_tracker_circle.clear();
163  hurricane_tracker_circle.reserve(n_points);
164 
165  // Fill the circle points
166  for (int i = 0; i < n_points; ++i) {
167  Real theta = two * static_cast<Real>(M_PI) * static_cast<Real>(i) / static_cast<Real>(n_points);
168  Real x = x_last + radius * std::cos(theta);
169  Real y = y_last + radius * std::sin(theta);
170  hurricane_tracker_circle.push_back({x, y});
171  }
172 }
M_PI
#define M_PI
Definition: ERF_HurricaneDiagnostics.cpp:26
ERF::hurricane_tracker_circle
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_tracker_circle
Definition: ERF.H:182

◆ ImposeBCsOnPhi()

void ERF::ImposeBCsOnPhi ( int  lev,
amrex::MultiFab &  phi,
const amrex::Box &  subdomain 
)

Impose bc's on the pressure that comes out of the solve

13 {
14  BL_PROFILE("ERF::ImposeBCsOnPhi()");
15 
16  auto const sub_lo = lbound(subdomain);
17  auto const sub_hi = ubound(subdomain);
18 
19  auto const dom_lo = lbound(geom[lev].Domain());
20  auto const dom_hi = ubound(geom[lev].Domain());
21 
22  phi.setBndry(Real(1.e25));
23  phi.FillBoundary(geom[lev].periodicity());
24 
25  // ****************************************************************************
26  // Impose bc's on pprime
27  // ****************************************************************************
28 #ifdef _OPENMP
29 #pragma omp parallel if (Gpu::notInLaunchRegion())
30 #endif
31  for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi)
32  {
33  Array4<Real> const& pp_arr = phi.array(mfi);
34  Box const& bx = mfi.tilebox();
35  auto const bx_lo = lbound(bx);
36  auto const bx_hi = ubound(bx);
37 
38  auto bc_type_xlo = domain_bc_type[Orientation(0,Orientation::low)];
39  auto bc_type_xhi = domain_bc_type[Orientation(0,Orientation::high)];
40  auto bc_type_ylo = domain_bc_type[Orientation(1,Orientation::low)];
41  auto bc_type_yhi = domain_bc_type[Orientation(1,Orientation::high)];
42  auto bc_type_zhi = domain_bc_type[Orientation(2,Orientation::high)];
43 
44  if ( (bx_lo.x == dom_lo.x) && (bc_type_xlo == "Outflow" || bc_type_xlo == "Open") && !solverChoice.use_real_bcs) {
45  ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
46  {
47  pp_arr(i-1,j,k) = -pp_arr(i,j,k);
48  });
49  } else if (bx_lo.x == sub_lo.x) {
50  ParallelFor(makeSlab(bx,0,sub_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
51  {
52  pp_arr(i-1,j,k) = pp_arr(i,j,k);
53  });
54  }
55 
56  if ( (bx_hi.x == dom_hi.x) && (bc_type_xhi == "Outflow" || bc_type_xhi == "Open") && !solverChoice.use_real_bcs) {
57  ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
58  {
59  pp_arr(i+1,j,k) = -pp_arr(i,j,k);
60  });
61  } else if (bx_hi.x == sub_hi.x) {
62  ParallelFor(makeSlab(bx,0,sub_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
63  {
64  pp_arr(i+1,j,k) = pp_arr(i,j,k);
65  });
66  }
67 
68  if ( (bx_lo.y == dom_lo.y) && (bc_type_ylo == "Outflow" || bc_type_ylo == "Open") && !solverChoice.use_real_bcs) {
69  ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
70  {
71  pp_arr(i,j-1,k) = -pp_arr(i,j,k);
72  });
73  } else if (bx_lo.y == sub_lo.y) {
74  ParallelFor(makeSlab(bx,1,sub_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
75  {
76  pp_arr(i,j-1,k) = pp_arr(i,j,k);
77  });
78  }
79 
80  if ( (bx_hi.y == dom_hi.y) && (bc_type_yhi == "Outflow" || bc_type_yhi == "Open") && !solverChoice.use_real_bcs) {
81  ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
82  {
83  pp_arr(i,j+1,k) = -pp_arr(i,j,k);
84  });
85  } else if (bx_hi.y == sub_hi.y) {
86  ParallelFor(makeSlab(bx,1,sub_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
87  {
88  pp_arr(i,j+1,k) = pp_arr(i,j,k);
89  });
90  }
91 
92  // At low z we are always Neumann whether the box touches the bottom boundary or not
93  Box zbx(bx); zbx.grow(0,1); zbx.grow(1,1); // Grow in x-dir and y-dir because we have filled that above
94  if (bx_lo.z == sub_lo.z) {
95  ParallelFor(makeSlab(zbx,2,dom_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
96  {
97  pp_arr(i,j,k-1) = pp_arr(i,j,k);
98  });
99  }
100 
101  if ( (bx_hi.z == dom_hi.z) && (bc_type_zhi == "Outflow" || bc_type_zhi == "Open") ) {
102  ParallelFor(makeSlab(bx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
103  {
104  pp_arr(i,j,k+1) = -pp_arr(i,j,k);
105  });
106  } else if (bx_hi.z == sub_hi.z) {
107  ParallelFor(makeSlab(bx,2,sub_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
108  {
109  pp_arr(i,j,k+1) = pp_arr(i,j,k);
110  });
111  }
112  } // mfi
113 
114  // Now overwrite with periodic fill outside domain and fine-fine fill inside
115  phi.FillBoundary(geom[lev].periodicity());
116 }
ERF::domain_bc_type
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
Definition: ERF.H:1105
Here is the call graph for this function:

◆ init1DArrays()

void ERF::init1DArrays ( )
private

◆ init_bcs()

void ERF::init_bcs ( )
private
300 {
301  bool rho_read = false;
302  bool read_prim_theta = true;
303  bool use_surfacelayer = false;
304 
305  init_phys_bcs(rho_read, read_prim_theta);
306 
307  Vector<Real> cons_dir_init(NBCVAR_max,zero);
308  cons_dir_init[BCVars::Rho_bc_comp] = one;
309  cons_dir_init[BCVars::RhoTheta_bc_comp] = -one;
310 
311  bool keqn_dir = (solverChoice.turbChoice[max_level].rans_type == RANSType::kEqn &&
312  solverChoice.turbChoice[max_level].dirichlet_k == true);
313  if (keqn_dir) {
314  // Need to change wall BC type, assume for now that all levels are RANS
315  for (int lev = 0; lev < max_level; ++lev) {
316  if (solverChoice.turbChoice[lev].rans_type != RANSType::kEqn) {
317  Error("If using one-eqn RANS, all levels must be RANS for now");
318  }
319  }
320  Print() << "Using dirichlet BC for k equation" << std::endl;
321  }
322 
323  // *****************************************************************************
324  //
325  // Here we translate the physical boundary conditions -- one type per face --
326  // into logical boundary conditions for each velocity component
327  //
328  // *****************************************************************************
329  {
330  domain_bcs_type.resize(AMREX_SPACEDIM+NBCVAR_max);
331  domain_bcs_type_d.resize(AMREX_SPACEDIM+NBCVAR_max);
332 
333  for (OrientationIter oit; oit; ++oit) {
334  Orientation ori = oit();
335  int dir = ori.coordDir();
336  Orientation::Side side = ori.faceDir();
337  auto const bct = phys_bc_type[ori];
338  if ( bct == ERF_BC::symmetry )
339  {
340  if (side == Orientation::low) {
341  for (int i = 0; i < AMREX_SPACEDIM; i++) {
342  domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::reflect_even);
343  }
344  domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::reflect_odd);
345  } else {
346  for (int i = 0; i < AMREX_SPACEDIM; i++) {
347  domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::reflect_even);
348  }
349  domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::reflect_odd);
350  }
351  }
352  else if (bct == ERF_BC::outflow or bct == ERF_BC::ho_outflow )
353  {
354  if (side == Orientation::low) {
355  for (int i = 0; i < AMREX_SPACEDIM; i++) {
356  domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
357  }
358  if (!solverChoice.anelastic[0]) {
359  domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::neumann_int);
360  }
361  } else {
362  for (int i = 0; i < AMREX_SPACEDIM; i++) {
363  domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
364  }
365  if (!solverChoice.anelastic[0]) {
366  domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::neumann_int);
367  }
368  }
369  }
370  else if (bct == ERF_BC::open)
371  {
372  if (side == Orientation::low) {
373  for (int i = 0; i < AMREX_SPACEDIM; i++)
374  domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::open);
375  } else {
376  for (int i = 0; i < AMREX_SPACEDIM; i++)
377  domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::open);
378  }
379  }
380  else if (bct == ERF_BC::inflow)
381  {
382  if (side == Orientation::low) {
383  for (int i = 0; i < AMREX_SPACEDIM; i++) {
384  domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
385  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
386  domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
387  }
388  }
389  } else {
390  for (int i = 0; i < AMREX_SPACEDIM; i++) {
391  domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
392  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
393  domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
394  }
395  }
396  }
397  }
398  else if (bct == ERF_BC::inflow_outflow)
399  {
400  if (side == Orientation::low) {
401  for (int i = 0; i < AMREX_SPACEDIM; i++) {
402  domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir_upwind);
403  }
404  } else {
405  for (int i = 0; i < AMREX_SPACEDIM; i++) {
406  domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir_upwind);
407  }
408  }
409  }
410  else if (bct == ERF_BC::no_slip_wall)
411  {
412  if (side == Orientation::low) {
413  for (int i = 0; i < AMREX_SPACEDIM; i++) {
414  domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::ext_dir);
415  }
416  } else {
417  for (int i = 0; i < AMREX_SPACEDIM; i++) {
418  domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::ext_dir);
419  }
420  }
421  }
422  else if (bct == ERF_BC::slip_wall)
423  {
424  if (side == Orientation::low) {
425  for (int i = 0; i < AMREX_SPACEDIM; i++) {
426  domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::foextrap);
427  }
428  // Only normal direction has ext_dir
429  domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, ERFBCType::ext_dir);
430 
431  } else {
432  for (int i = 0; i < AMREX_SPACEDIM; i++) {
433  domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::foextrap);
434  }
435  // Only normal direction has ext_dir
436  domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, ERFBCType::ext_dir);
437  }
438  }
439  else if (bct == ERF_BC::periodic)
440  {
441  if (side == Orientation::low) {
442  for (int i = 0; i < AMREX_SPACEDIM; i++) {
443  domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, ERFBCType::int_dir);
444  }
445  } else {
446  for (int i = 0; i < AMREX_SPACEDIM; i++) {
447  domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, ERFBCType::int_dir);
448  }
449  }
450  }
451  else if ( bct == ERF_BC::surface_layer )
452  {
453  use_surfacelayer = true;
454  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
455  domain_bcs_type[BCVars::xvel_bc+0].setLo(dir, ERFBCType::hoextrap);
456  domain_bcs_type[BCVars::xvel_bc+1].setLo(dir, ERFBCType::hoextrap);
457  domain_bcs_type[BCVars::xvel_bc+2].setLo(dir, ERFBCType::ext_dir);
458  }
459  }
460  }
461 
462  // *****************************************************************************
463  //
464  // Here we translate the physical boundary conditions -- one type per face --
465  // into logical boundary conditions for each cell-centered variable
466  // (including the base state variables)
467  // NOTE: all "scalars" share the same type of boundary condition
468  //
469  // *****************************************************************************
470  {
471  for (OrientationIter oit; oit; ++oit) {
472  Orientation ori = oit();
473  int dir = ori.coordDir();
474  Orientation::Side side = ori.faceDir();
475  auto const bct = phys_bc_type[ori];
476  if ( bct == ERF_BC::symmetry )
477  {
478  if (side == Orientation::low) {
479  for (int i = 0; i < NBCVAR_max; i++) {
480  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::reflect_even);
481  }
482  } else {
483  for (int i = 0; i < NBCVAR_max; i++) {
484  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::reflect_even);
485  }
486  }
487  }
488  else if ( bct == ERF_BC::outflow )
489  {
490  if (side == Orientation::low) {
491  for (int i = 0; i < NBCVAR_max; i++) {
492  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
493  }
494  } else {
495  for (int i = 0; i < NBCVAR_max; i++) {
496  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
497  }
498  }
499  }
500  else if ( bct == ERF_BC::ho_outflow )
501  {
502  if (side == Orientation::low) {
503  for (int i = 0; i < NBCVAR_max; i++) {
504  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::hoextrap);
505  }
506  } else {
507  for (int i = 0; i < NBCVAR_max; i++) {
508  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::hoextrap);
509  }
510  }
511  }
512  else if ( bct == ERF_BC::open )
513  {
514  if (side == Orientation::low) {
515  for (int i = 0; i < NBCVAR_max; i++)
516  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::open);
517  } else {
518  for (int i = 0; i < NBCVAR_max; i++)
519  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::open);
520  }
521  }
522  else if ( bct == ERF_BC::no_slip_wall )
523  {
524  if (side == Orientation::low) {
525  for (int i = 0; i < NBCVAR_max; i++) {
526  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
527  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
528  if (rho_read) {
529  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
530  } else {
531  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
532  }
533  }
534  }
535  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > zero) {
536  domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
537  }
538  } else {
539  for (int i = 0; i < NBCVAR_max; i++) {
540  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
541  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
542  if (rho_read) {
543  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
544  } else {
545  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
546  }
547  }
548  }
549  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > zero) {
550  domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
551  }
552  }
553  }
554  else if (bct == ERF_BC::slip_wall)
555  {
556  if (side == Orientation::low) {
557  for (int i = 0; i < NBCVAR_max; i++) {
558  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
559  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
560  if (rho_read) {
561  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
562  } else {
563  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
564  }
565  }
566  }
567  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > zero) {
568  domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::neumann);
569  }
570  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > zero) {
571  domain_bcs_type[BCVars::Rho_bc_comp].setLo(dir, ERFBCType::neumann);
572  }
573  } else {
574  for (int i = 0; i < NBCVAR_max; i++) {
575  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
576  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
577  if (rho_read) {
578  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
579  } else {
580  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
581  }
582  }
583  }
584  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > zero) {
585  domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::neumann);
586  }
587  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > zero) {
588  domain_bcs_type[BCVars::Rho_bc_comp].setHi(dir, ERFBCType::neumann);
589  }
590  }
591  }
592  else if (bct == ERF_BC::inflow)
593  {
594  if (side == Orientation::low) {
595  for (int i = 0; i < NBCVAR_max; i++) {
596  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir);
597  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
598  (th_bc_data[0].data() != nullptr))
599  {
600  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
601  }
602  else if (input_bndry_planes && dir < 2 && (
603  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
604  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
605  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
606  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
607  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
608  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )) )
609  {
610  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_ingested);
611  }
612  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
613  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
614  }
615  }
616  // Non-reflecting: extrapolate RhoTheta from interior so
617  // pressure is not prescribed, allowing acoustic waves out
618  if (m_bc_nonreflecting[ori]) {
619  domain_bcs_type[BCVars::RhoTheta_bc_comp].setLo(dir, ERFBCType::foextrap);
620  }
621  } else {
622  for (int i = 0; i < NBCVAR_max; i++) {
623  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir);
624  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
625  (th_bc_data[0].data() != nullptr))
626  {
627  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
628  }
629  else if (input_bndry_planes && dir < 2 && (
630  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
631  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
632  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
633  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
634  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
635  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )
636  ) )
637  {
638  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_ingested);
639  }
640  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
641  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
642  }
643  }
644  // Non-reflecting: extrapolate RhoTheta from interior
645  if (m_bc_nonreflecting[ori]) {
646  domain_bcs_type[BCVars::RhoTheta_bc_comp].setHi(dir, ERFBCType::foextrap);
647  }
648  }
649  }
650  else if (bct == ERF_BC::inflow_outflow )
651  {
652  if (side == Orientation::low) {
653  for (int i = 0; i < NBCVAR_max; i++) {
654  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_upwind);
655  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
656  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
657  }
658  }
659  } else {
660  for (int i = 0; i < NBCVAR_max; i++) {
661  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_upwind);
662  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
663  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::foextrap);
664  }
665  }
666  }
667  }
668  else if (bct == ERF_BC::periodic)
669  {
670  if (side == Orientation::low) {
671  for (int i = 0; i < NBCVAR_max; i++) {
672  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::int_dir);
673  }
674  } else {
675  for (int i = 0; i < NBCVAR_max; i++) {
676  domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::int_dir);
677  }
678  }
679  }
680  else if ( bct == ERF_BC::surface_layer )
681  {
682  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
683  for (int i = 0; i < NBCVAR_max; i++) {
684  domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::foextrap);
685  }
686  if (keqn_dir) {
687  Print() << "Setting surface layer logical BC to dirichlet for RANS with k model" << std::endl;
688  domain_bcs_type[BCVars::RhoKE_bc_comp].setLo(dir, ERFBCType::ext_dir);
689  }
690  }
691  }
692  }
693 
694  // Sanity check that implicit diffusion is consistent with the BC types.
695  // Turn off implicit diffusion for a component if its BCs don't match
696  // those allowed by the tridiagonal solver.
697  const BCRec* bc_ptr = domain_bcs_type.data();
698  Vector<int> cc_comp_map = {BCVars::RhoTheta_bc_comp, BCVars::RhoKE_bc_comp, BCVars::RhoQ1_bc_comp};
699  Vector<std::string> cc_comp_name = {"Theta", "KE", "Qv"};
700  for (int icomp(0); icomp < cc_comp_map.size(); ++icomp) {
701  int bc_comp = cc_comp_map[icomp];
702  bool foextrap_on_zlo = (bc_ptr[bc_comp].lo(2) == ERFBCType::foextrap);
703  bool foextrap_on_zhi = (bc_ptr[bc_comp].hi(2) == ERFBCType::foextrap);
704  bool neumann_on_zlo = (bc_ptr[bc_comp].lo(2) == ERFBCType::neumann);
705  bool neumann_on_zhi = (bc_ptr[bc_comp].hi(2) == ERFBCType::neumann);
706  if ( (!foextrap_on_zlo && !neumann_on_zlo && !use_surfacelayer) ||
707  (!foextrap_on_zhi && !neumann_on_zhi) ) {
708  Print() << "WARNING: Selected BCs for " << cc_comp_name[icomp] << " are not supported with implicit diffusion. Turning off implicit diffusion for this component." << "\n";
709  if (bc_comp == BCVars::RhoTheta_bc_comp) { solverChoice.implicit_thermal_diffusion = false; }
710  if (bc_comp == BCVars::RhoKE_bc_comp) { solverChoice.implicit_ke_diffusion = false; }
711  if (bc_comp == BCVars::RhoQ1_bc_comp) { solverChoice.implicit_moisture_diffusion = false; }
712  }
713  }
714  Vector<int> vel_comp_map = {BCVars::xvel_bc, BCVars::yvel_bc};
715  Vector<std::string> vel_comp_name = {"xvel", "yvel"};
716  for (int icomp(0); icomp < vel_comp_map.size(); ++icomp) {
717  int bc_comp = vel_comp_map[icomp];
718  bool ext_dir_on_zlo = (bc_ptr[bc_comp].lo(2) == ERFBCType::ext_dir ||
719  bc_ptr[bc_comp].lo(2) == ERFBCType::ext_dir_prim);
720  bool ext_dir_on_zhi = (bc_ptr[bc_comp].hi(2) == ERFBCType::ext_dir ||
721  bc_ptr[bc_comp].hi(2) == ERFBCType::ext_dir_prim);
722  bool foextrap_on_zlo = (bc_ptr[bc_comp].lo(2) == ERFBCType::foextrap);
723  bool foextrap_on_zhi = (bc_ptr[bc_comp].hi(2) == ERFBCType::foextrap);
724  if ( (!foextrap_on_zlo && !ext_dir_on_zlo && !use_surfacelayer) ||
725  (!foextrap_on_zhi && !ext_dir_on_zhi) ) {
726  Print() << "WARNING: Selected BCs for " << vel_comp_name[icomp] << " are not supported with implicit diffusion. Turning off implicit diffusion for this component." << "\n";
727  solverChoice.implicit_momentum_diffusion = false;
728  }
729  }
730 
731  // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
732  Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
733 }
NBCVAR_max
#define NBCVAR_max
Definition: ERF_IndexDefines.H:29
ERF_BC::ho_outflow
@ ho_outflow
ERF_BC::open
@ open
ERF_BC::inflow_outflow
@ inflow_outflow
ERF_BC::outflow
@ outflow
ERF_BC::inflow
@ inflow
ERF_BC::periodic
@ periodic
ERF::m_bc_nonreflecting
amrex::Array< bool, AMREX_SPACEDIM *2 > m_bc_nonreflecting
Definition: ERF.H:1114
ERF::init_phys_bcs
void init_phys_bcs(bool &rho_read, bool &read_prim_theta)
Definition: ERF_InitBCs.cpp:20
ERF::m_bc_neumann_vals
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_neumann_vals
Definition: ERF.H:1111
BCVars::RhoQ1_bc_comp
@ RhoQ1_bc_comp
Definition: ERF_IndexDefines.H:91
BCVars::RhoKE_bc_comp
@ RhoKE_bc_comp
Definition: ERF_IndexDefines.H:89
BCVars::RhoTheta_bc_comp
@ RhoTheta_bc_comp
Definition: ERF_IndexDefines.H:88
BCVars::RhoQ2_bc_comp
@ RhoQ2_bc_comp
Definition: ERF_IndexDefines.H:92
BCVars::Rho_bc_comp
@ Rho_bc_comp
Definition: ERF_IndexDefines.H:87
ERFBCType::neumann
@ neumann
Definition: ERF_IndexDefines.H:253
ERFBCType::open
@ open
Definition: ERF_IndexDefines.H:255
ERFBCType::reflect_odd
@ reflect_odd
Definition: ERF_IndexDefines.H:244
ERFBCType::hoextrap
@ hoextrap
Definition: ERF_IndexDefines.H:249
ERFBCType::foextrap
@ foextrap
Definition: ERF_IndexDefines.H:247
ERFBCType::ext_dir
@ ext_dir
Definition: ERF_IndexDefines.H:248
ERFBCType::ext_dir_prim
@ ext_dir_prim
Definition: ERF_IndexDefines.H:251
ERFBCType::ext_dir_upwind
@ ext_dir_upwind
Definition: ERF_IndexDefines.H:256
ERFBCType::int_dir
@ int_dir
Definition: ERF_IndexDefines.H:245
ERFBCType::neumann_int
@ neumann_int
Definition: ERF_IndexDefines.H:254
ERFBCType::reflect_even
@ reflect_even
Definition: ERF_IndexDefines.H:246
SolverChoice::implicit_ke_diffusion
bool implicit_ke_diffusion
Definition: ERF_DataStruct.H:1279
SolverChoice::implicit_moisture_diffusion
bool implicit_moisture_diffusion
Definition: ERF_DataStruct.H:1278
SolverChoice::implicit_thermal_diffusion
bool implicit_thermal_diffusion
Definition: ERF_DataStruct.H:1277
SolverChoice::implicit_momentum_diffusion
bool implicit_momentum_diffusion
Definition: ERF_DataStruct.H:1280
Here is the call graph for this function:

◆ init_custom()

void ERF::init_custom ( int  lev)

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters
levInteger specifying the current level
27 {
28  auto& lev_new = vars_new[lev];
29 
30  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
31  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
32 
33  MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
34  lev_new[Vars::cons].nComp() , lev_new[Vars::cons].nGrow());
35  MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
36  MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
37  MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
38 
39  // Default all perturbations to zero
40  cons_pert.setVal(0.);
41  xvel_pert.setVal(0.);
42  yvel_pert.setVal(0.);
43  zvel_pert.setVal(0.);
44 
45 
46 #ifdef _OPENMP
47 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
48 #endif
49  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
50  {
51  const Box &bx = mfi.tilebox();
52  const Box &xbx = mfi.tilebox(IntVect(1,0,0));
53  const Box &ybx = mfi.tilebox(IntVect(0,1,0));
54  const Box &zbx = mfi.tilebox(IntVect(0,0,1));
55 
56  const auto &cons_pert_arr = cons_pert.array(mfi);
57  const auto &xvel_pert_arr = xvel_pert.array(mfi);
58  const auto &yvel_pert_arr = yvel_pert.array(mfi);
59  const auto &zvel_pert_arr = zvel_pert.array(mfi);
60 
61  Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
62  Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
63  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
64 
65  // Here we arbitrarily choose the x-oriented map factor -- this should be generalized
66  Array4<Real const> mf_m = mapfac[lev][MapFacType::m_x]->const_array(mfi);
67  Array4<Real const> mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
68  Array4<Real const> mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
69 
70  Array4<Real> r_hse_arr = r_hse.array(mfi);
71  Array4<Real> p_hse_arr = p_hse.array(mfi);
72 
73  prob->init_custom_pert(bx, cons_arr, cons_pert_arr,
74  r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
75  geom[lev].data(), mf_m, solverChoice, lev);
76  prob->init_custom_pert_vels(xbx, ybx, zbx,
77  xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
78  z_nd_arr, geom[lev].data(), mf_u, mf_v,
79  solverChoice, lev);
80 
81  // Zero out perturbations in covered cells in EB
82  if (solverChoice.terrain_type == TerrainType::EB) {
83 
84  Array4<const EBCellFlag> c_cellflg = (get_eb(lev).get_const_factory())->getMultiEBCellFlagFab()[mfi].const_array();
85  Array4<const EBCellFlag> u_cellflg = (get_eb(lev).get_u_const_factory())->getMultiEBCellFlagFab()[mfi].const_array();
86  Array4<const EBCellFlag> v_cellflg = (get_eb(lev).get_v_const_factory())->getMultiEBCellFlagFab()[mfi].const_array();
87  Array4<const EBCellFlag> w_cellflg = (get_eb(lev).get_w_const_factory())->getMultiEBCellFlagFab()[mfi].const_array();
88 
89  ParallelFor(bx,
90  [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
91  if (c_cellflg(i,j,k).isCovered()) {
92  cons_pert_arr(i,j,k,RhoTheta_comp) = 0.0;
93  }
94  });
95 
96  ParallelFor(xbx, ybx, zbx,
97  [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
98  if (u_cellflg(i,j,k).isCovered()) {
99  xvel_pert_arr(i,j,k) = 0.0;
100  }
101  },
102  [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
103  if (v_cellflg(i,j,k).isCovered()) {
104  yvel_pert_arr(i,j,k) = 0.0;
105  }
106  },
107  [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
108  if (w_cellflg(i,j,k).isCovered()) {
109  zvel_pert_arr(i,j,k) = 0.0;
110  }
111  });
112  }
113 
114  } //mfi
115 
116 
117  // Add problem-specific perturbation to background flow if not doing anelastic with fixed-in-time density
118  if (!solverChoice.fixed_density[lev]) {
119  MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp, Rho_comp, 1, cons_pert.nGrow());
120  }
121  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp, 1, cons_pert.nGrow());
122  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,NSCALARS, cons_pert.nGrow());
123 
124  // RhoKE is relevant if using Deardorff with LES, k-equation for RANS, MYJ, SHOC, MYNN2.5 or MYNN-EDMF
125  // Here we initialize TKE to the tke_min value and then add problem-specific perturbations
126  if (solverChoice.turbChoice[lev].use_tke) {
127  lev_new[Vars::cons].setVal(solverChoice.turbChoice[lev].tke_min, RhoKE_comp, 1);
128  MultiFab::Multiply(lev_new[Vars::cons],lev_new[Vars::cons],Rho_comp,RhoKE_comp,1,lev_new[Vars::cons].nGrowVect());
129  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp, RhoKE_comp, 1, cons_pert.nGrow());
130  }
131 
132  if (solverChoice.moisture_type != MoistureType::None) {
133  int qstate_size = micro->Get_Qstate_Size();
134  for (int q_offset(0); q_offset<qstate_size; ++q_offset) {
135  int q_idx = RhoQ1_comp+q_offset;
136  MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
137  }
138  }
139 
140  // Should we initialize the velocities from a checkpoint file?
141  static std::string init_vels_from_checkpoint;
142  ParmParse pp("erf");
143  if (pp.query("init_vels_from_checkpoint",init_vels_from_checkpoint)) {
144  ReadVelsOnlyFromCheckpointFile(lev,init_vels_from_checkpoint);
145  } else {
146  MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0, 0, 1, xvel_pert.nGrowVect());
147  MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0, 0, 1, yvel_pert.nGrowVect());
148  MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0, 0, 1, zvel_pert.nGrowVect());
149  }
150 
151  // If initializing for ensemble simluations, then
152  // 1. Create cell-centered random perturbations
153  // 2. Create cell-centered spatially correlated perturbations
154  // 3. Read in the coarse background state from the coarse data file,
155  // interpolate the state data onto the current mesh, and
156  // add the perturbations to the background state and then populate the "pert variables
157 
158  if(solverChoice.is_init_for_ensemble) {
159  MultiFab mf_cc_pert;
160  create_random_perturbations(lev, mf_cc_pert);
161  apply_gaussian_smoothing_to_perturbations(lev, mf_cc_pert);
162  create_background_state_for_ensemble(lev, mf_cc_pert, lev_new[Vars::cons], lev_new[Vars::xvel], lev_new[Vars::yvel], lev_new[Vars::zvel]);
163  }
164 }
xbx
const Box xbx
Definition: ERF_SetupDiff.H:7
ybx
const Box ybx
Definition: ERF_SetupDiff.H:8
ERF::create_background_state_for_ensemble
void create_background_state_for_ensemble(int lev, amrex::MultiFab &mf_cc_pert, amrex::MultiFab &cons_pert, amrex::MultiFab &xvel_pert, amrex::MultiFab &yvel_pert, amrex::MultiFab &zvel_pert)
Definition: ERF_InitForEnsemble.cpp:516
ERF::apply_gaussian_smoothing_to_perturbations
void apply_gaussian_smoothing_to_perturbations(const int lev, amrex::MultiFab &mf_cc_pert)
Definition: ERF_InitForEnsemble.cpp:86
ERF::ReadVelsOnlyFromCheckpointFile
void ReadVelsOnlyFromCheckpointFile(int lev_to_fill, std::string &chkfile)
Definition: ERF_Checkpoint.cpp:1232
ERF::create_random_perturbations
void create_random_perturbations(const int lev, amrex::MultiFab &mf_cc_pert)
Definition: ERF_InitForEnsemble.cpp:14
eb_::get_v_const_factory
eb_aux_ const * get_v_const_factory() const noexcept
Definition: ERF_EB.H:51
eb_::get_u_const_factory
eb_aux_ const * get_u_const_factory() const noexcept
Definition: ERF_EB.H:50
SolverChoice::fixed_density
amrex::Vector< int > fixed_density
Definition: ERF_DataStruct.H:1258
SolverChoice::is_init_for_ensemble
bool is_init_for_ensemble
Definition: ERF_DataStruct.H:1481
Here is the call graph for this function:

◆ init_Dirichlet_bc_data()

void ERF::init_Dirichlet_bc_data ( const std::string  input_file)
private
736 {
737  // Read the dirichlet_input file
738  Print() << "dirichlet_input file location : " << input_file << std::endl;
739  std::ifstream input_reader(input_file);
740  if (!input_reader.is_open()) {
741  amrex::Abort("Error opening the dirichlet_input file.\n");
742  }
743 
744  Print() << "Successfully opened the dirichlet_input file. Now reading... " << std::endl;
745  std::string line;
746 
747  // Size of Ninp (number of z points in input file)
748  Vector<Real> z_inp_tmp, u_inp_tmp, v_inp_tmp, w_inp_tmp, th_inp_tmp;
749 
750  // Top and bot for domain
751  const int klo = geom[0].Domain().smallEnd()[2];
752  const int khi = geom[0].Domain().bigEnd()[2];
753  const Real zbot = zlevels_stag[0][klo];
754  const Real ztop = zlevels_stag[0][khi+1];
755 
756  // Flag if theta input
757  Real th_init = -Real(300.0);
758  bool th_read{false};
759 
760  // Add surface
761  z_inp_tmp.push_back(zbot); // height above sea level [m]
762  u_inp_tmp.push_back(zero);
763  v_inp_tmp.push_back(zero);
764  w_inp_tmp.push_back(zero);
765  th_inp_tmp.push_back(th_init);
766 
767  // Read the vertical profile at each given height
768  Real z, u, v, w, th;
769  while(std::getline(input_reader, line)) {
770  std::istringstream iss_z(line);
771 
772  Vector<Real> rval_v;
773  Real rval;
774  while (iss_z >> rval) {
775  rval_v.push_back(rval);
776  }
777  if ((rval_v.size() != 4) && (rval_v.size() != 5)) {
778  Abort("Unknown inflow file format!");
779  }
780  z = rval_v[0];
781  u = rval_v[1];
782  v = rval_v[2];
783  w = rval_v[3];
784 
785  // Format without theta
786  if (rval_v.size() == 4) {
787  if (z == zbot) {
788  u_inp_tmp[0] = u;
789  v_inp_tmp[0] = v;
790  w_inp_tmp[0] = w;
791  } else {
792  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
793  z_inp_tmp.push_back(z);
794  u_inp_tmp.push_back(u);
795  v_inp_tmp.push_back(v);
796  w_inp_tmp.push_back(w);
797  if (z >= ztop) break;
798  }
799  } else if (rval_v.size() == 5) {
800  th_read = true;
801  th = rval_v[4];
802  if (z == zbot) {
803  u_inp_tmp[0] = u;
804  v_inp_tmp[0] = v;
805  w_inp_tmp[0] = w;
806  th_inp_tmp[0] = th;
807  } else {
808  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
809  z_inp_tmp.push_back(z);
810  u_inp_tmp.push_back(u);
811  v_inp_tmp.push_back(v);
812  w_inp_tmp.push_back(w);
813  th_inp_tmp.push_back(th);
814  if (z >= ztop) break;
815  }
816  } else {
817  Abort("Unknown inflow file format!");
818  }
819  }
820 
821  // Ensure we set a reasonable theta surface
822  if (th_read) {
823  if (th_inp_tmp[0] == th_init) {
824  AMREX_ALWAYS_ASSERT_WITH_MESSAGE((th_inp_tmp.size() > 2) && (z_inp_tmp.size() > 2),
825  "Need at least 3 theta profile points to extrapolate surface theta");
826  Real slope = (th_inp_tmp[2] - th_inp_tmp[1]) / (z_inp_tmp[2] - z_inp_tmp[1]);
827  Real dz = z_inp_tmp[0] - z_inp_tmp[1];
828  th_inp_tmp[0] = slope * dz + th_inp_tmp[1];
829  }
830  }
831 
832  amrex::Print() << "Successfully read and interpolated the dirichlet_input file..." << std::endl;
833  input_reader.close();
834 
835  for (int lev = 0; lev <= max_level; lev++) {
836 
837  const int Nz = geom[lev].Domain().size()[2];
838 
839  // Size of Nz (domain grid)
840  Vector<Real> zcc_inp(Nz );
841  Vector<Real> znd_inp(Nz+1);
842  Vector<Real> u_inp(Nz ); xvel_bc_data[lev].resize(Nz ,zero);
843  Vector<Real> v_inp(Nz ); yvel_bc_data[lev].resize(Nz ,zero);
844  Vector<Real> w_inp(Nz+1); zvel_bc_data[lev].resize(Nz+1,zero);
845  Vector<Real> th_inp;
846  if (th_read) {
847  th_inp.resize(Nz);
848  th_bc_data[lev].resize(Nz, zero);
849  }
850 
851  // At this point, we have an input from zbot up to
852  // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights
853  const int Ninp = z_inp_tmp.size();
854  for (int k(0); k<Nz; ++k) {
855  zcc_inp[k] = myhalf * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
856  znd_inp[k] = zlevels_stag[lev][k+1];
857  u_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), u_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
858  v_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), v_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
859  w_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), znd_inp[k], Ninp);
860  if (th_read) {
861  th_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), th_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
862  }
863  }
864  znd_inp[Nz] = ztop;
865  w_inp[Nz] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), ztop, Ninp);
866 
867  // Copy host data to the device
868  Gpu::copy(Gpu::hostToDevice, u_inp.begin(), u_inp.end(), xvel_bc_data[lev].begin());
869  Gpu::copy(Gpu::hostToDevice, v_inp.begin(), v_inp.end(), yvel_bc_data[lev].begin());
870  Gpu::copy(Gpu::hostToDevice, w_inp.begin(), w_inp.end(), zvel_bc_data[lev].begin());
871  if (th_read) {
872  Gpu::copy(Gpu::hostToDevice, th_inp.begin(), th_inp.end(), th_bc_data[lev].begin());
873  }
874 
875  // NOTE: These device vectors are passed to the PhysBC constructors when that
876  // class is instantiated in ERF_MakeNewArrays.cpp.
877  } // lev
878 }
ztop
const Real ztop
Definition: ERF_InitCustomPertVels_ParticleTests.H:4
interpolate_1d
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real interpolate_1d(const amrex::Real *alpha, const amrex::Real *beta, const amrex::Real alpha_interp, const int alpha_size)
Definition: ERF_Interpolation_1D.H:14
Here is the call graph for this function:

◆ init_from_hse()

void ERF::init_from_hse ( int  lev)

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

  • call prob->erf_init_dens_hse_dry(...)
    • call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
    • save r_hse
  • call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse and th_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation
Parameters
levInteger specifying the current level
33 {
34  auto& lev_new = vars_new[lev];
35 
36  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
37  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
38 
39 #ifdef _OPENMP
40 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
41 #endif
42  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
43  {
44  const Box &gbx = mfi.growntilebox(1);
45  const Array4<Real >& cons_arr = lev_new[Vars::cons].array(mfi);
46  const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
47  const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
48 
49  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
50  {
51  cons_arr(i,j,k,Rho_comp) = r_hse_arr(i,j,k);
52  cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
53  });
54  } //mfi
55 }
Here is the call graph for this function:

◆ init_from_input_sounding()

void ERF::init_from_input_sounding ( int  lev)

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters
levInteger specifying the current level
54 {
55  // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
56  if (lev == 0) {
57  if (input_sounding_data.input_sounding_file.empty()) {
58  Error("input_sounding file name must be provided via input");
59  }
60 
61  input_sounding_data.resize_arrays();
62 
63  bool is_moist = (solverChoice.moisture_type != MoistureType::None);
64 
65  // this will interpolate the input profiles to the nominal height levels
66  // (ranging from 0 to the domain top)
67  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
68  input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n, is_moist);
69  }
70 
71  // this will calculate the hydrostatically balanced density and pressure
72  // profiles following WRF ideal.exe
73  if (solverChoice.sounding_type == SoundingType::Ideal) {
74  input_sounding_data.calc_rho_p(0);
75  } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
76  solverChoice.sounding_type == SoundingType::DryIsentropic) {
77  input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
78  input_sounding_data.calc_rho_p_isentropic(0);
79  }
80 
81  } else {
82  //
83  // We need to do this interp from coarse level in order to set the values of
84  // the base state inside the domain but outside of the fine region
85  //
86  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
87  //
88  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
89  // have been pre-filled - this includes ghost cells both inside and outside
90  // the domain
91  //
92  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
93  IntVect(0,0,0), // do not fill ghost cells outside the domain
94  base_state[lev-1], 0, 0, base_state[lev].nComp(),
95  geom[lev-1], geom[lev],
96  refRatio(lev-1), &cell_cons_interp,
97  domain_bcs_type, BaseBCVars::rho0_bc_comp);
98 
99  // We need to do this here because the interpolation above may leave corners unfilled
100  // when the corners need to be filled by, for example, reflection of the fine ghost
101  // cell outside the fine region but inide the domain.
102  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
103  }
104 
105  auto& lev_new = vars_new[lev];
106 
107  const bool l_isentropic = (solverChoice.sounding_type == SoundingType::Isentropic ||
108  solverChoice.sounding_type == SoundingType::DryIsentropic);
109 
110  const bool constant_density_sounding = (solverChoice.sounding_type == SoundingType::ConstantDensity);
111 
112  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
113  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
114  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
115  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
116  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
117 
118  const Real l_gravity = solverChoice.gravity;
119  const Real l_rdOcp = solverChoice.rdOcp;
120  const bool l_moist = (solverChoice.moisture_type != MoistureType::None);
121 
122  int ngz = r_hse.nGrow(2);
123 
124 #ifdef _OPENMP
125 #pragma omp parallel if (Gpu::notInLaunchRegion())
126 #endif
127  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
128  const Box &bx = mfi.tilebox();
129  const auto &cons_arr = lev_new[Vars::cons].array(mfi);
130  Array4<Real> r_hse_arr = r_hse.array(mfi);
131  Array4<Real> p_hse_arr = p_hse.array(mfi);
132  Array4<Real> pi_hse_arr = pi_hse.array(mfi);
133  Array4<Real> th_hse_arr = th_hse.array(mfi);
134  Array4<Real> qv_hse_arr = qv_hse.array(mfi);
135 
136  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
137 
138  if (constant_density_sounding) {
139  // This assumes rho_0 = one
140  // HSE will be calculated later with call to initHSE
141  init_state_from_input_sounding(bx, cons_arr, geom[lev].data(), z_cc_arr,
142  l_moist, input_sounding_data);
143  }
144  else
145  {
146  // HSE will be initialized here, interpolated from values previously
147  // calculated by calc_rho_p or calc_rho_p_isentropic
148  init_state_from_input_sounding_hse(bx, cons_arr,
149  r_hse_arr, p_hse_arr, pi_hse_arr, th_hse_arr, qv_hse_arr,
150  geom[lev].data(), z_cc_arr,
151  l_gravity, l_rdOcp, l_moist, input_sounding_data,
152  l_isentropic, ngz);
153  }
154  }
155 
156  if (!constant_density_sounding) {
157  // Enforce HSE on the base state -- holding th_hse and qv_hse constant
158  bool maintain_Th = true;
159  rebalance_columns(r_hse, th_hse, qv_hse, qv_hse, z_phys_nd[lev].get(), geom[lev], maintain_Th);
160 
161  // Update rho in the state from base state
162  MultiFab::Copy(lev_new[Vars::cons], r_hse, 0, Rho_comp, 1, 1);
163 
164  // Update (rho theta) in the state from base state
165  MultiFab::Copy(lev_new[Vars::cons], th_hse, 0, RhoTheta_comp, 1, 1);
166  MultiFab::Multiply(lev_new[Vars::cons], lev_new[Vars::cons], Rho_comp, RhoTheta_comp, 1, 1);
167 
168  if (l_moist) {
169  // Update (rho qv) in the state from base state
170  MultiFab::Copy(lev_new[Vars::cons], qv_hse, 0, RhoQ1_comp, 1, 1);
171  MultiFab::Multiply(lev_new[Vars::cons], lev_new[Vars::cons], Rho_comp, RhoQ1_comp, 1, 1);
172  }
173  }
174 
175  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi)
176  {
177  const Box &bx = mfi.tilebox();
178  const auto& xvel_arr = lev_new[Vars::xvel].array(mfi);
179  const auto& yvel_arr = lev_new[Vars::yvel].array(mfi);
180  const auto& zvel_arr = lev_new[Vars::zvel].array(mfi);
181  const auto& z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
182  init_velocities_from_input_sounding(bx, xvel_arr, yvel_arr, zvel_arr,
183  geom[lev].data(), z_nd_arr, input_sounding_data);
184  }
185 
186  // *****************************************************************************
187  // Re-compute p_hse and pi_hse
188  // *****************************************************************************
189  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi)
190  {
191  Box bx = mfi.tilebox();
192 
193  const Array4<const Real>& r_hse_arr = r_hse.const_array(mfi);
194  const Array4<const Real>& th_hse_arr = th_hse.const_array(mfi);
195  const Array4<const Real>& qv_hse_arr = qv_hse.const_array(mfi);
196 
197  const Array4< Real>& p_hse_arr = p_hse.array(mfi);
198  const Array4< Real>& pi_hse_arr = pi_hse.array(mfi);
199 
200  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
201  {
202  Real rhotheta = r_hse_arr(i,j,k) * th_hse_arr(i,j,k);
203  p_hse_arr(i,j,k) = getPgivenRTh(rhotheta,qv_hse_arr(i,j,k));
204  pi_hse_arr(i,j,k) = getExnergivenRTh(rhotheta, l_rdOcp, qv_hse_arr(i,j,k));
205  });
206  }
207 }
getExnergivenRTh
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getExnergivenRTh(const amrex::Real rhotheta, const amrex::Real rdOcp, const amrex::Real qv=amrex::Real(0))
Definition: ERF_EOS.H:156
init_state_from_input_sounding_hse
void init_state_from_input_sounding_hse(const Box &bx, Array4< Real > const &state, Array4< Real > const &r_hse_arr, Array4< Real > const &p_hse_arr, Array4< Real > const &pi_hse_arr, Array4< Real > const &th_hse_arr, Array4< Real > const &qv_hse_arr, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const Real &l_gravity, const Real &l_rdOcp, const bool &l_moist, InputSoundingData const &inputSoundingData, const bool &l_isentropic, const int &ngz)
Definition: ERF_InitFromInputSounding.cpp:275
init_velocities_from_input_sounding
void init_velocities_from_input_sounding(const Box &bx, Array4< Real > const &x_vel, Array4< Real > const &y_vel, Array4< Real > const &z_vel, GeometryData const &geomdata, Array4< const Real > const &z_nd_arr, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:405
init_state_from_input_sounding
void init_state_from_input_sounding(const Box &bx, Array4< Real > const &state, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const bool &l_moist, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:219
rebalance_columns
void rebalance_columns(MultiFab &rho, MultiFab &theta, const MultiFab &qv, const MultiFab &qt, const MultiFab *z_phys, const Geometry &geom, const bool &maintain_Th, bool use_sfc)
Definition: ERF_Rebalance.cpp:7
ERF::input_sounding_data
InputSoundingData input_sounding_data
Definition: ERF.H:877
BaseBCVars::rho0_bc_comp
@ rho0_bc_comp
Definition: ERF_IndexDefines.H:113
InputSoundingData::resize_arrays
void resize_arrays()
Definition: ERF_InputSoundingData.H:60
InputSoundingData::n_sounding_files
int n_sounding_files
Definition: ERF_InputSoundingData.H:406
InputSoundingData::input_sounding_file
amrex::Vector< std::string > input_sounding_file
Definition: ERF_InputSoundingData.H:404
InputSoundingData::calc_rho_p
void calc_rho_p(int itime)
Definition: ERF_InputSoundingData.H:178
InputSoundingData::calc_rho_p_isentropic
void calc_rho_p_isentropic(int itime)
Definition: ERF_InputSoundingData.H:270
InputSoundingData::read_from_file
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag, int itime, bool is_moist)
Definition: ERF_InputSoundingData.H:77
InputSoundingData::assume_dry
bool assume_dry
Definition: ERF_InputSoundingData.H:409
SolverChoice::sounding_type
static SoundingType sounding_type
Definition: ERF_DataStruct.H:1227
Here is the call graph for this function:

◆ init_geo_wind_profile()

void ERF::init_geo_wind_profile ( const std::string  input_file,
amrex::Vector< amrex::Real > &  u_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  u_geos_d,
amrex::Vector< amrex::Real > &  v_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  v_geos_d,
const amrex::Geometry &  lgeom,
const amrex::Vector< amrex::Real > &  zlev_stag 
)
private
17 {
18  const int klo = 0;
19  const int khi = lgeom.Domain().bigEnd()[AMREX_SPACEDIM-1];
20  const amrex::Real dz = lgeom.CellSize()[AMREX_SPACEDIM-1];
21 
22  const bool grid_stretch = (zlev_stag.size() > 0);
23  const Real zbot = (grid_stretch) ? zlev_stag[klo] : lgeom.ProbLo(AMREX_SPACEDIM-1);
24  const Real ztop = (grid_stretch) ? zlev_stag[khi+1] : lgeom.ProbHi(AMREX_SPACEDIM-1);
25 
26  amrex::Print() << "Reading geostrophic wind profile from " << input_file << std::endl;
27  std::ifstream profile_reader(input_file);
28  if(!profile_reader.is_open()) {
29  amrex::Error("Error opening the abl_geo_wind_table\n");
30  }
31 
32  // First, read the input data into temp vectors
33  std::string line;
34  Vector<Real> z_inp, Ug_inp, Vg_inp;
35  Real z, Ug, Vg;
36  amrex::Print() << "z Ug Vg" << std::endl;
37  while(std::getline(profile_reader, line)) {
38  std::istringstream iss(line);
39  iss >> z >> Ug >> Vg;
40  amrex::Print() << z << " " << Ug << " " << Vg << std::endl;
41  z_inp.push_back(z);
42  Ug_inp.push_back(Ug);
43  Vg_inp.push_back(Vg);
44  if (z >= ztop) break;
45  }
46 
47  const int Ninp = z_inp.size();
48  AMREX_ALWAYS_ASSERT(z_inp[0] <= zbot);
49  AMREX_ALWAYS_ASSERT(z_inp[Ninp-1] >= ztop);
50 
51  // Now, interpolate vectors to the cell centers
52  for (int k = 0; k <= khi; k++) {
53  z = (grid_stretch) ? myhalf * (zlev_stag[k] + zlev_stag[k+1])
54  : zbot + (k + myhalf) * dz;
55  u_geos[k] = interpolate_1d(z_inp.dataPtr(), Ug_inp.dataPtr(), z, Ninp);
56  v_geos[k] = interpolate_1d(z_inp.dataPtr(), Vg_inp.dataPtr(), z, Ninp);
57  }
58 
59  // Copy from host version to device version
60  Gpu::copy(Gpu::hostToDevice, u_geos.begin(), u_geos.end(), u_geos_d.begin());
61  Gpu::copy(Gpu::hostToDevice, v_geos.begin(), v_geos.end(), v_geos_d.begin());
62 
63  profile_reader.close();
64 }
Here is the call graph for this function:

◆ init_immersed_forcing()

void ERF::init_immersed_forcing ( int  lev)

Set velocities in cells that are immersed to be 0 (or a very small number)

Parameters
levInteger specifying the current level
16 {
17  auto& lev_new = vars_new[lev];
18  MultiFab* terrain_blank = terrain_blanking[lev].get();
19 
20 #ifdef _OPENMP
21 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
22 #endif
23  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
24  {
25  const Box &xbx = mfi.tilebox(IntVect(1,0,0));
26  const Box &ybx = mfi.tilebox(IntVect(0,1,0));
27  const Box &zbx = mfi.tilebox(IntVect(0,0,1));
28  const Real epsilon = Real(1e-2);
29 
30  const Array4<const Real>& t_blank_arr = terrain_blank->const_array(mfi);
31 
32  const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
33  const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
34  const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
35 
36  // Set the x,y,z-velocities
37  ParallelFor(xbx, ybx, zbx,
38  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
39  const Real t_blank = myhalf * (t_blank_arr(i, j, k) + t_blank_arr(i-1, j, k));
40  if (t_blank == one) { xvel_arr(i, j, k) = epsilon; }
41  },
42  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
43  const Real t_blank = myhalf * (t_blank_arr(i, j, k) + t_blank_arr(i, j-1, k));
44  if (t_blank == one) { yvel_arr(i, j, k) = epsilon; }
45  },
46  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
47  const Real t_blank = myhalf * (t_blank_arr(i, j, k) + t_blank_arr(i, j, k-1));
48  if (t_blank == one) { zvel_arr(i, j, k) = epsilon; }
49  });
50  } //mfi
51 }
module_model_constants::epsilon
real(c_double), parameter epsilon
Definition: ERF_module_model_constants.F90:12
Here is the call graph for this function:

◆ init_only()

void ERF::init_only ( int  lev,
double  time 
)
1814 {
1815  t_new[lev] = elapsed_time;
1816  t_old[lev] = elapsed_time - bogus_large_value;
1817 
1818  auto& lev_new = vars_new[lev];
1819  auto& lev_old = vars_old[lev];
1820 
1821  // Loop over grids at this level to initialize our grid data
1822  lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
1823  lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
1824  lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
1825  lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
1826 
1827  // Initialize background flow (optional)
1828  if (solverChoice.init_type == InitType::Input_Sounding) {
1829  // The physbc's need the terrain but are needed for initHSE
1830  // We have already made the terrain in the call to init_zphys
1831  // in MakeNewLevelFromScratch
1832  make_physbcs(lev);
1833 
1834  // Now init the base state and the data itself
1835  init_from_input_sounding(lev);
1836 
1837  // The base state has been initialized by integrating vertically
1838  // through the sounding for ideal (like WRF) or isentropic approaches
1839  if (solverChoice.sounding_type == SoundingType::Ideal ||
1840  solverChoice.sounding_type == SoundingType::Isentropic ||
1841  solverChoice.sounding_type == SoundingType::DryIsentropic) {
1842  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
1843  "Gravity should be on to be consistent with sounding initialization.");
1844  } else { // SoundingType::ConstantDensity
1845  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!solverChoice.use_gravity || (solverChoice.anelastic[lev] == 1),
1846  "Constant density probably doesn't make sense for compressible flow with gravity");
1847  initHSE();
1848  }
1849 
1850 #ifdef ERF_USE_NETCDF
1851  }
1852  else if (solverChoice.init_type == InitType::WRFInput && !nc_init_file[lev].empty())
1853  {
1854  // The base state is initialized from WRF wrfinput data, output by
1855  // ideal.exe or real.exe
1856 
1857  init_from_wrfinput(lev, *mf_PSFC[lev]);
1858 
1859  // The physbc's need the terrain but are needed for initHSE
1860  make_physbcs(lev);
1861  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
1862  }
1863  else if (solverChoice.init_type == InitType::WRFInput && nc_init_file[lev].empty())
1864  {
1865  amrex::Abort("This pathway is not quite implemented yet");
1866  }
1867  else if (solverChoice.init_type == InitType::NCFile)
1868  {
1869  // The state is initialized by reading from a Netcdf file
1870  init_from_ncfile(lev);
1871 
1872  // The physbc's need the terrain but are needed for initHSE
1873  make_physbcs(lev);
1874  }
1875  else if (solverChoice.init_type == InitType::Metgrid)
1876  {
1877  // The base state is initialized from data output by WPS metgrid;
1878  // we will rebalance after interpolation
1879  init_from_metgrid(lev);
1880 #endif
1881  } else if ( (solverChoice.init_type == InitType::Uniform ) ||
1882  (solverChoice.init_type == InitType::ConstantDensity) ||
1883  (solverChoice.init_type == InitType::Isentropic ) ||
1884  (solverChoice.init_type == InitType::ConstantDensityLinearTheta ) ||
1885  (solverChoice.init_type == InitType::HindCast ) ||
1886  (solverChoice.init_type == InitType::MoistBaseState ) ) {
1887  // Initialize a uniform density/entropy background field and base state
1888  // based on the problem-specified reference density and temperature
1889 
1890  // The physbc's need the terrain but are needed for initHSE
1891  make_physbcs(lev);
1892 
1893  // We will initialize the state from the background state so must set that first
1894  // The choice between constant rho and constant theta will be made inside initHSE
1895  initHSE(lev);
1896 
1897  // Copy rho and rhotheta from rho_hse and p_hse
1898  init_from_hse(lev);
1899 
1900  } else {
1901  Abort("Unknown init_type!");
1902  }
1903 
1904  // Add problem-specific flow features
1905  //
1906  // Notes:
1907  // - This calls init_custom_pert that is defined for each problem
1908  // - This may modify the base state
1909  // - The fields set by init_custom_pert are **perturbations** to the
1910  // background flow set based on init_type
1911  if (solverChoice.init_type != InitType::NCFile) {
1912  init_custom(lev);
1913  }
1914 
1915  // Ensure that the face-based data are the same on both sides of a periodic domain.
1916  // The data associated with the lower grid ID is considered the correct value.
1917  lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
1918  lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
1919  lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
1920 
1921  if (solverChoice.spongeChoice.sponge_type == SpongeType::Input_Sponge)
1922  {
1923  input_sponge(lev);
1924  }
1925 
1926  // Initialize turbulent perturbation
1927  if (solverChoice.use_perturbation(lev)) {
1928  turbPert_update(lev, zero);
1929  if (solverChoice.use_wvel_perturbation(lev)) {
1930  turbPert_amplitude_w(lev);
1931  } else {
1932  turbPert_amplitude(lev);
1933  }
1934  }
1935 
1936  // Set initial velocity field for immersed cells to be close to 0
1937  if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
1938  solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
1939  init_immersed_forcing(lev);
1940  }
1941 }
ERF::turbPert_amplitude_w
void turbPert_amplitude_w(const int lev)
Definition: ERF_InitTurbPert.cpp:60
ERF::init_from_input_sounding
void init_from_input_sounding(int lev)
Definition: ERF_InitFromInputSounding.cpp:53
ERF::turbPert_update
void turbPert_update(const int lev, const double dt)
Definition: ERF_InitTurbPert.cpp:12
ERF::init_custom
void init_custom(int lev)
Definition: ERF_InitCustomPertState.cpp:26
ERF::init_from_hse
void init_from_hse(int lev)
Definition: ERF_InitFromHSE.cpp:32
ERF::initHSE
void initHSE()
Initialize HSE.
Definition: ERF_Init1D.cpp:192
ERF::input_sponge
void input_sponge(int lev)
Definition: ERF_InitSponge.cpp:17
ERF::make_physbcs
void make_physbcs(int lev)
Definition: ERF_MakeNewArrays.cpp:920
ERF::init_immersed_forcing
void init_immersed_forcing(int lev)
Definition: ERF_InitImmersedForcing.cpp:15
ERF::turbPert_amplitude
void turbPert_amplitude(const int lev)
Definition: ERF_InitTurbPert.cpp:33
SolverChoice::use_gravity
bool use_gravity
Definition: ERF_DataStruct.H:1299
SolverChoice::spongeChoice
SpongeChoice spongeChoice
Definition: ERF_DataStruct.H:1250

◆ init_phys_bcs()

void ERF::init_phys_bcs ( bool &  rho_read,
bool &  read_prim_theta 
)
private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, InflowOutflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

21 {
22  auto f = [this,&rho_read,&read_prim_theta,one_d=one,zero_d=zero] (std::string const& bcid, Orientation ori)
23  {
24  // These are simply defaults for Dirichlet faces -- they should be over-written below
25  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = one_d;
26  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = -one_d; // It is important to set this negative
27  // because the sign is tested on below
28  for (int n = BCVars::RhoKE_bc_comp; n < BCVars::xvel_bc; n++) {
29  m_bc_extdir_vals[n][ori] = zero_d;
30  }
31 
32  m_bc_extdir_vals[BCVars::xvel_bc][ori] = zero_d; // default
33  m_bc_extdir_vals[BCVars::yvel_bc][ori] = zero_d;
34  m_bc_extdir_vals[BCVars::zvel_bc][ori] = zero_d;
35 
36  // These are simply defaults for Neumann gradients -- they should be over-written below
37  m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = zero_d;
38  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = zero_d;
39 
40  m_bc_neumann_vals[BCVars::RhoKE_bc_comp][ori] = zero_d;
41  m_bc_neumann_vals[BCVars::RhoScalar_bc_comp][ori] = zero_d;
42  m_bc_neumann_vals[BCVars::RhoQ1_bc_comp][ori] = zero_d;
43  m_bc_neumann_vals[BCVars::RhoQ2_bc_comp][ori] = zero_d;
44  m_bc_neumann_vals[BCVars::RhoQ3_bc_comp][ori] = zero_d;
45  m_bc_neumann_vals[BCVars::RhoQ4_bc_comp][ori] = zero_d;
46  m_bc_neumann_vals[BCVars::RhoQ5_bc_comp][ori] = zero_d;
47  m_bc_neumann_vals[BCVars::RhoQ6_bc_comp][ori] = zero_d;
48 
49  m_bc_neumann_vals[BCVars::xvel_bc][ori] = zero_d;
50  m_bc_neumann_vals[BCVars::yvel_bc][ori] = zero_d;
51  m_bc_neumann_vals[BCVars::zvel_bc][ori] = zero_d;
52 
53  std::string pp_text = pp_prefix + "." + bcid;
54  ParmParse pp(pp_text);
55 
56  std::string bc_type_in;
57  if (pp.query("type", bc_type_in) <= 0)
58  {
59  pp_text = bcid;
60  pp = ParmParse(pp_text);
61  pp.query("type", bc_type_in);
62  }
63 
64  std::string bc_type = amrex::toLower(bc_type_in);
65 
66  if (bc_type == "symmetry")
67  {
68  // Print() << bcid << " set to symmetry.\n";
69  phys_bc_type[ori] = ERF_BC::symmetry;
70  domain_bc_type[ori] = "Symmetry";
71  }
72  else if ( (ori.coordDir() != 2 && solverChoice.use_real_bcs) || (bc_type == "outflow") )
73  {
74  // Print() << bcid << " set to outflow.\n";
75  phys_bc_type[ori] = ERF_BC::outflow;
76  domain_bc_type[ori] = "Outflow";
77  }
78  else if (bc_type == "open")
79  {
80  // Print() << bcid << " set to open.\n";
81  AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
82  phys_bc_type[ori] = ERF_BC::open;
83  domain_bc_type[ori] = "Open";
84  }
85  else if (bc_type == "ho_outflow")
86  {
87  phys_bc_type[ori] = ERF_BC::ho_outflow;
88  domain_bc_type[ori] = "HO_Outflow";
89  }
90 
91  else if (bc_type == "inflow" || bc_type == "inflow_outflow")
92  {
93  if (bc_type == "inflow") {
94  // Print() << bcid << " set to inflow.\n";
95  phys_bc_type[ori] = ERF_BC::inflow;
96  domain_bc_type[ori] = "Inflow";
97  } else {
98  // Print() << bcid << " set to inflow_outflow.\n";
99  phys_bc_type[ori] = ERF_BC::inflow_outflow;
100  domain_bc_type[ori] = "InflowOutflow";
101  }
102 
103  std::vector<Real> v;
104  if (input_bndry_planes && m_r2d->ingested_velocity()) {
105  m_bc_extdir_vals[BCVars::xvel_bc][ori] = zero_d;
106  m_bc_extdir_vals[BCVars::yvel_bc][ori] = zero_d;
107  m_bc_extdir_vals[BCVars::zvel_bc][ori] = zero_d;
108  } else {
109  // Test for input data file if at xlo face
110  std::string dirichlet_file;
111  auto file_exists = pp.query("dirichlet_file", dirichlet_file);
112  if (file_exists) {
113  pp.query("read_prim_theta", read_prim_theta);
114  init_Dirichlet_bc_data(dirichlet_file);
115  } else {
116  pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
117  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
118  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
119  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
120  }
121  }
122 
123  Real rho_in = zero_d;
124  if (input_bndry_planes && m_r2d->ingested_density()) {
125  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = zero_d;
126  } else {
127  if (!pp.query("density", rho_in)) {
128  amrex::Print() << "Using interior values to set conserved vars" << std::endl;
129  }
130  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
131  }
132 
133  bool th_read = (th_bc_data[0].data()!=nullptr);
134  Real theta_in = zero_d;
135  if (input_bndry_planes && m_r2d->ingested_theta()) {
136  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = zero_d;
137  } else if (!th_read) {
138  if (rho_in > 0) {
139  pp.get("theta", theta_in);
140  }
141  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
142  }
143 
144  // Non-reflecting inflow: prescribe velocity and density but
145  // extrapolate RhoTheta (and hence pressure) from the interior.
146  // This lets upstream-propagating acoustic waves exit the domain
147  // instead of reflecting off the rigid Dirichlet boundary.
148  bool nonreflecting = false;
149  pp.query("nonreflecting", nonreflecting);
150  m_bc_nonreflecting[ori] = nonreflecting;
151 
152  Real scalar_in = zero_d;
153  if (input_bndry_planes && m_r2d->ingested_scalar()) {
154  m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = zero_d;
155  } else {
156  if (pp.query("scalar", scalar_in))
157  m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
158  }
159 
160  if (solverChoice.moisture_type != MoistureType::None) {
161  Real qv_in = zero_d;
162  if (input_bndry_planes && m_r2d->ingested_q1()) {
163  m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = zero_d;
164  } else {
165  if (pp.query("qv", qv_in))
166  m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
167  }
168  Real qc_in = zero_d;
169  if (input_bndry_planes && m_r2d->ingested_q2()) {
170  m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = zero_d;
171  } else {
172  if (pp.query("qc", qc_in))
173  m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
174  }
175  }
176 
177  Real KE_in = zero_d;
178  if (input_bndry_planes && m_r2d->ingested_KE()) {
179  m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = zero_d;
180  } else {
181  if (pp.query("KE", KE_in))
182  m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
183  }
184  }
185  else if (bc_type == "noslipwall")
186  {
187  // Print() << bcid <<" set to no-slip wall.\n";
188  phys_bc_type[ori] = ERF_BC::no_slip_wall;
189  domain_bc_type[ori] = "NoSlipWall";
190 
191  std::vector<Real> v;
192 
193  // The values of m_bc_extdir_vals default to zero
194  // But if we find "velocity" in the inputs file, use those values instead.
195  if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
196  {
197  v[ori.coordDir()] = zero_d;
198  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
199  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
200  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
201  }
202 
203  Real rho_in;
204  rho_read = pp.query("density", rho_in);
205  if (rho_read)
206  {
207  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
208  }
209 
210  Real theta_in;
211  if (pp.query("theta", theta_in))
212  {
213  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
214  }
215 
216  Real theta_grad_in;
217  if (pp.query("theta_grad", theta_grad_in))
218  {
219  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
220  }
221 
222  Real qv_in;
223  if (pp.query("qv", qv_in))
224  {
225  m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = qv_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
226  }
227  }
228  else if (bc_type == "slipwall")
229  {
230  // Print() << bcid <<" set to slip wall.\n";
231 
232  phys_bc_type[ori] = ERF_BC::slip_wall;
233  domain_bc_type[ori] = "SlipWall";
234 
235  Real rho_in;
236  rho_read = pp.query("density", rho_in);
237  if (rho_read)
238  {
239  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
240  }
241 
242  Real theta_in;
243  if (pp.query("theta", theta_in))
244  {
245  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = theta_in*m_bc_extdir_vals[BCVars::Rho_bc_comp][ori];
246  }
247 
248  Real rho_grad_in;
249  if (pp.query("density_grad", rho_grad_in))
250  {
251  m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = rho_grad_in;
252  }
253 
254  Real theta_grad_in;
255  if (pp.query("theta_grad", theta_grad_in))
256  {
257  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
258  }
259  }
260  else if (bc_type == "surface_layer")
261  {
262  phys_bc_type[ori] = ERF_BC::surface_layer;
263  domain_bc_type[ori] = "surface_layer";
264  }
265  else
266  {
267  phys_bc_type[ori] = ERF_BC::undefined;
268  }
269 
270  if (geom[0].isPeriodic(ori.coordDir())) {
271  domain_bc_type[ori] = "Periodic";
272  if (phys_bc_type[ori] == ERF_BC::undefined)
273  {
274  phys_bc_type[ori] = ERF_BC::periodic;
275  } else {
276  Abort("Wrong BC type for periodic boundary");
277  }
278  }
279 
280  if (phys_bc_type[ori] == ERF_BC::undefined)
281  {
282  if (solverChoice.use_real_bcs) {
283  Print() << "We are using real bc's so don't need to set lateral bc's" << std::endl;
284  } else {
285  Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
286  Abort("This BC type is unknown");
287  }
288  }
289  };
290 
291  f("xlo", Orientation(Direction::x,Orientation::low));
292  f("xhi", Orientation(Direction::x,Orientation::high));
293  f("ylo", Orientation(Direction::y,Orientation::low));
294  f("yhi", Orientation(Direction::y,Orientation::high));
295  f("zlo", Orientation(Direction::z,Orientation::low));
296  f("zhi", Orientation(Direction::z,Orientation::high));
297 }
ERF_BC::undefined
@ undefined
AMREX_ASSERT_WITH_MESSAGE
AMREX_ASSERT_WITH_MESSAGE(wbar_cutoff_min > wbar_cutoff_max, "ERROR: wbar_cutoff_min < wbar_cutoff_max")
ERF::init_Dirichlet_bc_data
void init_Dirichlet_bc_data(const std::string input_file)
Definition: ERF_InitBCs.cpp:735
BCVars::RhoQ6_bc_comp
@ RhoQ6_bc_comp
Definition: ERF_IndexDefines.H:96
BCVars::RhoQ4_bc_comp
@ RhoQ4_bc_comp
Definition: ERF_IndexDefines.H:94
BCVars::RhoQ3_bc_comp
@ RhoQ3_bc_comp
Definition: ERF_IndexDefines.H:93
BCVars::RhoQ5_bc_comp
@ RhoQ5_bc_comp
Definition: ERF_IndexDefines.H:95
Here is the call graph for this function:

◆ init_stuff()

void ERF::init_stuff ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm,
amrex::Vector< amrex::MultiFab > &  lev_new,
amrex::Vector< amrex::MultiFab > &  lev_old,
amrex::MultiFab &  tmp_base_state,
std::unique_ptr< amrex::MultiFab > &  tmp_zphys_nd 
)
private
28 {
29  // ********************************************************************************************
30  // Base state holds r_0, pres_0, pi_0, th_0 (in that order)
31  //
32  // Here is where we set the number of ghost cells for the base state!
33  // ********************************************************************************************
34  int ngb = (solverChoice.terrain_type == TerrainType::EB) ? ComputeGhostCells(solverChoice)+1 : 3;
35  tmp_base_state.define(ba,dm,BaseState::num_comps,ngb);
36  tmp_base_state.setVal(zero);
37 
38  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
39  base_state_new[lev].define(ba,dm,BaseState::num_comps,base_state[lev].nGrowVect());
40  base_state_new[lev].setVal(zero);
41  }
42 
43  // ********************************************************************************************
44  // Allocate terrain arrays
45  // ********************************************************************************************
46 
47  BoxArray ba_nd(ba);
48  ba_nd.surroundingNodes();
49 
50  // NOTE: this is where we actually allocate z_phys_nd -- but here it's called "tmp_zphys_nd"
51  // We need this to be one greater than the ghost cells to handle levels > 0
52 
53  int ngrow = ComputeGhostCells(solverChoice) + 2;
54  tmp_zphys_nd = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
55 
56  // Offset z-coordinate for interpolate_1d when plane EB is used.
57  Real z_offset = zero;
58  if (solverChoice.terrain_type == TerrainType::EB) {
59  ParmParse pp_eb2("eb2");
60  std::string geometry;
61  pp_eb2.query("geometry", geometry);
62  if (geometry == "plane") {
63  RealArray plane_point{zero, zero, zero};
64  pp_eb2.query("plane_point", plane_point);
65  z_offset = plane_point[2];
66  }
67  }
68 
69  z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,2);
70  init_default_zphys(lev, geom[lev], *tmp_zphys_nd, *z_phys_cc[lev], z_offset);
71 
72  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh)
73  {
74  detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
75  detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
76 
77  ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
78  ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
79  az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
80 
81  z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
82 
83  z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
84  z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
85  z_phys_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
86  }
87  else
88  {
89  z_phys_nd_new[lev] = nullptr;
90  detJ_cc_new[lev] = nullptr;
91 
92  z_phys_nd_src[lev] = nullptr;
93  z_phys_cc_src[lev] = nullptr;
94  detJ_cc_src[lev] = nullptr;
95 
96  z_t_rk[lev] = nullptr;
97  }
98 
99  if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
100  solverChoice.buildings_type == BuildingsType::ImmersedForcing)
101  {
102  terrain_blanking[lev] = std::make_unique<MultiFab>(ba,dm,1,ngrow);
103  terrain_blanking[lev]->setVal(one);
104  }
105 
106  // We use these area arrays regardless of terrain, EB or none of the above
107  detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
108  ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
109  ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
110  az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
111 
112  detJ_cc[lev]->setVal(one);
113  ax[lev]->setVal(one);
114  ay[lev]->setVal(one);
115  az[lev]->setVal(one);
116 
117  // ********************************************************************************************
118  // Create wall distance array for RANS modeling
119  // ********************************************************************************************
120  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
121  walldist[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
122  walldist[lev]->setVal(bogus_large_value);
123  } else {
124  walldist[lev] = nullptr;
125  }
126 
127  // ********************************************************************************************
128  // These are the persistent containers for the old and new data
129  // ********************************************************************************************
130  int ncomp;
131  if (lev > 0) {
132  ncomp = vars_new[lev-1][Vars::cons].nComp();
133  } else {
134  int n_qstate = micro->Get_Qstate_Size();
135  ncomp = NDRY + NSCALARS + n_qstate;
136  }
137 
138  // ********************************************************************************************
139  // The number of ghost cells for density must be 1 greater than that for velocity
140  // so that we can go back in forth between velocity and momentum on all faces
141  // ********************************************************************************************
142  int ngrow_state = ComputeGhostCells(solverChoice) + 1;
143  int ngrow_vels = ComputeGhostCells(solverChoice);
144 
145  // ********************************************************************************************
146  // New solution data containers
147  // ********************************************************************************************
148  if (solverChoice.terrain_type != TerrainType::EB) {
149  lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
150  lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
151  } else {
152  // EB: Define the MultiFabs with the EBFactory
153  lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
154  lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
155  }
156 
157  // Initialize all components to zero so we don't need to explicitly set
158  // scalars / moisture variables to zero in the initialization
159  lev_new[Vars::cons].setVal(zero);
160  lev_old[Vars::cons].setVal(zero);
161 
162  lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
163  lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
164 
165  lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
166  lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
167 
168  // Set these to avoid operations on uninitialized data
169  lev_new[Vars::xvel].setVal(bogus_large_value);
170  lev_old[Vars::xvel].setVal(bogus_large_value);
171  lev_new[Vars::yvel].setVal(bogus_large_value);
172  lev_old[Vars::yvel].setVal(bogus_large_value);
173 
174  // Note that we need the ghost cells in the z-direction if we are doing any
175  // kind of domain decomposition in the vertical (at level 0 or above)
176  lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
177  lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
178 
179  gradp[lev][GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp[lev][GpVars::gpx].setVal(zero);
180  gradp[lev][GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp[lev][GpVars::gpy].setVal(zero);
181  gradp[lev][GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp[lev][GpVars::gpz].setVal(zero);
182 
183  if ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) {
184  pp_inc[lev].define(ba, dm, 1, 1);
185  pp_inc[lev].setVal(zero);
186  }
187 
188  // We use this in the fast substepping only
189  if (solverChoice.anelastic[lev] == 0) {
190  lagged_delta_rt[lev].define(ba, dm, 1, 1);
191  lagged_delta_rt[lev].setVal(zero);
192  }
193 
194  // We use these for advecting the slow variables, whether anelastic or compressible
195  avg_xmom[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, 1);
196  avg_ymom[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, 1);
197  avg_zmom[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, 1);
198  avg_xmom[lev].setVal(0); avg_ymom[lev].setVal(0); avg_zmom[lev].setVal(zero);
199 
200  // ********************************************************************************************
201  // These are just used for scratch in the time integrator but we might as well define them here
202  // ********************************************************************************************
203  if (solverChoice.terrain_type != TerrainType::EB) {
204  rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
205  rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
206 
207  rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
208  rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
209 
210  rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
211  rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
212  } else {
213  // EB: Define the MultiFabs with the EBFactory
214  rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
215  rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
216 
217  rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
218  rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
219 
220  rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
221  rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
222  }
223 
224  if (lev > 0) {
225  //xmom_crse_rhs[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, IntVect{0});
226  //ymom_crse_rhs[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, IntVect{0});
227  zmom_crse_rhs[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect{0});
228  }
229 
230  // We do this here just so they won't be undefined in the initial FillPatch
231  rU_old[lev].setVal(bogus_large_value);
232  rV_old[lev].setVal(bogus_large_value);
233  rW_old[lev].setVal(bogus_large_value);
234  rU_new[lev].setVal(bogus_large_value);
235  rV_new[lev].setVal(bogus_large_value);
236  rW_new[lev].setVal(bogus_large_value);
237 
238  // ********************************************************************************************
239  // These are just time averaged fields for diagnostics
240  // ********************************************************************************************
241 
242  // NOTE: We are not completing a fillpatch call on the time averaged data;
243  // which would copy on intersection and interpolate from coarse.
244  // Therefore, we are restarting the averaging when the ba changes,
245  // this may give poor statistics for dynamic mesh refinement.
246  vel_t_avg[lev] = nullptr;
247  if (solverChoice.time_avg_vel) {
248  vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
249  vel_t_avg[lev]->setVal(zero);
250  t_avg_cnt[lev] = zero;
251  }
252 
253  // ********************************************************************************************
254  // Initialize flux registers whenever we create/re-create a level
255  // ********************************************************************************************
256  if (solverChoice.coupling_type == CouplingType::TwoWay) {
257  if (lev == 0) {
258  advflux_reg[0] = nullptr;
259  } else {
260  int ncomp_reflux = vars_new[0][Vars::cons].nComp();
261  advflux_reg[lev] = new YAFluxRegister(ba , grids[lev-1],
262  dm , dmap[lev-1],
263  geom[lev], geom[lev-1],
264  ref_ratio[lev-1], lev, ncomp_reflux);
265  }
266  }
267 
268  // ********************************************************************************************
269  // Define Theta_prim storage if using surface_layer BC
270  // ********************************************************************************************
271  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
272  Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
273  if (solverChoice.moisture_type != MoistureType::None) {
274  Qv_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
275  Qr_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
276  } else {
277  Qv_prim[lev] = nullptr;
278  Qr_prim[lev] = nullptr;
279  }
280  } else {
281  Theta_prim[lev] = nullptr;
282  Qv_prim[lev] = nullptr;
283  Qr_prim[lev] = nullptr;
284  }
285 
286  // ********************************************************************************************
287  // Build 1D BA and 2D BA
288  // ********************************************************************************************
289 
290  // NOTE: By design, the compressed BAs have their compressed indices set to 0
291  // MFIters that need more detailed box information should be done over 3D MFs
292 
293  // Build 2D BA
294  BoxList bl2d = ba.boxList();
295  for (auto& b : bl2d) {
296  b.setRange(2,0);
297  }
298  ba2d[lev] = BoxArray(std::move(bl2d));
299 
300  // Build 1D BA
301  BoxList bl1d = ba.boxList();
302  for (auto& b : bl1d) {
303  b.setRange(0,0);
304  b.setRange(1,0);
305  }
306  ba1d[lev] = BoxArray(std::move(bl1d));
307 
308  // ********************************************************************************************
309  // Map factors
310  // ********************************************************************************************
311  mapfac[lev].resize(MapFacType::num);
312  mapfac[lev][MapFacType::m_x] = std::make_unique<MultiFab>( ba2d[lev],dm,1,IntVect(3,3,0));
313  mapfac[lev][MapFacType::u_x] = std::make_unique<MultiFab>(convert(ba2d[lev],IntVect(1,0,0)),dm,1,IntVect(3,3,0));
314  mapfac[lev][MapFacType::v_x] = std::make_unique<MultiFab>(convert(ba2d[lev],IntVect(0,1,0)),dm,1,IntVect(3,3,0));
315 
316 #if 0
317  // For now we comment this out to avoid CI failures but we will need to re-enable
318  // this if using non-conformal mappings
319  if (MapFacType::m_y != MapFacType::m_x) {
320  mapfac[lev][MapFacType::m_y] = std::make_unique<MultiFab>(ba2d[lev],dm,1,IntVect(3,3,0));
321  }
322  if (MapFacType::u_y != MapFacType::u_x) {
323  mapfac[lev][MapFacType::u_y] = std::make_unique<MultiFab>(convert(ba2d[lev],IntVect(1,0,0)),dm,1,IntVect(3,3,0));
324  }
325  if (MapFacType::v_y != MapFacType::v_x) {
326  mapfac[lev][MapFacType::v_y] = std::make_unique<MultiFab>(convert(ba2d[lev],IntVect(0,1,0)),dm,1,IntVect(3,3,0));
327  }
328 #endif
329 
330  if (solverChoice.test_mapfactor) {
331  for (int i = 0; i < 3; i++) {
332  mapfac[lev][i]->setVal(myhalf);
333  }
334  for (int i = 3; i < mapfac[lev].size(); i++) {
335  mapfac[lev][i]->setVal(fourth);
336  }
337  } else {
338  for (int i = 0; i < mapfac[lev].size(); i++) {
339  mapfac[lev][i]->setVal(one);
340  }
341  }
342 
343  // ********************************************************************************************
344  // Build WRF data structures
345  // ********************************************************************************************
346  IntVect ng = vars_new[lev][Vars::cons].nGrowVect();
347 
348  if (lev == 0) {
349  wrf_C1H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
350  wrf_C2H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
351  wrf_RDNW = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
352  wrf_MUB = std::make_unique<MultiFab>(ba2d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
353  wrf_PHB = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,IntVect(ngrow+1,ngrow+1,0));
354  }
355 
356  mf_PSFC[lev] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ng);
357 
358  //*********************************************************
359  // Variables for Fitch model for windfarm parametrization
360  //*********************************************************
361 #if defined(ERF_USE_WINDFARM)
362  if (solverChoice.windfarm_type == WindFarmType::Fitch){
363  vars_windfarm[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
364  }
365  if (solverChoice.windfarm_type == WindFarmType::EWP){
366  vars_windfarm[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
367  }
368  if (solverChoice.windfarm_type == WindFarmType::SimpleAD) {
369  vars_windfarm[lev].define(ba, dm, 2, ngrow_state);// dudt, dvdt
370  }
371  if (solverChoice.windfarm_type == WindFarmType::GeneralAD) {
372  vars_windfarm[lev].define(ba, dm, 3, ngrow_state);// dudt, dvdt, dwdt
373  }
374  Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
375  SMark[lev].define(ba, dm, 2, 1); // Free stream velocity/source term
376  // sampling marker in a cell - 2 components
377 #endif
378 
379  if(solverChoice.init_type == InitType::HindCast and
380  solverChoice.hindcast_lateral_forcing) {
381 
382  int ncomp_extra = 2;
383  int nvars = vars_new[lev].size();
384 
385  // Resize all containers
386  forecast_state_1[lev].resize(nvars + 1);
387  forecast_state_2[lev].resize(nvars + 1);
388  forecast_state_interp[lev].resize(nvars + 1);
389 
390  // Define the "normal" components
391  for (int comp = 0; comp < nvars; ++comp) {
392  const MultiFab& src = vars_new[lev][comp];
393  ncomp = src.nComp();
394  ngrow = src.nGrow();
395 
396  forecast_state_1[lev][comp].define(ba, dm, ncomp, ng);
397  forecast_state_2[lev][comp].define(ba, dm, ncomp, ng);
398  forecast_state_interp[lev][comp].define(ba, dm, ncomp, ng);
399  }
400 
401  // Define the "extra" component (last slot)
402  {
403  const MultiFab& src0 = vars_new[lev][0];
404  ngrow = src0.nGrow();
405  int idx = nvars;
406 
407  forecast_state_1[lev][idx].define(ba, dm, ncomp_extra, ngrow);
408  forecast_state_2[lev][idx].define(ba, dm, ncomp_extra, ngrow);
409  forecast_state_interp[lev][idx].define(ba, dm, ncomp_extra, ngrow);
410  }
411  bool regrid_forces_file_read = true;
412  WeatherDataInterpolation(lev, t_new[0],z_phys_nd, regrid_forces_file_read);
413  }
414 
415 
416  if(solverChoice.init_type == InitType::HindCast and
417  solverChoice.hindcast_surface_bcs)
418 
419  {
420  const MultiFab& src = vars_new[lev][0];
421  const amrex::DistributionMapping& dm_hc = src.DistributionMap();
422 
423  surface_state_1[lev].define(ba2d[lev], dm_hc, 2, src.nGrow());
424  surface_state_2[lev].define(ba2d[lev], dm_hc, 2, src.nGrow());
425  surface_state_interp[lev].define(ba2d[lev], dm_hc, 2, src.nGrow());
426 
427  bool regrid_forces_file_read = true;
428  SurfaceDataInterpolation(lev, t_new[0], z_phys_nd, regrid_forces_file_read);
429  }
430 
431 #ifdef ERF_USE_WW3_COUPLING
432  // create a new BoxArray and DistributionMapping for a MultiFab with 1 box
433  BoxArray ba_onegrid(geom[lev].Domain());
434  BoxList bl2d_onegrid = ba_onegrid.boxList();
435  for (auto& b : bl2d_onegrid) { b.setRange(2,b.smallEnd(2)); }
436  BoxArray ba2d_onegrid(std::move(bl2d_onegrid));
437  Vector<int> pmap;
438  pmap.resize(1);
439  pmap[0]=0;
440  DistributionMapping dm_onegrid(ba2d_onegrid);
441  dm_onegrid.define(pmap);
442 
443  Hwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
444  Lwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
445 
446  BoxList bl2d_wave = ba.boxList();
447  for (auto& b : bl2d_wave) { b.setRange(2,b.smallEnd(2)); }
448  BoxArray ba2d_wave(std::move(bl2d_wave));
449 
450  Hwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
451  Lwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
452 
453  std::cout<<ba_onegrid<<std::endl;
454  std::cout<<ba2d_onegrid<<std::endl;
455  std::cout<<dm_onegrid<<std::endl;
456 #endif
457 
458 
459  //*********************************************************
460  // Radiation heating source terms
461  //*********************************************************
462  if (solverChoice.rad_type != RadiationType::None)
463  {
464  qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, 0);
465  rad_fluxes[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0);
466  qheating_rates[lev]->setVal(zero);
467  rad_fluxes[lev]->setVal(zero);
468  }
469 
470  //*********************************************************
471  // Turbulent perturbation region initialization
472  //*********************************************************
473  if (solverChoice.use_perturbation(lev))
474  {
475  turbPert.init_tpi_type(lev, solverChoice.pert_type[lev], max_level);
476  turbPert.init_tpi(lev, subdomains[lev], geom[lev].CellSizeArray(),
477  ba, dm, ngrow_state, pp_prefix, refRatio(), max_level);
478  }
479 
480  //
481  // Define the land mask here and set it to all land by default
482  // NOTE: the logic below will BREAK if we have any grids not touching the bottom boundary
483  //
484  {
485  lmask_lev[lev].resize(1);
486  auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0;
487  lmask_lev[lev][0] = std::make_unique<iMultiFab>(ba2d[lev],dm,1,ngv);
488  lmask_lev[lev][0]->setVal(solverChoice.is_land[lev]);
489  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
490 
491  land_type_lev[lev].resize(1);
492  land_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d[lev],dm,1,ngv);
493  land_type_lev[lev][0]->setVal(0);
494  land_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
495 
496  soil_type_lev[lev].resize(1);
497  soil_type_lev[lev][0] = std::make_unique<iMultiFab>(ba2d[lev],dm,1,ngv);
498  soil_type_lev[lev][0]->setVal(0);
499  soil_type_lev[lev][0]->FillBoundary(geom[lev].periodicity());
500 
501  urb_frac_lev[lev].resize(1);
502  urb_frac_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ngv);
503  urb_frac_lev[lev][0]->setVal(one);
504  urb_frac_lev[lev][0]->FillBoundary(geom[lev].periodicity());
505  }
506 
507  // Read in tables needed for windfarm simulations
508  // fill in Nturb multifab - number of turbines in each mesh cell
509  // write out the vtk files for wind turbine location and/or
510  // actuator disks
511  #ifdef ERF_USE_WINDFARM
512  //init_windfarm(lev);
513  #endif
514 
515  if (lev > 0) {
516  fine_mask[lev] = std::make_unique<MultiFab>(grids[lev-1], dmap[lev-1], 1, 0);
517  build_fine_mask(lev, *fine_mask[lev].get());
518  }
519 
520 #ifdef ERF_USE_FFT
521  if ( ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) &&
522  ( (solverChoice.mesh_type == MeshType::ConstantDz) || (solverChoice.mesh_type == MeshType::StretchedDz) ) ) {
523  build_fft_solvers(lev);
524  }
525 #endif
526 }
num
@ num
Definition: ERF_DataStruct.H:24
NDRY
#define NDRY
Definition: ERF_IndexDefines.H:13
init_default_zphys
void init_default_zphys(int, const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc, Real z_offset)
Definition: ERF_TerrainMetrics.cpp:15
ERF::wrf_MUB
std::unique_ptr< amrex::MultiFab > wrf_MUB
Definition: ERF.H:1390
ERF::wrf_C1H
std::unique_ptr< amrex::MultiFab > wrf_C1H
Definition: ERF.H:1388
ERF::wrf_PHB
std::unique_ptr< amrex::MultiFab > wrf_PHB
Definition: ERF.H:1392
ERF::build_fine_mask
void build_fine_mask(int lev, amrex::MultiFab &fine_mask)
Definition: ERF_VolWgtSum.cpp:125
ERF::ComputeGhostCells
static AMREX_FORCE_INLINE int ComputeGhostCells(const SolverChoice &sc)
Definition: ERF.H:1484
ERF::wrf_RDNW
std::unique_ptr< amrex::MultiFab > wrf_RDNW
Definition: ERF.H:1391
ERF::EBFactory
amrex::EBFArrayBoxFactory const & EBFactory(int lev) const noexcept
Definition: ERF.H:1763
ERF::wrf_C2H
std::unique_ptr< amrex::MultiFab > wrf_C2H
Definition: ERF.H:1389
BaseState::num_comps
@ num_comps
Definition: ERF_IndexDefines.H:78
GpVars::gpz
@ gpz
Definition: ERF_IndexDefines.H:187
GpVars::gpy
@ gpy
Definition: ERF_IndexDefines.H:186
GpVars::gpx
@ gpx
Definition: ERF_IndexDefines.H:185
SolverChoice::pert_type
amrex::Vector< PerturbationType > pert_type
Definition: ERF_DataStruct.H:1417
SolverChoice::is_land
amrex::Vector< int > is_land
Definition: ERF_DataStruct.H:1261
SolverChoice::test_mapfactor
bool test_mapfactor
Definition: ERF_DataStruct.H:1292
TurbulentPerturbation::init_tpi_type
void init_tpi_type(const int lev, const PerturbationType &pert_type, const int max_level)
Definition: ERF_TurbPertStruct.H:28
TurbulentPerturbation::init_tpi
void init_tpi(const int lev, const amrex::Vector< amrex::BoxArray > &subdomains_lev, const amrex::GpuArray< amrex::Real, 3 > dx, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, const int ngrow_state, std::string pp_prefix, const amrex::Vector< amrex::IntVect > refRatio, const int max_level)
Definition: ERF_TurbPertStruct.H:54
Here is the call graph for this function:

◆ init_thin_body()

void ERF::init_thin_body ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
926 {
927  //********************************************************************************************
928  // Thin immersed body
929  // *******************************************************************************************
930 #if 0
931  if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
932  (solverChoice.advChoice.zero_yflux.size() > 0) ||
933  (solverChoice.advChoice.zero_zflux.size() > 0))
934  {
935  overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
936  overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
937  }
938 #endif
939 
940  if (solverChoice.advChoice.zero_xflux.size() > 0) {
941  amrex::Print() << "Setting up thin immersed body for "
942  << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
943  BoxArray ba_xf(ba);
944  ba_xf.surroundingNodes(0);
945  thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
946  thin_xforce[lev]->setVal(0.0);
947  xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
948  xflux_imask[lev]->setVal(1);
949  for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
950  {
951  Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
952  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
953  Box xbx = mfi.nodaltilebox(0);
954  for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
955  const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
956  if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
957  (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
958  (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
959  {
960  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
961  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
962  //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
963  amrex::AllPrint() << " mask xface at " << faceidx << std::endl;
964  }
965  }
966  }
967  } else {
968  thin_xforce[lev] = nullptr;
969  xflux_imask[lev] = nullptr;
970  }
971 
972  if (solverChoice.advChoice.zero_yflux.size() > 0) {
973  amrex::Print() << "Setting up thin immersed body for "
974  << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
975  BoxArray ba_yf(ba);
976  ba_yf.surroundingNodes(1);
977  thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
978  thin_yforce[lev]->setVal(0.0);
979  yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
980  yflux_imask[lev]->setVal(1);
981  for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
982  {
983  Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
984  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
985  Box ybx = mfi.nodaltilebox(1);
986  for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
987  const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
988  if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
989  (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
990  (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
991  {
992  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
993  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
994  //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
995  amrex::AllPrint() << " mask yface at " << faceidx << std::endl;
996  }
997  }
998  }
999  } else {
1000  thin_yforce[lev] = nullptr;
1001  yflux_imask[lev] = nullptr;
1002  }
1003 
1004  if (solverChoice.advChoice.zero_zflux.size() > 0) {
1005  amrex::Print() << "Setting up thin immersed body for "
1006  << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
1007  BoxArray ba_zf(ba);
1008  ba_zf.surroundingNodes(2);
1009  thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
1010  thin_zforce[lev]->setVal(0.0);
1011  zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
1012  zflux_imask[lev]->setVal(1);
1013  for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
1014  {
1015  Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
1016  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
1017  Box zbx = mfi.nodaltilebox(2);
1018  for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
1019  const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
1020  if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
1021  (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
1022  (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
1023  {
1024  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
1025  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
1026  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
1027  amrex::AllPrint() << " mask zface at " << faceidx << std::endl;
1028  }
1029  }
1030  }
1031  } else {
1032  thin_zforce[lev] = nullptr;
1033  zflux_imask[lev] = nullptr;
1034  }
1035 }
AdvChoice::zero_yflux
amrex::Vector< amrex::IntVect > zero_yflux
Definition: ERF_AdvStruct.H:438
AdvChoice::zero_xflux
amrex::Vector< amrex::IntVect > zero_xflux
Definition: ERF_AdvStruct.H:437
AdvChoice::zero_zflux
amrex::Vector< amrex::IntVect > zero_zflux
Definition: ERF_AdvStruct.H:439

◆ init_zphys()

void ERF::init_zphys ( int  lev,
double  elapsed_time 
)
689 {
690  // For EB, z_phys_nd was already initialized with the correct z_offset by init_default_zphys.
691  // The terrain-fitting (BTF) done below is irrelevant for a flat EB mesh and would clobber
692  // the offset, so return early here.
693  if (solverChoice.terrain_type == TerrainType::EB) {
694  Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
695  micro->Set_dzmin(lev, dzmin);
696  return;
697  }
698 
699  if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
700  {
701  if (lev > 0) {
702  //
703  // First interpolate from coarser level if there is one
704  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
705  // have been pre-filled - this includes ghost cells both inside and outside
706  // the domain
707  //
708  InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
709  IntVect(0,0,0), // do NOT fill ghost cells outside the domain
710  *z_phys_nd[lev-1], 0, 0, 1,
711  geom[lev-1], geom[lev],
712  refRatio(lev-1), &node_bilinear_interp,
713  domain_bcs_type, BCVars::cons_bc);
714  }
715 
716  int ngrow = ComputeGhostCells(solverChoice) + 2;
717  Box bx(surroundingNodes(Geom(lev).Domain())); bx.grow(ngrow);
718  FArrayBox terrain_fab(makeSlab(bx,2,0),1);
719 
720  //
721  // If we are using fitted mesh then we use the surface as defined above
722  // If we are not using fitted mesh but are using z_levels, we still need z_phys (for now)
723  // but we need to use a flat terrain for the mesh itself (the EB data has already been made
724  // from the correct terrain)
725  //
726  if (solverChoice.terrain_type != TerrainType::StaticFittedMesh &&
727  solverChoice.terrain_type != TerrainType::MovingFittedMesh) {
728  terrain_fab.template setVal<RunOn::Device>(zero);
729  } else {
730  //
731  // Fill the values of the terrain height at k=0 only
732  //
733  prob->init_terrain_surface(geom[lev],terrain_fab,elapsed_time);
734  }
735 
736  for (MFIter mfi(*z_phys_nd[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
737  {
738  Box isect = terrain_fab.box() & (*z_phys_nd[lev])[mfi].box();
739  if (!isect.isEmpty()) {
740  (*z_phys_nd[lev])[mfi].template copy<RunOn::Device>(terrain_fab,isect,0,isect,0,1);
741  }
742  }
743 
744  make_terrain_fitted_coords(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type);
745 
746  z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
747 
748  if (lev == 0) {
749  Real zmax = z_phys_nd[0]->max(0,0,false);
750  Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
751  if (rel_diff < Real(1.e-8)) {
752  amrex::Print() << "max of zphys_nd " << zmax << std::endl;
753  amrex::Print() << "max of zlevels " << zlevels_stag[0][zlevels_stag[0].size()-1] << std::endl;
754  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < Real(1.e-8), "Terrain is taller than domain top!");
755  }
756 
757 #if 0
758  // This remains commented out until we verify that the stretched and variable dz pathways
759  // in fact give the same answer when appropriate
760  if (SolverChoice::mesh_type == MeshType::VariableDz)
761  {
762  check_mesh_type(lev);
763  }
764 #endif
765  } // lev == 0
766 
767  } else {
768  // NOTE: If a WRFInput file is NOT provided for a finer level,
769  // we simply interpolate from the coarse. This is necessary
770  // since we average_down the terrain (ERF_MakeNewLevel.cpp L351).
771  // If a WRFInput file IS present, it overwrites the terrain data.
772  if (lev > 0) {
773  //
774  // First interpolate from coarser level if there is one
775  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
776  // have been pre-filled - this includes ghost cells both inside and outside
777  // the domain
778  //
779  InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
780  z_phys_nd[lev]->nGrowVect(), // DO fill ghost cells outside the domain
781  *z_phys_nd[lev-1], 0, 0, 1,
782  geom[lev-1], geom[lev],
783  refRatio(lev-1), &node_bilinear_interp,
784  domain_bcs_type, BCVars::cons_bc);
785  }
786  } // init_type
787 
788  if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
789  solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
790  terrain_blanking[lev]->setVal(one);
791  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ComputeGhostCells(solverChoice) + 2);
792  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
793  init_immersed_forcing(lev); // needed for real cases
794 
795  // buildings are landmask = 2
796  for (MFIter mfi(*lmask_lev[lev][0]); mfi.isValid(); ++mfi) {
797  const Box& bx2d = mfi.growntilebox();
798  auto lmask_arr = lmask_lev[lev][0]->array(mfi);
799  const auto& t_blank_arr = terrain_blanking[lev]->array(mfi);
800 
801  amrex::ParallelFor(bx2d, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
802  // Use k=0 for the terrain_blanking field
803  if (t_blank_arr(i, j, 0) > 0.0) {
804  lmask_arr(i, j, k) = 2;
805  }
806  });
807  }
808  }
809 
810  // Compute the min dz and pass to the micro model
811  Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
812  micro->Set_dzmin(lev, dzmin);
813 }
get_dzmin_terrain
Real get_dzmin_terrain(MultiFab &z_phys_nd)
Definition: ERF_TerrainMetrics.cpp:653
make_terrain_fitted_coords
void make_terrain_fitted_coords(int lev, const Geometry &geom, MultiFab &z_phys_nd, Vector< Real > const &z_levels_h, GpuArray< ERF_BC, AMREX_SPACEDIM *2 > &phys_bc_type)
Definition: ERF_TerrainMetrics.cpp:47
ERF::check_mesh_type
void check_mesh_type(int lev)
Definition: ERF.cpp:2953
Here is the call graph for this function:

◆ InitData()

void ERF::InitData ( )
513 {
514  BL_PROFILE_VAR("ERF::InitData()", InitData);
515  InitData_pre();
516  InitData_post();
517  BL_PROFILE_VAR_STOP(InitData);
518 }
ERF::InitData_pre
void InitData_pre()
Definition: ERF.cpp:521
ERF::InitData_post
void InitData_post()
Definition: ERF.cpp:553
ERF::InitData
void InitData()
Definition: ERF.cpp:512

Referenced by main().

Here is the caller graph for this function:

◆ InitData_post()

void ERF::InitData_post ( )
554 {
555  if (solverChoice.advChoice.have_zero_flux_faces)
556  {
557  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
558  "Thin immersed body with refinement not currently supported.");
559  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
560  amrex::Print() << "NOTE: Thin immersed body with non-constant dz has not been tested." << std::endl;
561  }
562  }
563 
564  if (!restart_chkfile.empty()) {
565  restart();
566  }
567 
568  // Select 2-D variables after the active LSM has initialized its runtime
569  // field inventory, including provider-specific soil layers.
570  setPlotVariables2D("plot2d_vars_1", plot2d_var_names_1);
571  setPlotVariables2D("plot2d_vars_2", plot2d_var_names_2);
572 
573  //
574  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one and if two way coupling
575  //
576  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
577  if (solverChoice.coupling_type == CouplingType::TwoWay) {
578  for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
579  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
580  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
581  }
582  }
583  for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
584  detJ_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
585  z_phys_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
586  }
587  }
588 
589 #ifdef ERF_IMPLICIT_W
590  if ( (SolverChoice::mesh_type == MeshType::VariableDz) &&
591  (solverChoice.implicit_momentum_diffusion) ) {
592  for (int lev = 0; lev <= finest_level; lev++) {
593  if ( (solverChoice.vert_implicit_fac[lev][0] > 0) ||
594  (solverChoice.vert_implicit_fac[lev][1] > 0) ||
595  (solverChoice.vert_implicit_fac[lev][2] > 0) )
596  {
597  Warning("Doing implicit solve for u, v, and w with terrain at level " << lev << " -- this has not been tested");
598  }
599  }
600  }
601 #endif
602 
603  //
604  // Copy vars_new into vars_old, then use vars_old to fill covered cells in vars_new during AverageDown
605  //
606  if (SolverChoice::terrain_type == TerrainType::EB) {
607  for (int lev = 0; lev <= finest_level; lev++) {
608  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
609  MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,ncomp_cons,vars_new[lev][Vars::cons].nGrowVect());
610  }
611  }
612 
613  if (restart_chkfile.empty()) {
614  if (solverChoice.coupling_type == CouplingType::TwoWay) {
615  AverageDown();
616  }
617  }
618 
619 #ifdef ERF_USE_PARTICLES
620  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
621  // Promote the Lagrangian PC to the multi-level ParGDB before init so
622  // per-level addParticles() can use ParticleBoxArray(lev)/DistributionMap(lev).
623  auto* pc_ptr = dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer();
624  AMREX_ALWAYS_ASSERT(pc_ptr != nullptr);
625  pc_ptr->Define(static_cast<amrex::ParGDBBase*>(GetParGDB()));
626 
627  if (restart_chkfile.empty()) {
628  if (solverChoice.moisture_tight_coupling) {
629  Warning("Tight coupling has not been tested with Lagrangian microphysics");
630  }
631 
632  for (int lev = 0; lev <= finest_level; lev++) {
633  dynamic_cast<LagrangianMicrophysics&>(*micro).initParticles(lev, z_phys_nd[lev]);
634  }
635  }
636  }
637 #endif
638 
639  if (!restart_chkfile.empty()) { // Restart from a checkpoint
640 
641  // Create the physbc objects for {cons, u, v, w, base state}
642  // We fill the additional base state ghost cells just in case we have read the old format
643  for (int lev(0); lev <= finest_level; ++lev) {
644  make_physbcs(lev);
645  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
646  }
647 
648  if (solverChoice.do_forest_drag) {
649  for (int lev(0); lev <= finest_level; ++lev) {
650  m_forest_drag[lev]->define_drag_field(grids[lev], dmap[lev], geom[lev],
651  z_phys_cc[lev].get(), z_phys_nd[lev].get());
652  }
653  }
654 
655 #ifdef ERF_USE_NETCDF
656  //
657  // Create the needed bdy_data_xlo etc ... since we don't read it in from checkpoint any more
658  // This follows init_from_wrfinput()
659  //
660  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
661  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
662 
663  if ( geom[0].isPeriodic(0) || geom[0].isPeriodic(1) ) {
664  amrex::Error("Cannot set periodic lateral boundary conditions when reading in real boundary values");
665  }
666 
667  bdy_time_interval = read_times_from_wrfbdy(nc_bdy_file,
668  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
669  start_bdy_time, final_bdy_time);
670 
671  double time_since_start_bdy = t_new[0] + start_time - start_bdy_time;
672  int n_time_old = static_cast<int>(time_since_start_bdy / bdy_time_interval);
673  MultiFab r_hse(base_state[0], make_alias, BaseState::r0_comp, 1);
674  Array<MultiFab*, AMREX_SPACEDIM> area_vec = {ax[0].get(), ay[0].get(), az[0].get()};
675 
676  // Need itime=0 for vertical interpolation
677  if (n_time_old > 0) {
678  int itime = 0;
679  bool is_anelastic = (solverChoice.anelastic[0] == 1);
680  read_and_convert_from_wrfbdy(itime,nc_bdy_file,
681  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
682  wrf_MUB, wrf_C1H, wrf_C2H, wrf_RDNW, wrf_PHB, z_phys_nd[0],
683  vars_new[0][Vars::xvel], vars_new[0][Vars::yvel], vars_new[0][Vars::cons],
684  r_hse, area_vec, geom[0], use_moist, solverChoice.rebalance_wrf_input, domain_bcs_type,
685  real_width, bdy_time_interval, is_anelastic);
686  }
687 
688  int ntimes = std::min(n_time_old+3, static_cast<int>(bdy_data_xlo.size()));
689 
690  for (int itime = n_time_old; itime < ntimes; itime++)
691  {
692  bool is_anelastic = (solverChoice.anelastic[0] == 1);
693  read_and_convert_from_wrfbdy(itime,nc_bdy_file,
694  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
695  wrf_MUB, wrf_C1H, wrf_C2H, wrf_RDNW, wrf_PHB, z_phys_nd[0],
696  vars_new[0][Vars::xvel], vars_new[0][Vars::yvel], vars_new[0][Vars::cons],
697  r_hse, area_vec, geom[0], use_moist, solverChoice.rebalance_wrf_input, domain_bcs_type,
698  real_width, bdy_time_interval, is_anelastic);
699  } // itime
700  } // use_real_bcs
701 
702  if (!nc_low_file.empty())
703  {
704  low_time_interval = read_times_from_wrflow(nc_low_file, low_data_zlo, start_low_time, final_low_time);
705 
706  int lev = 0;
707  sst_lev[lev].resize(low_data_zlo.size());
708  tsk_lev[lev].resize(low_data_zlo.size());
709 
710  double time_since_start_low = t_new[0] + start_time - start_low_time;
711  int n_time_old = static_cast<int>(time_since_start_low / low_time_interval);
712 
713  int ntimes = std::min(n_time_old+3, static_cast<int>(low_data_zlo.size()));
714 
715  for (int itime = n_time_old; itime < ntimes; itime++)
716  {
717  read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
718 
719  // Need to read PSFC
720  FArrayBox NC_fab_var_file;
721  for (int idx = 0; idx < num_boxes_at_level[lev]; idx++) {
722  int success, use_theta_m;
723  read_from_wrfinput(lev, boxes_at_level[lev][idx], nc_init_file[lev][0],
724  NC_fab_var_file, "PSFC", geom[lev],
725  use_theta_m, success);
726  auto& var_fab = NC_fab_var_file;
727 #ifdef _OPENMP
728 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
729 #endif
730  for ( MFIter mfi(*mf_PSFC[lev], false); mfi.isValid(); ++mfi )
731  {
732  FArrayBox &cur_fab = (*mf_PSFC[lev])[mfi];
733  cur_fab.template copy<RunOn::Device>(var_fab, 0, 0, 1);
734  }
735  var_fab.clear();
736  }
737 
738  update_sst_tsk(itime, geom[lev], ba2d[lev],
739  sst_lev[lev], tsk_lev[lev],
740  m_SurfaceLayer, low_data_zlo,
741  vars_new[lev][Vars::cons], *mf_PSFC[lev],
742  solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
743  } // itime
744  }
745 #endif
746 #ifdef ERF_USE_FFT
747  for (int lev = 0; lev <= finest_level; lev++) {
748  // rebuild fft solvers here in case mesh type was changed when reading the checkpoint file
749  if ( ( (solverChoice.anelastic[lev] == 1) || (solverChoice.project_initial_velocity[lev] == 1) ) &&
750  ( (solverChoice.mesh_type == MeshType::ConstantDz) || (solverChoice.mesh_type == MeshType::StretchedDz) ) ) {
751  build_fft_solvers(lev);
752  }
753  }
754 #endif
755  } // end restart
756 
757 #ifdef ERF_USE_PARTICLES
758  /* If using a Lagrangian microphysics model, its particle container has now been
759  constructed and initialized (calls to micro->Init). So, add its pointer to
760  ERF::particleData and remove its name from list of unallocated particle containers. */
761  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
762  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
763  const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
764  AMREX_ALWAYS_ASSERT(pc_ptr != nullptr);
765  particleData.pushBack(pc_name, pc_ptr);
766  particleData.getNamesUnalloc().remove(pc_name);
767  }
768 #endif
769 
770  if (input_bndry_planes) {
771  // Read the "time.dat" file to know what data is available
772  m_r2d->read_time_file();
773 
774  // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
775  double dt_dummy = 0.0;
776  m_r2d->read_input_files(t_new[0]+start_time,dt_dummy,m_bc_extdir_vals);
777  }
778 
779  if (solverChoice.custom_rhotheta_forcing)
780  {
781  rhotheta_src.resize(max_level+1);
782  for (int lev = 0; lev <= finest_level; lev++) {
783  BoxList bl_src = vars_new[lev][Vars::cons].boxArray().boxList();
784  for (auto& b : bl_src) {
785  if (!solverChoice.spatial_rhotheta_forcing)
786  {
787  // source is only defined in Z
788  b.setRange(0, 0, 1);
789  b.setRange(1, 0, 1);
790  }
791  }
792  BoxArray ba_src(std::move(bl_src));
793  rhotheta_src[lev] = std::make_unique<MultiFab>(ba_src, vars_new[lev][Vars::cons].DistributionMap(), 1, 0);
794  rhotheta_src[lev]->setVal(0.);
795  prob->update_rhotheta_sources(t_new[0],
796  rhotheta_src[lev].get(),
797  geom[lev], z_phys_cc[lev]);
798  }
799  }
800 
801  if (solverChoice.have_geo_wind_profile)
802  {
803  h_u_geos.resize(max_level+1, Vector<Real>(0));
804  d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
805  h_v_geos.resize(max_level+1, Vector<Real>(0));
806  d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
807  for (int lev = 0; lev <= finest_level; lev++) {
808  const int domlen = geom[lev].Domain().length(2);
809  h_u_geos[lev].resize(domlen, 0.0_rt);
810  d_u_geos[lev].resize(domlen, 0.0_rt);
811  h_v_geos[lev].resize(domlen, 0.0_rt);
812  d_v_geos[lev].resize(domlen, 0.0_rt);
813  if (solverChoice.custom_geostrophic_profile) {
814  prob->update_geostrophic_profile(t_new[0],
815  h_u_geos[lev], d_u_geos[lev],
816  h_v_geos[lev], d_v_geos[lev],
817  geom[lev], z_phys_cc[lev]);
818  } else {
819  if (SolverChoice::mesh_type == MeshType::VariableDz) {
820  amrex::Print() << "Note: 1-D geostrophic wind profile input is not defined for real terrain" << std::endl;
821  }
822  init_geo_wind_profile(solverChoice.abl_geo_wind_table,
823  h_u_geos[lev], d_u_geos[lev],
824  h_v_geos[lev], d_v_geos[lev],
825  geom[lev],
826  zlevels_stag[0]);
827  }
828  }
829  }
830 
831  if (solverChoice.custom_moisture_forcing)
832  {
833  rhoqt_src.resize(max_level+1);
834  for (int lev = 0; lev <= finest_level; lev++) {
835  BoxList bl_src = vars_new[lev][Vars::cons].boxArray().boxList();
836  for (auto& b : bl_src) {
837  if (!solverChoice.spatial_moisture_forcing)
838  {
839  // source is only defined in Z
840  b.setRange(0, 0, 1);
841  b.setRange(1, 0, 1);
842  }
843  }
844  BoxArray ba_src(std::move(bl_src));
845  rhoqt_src[lev] = std::make_unique<MultiFab>(ba_src, vars_new[lev][Vars::cons].DistributionMap(), 1, 0);
846  rhoqt_src[lev]->setVal(0.);
847  prob->update_rhoqt_sources(t_new[0],
848  rhoqt_src[lev].get(),
849  geom[lev], z_phys_cc[lev]);
850  }
851  }
852 
853  if (solverChoice.custom_w_subsidence)
854  {
855  h_w_subsid.resize(max_level+1, Vector<Real>(0));
856  d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
857  for (int lev = 0; lev <= finest_level; lev++) {
858  const int domlen = geom[lev].Domain().length(2) + 1; // lives on z-faces
859  h_w_subsid[lev].resize(domlen, 0.0_rt);
860  d_w_subsid[lev].resize(domlen, 0.0_rt);
861  prob->update_w_subsidence(t_new[0],
862  h_w_subsid[lev], d_w_subsid[lev], base_state[lev],
863  geom[lev], z_phys_nd[lev]);
864  }
865  }
866 
867  if (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
868  solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T)
869  {
870  for (int lev = 0; lev <= finest_level; lev++) {
871  initRayleigh_at_level(lev);
872  }
873  if (solverChoice.init_type == InitType::Input_Sounding)
874  {
875  // Overwrite ubar, vbar, and thetabar with input profiles;
876  // wbar is assumed to be zero Note: the tau coefficient set by
877  // prob->erf_init_rayleigh() is still used
878  bool restarting = (!restart_chkfile.empty());
879  setRayleighRefFromSounding(restarting);
880  }
881  }
882 
883  // Read in sponge data from input file
884  if(solverChoice.spongeChoice.sponge_type == SpongeType::Input_Sponge)
885  {
886  initSponge();
887  bool restarting = (!restart_chkfile.empty());
888  setSpongeRefFromSounding(restarting);
889  }
890 
891  if (solverChoice.any_perturbation()) {
892  if (is_it_time_for_action(istep[0], t_new[0], dt[0], pert_interval, -one)) {
893  turbPert.debug(t_new[0]);
894  }
895  }
896 
897  // We only write the file at level 0 for now
898  if (output_bndry_planes)
899  {
900  // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
901  m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
902 
903  double tot_time = t_new[0]+start_time;
904  if (tot_time >= bndry_output_planes_start_time) {
905  bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
906  m_w2d->write_planes(0, tot_time, vars_new, is_moist);
907  }
908  }
909 
910  // Fill boundary conditions in vars_new
911  for (int lev = 0; lev <= finest_level; ++lev)
912  {
913  auto& lev_new = vars_new[lev];
914 
915  // ***************************************************************************
916  // Physical bc's at domain boundary
917  // ***************************************************************************
918  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
919  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
920 
921  int ncomp_cons = lev_new[Vars::cons].nComp();
922  bool do_fb = true;
923 
924 #ifdef ERF_USE_NETCDF
925  if (solverChoice.use_real_bcs && (lev==0)) {
926  int icomp_cons = 0;
927  bool cons_only = false;
928  Vector<MultiFab*> mfs_vec = {&lev_new[Vars::cons],&lev_new[Vars::xvel],
929  &lev_new[Vars::yvel],&lev_new[Vars::zvel]};
930  fill_from_realbdy(mfs_vec,t_new[lev],cons_only,icomp_cons,
931  ncomp_cons,ngvect_cons,ngvect_vels);
932  do_fb = false;
933  }
934 #endif
935 
936  (*physbcs_cons[lev])(lev_new[Vars::cons],lev_new[Vars::xvel],lev_new[Vars::yvel],0,ncomp_cons,
937  ngvect_cons,t_new[lev],BCVars::cons_bc,do_fb);
938  ( *physbcs_u[lev])(lev_new[Vars::xvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
939  ngvect_vels,t_new[lev],BCVars::xvel_bc,do_fb);
940  ( *physbcs_v[lev])(lev_new[Vars::yvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
941  ngvect_vels,t_new[lev],BCVars::yvel_bc,do_fb);
942  ( *physbcs_w[lev])(lev_new[Vars::zvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
943  ngvect_vels,t_new[lev],BCVars::zvel_bc,do_fb);
944  }
945 
946  //
947  // If we are starting from scratch, we have the option to project the initial velocity field
948  // regardless of how we initialized. Note that project_initial_velocity operates on vars_new.
949  // pp_inc is used as scratch space here; we zero it out after the projection
950  //
951  if (restart_chkfile == "")
952  {
953  for (int lev = 0; lev <= finest_level; ++lev)
954  {
955  if (solverChoice.project_initial_velocity[lev] == 1) {
956  double dummy_dt = 1.0;
957  if (verbose > 0) {
958  amrex::Print() << "Projecting initial velocity field at level " << lev << std::endl;
959  }
960 
961  project_initial_velocity(lev, t_new[lev], dummy_dt);
962 
963  pp_inc[lev].setVal(0.);
964  gradp[lev][GpVars::gpx].setVal(0.);
965  gradp[lev][GpVars::gpy].setVal(0.);
966  gradp[lev][GpVars::gpz].setVal(0.);
967  } // project
968  } // lev
969  }
970 
971  // Copy from new into old just in case (after filling boundary conditions and possibly projecting)
972  for (int lev = 0; lev <= finest_level; ++lev)
973  {
974  int nc = vars_new[lev][Vars::cons].nComp();
975 
976  MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,nc,vars_new[lev][Vars::cons].nGrowVect());
977  MultiFab::Copy(vars_old[lev][Vars::xvel],vars_new[lev][Vars::xvel],0,0, 1,vars_new[lev][Vars::xvel].nGrowVect());
978  MultiFab::Copy(vars_old[lev][Vars::yvel],vars_new[lev][Vars::yvel],0,0, 1,vars_new[lev][Vars::yvel].nGrowVect());
979  MultiFab::Copy(vars_old[lev][Vars::zvel],vars_new[lev][Vars::zvel],0,0, 1,vars_new[lev][Vars::zvel].nGrowVect());
980  }
981 
982  // Compute the minimum dz in the domain at each level (to be used for setting the timestep)
983  dz_min.resize(max_level+1);
984  for (int lev = 0; lev <= finest_level; ++lev)
985  {
986  dz_min[lev] = geom[lev].CellSize(2);
987  if ( SolverChoice::mesh_type != MeshType::ConstantDz ) {
988  dz_min[lev] *= (*detJ_cc[lev]).min(0);
989  }
990  }
991 
992 
993  // We don't need to recompute dt[lev] on restart because we read it in from the checkpoint file.
994  if (restart_chkfile.empty()) {
995  ComputeDt();
996  }
997 
998  // Check the viscous limit
999  DiffChoice dc = solverChoice.diffChoice;
1000  if (dc.molec_diff_type == MolecDiffType::Constant ||
1001  dc.molec_diff_type == MolecDiffType::ConstantAlpha) {
1002  Real delta = std::min({geom[finest_level].CellSize(0),
1003  geom[finest_level].CellSize(1),
1004  dz_min[finest_level]});
1005  if (dc.dynamic_viscosity == 0) {
1006  Print() << "Note: Molecular diffusion specified but dynamic_viscosity has not been specified" << std::endl;
1007  } else {
1008  Real nu = dc.dynamic_viscosity / dc.rho0_trans;
1009  Real viscous_limit = two * delta*delta / nu;
1010  Print() << "smallest grid spacing at level " << finest_level << " = " << delta << std::endl;
1011  Print() << "dt at level " << finest_level << " = " << dt[finest_level] << std::endl;
1012  Print() << "Viscous CFL is " << dt[finest_level] / viscous_limit << std::endl;
1013  if (fixed_dt[finest_level] >= viscous_limit) {
1014  Warning("Specified fixed_dt is above the viscous limit");
1015  } else if (dt[finest_level] >= viscous_limit) {
1016  Warning("Adaptive dt based on convective CFL only is above the viscous limit");
1017  }
1018  }
1019  }
1020 
1021  // Fill ghost cells/faces
1022  for (int lev = 0; lev <= finest_level; ++lev)
1023  {
1024  if (lev > 0 && cf_width >= 0) {
1025  Construct_ERFFillPatchers(lev);
1026  }
1027 
1028  auto& lev_new = vars_new[lev];
1029 
1030  //
1031  // Fill boundary conditions -- not sure why we need this here
1032  //
1033  bool fillset = false;
1034  if (lev == 0) {
1035  FillPatchCrseLevel(lev, t_new[lev],
1036  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]});
1037  } else {
1038  FillPatchFineLevel(lev, t_new[lev],
1039  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
1040  {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
1041  base_state[lev], base_state[lev],
1042  fillset);
1043  }
1044 
1045  //
1046  // We do this here to make sure level (lev-1) boundary conditions are filled
1047  // before we interpolate to level (lev) ghost cells
1048  //
1049  if (lev < finest_level) {
1050  auto& lev_old = vars_old[lev];
1051  MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
1052  MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
1053  MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
1054  MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
1055  }
1056 
1057  //
1058  // We fill the ghost cell values of the base state in case it wasn't done in the initialization
1059  //
1060  base_state[lev].FillBoundary(geom[lev].periodicity());
1061 
1062  // For moving terrain only
1063  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
1064  MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
1065  base_state_new[lev].FillBoundary(geom[lev].periodicity());
1066  }
1067 
1068  }
1069 
1070  // If lev > 0, we need to fill bc's by interpolation from coarser grid
1071  for (int lev = 1; lev <= finest_level; ++lev)
1072  {
1073  Interp2DArrays(lev,ba2d[lev],dmap[lev]);
1074  } // lev
1075 
1076 #ifdef ERF_USE_WW3_COUPLING
1077  int my_lev = 0;
1078  amrex::Print() << " About to call send_to_ww3 from ERF.cpp" << std::endl;
1079  send_to_ww3(my_lev);
1080  amrex::Print() << " About to call read_waves from ERF.cpp" << std::endl;
1081  read_waves(my_lev);
1082  // send_to_ww3(my_lev);
1083 #endif
1084 
1085  // Create wall distance field for RANS model
1086  for (int lev = 0; lev <= finest_level; lev++) {
1087  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
1088  // Handle bottom boundary
1089  poisson_wall_dist(lev);
1090 
1091  // Correct the wall distance for immersed bodies
1092  if (solverChoice.advChoice.have_zero_flux_faces) {
1093  thinbody_wall_dist(walldist[lev],
1094  solverChoice.advChoice.zero_xflux,
1095  solverChoice.advChoice.zero_yflux,
1096  solverChoice.advChoice.zero_zflux,
1097  geom[lev],
1098  z_phys_cc[lev]);
1099  }
1100  }
1101  }
1102 
1103  // Configure SurfaceLayer params if used
1104  // NOTE: we must set up the MOST routine after calling FillPatch
1105  // in order to have lateral ghost cells filled (MOST + terrain interp).
1106  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
1107  {
1108  bool has_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
1109  (solverChoice.turbChoice[0].les_type != LESType::None) ||
1110  (solverChoice.turbChoice[0].rans_type != RANSType::None) ||
1111  (solverChoice.turbChoice[0].pbl_type != PBLType::None) );
1112  AMREX_ALWAYS_ASSERT(has_diff);
1113 
1114  bool rotate = solverChoice.use_rotate_surface_flux;
1115  if (rotate) {
1116  Print() << "Using surface layer model with stress rotations" << std::endl;
1117  }
1118 
1119  //
1120  // This constructor will make the SurfaceLayer object but not allocate the arrays at each level.
1121  //
1122  // Build vector of eb pointers for all levels
1123  amrex::Vector<const eb_*> eb_ptrs;
1124  eb_ptrs.resize(finest_level + 1, nullptr);
1125  if (solverChoice.terrain_type == TerrainType::EB) {
1126  for (int lev = 0; lev <= finest_level; lev++) {
1127  eb_ptrs[lev] = eb[lev] ? eb[lev].get() : nullptr;
1128  }
1129  }
1130 
1131  m_SurfaceLayer = std::make_unique<SurfaceLayer>(geom, rotate, pp_prefix, Qv_prim,
1132  z_phys_nd,
1133  solverChoice.mesh_type,
1134  solverChoice.terrain_type,
1135  solverChoice.turbChoice[finest_level],
1136 #ifdef ERF_USE_NETCDF
1137  start_low_time, final_low_time, low_time_interval,
1138 #else
1139  zero, zero, zero,
1140 #endif
1141  eb_ptrs);
1142  // This call will allocate the arrays at each level. If we regrid later, either changing
1143  // the number of levels or just the grids at each existing level, we will call an update routine
1144  // to redefine the internal arrays in m_SurfaceLayer.
1145  for (int lev = 0; lev <= finest_level; lev++)
1146  {
1147  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
1148  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
1149  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,finest_level+1,
1150  mfv_old, Theta_prim[lev], Qv_prim[lev],
1151  Qr_prim[lev], z_phys_nd[lev],
1152  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
1153  lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
1154  sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
1155  }
1156 
1157  // If initializing from an input_sounding, make sure the surface layer
1158  // is using the same surface conditions
1159  if (solverChoice.init_type == InitType::Input_Sounding) {
1160  const Real theta0 = input_sounding_data.theta_ref_inp_sound;
1161  const Real qv0 = input_sounding_data.qv_ref_inp_sound;
1162  for (int lev = 0; lev <= finest_level; lev++) {
1163  m_SurfaceLayer->set_t_surf(lev, theta0);
1164  m_SurfaceLayer->set_q_surf(lev, qv0);
1165  }
1166  }
1167 
1168  if (restart_chkfile != "") {
1169  // Update surface fields if needed (and available)
1170  ReadCheckpointFileSurfaceLayer();
1171  }
1172 
1173  // We now configure ABLMost params here so that we can print the averages at t=0
1174  // Note we don't fill ghost cells here because this is just for diagnostics
1175  for (int lev = 0; lev <= finest_level; ++lev)
1176  {
1177  IntVect ng = Theta_prim[lev]->nGrowVect();
1178 
1179  MultiFab::Copy( *Theta_prim[lev], vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, ng);
1180  MultiFab::Divide(*Theta_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1181 
1182  if (solverChoice.moisture_type != MoistureType::None) {
1183  ng = Qv_prim[lev]->nGrowVect();
1184 
1185  MultiFab::Copy( *Qv_prim[lev], vars_new[lev][Vars::cons], RhoQ1_comp, 0, 1, ng);
1186  MultiFab::Divide(*Qv_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1187 
1188  int rhoqr_comp = solverChoice.moisture_indices.qr;
1189  if (rhoqr_comp > -1) {
1190  MultiFab::Copy( *Qr_prim[lev], vars_new[lev][Vars::cons], rhoqr_comp, 0, 1, ng);
1191  MultiFab::Divide(*Qr_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1192  } else {
1193  Qr_prim[lev]->setVal(0.0);
1194  }
1195  }
1196  m_SurfaceLayer->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim, Qr_prim);
1197 
1198  if (restart_chkfile == "") {
1199  // Only do this if starting from scratch; if restarting, then
1200  // we don't want to call update_fluxes multiple times because
1201  // it will change u* and theta* from their previous values
1202  m_SurfaceLayer->update_pblh(lev, vars_new, z_phys_cc[lev].get(),
1203  solverChoice.moisture_indices);
1204 #ifdef ERF_USE_NETCDF
1205  double elapsed_time_since_start_low = t_new[lev] + (start_time - start_low_time);
1206 #else
1207  double elapsed_time_since_start_low = t_new[lev] + start_time;
1208 #endif
1209  m_SurfaceLayer->update_fluxes(lev, t_new[lev], elapsed_time_since_start_low,
1210  vars_new[lev][Vars::cons],
1211  z_phys_nd[lev],
1212  walldist[lev]);
1213 
1214  // Initialize tke(x,y,z) as a function of u*(x,y)
1215  if (solverChoice.turbChoice[lev].init_tke_from_ustar) {
1216  Real qkefac = one;
1217  if (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25 ||
1218  solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF)
1219  {
1220  // https://github.com/NCAR/MYNN-EDMF/blob/90f36c25259ec1960b24325f5b29ac7c5adeac73/module_bl_mynnedmf.F90#L1325-L1333
1221  const Real B1 = solverChoice.turbChoice[lev].pbl_mynn.B1;
1222  qkefac = Real(1.5) * std::pow(B1, two/three);
1223  }
1224  m_SurfaceLayer->init_tke_from_ustar(lev, vars_new[lev][Vars::cons], z_phys_nd[lev], qkefac);
1225  }
1226  }
1227  }
1228  } // end if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
1229 
1230  // Update micro vars and finish moisture model initializations before first plot file
1231  if (solverChoice.moisture_type != MoistureType::None) {
1232  for (int lev = 0; lev <= finest_level; ++lev) {
1233  micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons]);
1234  micro->FinishInit(lev, vars_new[lev][Vars::cons], z_phys_nd);
1235  }
1236  }
1237 
1238  // Fill time averaged velocities before first plot file
1239  if (solverChoice.time_avg_vel) {
1240  for (int lev = 0; lev <= finest_level; ++lev) {
1241  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
1242  vars_new[lev][Vars::xvel],
1243  vars_new[lev][Vars::yvel],
1244  vars_new[lev][Vars::zvel]);
1245  }
1246  }
1247 
1248 #ifdef ERF_USE_PARTICLES
1249  // Redistribute particles so the container has valid data at all AMR levels
1250  // before the initial plotfile write
1251  if (finest_level > 0) {
1252  particleData.Redistribute(z_phys_nd);
1253  }
1254 #endif
1255 
1256  // Print max values of lateral gradients of base state pressure at level 0
1257  if (verbose > 0) {
1258  for (int lev = 0; lev <= finest_level; ++lev) {
1259  if (lev == 0) {
1260  compute_max_pressure_gradient_diagnostic(lev);
1261  }
1262  }
1263  }
1264 
1265  // check for additional plotting variables that are available after particle containers
1266  // are setup.
1267  const std::string& pv3d_1 = "plot_vars_1" ; appendPlotVariables(pv3d_1,plot3d_var_names_1);
1268  const std::string& pv3d_2 = "plot_vars_2" ; appendPlotVariables(pv3d_2,plot3d_var_names_2);
1269  const std::string& pv2d_1 = "plot2d_vars_1"; appendPlotVariables(pv2d_1,plot2d_var_names_1);
1270  const std::string& pv2d_2 = "plot2d_vars_2"; appendPlotVariables(pv2d_2,plot2d_var_names_2);
1271 
1272  if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > zero) )
1273  {
1274  WriteCheckpointFile();
1275  last_check_file_step = 0;
1276  if (m_check_per > zero) {last_check_file_time += m_check_per;}
1277  }
1278 
1279  if ( (restart_chkfile.empty()) ||
1280  (!restart_chkfile.empty() && plot_file_on_restart) )
1281  {
1282  if (m_plot3d_int_1 > 0 || m_plot3d_per_1 > zero)
1283  {
1284  Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
1285  if (m_plot3d_per_1 > zero) {last_plot3d_file_time_1 += m_plot3d_per_1;}
1286  last_plot3d_file_step_1 = istep[0];
1287  }
1288  if (m_plot3d_int_2 > 0 || m_plot3d_per_2 > zero)
1289  {
1290  Write3DPlotFile(2,plotfile3d_type_2,plot3d_var_names_2);
1291  if (m_plot3d_per_2 > zero) {last_plot3d_file_time_2 += m_plot3d_per_2;}
1292  last_plot3d_file_step_2 = istep[0];
1293  }
1294  if (m_plot2d_int_1 > 0 || m_plot2d_per_1 > zero)
1295  {
1296  Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
1297  if (m_plot2d_per_1 > zero) {last_plot2d_file_time_1 += m_plot2d_per_1;}
1298  last_plot2d_file_step_1 = istep[0];
1299  }
1300  if (m_plot2d_int_2 > 0 || m_plot2d_per_2 > zero)
1301  {
1302  Write2DPlotFile(2,plotfile2d_type_2,plot2d_var_names_2);
1303  if (m_plot2d_per_2 > zero) {last_plot2d_file_time_2 += m_plot2d_per_2;}
1304  last_plot2d_file_step_2 = istep[0];
1305  }
1306  for (int i = 0; i < m_subvol_int.size(); i++) {
1307  if (m_subvol_int[i] > 0 || m_subvol_per[i] > zero) {
1308  WriteSubvolume(i,subvol3d_var_names);
1309  last_subvol_step[i] = istep[0];
1310  if (m_subvol_per[i] > zero) {last_subvol_time[i] += m_subvol_per[i];}
1311  }
1312  }
1313  }
1314 
1315  // Set these up here because we need to know which MPI rank "cell" is on...
1316  ParmParse pp("erf");
1317  if (pp.contains("data_log"))
1318  {
1319  int num_datalogs = pp.countval("data_log");
1320  datalog.resize(num_datalogs);
1321  datalogname.resize(num_datalogs);
1322  pp.queryarr("data_log",datalogname,0,num_datalogs);
1323  for (int i = 0; i < num_datalogs; i++) {
1324  setRecordDataInfo(i,datalogname[i]);
1325  }
1326  }
1327 
1328  if (pp.contains("der_data_log"))
1329  {
1330  int num_der_datalogs = pp.countval("der_data_log");
1331  der_datalog.resize(num_der_datalogs);
1332  der_datalogname.resize(num_der_datalogs);
1333  pp.queryarr("der_data_log",der_datalogname,0,num_der_datalogs);
1334  for (int i = 0; i < num_der_datalogs; i++) {
1335  setRecordDerDataInfo(i,der_datalogname[i]);
1336  }
1337  }
1338 
1339  if (pp.contains("energy_data_log"))
1340  {
1341  int num_energy_datalogs = pp.countval("energy_data_log");
1342  tot_e_datalog.resize(num_energy_datalogs);
1343  tot_e_datalogname.resize(num_energy_datalogs);
1344  pp.queryarr("energy_data_log",tot_e_datalogname,0,num_energy_datalogs);
1345  for (int i = 0; i < num_energy_datalogs; i++) {
1346  setRecordEnergyDataInfo(i,tot_e_datalogname[i]);
1347  }
1348  }
1349 
1350  if (solverChoice.rad_type != RadiationType::None)
1351  {
1352  // Create data log for radiation model if requested
1353  rad[0]->setupDataLog();
1354  }
1355 
1356 
1357  if (restart_chkfile.empty() && profile_int > 0) {
1358  if (destag_profiles) {
1359  // all variables cell-centered
1360  write_1D_profiles(t_new[0]);
1361  } else {
1362  // some variables staggered
1363  write_1D_profiles_stag(t_new[0]);
1364  }
1365  }
1366 
1367  if (pp.contains("sample_point_log") && pp.contains("sample_point"))
1368  {
1369  int lev = 0;
1370 
1371  int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
1372  if (num_samplepts > 0) {
1373  Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
1374  samplepoint.resize(num_samplepts);
1375 
1376  pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
1377  for (int i = 0; i < num_samplepts; i++) {
1378  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1379  samplepoint[i] = iv;
1380  }
1381  }
1382 
1383  int num_sampleptlogs = pp.countval("sample_point_log");
1384  AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
1385  if (num_sampleptlogs > 0) {
1386  sampleptlog.resize(num_sampleptlogs);
1387  sampleptlogname.resize(num_sampleptlogs);
1388  pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
1389 
1390  for (int i = 0; i < num_sampleptlogs; i++) {
1391  setRecordSamplePointInfo(i,lev,samplepoint[i],sampleptlogname[i]);
1392  }
1393  }
1394 
1395  }
1396 
1397  bool has_sample_line = pp.contains("sample_line");
1398  bool has_sample_line_real = pp.contains("sample_line_real");
1399  if (has_sample_line && has_sample_line_real) {
1400  Abort("Specify only one of erf.sample_line or erf.sample_line_real");
1401  }
1402 
1403  if (pp.contains("sample_line_log") && (has_sample_line || has_sample_line_real))
1404  {
1405  int lev = 0;
1406 
1407  int num_samplelines = 0;
1408  if (has_sample_line) {
1409  num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
1410  if (num_samplelines > 0) {
1411  Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
1412  sampleline.resize(num_samplelines);
1413 
1414  pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
1415  for (int i = 0; i < num_samplelines; i++) {
1416  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1417  sampleline[i] = iv;
1418  }
1419  }
1420  } else {
1421  int num_real_vals = pp.countval("sample_line_real");
1422  if (num_real_vals % AMREX_SPACEDIM != 0) {
1423  Abort("erf.sample_line_real must be specified as (x,y,z) triples");
1424  }
1425 
1426  num_samplelines = num_real_vals / AMREX_SPACEDIM;
1427  if (num_samplelines > 0) {
1428  Vector<Real> location; location.resize(num_real_vals);
1429  sampleline.resize(num_samplelines);
1430 
1431  pp.queryarr("sample_line_real",location,0,num_real_vals);
1432 
1433  const Box& domain = geom[lev].Domain();
1434  const auto* prob_lo = geom[lev].ProbLo();
1435  const auto* prob_hi = geom[lev].ProbHi();
1436  const auto* dx = geom[lev].CellSize();
1437 
1438  for (int i = 0; i < num_samplelines; i++) {
1439  Real xloc = location[AMREX_SPACEDIM*i+0];
1440  Real yloc = location[AMREX_SPACEDIM*i+1];
1441  Real zloc = location[AMREX_SPACEDIM*i+2];
1442 
1443  if (xloc < prob_lo[0] || xloc > prob_hi[0] ||
1444  yloc < prob_lo[1] || yloc > prob_hi[1] ||
1445  zloc < prob_lo[2] || zloc > prob_hi[2]) {
1446  Abort("erf.sample_line_real must lie within the level-0 domain");
1447  }
1448 
1449  int i_cell = domain.smallEnd(0) + static_cast<int>(std::floor((xloc - prob_lo[0]) / dx[0]));
1450  int j_cell = domain.smallEnd(1) + static_cast<int>(std::floor((yloc - prob_lo[1]) / dx[1]));
1451  int k_cell = domain.smallEnd(2) + static_cast<int>(std::floor((zloc - prob_lo[2]) / dx[2]));
1452 
1453  i_cell = std::min(i_cell, domain.bigEnd(0));
1454  j_cell = std::min(j_cell, domain.bigEnd(1));
1455  k_cell = std::min(k_cell, domain.bigEnd(2));
1456 
1457  sampleline[i] = IntVect(i_cell, j_cell, k_cell);
1458  }
1459  }
1460  }
1461 
1462  int num_samplelinelogs = pp.countval("sample_line_log");
1463  AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
1464  if (num_samplelinelogs > 0) {
1465  samplelinelog.resize(num_samplelinelogs);
1466  samplelinelogname.resize(num_samplelinelogs);
1467  pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
1468 
1469  for (int i = 0; i < num_samplelinelogs; i++) {
1470  setRecordSampleLineInfo(i,lev,sampleline[i],samplelinelogname[i]);
1471  }
1472  }
1473 
1474  }
1475 
1476  if (is_it_time_for_action(istep[0], t_new[0], dt[0], sum_interval, sum_per)) {
1477  sum_integrated_quantities(t_new[0]);
1478  sum_derived_quantities(t_new[0]);
1479  sum_energy_quantities(t_new[0]);
1480  }
1481 
1482  // Create object to do line and plane sampling if needed
1483  bool do_line = false; bool do_plane = false;
1484  pp.query("do_line_sampling",do_line); pp.query("do_plane_sampling",do_plane);
1485  if (do_line) {
1486  if (line_sampling_interval < 0 && line_sampling_per < 0) {
1487  Abort("Need to specify line_sampling_interval or line_sampling_per");
1488  }
1489  line_sampler = std::make_unique<LineSampler>();
1490  line_sampler->write_coords(z_phys_cc, geom);
1491  }
1492  if (do_plane) {
1493  if (plane_sampling_interval < 0 && plane_sampling_per < 0) {
1494  Abort("Need to specify plane_sampling_interval or plane_sampling_per");
1495  }
1496  plane_sampler = std::make_unique<PlaneSampler>();
1497  }
1498 
1499  if ( solverChoice.terrain_type == TerrainType::EB ||
1500  solverChoice.terrain_type == TerrainType::ImmersedForcing ||
1501  solverChoice.buildings_type == BuildingsType::ImmersedForcing )
1502  {
1503  bool write_eb_surface = false;
1504  pp.query("write_eb_surface", write_eb_surface);
1505  if (write_eb_surface) {
1506  if (verbose > 0) {
1507  amrex::Print() << "Writing the geometry to a vtp file.\n" << std::endl;
1508  }
1509  WriteEBSurface(grids[finest_level],dmap[finest_level],Geom(finest_level),&EBFactory(finest_level));
1510  }
1511  }
1512 }
three
constexpr amrex::Real three
Definition: ERF_Constants.H:11
MolecDiffType::ConstantAlpha
@ ConstantAlpha
MolecDiffType::Constant
@ Constant
prob_hi
const amrex::Real * prob_hi
Definition: ERF_InitCustomPert_DataAssimilation_ISV.H:17
thinbody_wall_dist
void thinbody_wall_dist(std::unique_ptr< MultiFab > &wdist, Vector< IntVect > &xfaces, Vector< IntVect > &yfaces, Vector< IntVect > &zfaces, const Geometry &geomdata, std::unique_ptr< MultiFab > &z_phys_cc)
Definition: ERF_ThinBodyWallDist.cpp:12
ERF::last_check_file_step
static int last_check_file_step
Definition: ERF.H:1134
ERF::samplelinelogname
amrex::Vector< std::string > samplelinelogname
Definition: ERF.H:1741
ERF::setRayleighRefFromSounding
void setRayleighRefFromSounding(bool restarting)
Set Rayleigh mean profiles from input sounding.
Definition: ERF_InitRayleigh.cpp:83
ERF::sampleline
amrex::Vector< amrex::IntVect > sampleline
Definition: ERF.H:1742
ERF::subvol3d_var_names
amrex::Vector< std::string > subvol3d_var_names
Definition: ERF.H:1226
ERF::last_plot2d_file_time_1
static double last_plot2d_file_time_1
Definition: ERF.H:1138
ERF::plane_sampling_per
amrex::Real plane_sampling_per
Definition: ERF.H:1725
ERF::m_plot3d_per_2
double m_plot3d_per_2
Definition: ERF.H:1210
ERF::last_plot2d_file_step_2
static int last_plot2d_file_step_2
Definition: ERF.H:1133
ERF::setRecordDataInfo
void setRecordDataInfo(int i, const std::string &filename)
Definition: ERF.H:1648
ERF::plot_file_on_restart
static bool plot_file_on_restart
Definition: ERF.H:1145
ERF::m_check_per
double m_check_per
Definition: ERF.H:1224
ERF::sum_per
static double sum_per
Definition: ERF.H:1339
ERF::lsm_flux_name
amrex::Vector< std::string > lsm_flux_name
Definition: ERF.H:991
ERF::sum_derived_quantities
void sum_derived_quantities(double time)
Definition: ERF_WriteScalarProfiles.cpp:178
ERF::last_subvol_time
amrex::Vector< double > last_subvol_time
Definition: ERF.H:1143
ERF::samplelinelog
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
Definition: ERF.H:1740
ERF::m_plot2d_int_2
int m_plot2d_int_2
Definition: ERF.H:1204
ERF::m_plot3d_int_1
int m_plot3d_int_1
Definition: ERF.H:1201
ERF::last_plot2d_file_time_2
static double last_plot2d_file_time_2
Definition: ERF.H:1139
ERF::sum_interval
static int sum_interval
Definition: ERF.H:1337
ERF::pert_interval
static int pert_interval
Definition: ERF.H:1338
ERF::line_sampling_per
amrex::Real line_sampling_per
Definition: ERF.H:1724
ERF::restart
void restart()
Definition: ERF.cpp:1747
ERF::last_plot3d_file_step_2
static int last_plot3d_file_step_2
Definition: ERF.H:1131
ERF::plot2d_var_names_1
amrex::Vector< std::string > plot2d_var_names_1
Definition: ERF.H:1230
ERF::setPlotVariables2D
void setPlotVariables2D(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile2D.cpp:109
ERF::last_plot3d_file_time_2
static double last_plot3d_file_time_2
Definition: ERF.H:1137
ERF::profile_int
int profile_int
Definition: ERF.H:1218
ERF::destag_profiles
bool destag_profiles
Definition: ERF.H:1219
ERF::compute_max_pressure_gradient_diagnostic
void compute_max_pressure_gradient_diagnostic(int lev)
Definition: ERF_Diagnostics.cpp:12
ERF::m_check_int
int m_check_int
Definition: ERF.H:1223
ERF::last_plot3d_file_time_1
static double last_plot3d_file_time_1
Definition: ERF.H:1136
ERF::m_plot2d_per_2
double m_plot2d_per_2
Definition: ERF.H:1212
ERF::write_1D_profiles
void write_1D_profiles(double time)
Definition: ERF_Write1DProfiles.cpp:17
ERF::initRayleigh_at_level
void initRayleigh_at_level(const int &lev)
Initialize Rayleigh damping profiles at a level.
Definition: ERF_InitRayleigh.cpp:14
ERF::appendPlotVariables
void appendPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:176
ERF::tot_e_datalogname
amrex::Vector< std::string > tot_e_datalogname
Definition: ERF.H:1734
ERF::output_bndry_planes
static int output_bndry_planes
Definition: ERF.H:1405
ERF::nc_bdy_file
static std::string nc_bdy_file
Definition: ERF.H:1357
ERF::AverageDown
void AverageDown()
Definition: ERF_AverageDown.cpp:16
ERF::write_1D_profiles_stag
void write_1D_profiles_stag(double time)
Definition: ERF_Write1DProfiles_stag.cpp:25
ERF::Write2DPlotFile
void Write2DPlotFile(int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile2D.cpp:151
ERF::sampleptlogname
amrex::Vector< std::string > sampleptlogname
Definition: ERF.H:1737
ERF::setRecordDerDataInfo
void setRecordDerDataInfo(int i, const std::string &filename)
Definition: ERF.H:1661
ERF::sampleptlog
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
Definition: ERF.H:1736
ERF::poisson_wall_dist
void poisson_wall_dist(int lev)
Definition: ERF_PoissonWallDist.cpp:22
ERF::is_it_time_for_action
static bool is_it_time_for_action(int nstep, double time, double dt, int action_interval, double action_per)
Definition: ERF_WriteScalarProfiles.cpp:659
ERF::sum_integrated_quantities
void sum_integrated_quantities(double time)
Definition: ERF_WriteScalarProfiles.cpp:15
ERF::m_subvol_per
amrex::Vector< double > m_subvol_per
Definition: ERF.H:1207
ERF::m_w2d
std::unique_ptr< WriteBndryPlanes > m_w2d
Definition: ERF.H:1468
ERF::init_geo_wind_profile
void init_geo_wind_profile(const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
Definition: ERF_InitGeowind.cpp:10
ERF::plot2d_var_names_2
amrex::Vector< std::string > plot2d_var_names_2
Definition: ERF.H:1231
ERF::WriteSubvolume
void WriteSubvolume(int isub, amrex::Vector< std::string > subvol_var_names)
Definition: ERF_WriteSubvolume.cpp:145
ERF::sum_energy_quantities
void sum_energy_quantities(double time)
Definition: ERF_WriteScalarProfiles.cpp:317
ERF::lsm_data_name
amrex::Vector< std::string > lsm_data_name
Definition: ERF.H:989
ERF::initSponge
void initSponge()
Initialize sponge profiles.
Definition: ERF_InitSponge.cpp:35
ERF::plane_sampler
std::unique_ptr< PlaneSampler > plane_sampler
Definition: ERF.H:1727
ERF::tot_e_datalog
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
Definition: ERF.H:1731
ERF::m_plot3d_per_1
double m_plot3d_per_1
Definition: ERF.H:1209
ERF::real_width
int real_width
Definition: ERF.H:1358
ERF::last_subvol_step
amrex::Vector< int > last_subvol_step
Definition: ERF.H:1142
ERF::plotfile3d_type_2
static PlotFileType plotfile3d_type_2
Definition: ERF.H:1343
ERF::setRecordEnergyDataInfo
void setRecordEnergyDataInfo(int i, const std::string &filename)
Definition: ERF.H:1674
ERF::Interp2DArrays
void Interp2DArrays(int lev, const amrex::BoxArray &my_ba2d, const amrex::DistributionMapping &my_dm)
Definition: ERF.cpp:1515
ERF::plotfile2d_type_2
static PlotFileType plotfile2d_type_2
Definition: ERF.H:1345
ERF::plane_sampling_interval
int plane_sampling_interval
Definition: ERF.H:1723
ERF::m_plot2d_int_1
int m_plot2d_int_1
Definition: ERF.H:1203
ERF::WriteCheckpointFile
void WriteCheckpointFile() const
Definition: ERF_Checkpoint.cpp:35
ERF::Write3DPlotFile
void Write3DPlotFile(int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:265
ERF::last_check_file_time
static double last_check_file_time
Definition: ERF.H:1140
ERF::nc_low_file
static std::string nc_low_file
Definition: ERF.H:1362
ERF::last_plot2d_file_step_1
static int last_plot2d_file_step_1
Definition: ERF.H:1132
ERF::Construct_ERFFillPatchers
void Construct_ERFFillPatchers(int lev)
Definition: ERF.cpp:2713
ERF::setRecordSampleLineInfo
void setRecordSampleLineInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1704
ERF::setSpongeRefFromSounding
void setSpongeRefFromSounding(bool restarting)
Set sponge mean profiles from input sounding.
Definition: ERF_InitSponge.cpp:65
ERF::line_sampling_interval
int line_sampling_interval
Definition: ERF.H:1722
ERF::project_initial_velocity
void project_initial_velocity(int lev, double time, double dt)
Definition: ERF_PoissonSolve.cpp:31
ERF::samplepoint
amrex::Vector< amrex::IntVect > samplepoint
Definition: ERF.H:1738
ERF::last_plot3d_file_step_1
static int last_plot3d_file_step_1
Definition: ERF.H:1130
ERF::line_sampler
std::unique_ptr< LineSampler > line_sampler
Definition: ERF.H:1726
ERF::plotfile2d_type_1
static PlotFileType plotfile2d_type_1
Definition: ERF.H:1344
ERF::plotfile3d_type_1
static PlotFileType plotfile3d_type_1
Definition: ERF.H:1342
ERF::m_subvol_int
amrex::Vector< int > m_subvol_int
Definition: ERF.H:1206
ERF::setRecordSamplePointInfo
void setRecordSamplePointInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1687
ERF::m_plot3d_int_2
int m_plot3d_int_2
Definition: ERF.H:1202
ERF::bndry_output_planes_start_time
static double bndry_output_planes_start_time
Definition: ERF.H:1408
ERF::ReadCheckpointFileSurfaceLayer
void ReadCheckpointFileSurfaceLayer()
Definition: ERF_Checkpoint.cpp:1272
ERF::m_plot2d_per_1
double m_plot2d_per_1
Definition: ERF.H:1211
MicVar_Morr::nc
@ nc
Definition: ERF_Morrison.H:45
AdvChoice::have_zero_flux_faces
bool have_zero_flux_faces
Definition: ERF_AdvStruct.H:440
DiffChoice::rho0_trans
amrex::Real rho0_trans
Definition: ERF_DiffStruct.H:91
DiffChoice::dynamic_viscosity
amrex::Real dynamic_viscosity
Definition: ERF_DiffStruct.H:96
InputSoundingData::theta_ref_inp_sound
amrex::Real theta_ref_inp_sound
Definition: ERF_InputSoundingData.H:412
InputSoundingData::qv_ref_inp_sound
amrex::Real qv_ref_inp_sound
Definition: ERF_InputSoundingData.H:412
SolverChoice::rebalance_wrf_input
bool rebalance_wrf_input
Definition: ERF_DataStruct.H:1324
SolverChoice::spatial_moisture_forcing
bool spatial_moisture_forcing
Definition: ERF_DataStruct.H:1350
SolverChoice::any_perturbation
bool any_perturbation() const
Definition: ERF_DataStruct.H:1404
SolverChoice::have_geo_wind_profile
bool have_geo_wind_profile
Definition: ERF_DataStruct.H:1434
SolverChoice::abl_geo_wind_table
std::string abl_geo_wind_table
Definition: ERF_DataStruct.H:1433
SolverChoice::vert_implicit_fac
amrex::Vector< amrex::Vector< amrex::Real > > vert_implicit_fac
Definition: ERF_DataStruct.H:1274
SolverChoice::use_rotate_surface_flux
bool use_rotate_surface_flux
Definition: ERF_DataStruct.H:1357
SolverChoice::do_forest_drag
bool do_forest_drag
Definition: ERF_DataStruct.H:1459
SolverChoice::spatial_rhotheta_forcing
bool spatial_rhotheta_forcing
Definition: ERF_DataStruct.H:1349
TurbulentPerturbation::debug
void debug(double)
Definition: ERF_TurbPertStruct.H:672
Here is the call graph for this function:

◆ InitData_pre()

void ERF::InitData_pre ( )
522 {
523  if (m_driver_has_atm2ocn_coupling && verbose > 0) {
524  amrex::Print() << "ERF InitData_pre: driver-managed atm2ocn coupling enabled"
525  << " two_way=" << (m_driver_uses_two_way_coupling ? 1 : 0)
526  << " active_contract="
527  << (m_driver_atm2ocn_uses_state_contract ? "state" : "flux")
528  << "\n";
529  }
530 
531  // Initialize the start time for our CPU-time tracker
532  startCPUTime = ParallelDescriptor::second();
533 
534  // Create the ReadBndryPlanes object so we can read boundary plane data
535  // m_r2d is used by init_bcs so we must instantiate this class before
536  if (input_bndry_planes) {
537  Print() << "Defining r2d for the first time " << std::endl;
538  m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
539  }
540 
541  if (restart_chkfile.empty()) {
542  // Start simulation from the beginning
543  InitFromScratch(zero);
544  } else {
545  // For initialization this is done in init_only; it is done here for restart
546  init_bcs();
547  }
548 
549  solverChoice.check_params(max_level,geom,phys_bc_type);
550 }
ERF::init_bcs
void init_bcs()
Definition: ERF_InitBCs.cpp:299
SolverChoice::check_params
void check_params(int max_level, const amrex::Vector< amrex::Geometry > &geom_vect, amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type)
Definition: ERF_DataStruct.H:911

◆ initHSE() [1/2]

void ERF::initHSE ( )
private

Initialize HSE.

193 {
194  for (int lev = 0; lev <= finest_level; lev++)
195  {
196  initHSE(lev);
197  }
198 }

◆ initHSE() [2/2]

void ERF::initHSE ( int  lev)
private

Initialize density and pressure base state in hydrostatic equilibrium.

20 {
21  // This integrates up through column to update p_hse, pi_hse, th_hse;
22  // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
23 
24  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
25  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
26  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
27  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
28  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
29 
30  bool all_boxes_touch_bottom = true;
31  Box domain(geom[lev].Domain());
32 
33  int icomp = 0; int ncomp = BaseState::num_comps;
34 
35  if (lev == 0) {
36  BoxArray ba(base_state[lev].boxArray());
37  for (int i = 0; i < ba.size(); i++) {
38  if (ba[i].smallEnd(2) != domain.smallEnd(2)) {
39  all_boxes_touch_bottom = false;
40  }
41  }
42  }
43  else
44  {
45  //
46  // We need to do this interp from coarse level in order to set the values of
47  // the base state inside the domain but outside of the fine region
48  //
49  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
50  //
51  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
52  // have been pre-filled - this includes ghost cells both inside and outside
53  // the domain
54  //
55  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
56  IntVect(0,0,0), // do not fill ghost cells outside the domain
57  base_state[lev-1], icomp, icomp, ncomp,
58  geom[lev-1], geom[lev],
59  refRatio(lev-1), &cell_cons_interp,
60  domain_bcs_type, BCVars::cons_bc);
61 
62  // We need to do this here because the interpolation above may leave corners unfilled
63  // when the corners need to be filled by, for example, reflection of the fine ghost
64  // cell outside the fine region but inide the domain.
65  (*physbcs_base[lev])(base_state[lev],icomp,ncomp,base_state[lev].nGrowVect());
66  }
67 
68  bool is_constant_dz = (solverChoice.mesh_type == MeshType::ConstantDz);
69  bool is_stretched_dz = (solverChoice.mesh_type == MeshType::StretchedDz);
70 
71  if (all_boxes_touch_bottom || lev > 0) {
72 
73  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
74  if ( (solverChoice.init_type == InitType::MoistBaseState) ||
75  (solverChoice.init_type == InitType::HindCast) )
76  {
77  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
78  prob->erf_init_dens_hse_moist(r_hse, z_phys_nd[lev], geom[lev]);
79 
80  }
81  else if (solverChoice.init_type == InitType::ConstantDensity)
82  {
83  // In this case we set rho from user-specified values, then integrate
84  // to define p from HSE (even if gravity = 0), then compute theta from (p,rho)
85  prob->erf_init_const_dens_hse(r_hse);
86  }
87  else if (solverChoice.init_type == InitType::Uniform)
88  {
89  // In this case we set both rho and theta from user-specified values
90  AMREX_ALWAYS_ASSERT(!solverChoice.use_gravity || solverChoice.anelastic[lev]);
91  prob->erf_init_const_dens_and_th_hse(r_hse,p_hse,pi_hse,th_hse,qv_hse,solverChoice.rdOcp);
92  }
93  else if (solverChoice.init_type == InitType::ConstantDensityLinearTheta)
94  {
95  // In this case we set both rho and theta from user-specified values
96  AMREX_ALWAYS_ASSERT(!solverChoice.use_gravity || solverChoice.anelastic[lev]);
97  prob->erf_init_const_dens_and_linear_th_hse(r_hse,p_hse,pi_hse,th_hse,qv_hse,
98  solverChoice.rdOcp,z_phys_cc[lev]);
99  }
100  else
101  {
102  // In this case we set rho from user-specified values, then integrate
103  // to define p from HSE (even if gravity = 0), then compute theta from (p,rho)
104  prob->erf_init_dens_hse_dry(r_hse, z_phys_nd[lev], z_phys_cc[lev], geom[lev], stretched_dz_h[lev],
105  is_constant_dz, is_stretched_dz);
106  }
107 
108  if (solverChoice.init_type != InitType::Uniform && solverChoice.init_type !=InitType::ConstantDensityLinearTheta) {
109  erf_enforce_hse(lev, r_hse, p_hse, pi_hse, th_hse, qv_hse, z_phys_cc[lev]);
110  }
111 
112  //
113  // Impose physical bc's on the base state
114  //
115  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
116 
117  } else {
118 
119  BoxArray ba_new(domain);
120 
121  ChopGrids2D(ba_new, domain, ParallelDescriptor::NProcs());
122 
123  DistributionMapping dm_new(ba_new);
124 
125  MultiFab new_base_state(ba_new, dm_new, BaseState::num_comps, base_state[lev].nGrowVect());
126  new_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
127  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
128 
129  MultiFab new_r_hse (new_base_state, make_alias, BaseState::r0_comp, 1);
130  MultiFab new_p_hse (new_base_state, make_alias, BaseState::p0_comp, 1);
131  MultiFab new_pi_hse(new_base_state, make_alias, BaseState::pi0_comp, 1);
132  MultiFab new_th_hse(new_base_state, make_alias, BaseState::th0_comp, 1);
133  MultiFab new_qv_hse(new_base_state, make_alias, BaseState::qv0_comp, 1);
134 
135  std::unique_ptr<MultiFab> new_z_phys_cc;
136  std::unique_ptr<MultiFab> new_z_phys_nd;
137  if (solverChoice.mesh_type != MeshType::ConstantDz) {
138  new_z_phys_cc = std::make_unique<MultiFab>(ba_new,dm_new,1,1);
139  new_z_phys_cc->ParallelCopy(*z_phys_cc[lev],0,0,1,1,1);
140 
141  BoxArray ba_new_nd(ba_new);
142  ba_new_nd.surroundingNodes();
143  new_z_phys_nd = std::make_unique<MultiFab>(ba_new_nd,dm_new,1,1);
144  new_z_phys_nd->ParallelCopy(*z_phys_nd[lev],0,0,1,1,1);
145  }
146 
147  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
148  if (solverChoice.init_type == InitType::MoistBaseState) {
149  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
150  prob->erf_init_dens_hse_moist(new_r_hse, new_z_phys_nd, geom[lev]);
151  } else if (solverChoice.init_type == InitType::ConstantDensity) {
152 
153  // In this case we set rho from user-specified values, then integrate
154  // to define p from HSE (even if gravity = 0), then compute theta from (p,rho)
155  prob->erf_init_const_dens_hse(new_r_hse);
156 
157  } else if (solverChoice.init_type == InitType::Uniform) {
158 
159  // In this case we set both rho and theta from user-specified values
160  AMREX_ALWAYS_ASSERT(!solverChoice.use_gravity || solverChoice.anelastic[lev]);
161  prob->erf_init_const_dens_and_th_hse(new_r_hse,new_p_hse,new_pi_hse,new_th_hse,new_qv_hse,solverChoice.rdOcp);
162 
163  } else {
164  prob->erf_init_dens_hse_dry(new_r_hse, new_z_phys_nd, new_z_phys_cc, geom[lev], stretched_dz_h[lev],
165  is_constant_dz, is_stretched_dz);
166  }
167 
168  erf_enforce_hse(lev, new_r_hse, new_p_hse, new_pi_hse, new_th_hse, new_qv_hse, new_z_phys_cc);
169 
170  //
171  // Impose physical bc's on the base state (we must make new, temporary bcs object because the z_phys_nd is different)
172  //
173  ERFPhysBCFunct_base* temp_physbcs_base =
174  new ERFPhysBCFunct_base(lev, geom[lev], domain_bcs_type, domain_bcs_type_d, new_z_phys_nd,
175  (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
176  (*temp_physbcs_base)(new_base_state,0,new_base_state.nComp(),new_base_state.nGrowVect());
177  delete temp_physbcs_base;
178 
179  // Now copy back into the original arrays
180  base_state[lev].ParallelCopy(new_base_state,0,0,base_state[lev].nComp(),
181  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
182  }
183 
184  //
185  // Impose physical bc's on the base state -- the values outside the fine region
186  // but inside the domain have already been filled in the call above to InterpFromCoarseLevel
187  //
188  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
189 }
ChopGrids2D
void ChopGrids2D(BoxArray &ba, const Box &domain, int target_size)
Definition: ERF_ChopGrids.cpp:21
ERFPhysBCFunct_base
Definition: ERF_PhysBCFunct.H:286
ERF::erf_enforce_hse
void erf_enforce_hse(int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
Definition: ERF_Init1D.cpp:210
Here is the call graph for this function:

◆ initialize_integrator()

void ERF::initialize_integrator ( int  lev,
amrex::MultiFab &  cons_mf,
amrex::MultiFab &  vel_mf 
)
private
899 {
900  const BoxArray& ba(cons_mf.boxArray());
901  const DistributionMapping& dm(cons_mf.DistributionMap());
902 
903  int ncomp_cons = cons_mf.nComp();
904 
905  // Initialize the integrator memory
906  Vector<MultiFab> int_state; // integration state data structure example
907  int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons)); // cons
908  int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
909  int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
910  int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
911 
912  mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
913  mri_integrator_mem[lev]->setNoSubstepping((solverChoice.substepping_type[lev] == SubsteppingType::None));
914  mri_integrator_mem[lev]->setAnelastic(solverChoice.anelastic[lev]);
915  mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
916  mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
917 }

◆ InitializeFromFile()

void ERF::InitializeFromFile ( )
private

◆ InitializeLevelFromData()

void ERF::InitializeLevelFromData ( int  lev,
const amrex::MultiFab &  initial_data 
)
private

◆ initializeMicrophysics()

void ERF::initializeMicrophysics ( const int &  a_nlevsmax)
private
Parameters
a_nlevsmaxnumber of AMR levels
1714 {
1715  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
1716 
1717  micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1718 
1719  } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
1720 #ifdef ERF_USE_PARTICLES
1721  micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1722  /* Lagrangian microphysics models will have a particle container; it needs to be added
1723  to ERF::particleData */
1724  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
1725  /* The particle container has not yet been constructed and initialized, so just add
1726  its name here for now (so that functions to set plotting variables can see it). */
1727  particleData.addName( pc_name );
1728 
1729 #else
1730  Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
1731 #endif
1732  }
1733 
1734  qmoist.resize(a_nlevsmax);
1735  return;
1736 }
ERF::qmoist
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
Definition: ERF.H:973

Referenced by ERF_shared().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ initRayleigh_at_level()

void ERF::initRayleigh_at_level ( const int &  lev)
private

Initialize Rayleigh damping profiles at a level.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

15 {
16  const int khi = geom[0].Domain().bigEnd(2);
17  solverChoice.dampingChoice.rayleigh_ztop = (solverChoice.terrain_type == TerrainType::None) ? geom[0].ProbHi(2) : zlevels_stag[0][khi+1];
18 
19  // These have 4 components: ubar, vbar, wbar, thetabar
20  h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
21  d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
22 
23  const int zlen_rayleigh = geom[lev].Domain().length(2);
24 
25  // Allocate space for these 1D vectors
26  for (int n = 0; n < Rayleigh::nvars; n++) {
27  h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
28  d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
29  }
30 
31  h_sinesq_ptrs[lev].resize(zlen_rayleigh);
32  d_sinesq_ptrs[lev].resize(zlen_rayleigh);
33 
34  h_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
35  d_sinesq_stag_ptrs[lev].resize(zlen_rayleigh+1);
36 
37  Real ztop = solverChoice.dampingChoice.rayleigh_ztop;
38  Real zdamp = solverChoice.dampingChoice.rayleigh_zdamp;
39 
40  for (int k = 0; k < zlen_rayleigh; k++) {
41  Real z = myhalf * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1]);
42  if (z > (ztop - zdamp)) {
43  Real zfrac = one - (ztop - z) / zdamp;
44  Real s = std::sin(PIoTwo*zfrac);
45  h_sinesq_ptrs[lev][k] = s*s;
46  } else {
47  h_sinesq_ptrs[lev][k] = zero;
48  }
49  }
50 
51  for (int k = 0; k < zlen_rayleigh+1; k++) {
52  Real z = zlevels_stag[lev][k];
53  if (z > (ztop - zdamp)) {
54  Real zfrac = one - (ztop - z) / zdamp;
55  Real s = std::sin(PIoTwo*zfrac);
56  h_sinesq_stag_ptrs[lev][k] = s*s;
57  } else {
58  h_sinesq_stag_ptrs[lev][k] = zero;
59  }
60  }
61 
62  // Init the host vectors for the reference states
63  prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_nd[lev],
64  solverChoice.dampingChoice.rayleigh_zdamp);
65 
66  // Copy from host vectors to device vectors
67  for (int n = 0; n < Rayleigh::nvars; n++) {
68  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
69  d_rayleigh_ptrs[lev][n].begin());
70  }
71  Gpu::copy(Gpu::hostToDevice, h_sinesq_ptrs[lev].begin(), h_sinesq_ptrs[lev].end(), d_sinesq_ptrs[lev].begin());
72  Gpu::copy(Gpu::hostToDevice, h_sinesq_stag_ptrs[lev].begin(), h_sinesq_stag_ptrs[lev].end(), d_sinesq_stag_ptrs[lev].begin());
73 }
PIoTwo
constexpr amrex::Real PIoTwo
Definition: ERF_Constants.H:43
DampingChoice::rayleigh_ztop
amrex::Real rayleigh_ztop
Definition: ERF_DampingStruct.H:90
DampingChoice::rayleigh_zdamp
amrex::Real rayleigh_zdamp
Definition: ERF_DampingStruct.H:89

◆ initSponge()

void ERF::initSponge ( )
private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

36 {
37  h_sponge_ptrs.resize(max_level+1);
38  d_sponge_ptrs.resize(max_level+1);
39 
40  for (int lev = 0; lev <= finest_level; lev++)
41  {
42  // These have 2 components: ubar, vbar
43  h_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
44  d_sponge_ptrs[lev].resize(Sponge::nvars_sponge);
45 
46  const int zlen_sponge = geom[lev].Domain().length(2);
47 
48  // Allocate space for these 1D vectors
49  for (int n = 0; n < Sponge::nvars_sponge; n++) {
50  h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
51  d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
52  }
53 
54  }
55 }
ERF::h_sponge_ptrs
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
Definition: ERF.H:1440

◆ input_sponge()

void ERF::input_sponge ( int  lev)

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters
levInteger specifying the current level
18 {
19  // We only want to read the file once
20  if (lev == 0) {
21  if (input_sponge_data.input_sponge_file.empty())
22  Error("input_sounding file name must be provided via input");
23 
24  // this will interpolate the input profiles to the nominal height levels
25  // (ranging from 0 to the domain top)
26  input_sponge_data.read_from_file(geom[lev], zlevels_stag[lev]);
27  }
28 }
ERF::input_sponge_data
InputSpongeData input_sponge_data
Definition: ERF.H:880
InputSpongeData::read_from_file
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag)
Definition: ERF_InputSpongeData.H:28
InputSpongeData::input_sponge_file
std::string input_sponge_file
Definition: ERF_InputSpongeData.H:108

◆ Interp2DArrays()

void ERF::Interp2DArrays ( int  lev,
const amrex::BoxArray &  my_ba2d,
const amrex::DistributionMapping &  my_dm 
)
1516 {
1517  if (lev == 0) { return; }
1518 
1519  if (lon_m[lev-1] && !lon_m[lev]) {
1520  auto ngv = lon_m[lev-1]->nGrowVect(); ngv[2] = 0;
1521  lon_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
1522  InterpFromCoarseLevel(*lon_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
1523  *lon_m[lev-1], 0, 0, 1,
1524  geom[lev-1], geom[lev],
1525  refRatio(lev-1), &cell_cons_interp,
1526  domain_bcs_type, BCVars::cons_bc);
1527  }
1528  if (lat_m[lev-1] && !lat_m[lev]) {
1529  auto ngv = lat_m[lev-1]->nGrowVect(); ngv[2] = 0;
1530  lat_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
1531  InterpFromCoarseLevel(*lat_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
1532  *lat_m[lev-1], 0, 0, 1,
1533  geom[lev-1], geom[lev],
1534  refRatio(lev-1), &cell_cons_interp,
1535  domain_bcs_type, BCVars::cons_bc);
1536  }
1537  if (sinPhi_m[lev-1] && !sinPhi_m[lev]) {
1538  auto ngv = sinPhi_m[lev-1]->nGrowVect(); ngv[2] = 0;
1539  sinPhi_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
1540  InterpFromCoarseLevel(*sinPhi_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
1541  *sinPhi_m[lev-1], 0, 0, 1,
1542  geom[lev-1], geom[lev],
1543  refRatio(lev-1), &cell_cons_interp,
1544  domain_bcs_type, BCVars::cons_bc);
1545  }
1546  if (cosPhi_m[lev-1] && !cosPhi_m[lev]) {
1547  auto ngv = cosPhi_m[lev-1]->nGrowVect(); ngv[2] = 0;
1548  cosPhi_m[lev] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
1549  InterpFromCoarseLevel(*cosPhi_m[lev], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
1550  *cosPhi_m[lev-1], 0, 0, 1,
1551  geom[lev-1], geom[lev],
1552  refRatio(lev-1), &cell_cons_interp,
1553  domain_bcs_type, BCVars::cons_bc);
1554  }
1555  if (sst_lev[lev-1][0]) {
1556  if (sst_lev[lev].size() < sst_lev[lev-1].size()) {
1557  sst_lev[lev].resize(sst_lev[lev-1].size());
1558  }
1559 #ifdef ERF_USE_NETCDF
1560  double time_since_start_low = t_new[0] + start_time - start_low_time;
1561  int n_time_old = static_cast<int>(time_since_start_low / low_time_interval);
1562  int ntimes_to_interp = std::min(n_time_old+3, static_cast<int>(sst_lev[lev-1].size()));
1563 #else
1564  // TODO: Fix if SST is provided without NETCDF
1565  int n_time_old = 0;
1566  int ntimes_to_interp = 1;
1567 #endif
1568  auto ngv = sst_lev[lev-1][0]->nGrowVect(); ngv[2] = 0;
1569 
1570  for (int n = n_time_old; n < ntimes_to_interp; n++) {
1571  if (!sst_lev[lev-1][n]) { continue; }
1572  if (!sst_lev[lev][n]) {
1573  sst_lev[lev][n] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
1574  InterpFromCoarseLevel(*sst_lev[lev][n], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
1575  *sst_lev[lev-1][n], 0, 0, 1,
1576  geom[lev-1], geom[lev],
1577  refRatio(lev-1), &cell_cons_interp,
1578  domain_bcs_type, BCVars::cons_bc);
1579  }
1580  }
1581  }
1582  if (tsk_lev[lev-1][0]) {
1583  if (tsk_lev[lev].size() < tsk_lev[lev-1].size()) {
1584  tsk_lev[lev].resize(tsk_lev[lev-1].size());
1585  }
1586 #ifdef ERF_USE_NETCDF
1587  double time_since_start_low = t_new[0] + start_time - start_low_time;
1588  int n_time_old = static_cast<int>(time_since_start_low / low_time_interval);
1589  int ntimes_to_interp = std::min(n_time_old+3, static_cast<int>(tsk_lev[lev-1].size()));
1590 #else
1591  // TODO: Fix if TSK is provided without NETCDF
1592  int n_time_old = 0;
1593  int ntimes_to_interp = 1;
1594 #endif
1595  auto ngv = tsk_lev[lev-1][0]->nGrowVect(); ngv[2] = 0;
1596 
1597  for (int n = n_time_old; n < ntimes_to_interp; n++) {
1598  if (!tsk_lev[lev-1][n]) { continue; }
1599  if (!tsk_lev[lev][n]) {
1600  tsk_lev[lev][n] = std::make_unique<MultiFab>(my_ba2d,my_dm,1,ngv);
1601  InterpFromCoarseLevel(*tsk_lev[lev][n], ngv, IntVect(0,0,0), // do not fill ghost cells outside the domain
1602  *tsk_lev[lev-1][n], 0, 0, 1,
1603  geom[lev-1], geom[lev],
1604  refRatio(lev-1), &cell_cons_interp,
1605  domain_bcs_type, BCVars::cons_bc);
1606  }
1607  }
1608  }
1609 
1610  Real time_for_fp = 0.0; // This is not actually used
1611  Vector<Real> ftime = {static_cast<Real>(time_for_fp), static_cast<Real>(time_for_fp)};
1612  Vector<Real> ctime = {static_cast<Real>(time_for_fp), static_cast<Real>(time_for_fp)};
1613  if (lat_m[lev]) {
1614  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1615  Vector<MultiFab*> fmf = {lat_m[lev ].get(), lat_m[lev ].get()};
1616  Vector<MultiFab*> cmf = {lat_m[lev-1].get(), lat_m[lev-1].get()};
1617  IntVect ngv = lat_m[lev]->nGrowVect(); ngv[2] = 0;
1618  Interpolater* mapper = &cell_cons_interp;
1619  FillPatchTwoLevels(*lat_m[lev].get(), ngv, IntVect(0,0,0),
1620  time_for_fp, cmf, ctime, fmf, ftime,
1621  0, 0, 1, geom[lev-1], geom[lev],
1622  refRatio(lev-1), mapper, domain_bcs_type,
1623  BCVars::cons_bc);
1624  }
1625  if (lon_m[lev]) {
1626  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1627  Vector<MultiFab*> fmf = {lon_m[lev ].get(), lon_m[lev ].get()};
1628  Vector<MultiFab*> cmf = {lon_m[lev-1].get(), lon_m[lev-1].get()};
1629  IntVect ngv = lon_m[lev]->nGrowVect(); ngv[2] = 0;
1630  Interpolater* mapper = &cell_cons_interp;
1631  FillPatchTwoLevels(*lon_m[lev].get(), ngv, IntVect(0,0,0),
1632  time_for_fp, cmf, ctime, fmf, ftime,
1633  0, 0, 1, geom[lev-1], geom[lev],
1634  refRatio(lev-1), mapper, domain_bcs_type,
1635  BCVars::cons_bc);
1636  } // lon_m
1637  if (sinPhi_m[lev]) {
1638  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1639  Vector<MultiFab*> fmf = {sinPhi_m[lev ].get(), sinPhi_m[lev ].get()};
1640  Vector<MultiFab*> cmf = {sinPhi_m[lev-1].get(), sinPhi_m[lev-1].get()};
1641  IntVect ngv = sinPhi_m[lev]->nGrowVect(); ngv[2] = 0;
1642  Interpolater* mapper = &cell_cons_interp;
1643  FillPatchTwoLevels(*sinPhi_m[lev].get(), ngv, IntVect(0,0,0),
1644  time_for_fp, cmf, ctime, fmf, ftime,
1645  0, 0, 1, geom[lev-1], geom[lev],
1646  refRatio(lev-1), mapper, domain_bcs_type,
1647  BCVars::cons_bc);
1648  } // sinPhi
1649  if (cosPhi_m[lev]) {
1650  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1651  Vector<MultiFab*> fmf = {cosPhi_m[lev ].get(), cosPhi_m[lev ].get()};
1652  Vector<MultiFab*> cmf = {cosPhi_m[lev-1].get(), cosPhi_m[lev-1].get()};
1653  IntVect ngv = cosPhi_m[lev]->nGrowVect(); ngv[2] = 0;
1654  Interpolater* mapper = &cell_cons_interp;
1655  FillPatchTwoLevels(*cosPhi_m[lev].get(), ngv, IntVect(0,0,0),
1656  time_for_fp, cmf, ctime, fmf, ftime,
1657  0, 0, 1, geom[lev-1], geom[lev],
1658  refRatio(lev-1), mapper, domain_bcs_type,
1659  BCVars::cons_bc);
1660  } // cosPhi
1661  if (sst_lev[lev][0]) {
1662  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1663 #ifdef ERF_USE_NETCDF
1664  double time_since_start_low = t_new[0] + start_time - start_low_time;
1665  int n_time_old = static_cast<int>(time_since_start_low / low_time_interval);
1666  int ntimes_to_interp = std::min(n_time_old+3, static_cast<int>(sst_lev[lev-1].size()));
1667 #else
1668  // TODO: Fix if SST is provided without NETCDF
1669  int n_time_old = 0;
1670  int ntimes_to_interp = 1;
1671 #endif
1672  for (int n = n_time_old; n < ntimes_to_interp; n++) {
1673  if (!sst_lev[lev][n] || !sst_lev[lev-1][n]) { continue; }
1674  Vector<MultiFab*> fmf = {sst_lev[lev ][n].get(), sst_lev[lev ][n].get()};
1675  Vector<MultiFab*> cmf = {sst_lev[lev-1][n].get(), sst_lev[lev-1][n].get()};
1676  IntVect ngv = sst_lev[lev][n]->nGrowVect(); ngv[2] = 0;
1677  Interpolater* mapper = &cell_cons_interp;
1678  FillPatchTwoLevels(*sst_lev[lev][n].get(), ngv, IntVect(0,0,0),
1679  time_for_fp, cmf, ctime, fmf, ftime,
1680  0, 0, 1, geom[lev-1], geom[lev],
1681  refRatio(lev-1), mapper, domain_bcs_type,
1682  BCVars::cons_bc);
1683  } // ntimes
1684  } // sst_lev
1685  if (tsk_lev[lev][0]) {
1686  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1687 #ifdef ERF_USE_NETCDF
1688  double time_since_start_low = t_new[0] + start_time - start_low_time;
1689  int n_time_old = static_cast<int>(time_since_start_low / low_time_interval);
1690  int ntimes_to_interp = std::min(n_time_old+3, static_cast<int>(tsk_lev[lev-1].size()));
1691 #else
1692  // TODO: Fix if TSK is provided without NETCDF
1693  int n_time_old = 0;
1694  int ntimes_to_interp = 1;
1695 #endif
1696  for (int n = n_time_old; n < ntimes_to_interp; n++) {
1697  if (!tsk_lev[lev][n] || !tsk_lev[lev-1][n]) { continue; }
1698  Vector<MultiFab*> fmf = {tsk_lev[lev ][n].get(), tsk_lev[lev ][n].get()};
1699  Vector<MultiFab*> cmf = {tsk_lev[lev-1][n].get(), tsk_lev[lev-1][n].get()};
1700  IntVect ngv = tsk_lev[lev][n]->nGrowVect(); ngv[2] = 0;
1701  Interpolater* mapper = &cell_cons_interp;
1702  FillPatchTwoLevels(*tsk_lev[lev][n].get(), ngv, IntVect(0,0,0),
1703  time_for_fp, cmf, ctime, fmf, ftime,
1704  0, 0, 1, geom[lev-1], geom[lev],
1705  refRatio(lev-1), mapper, domain_bcs_type,
1706  BCVars::cons_bc);
1707  } // ntimes
1708  } // tsk_lev
1709 }
Here is the call graph for this function:

◆ is_it_time_for_action()

bool ERF::is_it_time_for_action ( int  nstep,
double  time,
double  dtlev,
int  action_interval,
double  action_per 
)
static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters
nstepTimestep number
timeCurrent time
dtlevTimestep for the current level
action_intervalInterval in number of timesteps for taking action
action_perInterval in simulation time for taking action
660 {
661  bool int_test = (action_interval > 0 && nstep % action_interval == 0);
662 
663  bool per_test = false;
664  if (action_per > zero) {
665  const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
666  const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
667 
668  if (num_per_old != num_per_new) {
669  per_test = true;
670  }
671  }
672 
673  return int_test || per_test;
674 }

◆ make_eb_box()

void ERF::make_eb_box ( )

◆ make_eb_regular()

void ERF::make_eb_regular ( )

◆ make_physbcs()

void ERF::make_physbcs ( int  lev)
private
921 {
922  if (SolverChoice::mesh_type == MeshType::VariableDz) {
923  AMREX_ALWAYS_ASSERT(z_phys_nd[lev] != nullptr);
924  }
925 
926  physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
927  m_bc_extdir_vals, m_bc_neumann_vals,
928  z_phys_nd[lev], solverChoice.use_real_bcs, th_bc_data[lev].data());
929  physbcs_u[lev] = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
930  m_bc_extdir_vals, m_bc_neumann_vals,
931  z_phys_nd[lev], solverChoice.use_real_bcs, xvel_bc_data[lev].data());
932  physbcs_v[lev] = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
933  m_bc_extdir_vals, m_bc_neumann_vals,
934  z_phys_nd[lev], solverChoice.use_real_bcs, yvel_bc_data[lev].data());
935  physbcs_w[lev] = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
936  m_bc_extdir_vals, m_bc_neumann_vals,
937  solverChoice.terrain_type, mapfac[lev], z_phys_nd[lev],
938  solverChoice.use_real_bcs, zvel_bc_data[lev].data());
939  physbcs_base[lev] = std::make_unique<ERFPhysBCFunct_base> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d, z_phys_nd[lev],
940  (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
941 }
Here is the call graph for this function:

◆ make_subdomains()

void ERF::make_subdomains ( const amrex::BoxList &  ba,
amrex::Vector< amrex::BoxArray > &  bins 
)
7 {
8  Vector<BoxList> bins_bl;
9 
10  // Clear out any old bins
11  bins.clear();
12 
13  // Iterate over boxes
14  for (auto bx : bl)
15  {
16  bool added = false;
17 
18  // Try to add box to existing bin
19  for (int j = 0; j < bins_bl.size(); ++j) {
20  BoxList& bin = bins_bl[j];
21  bool touches = false;
22 
23  for (auto& b : bin)
24  {
25  Box gbx(bx); gbx.grow(1);
26  if (gbx.intersects(b)) {
27  touches = true;
28  break;
29  }
30  }
31 
32  if (touches) {
33  bin.push_back(bx);
34  added = true;
35  break;
36  }
37  }
38 
39  // If box couldn't be added to existing bin, create new bin
40  if (!added) {
41  BoxList new_bin;
42  new_bin.push_back(bx);
43  bins_bl.push_back(new_bin);
44  }
45  }
46 
47  // Convert the BoxLists to BoxArrays
48  for (int i = 0; i < bins_bl.size(); ++i) {
49  bins.push_back(BoxArray(bins_bl[i]));
50  }
51 }

◆ MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage ( amrex::Vector< amrex::Real > &  h_havg,
amrex::MultiFab &  S,
int  n 
)
2668 {
2669  // Get the number of cells in z at level 0
2670  int dir_z = AMREX_SPACEDIM-1;
2671  auto domain = geom[0].Domain();
2672  int size_z = domain.length(dir_z);
2673  int start_z = domain.smallEnd()[dir_z];
2674  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
2675 
2676  // resize the level 0 horizontal average vectors
2677  h_havg.resize(size_z, 0.0_rt);
2678 
2679  // Get the cell centered data and construct sums
2680 #ifdef _OPENMP
2681 #pragma omp parallel if (Gpu::notInLaunchRegion())
2682 #endif
2683  for (MFIter mfi(S); mfi.isValid(); ++mfi) {
2684  const Box& box = mfi.validbox();
2685  const IntVect& se = box.smallEnd();
2686  const IntVect& be = box.bigEnd();
2687 
2688  auto fab_arr = S[mfi].array();
2689 
2690  FArrayBox fab_reduce(box, 1, The_Async_Arena());
2691  auto arr_reduce = fab_reduce.array();
2692 
2693  ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2694  arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
2695  });
2696 
2697  for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
2698  Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
2699  h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
2700  }
2701  }
2702 
2703  // combine sums from different MPI ranks
2704  ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), static_cast<int>(h_havg.size()));
2705 
2706  // divide by the total number of cells we are averaging over
2707  for (int k = 0; k < size_z; ++k) {
2708  h_havg[k] /= area_z;
2709  }
2710 }
Here is the call graph for this function:

◆ MakeEBGeometry()

void ERF::MakeEBGeometry ( )

◆ MakeFilename_EyeTracker_latlon()

std::string ERF::MakeFilename_EyeTracker_latlon ( int  nstep)
52  {
53  // Ensure output directory exists
54  const std::string dir = "Output_StormTracker/latlon";
55  if (!fs::exists(dir)) {
56  fs::create_directories(dir);
57  }
58 
59  // Construct filename with zero-padded step
60  std::ostringstream oss;
61  oss << dir << "/storm_track_latlon" << std::setw(7) << std::setfill('0') << nstep << ".txt";
62  return oss.str();
63 }

◆ MakeFilename_EyeTracker_maxvel()

std::string ERF::MakeFilename_EyeTracker_maxvel ( int  nstep)
66  {
67  // Ensure output directory exists
68  const std::string dir = "Output_StormTracker/maxvel";
69  if (!fs::exists(dir)) {
70  fs::create_directories(dir);
71  }
72 
73  // Construct filename with zero-padded step
74  std::ostringstream oss;
75  oss << dir << "/storm_maxvel_" << std::setw(7) << std::setfill('0') << nstep << ".txt";
76  return oss.str();
77 }

◆ MakeFilename_EyeTracker_minpressure()

std::string ERF::MakeFilename_EyeTracker_minpressure ( int  nstep)
80  {
81  // Ensure output directory exists
82  const std::string dir = "Output_StormTracker/minpressure";
83  if (!fs::exists(dir)) {
84  fs::create_directories(dir);
85  }
86 
87  // Construct filename with zero-padded step
88  std::ostringstream oss;
89  oss << dir << "/storm_minpressure_" << std::setw(7) << std::setfill('0') << nstep << ".txt";
90  return oss.str();
91 }

◆ MakeHorizontalAverages()

void ERF::MakeHorizontalAverages ( )
2562 {
2563  int lev = 0;
2564 
2565  // First, average down all levels (if doing two-way coupling)
2566  if (solverChoice.coupling_type == CouplingType::TwoWay) {
2567  AverageDown();
2568  }
2569 
2570  MultiFab mf(grids[lev], dmap[lev], 5, 0);
2571 
2572  int zdir = 2;
2573  auto domain = geom[0].Domain();
2574 
2575  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
2576  bool is_anelastic = (solverChoice.anelastic[lev] == 1);
2577 
2578  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
2579  const Box& bx = mfi.validbox();
2580  auto fab_arr = mf.array(mfi);
2581  auto const hse_arr = base_state[lev].const_array(mfi);
2582  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
2583  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2584  Real dens = cons_arr(i, j, k, Rho_comp);
2585  fab_arr(i, j, k, 0) = dens;
2586  fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
2587  if (!use_moisture) {
2588  if (is_anelastic) {
2589  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
2590  } else {
2591  fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp));
2592  }
2593  }
2594  });
2595  }
2596 
2597  if (use_moisture)
2598  {
2599  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
2600  const Box& bx = mfi.validbox();
2601  auto fab_arr = mf.array(mfi);
2602  auto const hse_arr = base_state[lev].const_array(mfi);
2603  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
2604  int ncomp = vars_new[lev][Vars::cons].nComp();
2605 
2606  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2607  Real dens = cons_arr(i, j, k, Rho_comp);
2608  if (is_anelastic) {
2609  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
2610  } else {
2611  Real qv = cons_arr(i, j, k, RhoQ1_comp) / dens;
2612  fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
2613  }
2614  fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : zero);
2615  fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : zero);
2616  });
2617  }
2618 
2619  Gpu::HostVector<Real> h_avg_qv = sumToLine(mf,3,1,domain,zdir);
2620  Gpu::HostVector<Real> h_avg_qc = sumToLine(mf,4,1,domain,zdir);
2621  }
2622 
2623  // Sum in the horizontal plane
2624  Gpu::HostVector<Real> h_avg_density = sumToLine(mf,0,1,domain,zdir);
2625  Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
2626  Gpu::HostVector<Real> h_avg_pressure = sumToLine(mf,2,1,domain,zdir);
2627 
2628  // Divide by the total number of cells we are averaging over
2629  int size_z = domain.length(zdir);
2630  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
2631  int klen = static_cast<int>(h_avg_density.size());
2632 
2633  for (int k = 0; k < klen; ++k) {
2634  h_havg_density[k] /= area_z;
2635  h_havg_temperature[k] /= area_z;
2636  h_havg_pressure[k] /= area_z;
2637  if (solverChoice.moisture_type != MoistureType::None)
2638  {
2639  h_havg_qc[k] /= area_z;
2640  h_havg_qv[k] /= area_z;
2641  }
2642  } // k
2643 
2644  // resize device vectors
2645  d_havg_density.resize(size_z, 0.0_rt);
2646  d_havg_temperature.resize(size_z, 0.0_rt);
2647  d_havg_pressure.resize(size_z, 0.0_rt);
2648 
2649  // copy host vectors to device vectors
2650  Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
2651  Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
2652  Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
2653 
2654  if (solverChoice.moisture_type != MoistureType::None)
2655  {
2656  d_havg_qv.resize(size_z, 0.0_rt);
2657  d_havg_qc.resize(size_z, 0.0_rt);
2658  Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
2659  Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
2660  }
2661 }
ERF::d_havg_temperature
amrex::Gpu::DeviceVector< amrex::Real > d_havg_temperature
Definition: ERF.H:1461
ERF::d_havg_qv
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qv
Definition: ERF.H:1463
ERF::h_havg_pressure
amrex::Vector< amrex::Real > h_havg_pressure
Definition: ERF.H:1456
ERF::h_havg_qc
amrex::Vector< amrex::Real > h_havg_qc
Definition: ERF.H:1458
ERF::h_havg_density
amrex::Vector< amrex::Real > h_havg_density
Definition: ERF.H:1454
ERF::d_havg_qc
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qc
Definition: ERF.H:1464
ERF::d_havg_density
amrex::Gpu::DeviceVector< amrex::Real > d_havg_density
Definition: ERF.H:1460
ERF::h_havg_temperature
amrex::Vector< amrex::Real > h_havg_temperature
Definition: ERF.H:1455
ERF::d_havg_pressure
amrex::Gpu::DeviceVector< amrex::Real > d_havg_pressure
Definition: ERF.H:1462
ERF::h_havg_qv
amrex::Vector< amrex::Real > h_havg_qv
Definition: ERF.H:1457
Here is the call graph for this function:

◆ MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
280 {
281  //
282  // Note that "time" here is elapsed time
283  //
284  AMREX_ALWAYS_ASSERT(lev > 0);
285 
286  if (verbose) {
287  amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
288  }
289 
290  //
291  // Grow the subdomains vector and build the subdomains vector at this level
292  //
293  subdomains.resize(lev+1);
294  //
295  // Create subdomains at each level within the domain such that
296  // 1) all boxes in a given subdomain are "connected"
297  // 2) no boxes in a subdomain touch any boxes in any other subdomain
298  //
299  make_subdomains(ba.simplified_list(), subdomains[lev]);
300 
301  if (lev == 0) init_bcs();
302 
303  //********************************************************************************************
304  // This allocates all kinds of things, including but not limited to: solution arrays,
305  // terrain arrays, ba2d, metric terms and base state.
306  // *******************************************************************************************
307  init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev], base_state[lev], z_phys_nd[lev]);
308 
309  //
310  // Note that t_new = time here is elapsed time
311  //
312  t_new[lev] = time;
313  t_old[lev] = time - bogus_large_value;
314 
315  // ********************************************************************************************
316  // Build the data structures for metric quantities used with terrain-fitted coordinates
317  // ********************************************************************************************
318  if ( solverChoice.terrain_type == TerrainType::EB ||
319  solverChoice.terrain_type == TerrainType::ImmersedForcing ||
320  solverChoice.buildings_type == BuildingsType::ImmersedForcing)
321  {
322  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
323  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
324  if (solverChoice.terrain_type == TerrainType::EB) {
325  eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
326  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
327  solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
328  eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
329  }
330  }
331  init_zphys(lev, time);
332  update_terrain_arrays(lev);
333 
334  //
335  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
336  // *and* if there is two-way coupling
337  //
338  if ( (SolverChoice::mesh_type != MeshType::ConstantDz) && (solverChoice.coupling_type == CouplingType::TwoWay) ) {
339  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
340  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
341  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
342  }
343  }
344 
345  // ********************************************************************************************
346  // Build the data structures for canopy model (depends upon z_phys)
347  // ********************************************************************************************
348  if (solverChoice.do_forest_drag) {
349  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
350  }
351 
352  //********************************************************************************************
353  // Radiation
354  // *******************************************************************************************
355  if (solverChoice.rad_type != RadiationType::None)
356  {
357  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
358  }
359 
360  // *****************************************************************************************************
361  // Initialize the boundary conditions (after initializing the terrain but before calling
362  // initHSE or FillCoarsePatch)
363  // *****************************************************************************************************
364  make_physbcs(lev);
365 
366  // ********************************************************************************************
367  // Update the base state at this level by interpolation from coarser level
368  // ********************************************************************************************
369  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
370  IntVect(0,0,0), // do not fill ghost cells outside the domain
371  base_state[lev-1], 0, 0, base_state[lev].nComp(),
372  geom[lev-1], geom[lev],
373  refRatio(lev-1), &cell_cons_interp,
374  domain_bcs_type, BCVars::cons_bc);
375 
376  // Impose bc's outside the domain
377  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
378 
379  //********************************************************************************************
380  // Microphysics
381  // *******************************************************************************************
382  int q_size = micro->Get_Qmoist_Size(lev);
383  qmoist[lev].resize(q_size);
384  micro->Define(lev, solverChoice);
385  if (solverChoice.moisture_type != MoistureType::None)
386  {
387  micro->Init(lev, vars_new[lev][Vars::cons],
388  grids[lev], Geom(lev), zero,
389  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
390  }
391  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
392  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
393  }
394 
395  // ********************************************************************************************
396  // Build the data structures for calculating diffusive/turbulent terms
397  // ********************************************************************************************
398  update_diffusive_arrays(lev, ba, dm);
399 
400  // ********************************************************************************************
401  // Build the data structures for holding sea surface temps and skin temps
402  // ********************************************************************************************
403  sst_lev[lev].resize(1); sst_lev[lev][0] = nullptr;
404  tsk_lev[lev].resize(1); tsk_lev[lev][0] = nullptr;
405 
406  // ********************************************************************************************
407  // Fill data at the new level by interpolation from the coarser level
408  // Note that internal to FillCoarsePatch we will convert velocity to momentum,
409  // then interpolate momentum, then convert momentum back to velocity
410  // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
411  // ********************************************************************************************
412 
413 #ifdef ERF_USE_NETCDF
414  if ( ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) ) &&
415  !nc_init_file[lev].empty() )
416  {
417  // Just making sure that ghost cells aren't uninitialized...
418  vars_new[lev][Vars::cons].setVal(0.0); vars_old[lev][Vars::cons].setVal(0.0);
419  vars_new[lev][Vars::xvel].setVal(0.0); vars_old[lev][Vars::xvel].setVal(0.0);
420  vars_new[lev][Vars::yvel].setVal(0.0); vars_old[lev][Vars::yvel].setVal(0.0);
421  vars_new[lev][Vars::zvel].setVal(0.0); vars_old[lev][Vars::zvel].setVal(0.0);
422 
423  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
424  if (solverChoice.init_type == InitType::Metgrid) {
425  init_from_metgrid(lev);
426  } else if (solverChoice.init_type == InitType::WRFInput) {
427  init_from_wrfinput(lev, *mf_PSFC[lev]);
428  }
429  init_zphys(lev, time);
430  update_terrain_arrays(lev);
431  make_physbcs(lev);
432 
433  dz_min[lev] = (*detJ_cc[lev]).min(0) * geom[lev].CellSize(2);
434 
435  } else {
436 #endif
437  //
438  // Interpolate the solution data
439  //
440  FillCoarsePatch(lev, time);
441 
442  //
443  // Interpolate the 2D arrays at the lower boundary
444  // Note that ba2d is constructed already in init_stuff, but we have not yet defined dmap[lev]
445  // so we must explicitly pass dm.
446  Interp2DArrays(lev,ba2d[lev],dm);
447 
448  // Populate dz_min for dynamically-created fine levels (non-terrain path).
449  if (static_cast<int>(dz_min.size()) <= lev) { dz_min.resize(lev+1); }
450  dz_min[lev] = geom[lev].CellSize(2);
451  if ( SolverChoice::mesh_type != MeshType::ConstantDz && detJ_cc[lev] ) {
452  dz_min[lev] *= (*detJ_cc[lev]).min(0);
453  }
454 #ifdef ERF_USE_NETCDF
455  }
456 #endif
457 
458  // ********************************************************************************************
459  // Initialize the integrator class
460  // ********************************************************************************************
461  dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
462  initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
463 
464  // ********************************************************************************************
465  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
466  // ********************************************************************************************
467  if (lev > 0 && cf_width >= 0) {
468  Construct_ERFFillPatchers(lev);
469  Define_ERFFillPatchers(lev);
470  }
471 
472  // ********************************************************************************************
473  // For anelastic levels created from coarse (either on restart or during a run), project the
474  // interpolated velocity to enforce the divergence-free constraint. This Initializes gradp[lev]
475  // via the pressure projection, handling both the pure-anelastic case and the hybrid case
476  // (compressible lev-1, anelastic lev) where there is no coarse gradp to interpolate.
477  // FillPatchers must be constructed above before this call. pp_inc is scratch; zero afterward.
478  // ********************************************************************************************
479  if (solverChoice.anelastic[lev]) {
480  double dummy_dt = 1.0;
481 
482  // ****************************************************************************************
483  // Define grids[lev]/dmap[lev] to be the passed-in ba/dm *before* projecting.
484  //
485  // AmrCore::regrid does not call SetBoxArray/SetDistributionMap for this new level
486  // until MakeNewLevelFromCoarse returns, so grids[lev]/dmap[lev] are still empty here.
487  // project_momenta builds its per-subdomain RHS from grids[lev]/dmap[lev]; with those
488  // empty the subdomain box array is empty and the singular-solvability mean-subtraction
489  // divides by a zero-cell volume (0/0 -> NaN). That NaN traps with fpe_trap_invalid=1;
490  // with it off the projection is silently a no-op on the new level (empty RHS, early
491  // return) so the divergence-free constraint is never enforced. Setting them now
492  // (mirroring MakeNewLevelFromScratch) makes grids[lev] match the momentum MultiFabs
493  // built in init_stuff on the same ba/dm. This is idempotent with the
494  // SetBoxArray/SetDistributionMap that AmrCore performs with the identical ba/dm after
495  // this function returns.
496  // ****************************************************************************************
497  SetBoxArray(lev, ba);
498  SetDistributionMap(lev, dm);
499 
500  project_initial_velocity(lev, time, dummy_dt);
501  pp_inc[lev].setVal(0.0);
502  }
503 
504  //********************************************************************************************
505  // Land Surface Model
506  // *******************************************************************************************
507  int lsm_data_size = lsm.Get_Data_Size();
508  int lsm_flux_size = lsm.Get_Flux_Size();
509  lsm_data[lev].resize(lsm_data_size);
510  lsm_data_name.resize(lsm_data_size);
511  lsm_flux[lev].resize(lsm_flux_size);
512  lsm_flux_name.resize(lsm_flux_size);
513  lsm.Define(lev, solverChoice);
514  if (solverChoice.lsm_type != LandSurfaceType::None) {
515  IntVect RefRatio = (lev>0) ? refRatio(lev-1) : IntVect(1);
516  lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), Geom(0),
517  domain_bcs_type, RefRatio, zero, nc_init_file); // dummy dt value
518  }
519  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
520  lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
521  lsm_data_name[mvar] = lsm.Get_DataName(mvar);
522  }
523  for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
524  lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
525  lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
526  }
527  if (lev>0) {
528  lsm.Set_Lev0_Data_Ptr(lev);
529  lsm.Set_Lev0_Flux_Ptr(lev);
530  }
531 
532  // ********************************************************************************************
533  // Create the SurfaceLayer arrays at this (new) level
534  // ********************************************************************************************
535  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
536  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
537  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
538  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,lev+1,
539  mfv_old, Theta_prim[lev], Qv_prim[lev],
540  Qr_prim[lev], z_phys_nd[lev],
541  Hwave[lev].get(), Lwave[lev].get(), eddyDiffs_lev[lev].get(),
542  lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
543  sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
544  }
545 
546  // ********************************************************************************************
547  // Set up the Rayleigh damping vectors at this (new) level
548  // ********************************************************************************************
549  if (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
550  solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T)
551  {
552  initRayleigh_at_level(lev);
553  }
554 
555 }
ERF::update_diffusive_arrays
void update_diffusive_arrays(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewArrays.cpp:529
ERF::initialize_integrator
void initialize_integrator(int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
Definition: ERF_MakeNewArrays.cpp:898
ERF::make_subdomains
void make_subdomains(const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
Definition: ERF_MakeSubdomains.cpp:6
ERF::update_terrain_arrays
void update_terrain_arrays(int lev)
Definition: ERF_MakeNewArrays.cpp:881
ERF::init_stuff
void init_stuff(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:24
ERF::Define_ERFFillPatchers
void Define_ERFFillPatchers(int lev)
Definition: ERF.cpp:2739
ERF::init_zphys
void init_zphys(int lev, double elapsed_time)
Definition: ERF_MakeNewArrays.cpp:688
LandSurface::Init
void Init(const int &lev, const amrex::MultiFab &cons_in, const amrex::Geometry &geom, const amrex::Geometry &geom0, amrex::Vector< amrex::BCRec > &domain_bcs_type, amrex::IntVect &refRatio, const amrex::Real &dt_advance, amrex::Vector< amrex::Vector< std::string >> &nc_init_file)
Definition: ERF_LandSurface.H:46
LandSurface::Get_Data_Size
int Get_Data_Size()
Definition: ERF_LandSurface.H:127
LandSurface::Set_Lev0_Data_Ptr
void Set_Lev0_Data_Ptr(const int &lev)
Definition: ERF_LandSurface.H:154
LandSurface::Get_DataName
std::string Get_DataName(const int &varIdx)
Definition: ERF_LandSurface.H:133
LandSurface::Get_FluxName
std::string Get_FluxName(const int &varIdx)
Definition: ERF_LandSurface.H:168
LandSurface::Get_Flux_Ptr
amrex::MultiFab * Get_Flux_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:121
LandSurface::Set_Lev0_Flux_Ptr
void Set_Lev0_Flux_Ptr(const int &lev)
Definition: ERF_LandSurface.H:174
LandSurface::Define
void Define(const int &lev, SolverChoice &sc)
Definition: ERF_LandSurface.H:39
LandSurface::Get_Flux_Size
int Get_Flux_Size()
Definition: ERF_LandSurface.H:130
Here is the call graph for this function:

◆ MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
27 {
28  //
29  // Note that "time" here is elapsed time
30  //
31  BoxArray ba;
32  DistributionMapping dm;
33  Box domain(Geom(0).Domain());
34  if (lev == 0 && restart_chkfile.empty() &&
35  (max_grid_size[0][0] >= domain.length(0)) &&
36  (max_grid_size[0][1] >= domain.length(1)) &&
37  ba_in.size() != ParallelDescriptor::NProcs())
38  {
39  // We only decompose in z if max_grid_size_z indicates we should
40  bool decompose_in_z = (max_grid_size[0][2] < domain.length(2));
41 
42  ba = ERFPostProcessBaseGrids(Geom(0).Domain(),decompose_in_z);
43  dm = DistributionMapping(ba);
44  } else {
45  ba = ba_in;
46  dm = dm_in;
47  }
48 
49  // ********************************************************************************************
50  // Define grids[lev] to be ba
51  // ********************************************************************************************
52  SetBoxArray(lev, ba);
53 
54  // ********************************************************************************************
55  // Define dmap[lev] to be dm
56  // ********************************************************************************************
57  SetDistributionMap(lev, dm);
58 
59  if (verbose) {
60  amrex::Print() << "BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
61  // amrex::Print() <<" SIMPLIFIED BA FROM SCRATCH AT LEVEL " << lev << " " << ba.simplified_list() << std::endl;
62  }
63 
64  subdomains.resize(lev+1);
65  //
66  // Create subdomains at each level within the domain such that
67  // 1) all boxes in a given subdomain are "connected"
68  // 2) no boxes in a subdomain touch any boxes in any other subdomain
69  //
70  make_subdomains(ba.simplified_list(), subdomains[lev]);
71 
72  if (lev == 0) init_bcs();
73 
74  if ( solverChoice.terrain_type == TerrainType::EB ||
75  solverChoice.terrain_type == TerrainType::ImmersedForcing ||
76  solverChoice.buildings_type == BuildingsType::ImmersedForcing)
77  {
78  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
79  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
80  if (solverChoice.terrain_type == TerrainType::EB) {
81  eb[lev]->make_all_factories(lev, geom[lev], grids[lev], dmap[lev], eb_level);
82  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
83  solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
84  eb[lev]->make_cc_factory(lev, geom[lev], grids[lev], dmap[lev], eb_level);
85  }
86  }
87 
88  auto& lev_new = vars_new[lev];
89  auto& lev_old = vars_old[lev];
90 
91  //********************************************************************************************
92  // This allocates all kinds of things, including but not limited to: solution arrays,
93  // terrain arrays, metric terms and base state.
94  // *******************************************************************************************
95  init_stuff(lev, ba, dm, lev_new, lev_old, base_state[lev], z_phys_nd[lev]);
96 
97  //********************************************************************************************
98  // Land Surface Model
99  // *******************************************************************************************
100  int lsm_data_size = lsm.Get_Data_Size();
101  int lsm_flux_size = lsm.Get_Flux_Size();
102  lsm_data[lev].resize(lsm_data_size);
103  lsm_data_name.resize(lsm_data_size);
104  lsm_flux[lev].resize(lsm_flux_size);
105  lsm_flux_name.resize(lsm_flux_size);
106  lsm.Define(lev, solverChoice);
107  if (solverChoice.lsm_type != LandSurfaceType::None) {
108  IntVect RefRatio = (lev>0) ? refRatio(lev-1) : IntVect(1);
109  lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), Geom(0),
110  domain_bcs_type, RefRatio, zero, nc_init_file); // dummy dt value
111  }
112  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
113  lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
114  lsm_data_name[mvar] = lsm.Get_DataName(mvar);
115  }
116  for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
117  lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
118  lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
119  }
120  if (lev>0) {
121  lsm.Set_Lev0_Data_Ptr(lev);
122  lsm.Set_Lev0_Flux_Ptr(lev);
123  }
124 
125  // ********************************************************************************************
126  // Build the data structures for calculating diffusive/turbulent terms
127  // ********************************************************************************************
128  update_diffusive_arrays(lev, ba, dm);
129 
130  // ********************************************************************************************
131  // Build the data structures for holding sea surface temps and skin temps
132  // ********************************************************************************************
133  sst_lev[lev].resize(1); sst_lev[lev][0] = nullptr;
134  tsk_lev[lev].resize(1); tsk_lev[lev][0] = nullptr;
135 
136  // ********************************************************************************************
137  // Thin immersed body
138  // *******************************************************************************************
139  init_thin_body(lev, ba, dm);
140 
141  // ********************************************************************************************
142  // Initialize the integrator class
143  // ********************************************************************************************
144  initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
145 
146  // ********************************************************************************************
147  // Initialize the data itself
148  // If (init_type == InitType::WRFInput) then we are initializing terrain and the initial data in
149  // the same call so we must call init_only before update_terrain_arrays
150  // If (init_type != InitType::WRFInput) then we want to initialize the terrain before the initial data
151  // since we may need to use the grid information before constructing
152  // initial idealized data
153  // ********************************************************************************************
154  if (restart_chkfile.empty()) {
155  if ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) )
156  {
157  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
158  //
159  // Note that "time" here is elapsed time, and start_time is the start_time from wrfinput/metgrid files
160  //
161  init_only(lev, time);
162  init_zphys(lev, time);
163  update_terrain_arrays(lev);
164  make_physbcs(lev);
165  } else {
166  //
167  // Note that "time" here is elapsed time, and start_time = 0 when not using wrfinput/metgrid
168  //
169  init_zphys(lev, time);
170  update_terrain_arrays(lev);
171  // Note that for init_type != InitType::WRFInput and != InitType::Metgrid,
172  // make_physbcs is called inside init_only
173  init_only(lev, time);
174  }
175  } else {
176  // if restarting and nudging from input sounding, load the input sounding files
177  if (lev == 0 && solverChoice.init_type == InitType::Input_Sounding && solverChoice.nudging_from_input_sounding)
178  {
179  if (input_sounding_data.input_sounding_file.empty()) {
180  Error("input_sounding file name must be provided via input");
181  }
182 
183  input_sounding_data.resize_arrays();
184 
185  // this will interpolate the input profiles to the nominal height levels
186  // (ranging from 0 to the domain top)
187  bool is_moist = (solverChoice.moisture_type != MoistureType::None);
188  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
189  input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n, is_moist);
190  }
191 
192  // this will calculate the hydrostatically balanced density and pressure
193  // profiles following WRF ideal.exe
194  if (solverChoice.sounding_type == SoundingType::Ideal) {
195  input_sounding_data.calc_rho_p(0);
196  } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
197  solverChoice.sounding_type == SoundingType::DryIsentropic) {
198  input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
199  input_sounding_data.calc_rho_p_isentropic(0);
200  }
201  }
202 
203  // We re-create terrain_blanking on restart rather than storing it in the checkpoint
204  if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
205  solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
206  int ngrow = ComputeGhostCells(solverChoice) + 2;
207  terrain_blanking[lev]->setVal(1.0);
208  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
209  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
210  }
211  }
212 
213  // Read in tables needed for windfarm simulations
214  // fill in Nturb multifab - number of turbines in each mesh cell
215  // write out the vtk files for wind turbine location and/or
216  // actuator disks
217  #ifdef ERF_USE_WINDFARM
218  init_windfarm(lev);
219  #endif
220 
221  // ********************************************************************************************
222  // Build the data structures for canopy model (depends upon z_phys)
223  // ********************************************************************************************
224  if (restart_chkfile.empty()) {
225  if (solverChoice.do_forest_drag) {
226  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
227  }
228  }
229 
230  //********************************************************************************************
231  // Microphysics
232  // *******************************************************************************************
233  int q_size = micro->Get_Qmoist_Size(lev);
234  qmoist[lev].resize(q_size);
235  micro->Define(lev, solverChoice);
236  if (solverChoice.moisture_type != MoistureType::None)
237  {
238  micro->Init(lev, vars_new[lev][Vars::cons],
239  grids[lev], Geom(lev), zero,
240  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
241  }
242  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
243  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
244  }
245 
246  //********************************************************************************************
247  // Radiation
248  // *******************************************************************************************
249  if (solverChoice.rad_type != RadiationType::None)
250  {
251  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
252  }
253 
254  // ********************************************************************************************
255  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
256  // ********************************************************************************************
257  if (lev > 0 && cf_width >= 0) {
258  Construct_ERFFillPatchers(lev);
259  Define_ERFFillPatchers(lev);
260  }
261 
262 #ifdef ERF_USE_PARTICLES
263  if (restart_chkfile.empty()) {
264  if (lev == 0) {
265  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,time);
266  }
267  // For lev > 0: particle redistribute is handled in timeStep() AFTER
268  // regrid() completes, not here inside MakeNewLevelFromCoarse.
269  }
270 #endif
271 }
ERFPostProcessBaseGrids
BoxArray ERFPostProcessBaseGrids(const Box &domain, bool decompose_in_z)
Definition: ERF_ChopGrids.cpp:6
ERF::init_only
void init_only(int lev, double time)
Definition: ERF.cpp:1813
ERF::init_thin_body
void init_thin_body(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewLevel.cpp:925
SolverChoice::nudging_from_input_sounding
bool nudging_from_input_sounding
Definition: ERF_DataStruct.H:1354
Here is the call graph for this function:

◆ MakeVTKFilename()

std::string ERF::MakeVTKFilename ( int  nstep)
11  {
12  // Ensure output directory exists
13  const std::string dir = "Output_StormTracker";
14  if (!fs::exists(dir)) {
15  fs::create_directory(dir);
16  }
17 
18  std::ostringstream oss;
19  oss << dir << "/storm_track_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
20  return oss.str();
21 }

◆ MakeVTKFilename_EyeTracker_xy()

std::string ERF::MakeVTKFilename_EyeTracker_xy ( int  nstep)
38  {
39  // Ensure output directory exists
40  const std::string dir = "Output_StormTracker/xy";
41  if (!fs::exists(dir)) {
42  fs::create_directories(dir);
43  }
44 
45  // Construct filename with zero-padded step
46  std::ostringstream oss;
47  oss << dir << "/storm_track_xy_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
48  return oss.str();
49 }

◆ MakeVTKFilename_TrackerCircle()

std::string ERF::MakeVTKFilename_TrackerCircle ( int  nstep)
24  {
25  // Ensure output directory exists
26  const std::string dir = "Output_StormTracker/tracker_circle";
27  if (!fs::exists(dir)) {
28  fs::create_directories(dir);
29  }
30 
31  // Construct filename with zero-padded step
32  std::ostringstream oss;
33  oss << dir << "/storm_tracker_circle_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
34  return oss.str();
35 }

◆ nghost_eb_basic()

static int ERF::nghost_eb_basic ( )
inlinestaticprivate
1768  { return 5; }

◆ nghost_eb_full()

static int ERF::nghost_eb_full ( )
inlinestaticprivate
1775  { return 4; }

◆ nghost_eb_volume()

static int ERF::nghost_eb_volume ( )
inlinestaticprivate
1772  { return 5; }

◆ NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )
inlineprivatenoexcept
1564  {
1565  return static_cast<int>(datalog.size());
1566  }

◆ NumDerDataLogs()

AMREX_FORCE_INLINE int ERF::NumDerDataLogs ( )
inlineprivatenoexcept
1571  {
1572  return static_cast<int>(der_datalog.size());
1573  }

◆ NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )
inlineprivatenoexcept
1600  {
1601  return static_cast<int>(samplelinelog.size());
1602  }

◆ NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )
inlineprivatenoexcept
1626  {
1627  return static_cast<int>(sampleline.size());
1628  }

◆ NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )
inlineprivatenoexcept
1586  {
1587  return static_cast<int>(sampleptlog.size());
1588  }

◆ NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )
inlineprivatenoexcept
1613  {
1614  return static_cast<int>(samplepoint.size());
1615  }

◆ operator=() [1/2]

ERF& ERF::operator= ( const ERF &  other)
delete

◆ operator=() [2/2]

ERF& ERF::operator= ( ERF &&  other)
deletenoexcept

◆ PackAtmosphericStates()

void ERF::PackAtmosphericStates ( amrex::Vector< amrex::MultiFab * > &  states,
double  time 
)
171 {
172  using namespace amrex;
173 
174  // Contract slot indices (mirrors ERFRemoraCouplingContract.H; repeated here
175  // to avoid a driver→submodule header dependency).
176  constexpr int iUwind = 0, iVwind = 1, iPatm = 2, iRH = 3, iTair = 4;
177  constexpr int iCloud = 5, iRain = 6, iSWrad = 7, iLWrad = 8;
178  constexpr int nFluxLanes = 8;
179 
180  const int lev = 0;
181 
182  auto& cons = vars_new[lev][Vars::cons];
183  auto& xvel = vars_new[lev][Vars::xvel]; // XFace
184  auto& yvel = vars_new[lev][Vars::yvel]; // YFace
185 
186  const bool has_moisture = (solverChoice.moisture_type != MoistureType::None);
187  const bool has_radiation = (!rad_fluxes.empty() && rad_fluxes[lev] != nullptr);
188 
189  amrex::ignore_unused(has_moisture, has_radiation);
190 
191  const auto& ba = cons.boxArray();
192  const auto& dm = cons.DistributionMap();
193  const auto& ba2d_lev = ba2d[lev];
194  const auto& xba2d_lev = amrex::convert(ba2d_lev, IntVect(1,0,0));
195  const auto& yba2d_lev = amrex::convert(ba2d_lev, IntVect(0,1,0));
196  const int klo = ba.minimalBox().smallEnd(2);
197  const Box domain2d = ba2d_lev.minimalBox();
198 
199  const bool flux_mode = (states.size() == nFluxLanes);
200  if (flux_mode) {
201  constexpr int iTauX = 0, iTauY = 1, iSHflux = 2, iLHflux = 3;
202  constexpr int iFluxSWrad = 4, iFluxLWrad = 5, iFluxRain = 6, iFluxEvap = 7;
203  if (verbose) {
204  amrex::Print() << "ERF flux-pack: states.size()=" << states.size()
205  << " has_radiation=" << has_radiation
206  << " tau13_ptr=" << (Tau[lev][TauType::tau13] != nullptr)
207  << " tau23_ptr=" << (Tau[lev][TauType::tau23] != nullptr)
208  << " sfs_hfx3_ptr="
209  << (!SFS_hfx3_lev.empty() && SFS_hfx3_lev[lev] != nullptr)
210  << " sfs_q1fx3_ptr="
211  << (!SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev] != nullptr)
212  << " rad_fluxes_ptr="
213  << (!rad_fluxes.empty() && rad_fluxes[lev] != nullptr)
214  << "\n";
215  if (Tau[lev][TauType::tau13] != nullptr) {
216  PrintFluxLaneStats("tau13_src", *Tau[lev][TauType::tau13]);
217  }
218  if (Tau[lev][TauType::tau23] != nullptr) {
219  PrintFluxLaneStats("tau23_src", *Tau[lev][TauType::tau23]);
220  }
221  if (!SFS_hfx3_lev.empty() && SFS_hfx3_lev[lev] != nullptr) {
222  PrintFluxLaneStats("SFS_hfx3_src", *SFS_hfx3_lev[lev]);
223  }
224  if (!SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev] != nullptr) {
225  PrintFluxLaneStats("SFS_q1fx3_src", *SFS_q1fx3_lev[lev]);
226  }
227  if (has_radiation) {
228  PrintFluxLaneStats("rad_fluxes_swup_src", *rad_fluxes[lev], 0);
229  PrintFluxLaneStats("rad_fluxes_swdn_src", *rad_fluxes[lev], 1);
230  PrintFluxLaneStats("rad_fluxes_lwup_src", *rad_fluxes[lev], 2);
231  PrintFluxLaneStats("rad_fluxes_lwdn_src", *rad_fluxes[lev], 3);
232  }
233  }
234 
235  if (iTauX < static_cast<int>(states.size()) && states[iTauX] != nullptr) {
236  MultiFab tmp(xba2d_lev, dm, 1, 0);
237  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
238  Box bx = mfi.tilebox();
239  auto const& tau13 = Tau[lev][TauType::tau13]->const_array(mfi);
240  auto const& c = cons.const_array(mfi);
241  auto t = tmp.array(mfi);
242  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
243  const Real rho_face =
244  (i <= domain2d.smallEnd(0)) ? c(domain2d.smallEnd(0),j,klo,Rho_comp) :
245  (i > domain2d.bigEnd(0)) ? c(domain2d.bigEnd(0),j,klo,Rho_comp) :
246  Real(0.5) * (c(i-1,j,klo,Rho_comp) + c(i,j,klo,Rho_comp));
247  t(i,j,k) = rho_face * tau13(i,j,klo);
248  });
249  }
250  AverageDownThenRemap(tmp, *states[iTauX]);
251  if (verbose) { PrintFluxLaneStats("tau_x_lane", *states[iTauX]); }
252  }
253 
254  if (iTauY < static_cast<int>(states.size()) && states[iTauY] != nullptr) {
255  MultiFab tmp(yba2d_lev, dm, 1, 0);
256  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
257  Box bx = mfi.tilebox();
258  auto const& tau23 = Tau[lev][TauType::tau23]->const_array(mfi);
259  auto const& c = cons.const_array(mfi);
260  auto t = tmp.array(mfi);
261  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
262  const Real rho_face =
263  (j <= domain2d.smallEnd(1)) ? c(i,domain2d.smallEnd(1),klo,Rho_comp) :
264  (j > domain2d.bigEnd(1)) ? c(i,domain2d.bigEnd(1),klo,Rho_comp) :
265  Real(0.5) * (c(i,j-1,klo,Rho_comp) + c(i,j,klo,Rho_comp));
266  t(i,j,k) = rho_face * tau23(i,j,klo);
267  });
268  }
269  AverageDownThenRemap(tmp, *states[iTauY]);
270  if (verbose) { PrintFluxLaneStats("tau_y_lane", *states[iTauY]); }
271  }
272 
273  if (iSHflux < static_cast<int>(states.size()) && states[iSHflux] != nullptr &&
274  !SFS_hfx3_lev.empty() && SFS_hfx3_lev[lev] != nullptr) {
275  MultiFab tmp(ba2d_lev, dm, 1, 0);
276  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
277  Box bx = mfi.tilebox();
278  auto const& hfx = SFS_hfx3_lev[lev]->const_array(mfi);
279  auto t = tmp.array(mfi);
280  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
281  t(i,j,k) = surface_flux_diagnostics::sensible_heat_flux_wm2_from_rhotheta_flux(
282  hfx(i,j,klo));
283  });
284  }
285  AverageDownThenRemap(tmp, *states[iSHflux]);
286  if (verbose) { PrintFluxLaneStats("SHflux_lane", *states[iSHflux]); }
287  }
288 
289  if (iLHflux < static_cast<int>(states.size()) && states[iLHflux] != nullptr &&
290  !SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev] != nullptr) {
291  MultiFab tmp(ba2d_lev, dm, 1, 0);
292  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
293  Box bx = mfi.tilebox();
294  auto const& qfx = SFS_q1fx3_lev[lev]->const_array(mfi);
295  auto t = tmp.array(mfi);
296  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
297  t(i,j,k) = surface_flux_diagnostics::latent_heat_flux_wm2_from_rhoqv_flux(
298  qfx(i,j,klo));
299  });
300  }
301  AverageDownThenRemap(tmp, *states[iLHflux]);
302  if (verbose) { PrintFluxLaneStats("LHflux_lane", *states[iLHflux]); }
303  }
304 
305  if (has_radiation) {
306  if (iFluxSWrad < static_cast<int>(states.size()) && states[iFluxSWrad] != nullptr) {
307  MultiFab tmp(ba2d_lev, dm, 1, 0);
308  tmp.ParallelCopy(*rad_fluxes[lev], 1, 0, 1);
309  AverageDownThenRemap(tmp, *states[iFluxSWrad]);
310  if (verbose) { PrintFluxLaneStats("SWrad_lane", *states[iFluxSWrad]); }
311  }
312  if (iFluxLWrad < static_cast<int>(states.size()) && states[iFluxLWrad] != nullptr) {
313  MultiFab tmp(ba2d_lev, dm, 1, 0);
314  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
315  Box bx = mfi.tilebox();
316  auto const& rad_flux = rad_fluxes[lev]->const_array(mfi);
317  auto t = tmp.array(mfi);
318  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
319  t(i,j,k) = rad_flux(i,j,k,3) - rad_flux(i,j,k,2);
320  });
321  }
322  AverageDownThenRemap(tmp, *states[iFluxLWrad]);
323  if (verbose) { PrintFluxLaneStats("LWrad_lane", *states[iFluxLWrad]); }
324  }
325  }
326 
327  if (iFluxEvap < static_cast<int>(states.size()) && states[iFluxEvap] != nullptr &&
328  !SFS_q1fx3_lev.empty() && SFS_q1fx3_lev[lev] != nullptr) {
329  MultiFab tmp(ba2d_lev, dm, 1, 0);
330  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
331  Box bx = mfi.tilebox();
332  auto const& qfx = SFS_q1fx3_lev[lev]->const_array(mfi);
333  auto t = tmp.array(mfi);
334  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
335  t(i,j,k) = qfx(i,j,klo);
336  });
337  }
338  AverageDownThenRemap(tmp, *states[iFluxEvap]);
339  if (verbose) { PrintFluxLaneStats("evap_lane", *states[iFluxEvap]); }
340  }
341 
342  amrex::ignore_unused(iFluxRain);
343  return;
344  }
345 
346  // --- Uwind + Vwind: use AMReX's average_face_to_cellcenter which correctly
347  // handles tile boundaries via growntilebox(1) internally. ---
348  if ((iUwind < static_cast<int>(states.size()) && states[iUwind] != nullptr) ||
349  (iVwind < static_cast<int>(states.size()) && states[iVwind] != nullptr)) {
350 
351  auto& zvel = vars_new[lev][Vars::zvel];
352  MultiFab cc_vel(ba, dm, AMREX_SPACEDIM, 0);
353  amrex::average_face_to_cellcenter(cc_vel, 0,
354  Array<const MultiFab*, AMREX_SPACEDIM>{&xvel, &yvel, &zvel});
355 
356  // Collapse to 2D slab
357  MultiFab uv_slab(ba2d_lev, dm, 2, 0); // comp0=u, comp1=v
358  uv_slab.ParallelCopy(cc_vel, 0, 0, 2);
359 
360  if (iUwind < static_cast<int>(states.size()) && states[iUwind] != nullptr) {
361  MultiFab u_alias(uv_slab, amrex::make_alias, 0, 1); // alias u component
362  AverageDownThenRemap(u_alias, *states[iUwind]);
363  }
364 
365  if (iVwind < static_cast<int>(states.size()) && states[iVwind] != nullptr) {
366  MultiFab v_alias(uv_slab, amrex::make_alias, 1, 1); // alias v component
367  AverageDownThenRemap(v_alias, *states[iVwind]);
368  }
369  }
370 
371  // --- Patm: getPgivenRTh(RhoTheta, qv) at k=0 ---
372  if (iPatm < static_cast<int>(states.size()) && states[iPatm] != nullptr) {
373  MultiFab tmp(ba2d_lev, dm, 1, 0);
374  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
375  Box bx = mfi.tilebox();
376  auto const& c = cons.const_array(mfi);
377  auto t = tmp.array(mfi);
378  if (has_moisture) {
379  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
380  const Real qv = c(i,j,k,RhoQ1_comp) / c(i,j,k,Rho_comp);
381  t(i,j,k) = getPgivenRTh(c(i,j,k,RhoTheta_comp), qv);
382  });
383  } else {
384  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
385  t(i,j,k) = getPgivenRTh(c(i,j,k,RhoTheta_comp));
386  });
387  }
388  }
389  AverageDownThenRemap(tmp, *states[iPatm]);
390  }
391 
392  // --- Tair: getTgivenRandRTh(rho, RhoTheta, qv) at k=0 [K] ---
393  if (iTair < static_cast<int>(states.size()) && states[iTair] != nullptr) {
394  MultiFab tmp(ba2d_lev, dm, 1, 0);
395  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
396  Box bx = mfi.tilebox();
397  auto const& c = cons.const_array(mfi);
398  auto t = tmp.array(mfi);
399  if (has_moisture) {
400  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
401  const Real qv = c(i,j,k,RhoQ1_comp) / c(i,j,k,Rho_comp);
402  t(i,j,k) = getTgivenRandRTh(c(i,j,k,Rho_comp), c(i,j,k,RhoTheta_comp), qv);
403  });
404  } else {
405  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
406  t(i,j,k) = getTgivenRandRTh(c(i,j,k,Rho_comp), c(i,j,k,RhoTheta_comp));
407  });
408  }
409  }
410  AverageDownThenRemap(tmp, *states[iTair]);
411  }
412 
413  // --- Humidity lane: export relative humidity [0-1] for REMORA bulk fluxes ---
414  if (has_moisture) {
415 #if 0
416  if (iRH < static_cast<int>(states.size()) && states[iRH] != nullptr) {
417  MultiFab tmp(ba2d_lev, dm, 1, 0);
418  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
419  Box bx = mfi.tilebox();
420  auto const& c = cons.const_array(mfi);
421  auto t = tmp.array(mfi);
422  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
423  const Real qv = c(i,j,k,RhoQ1_comp) / c(i,j,k,Rho_comp);
424  const Real p_pa = getPgivenRTh(c(i,j,k,RhoTheta_comp), qv);
425  const Real temp = getTgivenRandRTh(c(i,j,k,Rho_comp), c(i,j,k,RhoTheta_comp), qv);
426  Real qsat = Real(0.0);
427  erf_qsatw(temp, p_pa * Real(0.01), qsat);
428  t(i,j,k) = amrex::max(Real(0.0), amrex::min(Real(1.0),
429  qv / amrex::max(qsat, Real(1.0e-12))));
430  });
431  }
432  IntVect ratio = ba2d_lev.minimalBox().length() / states[iRH]->boxArray().minimalBox().length();
433  amrex::average_down(tmp, *states[iRH], 0, 1, ratio);
434  }
435 #else
436  amrex::ignore_unused(iRH);
437  // Avoiding unexercised moist assumptions: retain the driver-prefilled
438  // humidity lane until live ERF->REMORA Qair semantics are made explicit.
439 #endif
440  if (iCloud < static_cast<int>(states.size()) && states[iCloud] != nullptr) {
441  const int qc_idx = solverChoice.moisture_indices.qc;
442  const int qi_idx = solverChoice.moisture_indices.qi;
443  if (qc_idx != -1 || qi_idx != -1) {
444  MultiFab tmp(ba2d_lev, dm, 1, 0);
445  const Real cf = amrex::max(Real(0.0), amrex::min(Real(1.0), cloud_fraction(0.0)));
446  amrex::ignore_unused(cf); // keep diagnostic computation active for consistency with ERF scalar stats
447  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
448  Box bx = mfi.tilebox();
449  auto const& c = cons.const_array(mfi);
450  auto t = tmp.array(mfi);
451  const int khi = ba.minimalBox().bigEnd(2);
452  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
453  int cloudy = 0;
454  for (int kk = klo; kk <= khi; ++kk) {
455  const Real rho = c(i,j,kk,Rho_comp);
456  const Real qc = (qc_idx != -1) ? c(i,j,kk,qc_idx) / rho : Real(0.0);
457  const Real qi = (qi_idx != -1) ? c(i,j,kk,qi_idx) / rho : Real(0.0);
458  if (qc + qi > Real(0.0)) { cloudy = 1; break; }
459  }
460  t(i,j,k) = static_cast<Real>(cloudy);
461  });
462  }
463  AverageDownThenRemap(tmp, *states[iCloud]);
464  }
465  }
466 #if 0
467  if (iRain < static_cast<int>(states.size()) && states[iRain] != nullptr) {
468  int qr_idx = solverChoice.moisture_indices.qr;
469  if (qr_idx != -1) {
470  MultiFab tmp(ba2d_lev, dm, 1, 0);
471  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
472  Box bx = mfi.tilebox();
473  auto const& c = cons.const_array(mfi);
474  auto t = tmp.array(mfi);
475  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
476  t(i,j,k) = c(i,j,k,qr_idx) / c(i,j,k,Rho_comp);
477  });
478  }
479  IntVect ratio = ba2d_lev.minimalBox().length() / states[iRain]->boxArray().minimalBox().length();
480  amrex::average_down(tmp, *states[iRain], 0, 1, ratio);
481  }
482  }
483 #else
484  amrex::ignore_unused(iRain);
485  // Avoiding unexercised moist assumptions: retain the driver-prefilled
486  // rain lane until live ERF->REMORA rain semantics are made explicit.
487 #endif
488  }
489  // No moisture: leave RH/Cloud/Rain slabs at their driver-pre-filled values.
490 
491  // --- Radiation: sw_flux_dn (comp=1) and lw_flux_dn (comp=3) from rad_fluxes ---
492  // When absent, leave slabs at their driver-pre-filled values.
493  if (has_radiation) {
494  if (iSWrad < static_cast<int>(states.size()) && states[iSWrad] != nullptr) {
495  MultiFab tmp(ba2d_lev, dm, 1, 0);
496  tmp.ParallelCopy(*rad_fluxes[lev], 1, 0, 1);
497  AverageDownThenRemap(tmp, *states[iSWrad]);
498  }
499  if (iLWrad < static_cast<int>(states.size()) && states[iLWrad] != nullptr) {
500  MultiFab tmp(ba2d_lev, dm, 1, 0);
501  tmp.ParallelCopy(*rad_fluxes[lev], 3, 0, 1);
502  AverageDownThenRemap(tmp, *states[iLWrad]);
503  }
504  }
505 }
erf_qsatw
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void erf_qsatw(amrex::Real t, amrex::Real p, amrex::Real &qsatw)
Definition: ERF_MicrophysicsUtils.H:228
ERF::cloud_fraction
amrex::Real cloud_fraction(double time)
Definition: ERF_WriteScalarProfiles.cpp:457
MicVar_WSM6::qi
@ qi
Definition: ERF_WSM6.H:27
amrex
Definition: ERF_ConsoleIO.cpp:12
surface_flux_diagnostics::sensible_heat_flux_wm2_from_rhotheta_flux
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real sensible_heat_flux_wm2_from_rhotheta_flux(amrex::Real rhotheta_flux) noexcept
Definition: ERF_SurfaceFluxDiagnostics.H:29
surface_flux_diagnostics::latent_heat_flux_wm2_from_rhoqv_flux
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real latent_heat_flux_wm2_from_rhoqv_flux(amrex::Real rhoqv_flux) noexcept
Definition: ERF_SurfaceFluxDiagnostics.H:36
MoistureComponentIndices::qi
int qi
Definition: ERF_DataStruct.H:109
Here is the call graph for this function:

◆ ParameterSanityChecks()

void ERF::ParameterSanityChecks ( )
private
2494 {
2495  AMREX_ALWAYS_ASSERT(cfl > zero || fixed_dt[0] > zero);
2496 
2497  // We don't allow use_real_bcs to be true if init_type is not either InitType::WRFInput or InitType::Metgrid
2498  AMREX_ALWAYS_ASSERT( !solverChoice.use_real_bcs ||
2499  ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) );
2500 
2501  AMREX_ALWAYS_ASSERT(real_width >= 0);
2502 
2503  if (cf_set_width != 0) {
2504  Abort("You must set cf_set_width == 0");
2505  }
2506  if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
2507  Abort("You must set cf_width >= cf_set_width >= 0");
2508  }
2509  if (max_level > 0 && cf_set_width > 0) {
2510  for (int lev = 1; lev <= max_level; lev++) {
2511  if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
2512  cf_set_width%ref_ratio[lev-1][1] != 0 ||
2513  cf_set_width%ref_ratio[lev-1][2] != 0 ) {
2514  Abort("You must set cf_width to be a multiple of ref_ratio");
2515  }
2516  }
2517  }
2518 
2519  // If fixed_mri_dt_ratio is set, it must be even
2520  if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
2521  {
2522  Abort("If you specify fixed_mri_dt_ratio, it must be even");
2523  }
2524 
2525  for (int lev = 0; lev <= max_level; lev++)
2526  {
2527  // We ignore fixed_fast_dt if not substepping
2528  if (solverChoice.substepping_type[lev] == SubsteppingType::None) {
2529  fixed_fast_dt[lev] = -one;
2530  }
2531 
2532  // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
2533  if (fixed_dt[lev] > zero && fixed_fast_dt[lev] > zero && fixed_mri_dt_ratio <= 0)
2534  {
2535  Real eps = Real(1.e-12);
2536  int ratio = static_cast<int>( ( (one+eps) * fixed_dt[lev] ) / fixed_fast_dt[lev] );
2537  if (fixed_dt[lev] / fixed_fast_dt[lev] != ratio)
2538  {
2539  Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
2540  }
2541  }
2542 
2543  // If all three are specified, they must be consistent
2544  if (fixed_dt[lev] > zero && fixed_fast_dt[lev] > zero && fixed_mri_dt_ratio > 0)
2545  {
2546  if (fixed_dt[lev] / fixed_fast_dt[lev] != fixed_mri_dt_ratio)
2547  {
2548  Abort("Dt is over-specfied");
2549  }
2550  }
2551  } // lev
2552 
2553  if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
2554  Abort("For two-way coupling you must set cf_width = 0");
2555  }
2556 }
Here is the call graph for this function:

◆ PerformDataAssimilation()

void ERF::PerformDataAssimilation ( int  da_iter)

◆ PlotFileName()

std::string ERF::PlotFileName ( int  lev) const
private

◆ PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string >  plot_var_names)
staticprivate
254 {
255  Vector<std::string> names;
256 
257  names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
258 
259  return names;
260 
261 }

◆ poisson_wall_dist()

void ERF::poisson_wall_dist ( int  lev)

Calculate wall distances using the Poisson equation

The zlo boundary is assumed to correspond to the land surface. If there are no boundary walls, then the other use case is to calculate wall distances for immersed boundaries (embedded or thin body).

See Tucker, P. G. (2003). Differential equation-based wall distance computation for DES and RANS. Journal of Computational Physics, 190(1), 229–Real(248.) https://doi.org/Real(10.1016)/S0021-9991(03)00272-9

23 {
24  BL_PROFILE("ERF::poisson_wall_dist()");
25 
26  bool havewall{false};
27  Orientation zlo(Direction::z, Orientation::low);
28  if ( ( phys_bc_type[zlo] == ERF_BC::surface_layer ) ||
29  ( phys_bc_type[zlo] == ERF_BC::no_slip_wall ) )/*||
30  ((phys_bc_type[zlo] == ERF_BC::slip_wall) && (dom_hi.z > dom_lo.z)) )*/
31  {
32  havewall = true;
33  }
34 
35  auto const& geomdata = geom[lev];
36  auto const& dxinv = geomdata.InvCellSizeArray();
37 
38  auto const& zphys_arr = z_phys_nd[lev]->const_arrays();
39 
40  if (havewall) {
41 #if 1
42  // Bypass wall dist calc in the trivial cases
43 
44  if (solverChoice.mesh_type == MeshType::ConstantDz) {
45  Print() << "Directly calculating direct wall distance for constant dz" << std::endl;
46  const Real* prob_lo = geomdata.ProbLo();
47  const Real* dx = geomdata.CellSize();
48  for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
49  const Box& bx = mfi.validbox();
50  auto dist_arr = walldist[lev]->array(mfi);
51  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
52  dist_arr(i, j, k) = prob_lo[2] + (k + myhalf) * dx[2];
53  });
54  }
55  return;
56  }
57 
58  if (solverChoice.mesh_type == MeshType::StretchedDz) {
59  Print() << "Directly calculating direct wall distance for stretched dz" << std::endl;
60  for (MFIter mfi(*walldist[lev],TileNoZ()); mfi.isValid(); ++mfi) {
61  const Box& bx = mfi.validbox();
62  auto dist_arr = walldist[lev]->array(mfi);
63  const auto zcc_arr = z_phys_cc[lev]->const_array(mfi);
64  const auto znd_arr = z_phys_nd[lev]->const_array(mfi);
65  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
66  dist_arr(i, j, k) = zcc_arr(i, j, k) - znd_arr(i, j, 0);
67  });
68  }
69  return;
70  }
71 #endif
72  }
73  else
74  {
75  Error("No solid boundaries in the computational domain");
76  }
77 
78  Print() << "Calculating Poisson wall distance for general terrain" << std::endl;
79 
80  // Make sure the solver only sees the levels over which we are solving
81  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
82  Vector<BoxArray> ba_tmp; ba_tmp.push_back(walldist[lev]->boxArray());
83  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(walldist[lev]->DistributionMap());
84 
85  Vector<MultiFab> rhs;
86  Vector<MultiFab> phi;
87 
88  if (solverChoice.terrain_type == TerrainType::EB) {
89  amrex::Error("Wall dist calc not implemented for EB");
90  } else {
91  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
92  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
93  }
94 
95  rhs[0].setVal(-1.0);
96 
97  auto const dom_lo = lbound(geom[lev].Domain());
98  auto const dom_hi = ubound(geom[lev].Domain());
99 
100  // ****************************************************************************
101  // Initialize phi
102  // (It is essential that we do this in order to fill the corners; this is
103  // used if we include blanking.)
104  // ****************************************************************************
105  phi[0].setVal(0.0);
106 
107  // ****************************************************************************
108  // Interior boundaries are marked with phi=0
109  // ****************************************************************************
110 #if 0
111  // Define an overset mask (0 or 1) to set dirichlet nodes on walls
112  // 1 means the node is an unknown. 0 means it's known.
113  iMultiFab mask(ba_tmp[0], dm_tmp[0], 1, 0);
114  Vector<const iMultiFab*> overset_mask = {&mask};
115 
116  mask.setVal(1);
117  if (solverChoice.advChoice.have_zero_flux_faces) {
118  Warning("Poisson distance is inaccurate for bodies in open domains that are small compared to the domain size, skipping");
119  return;
120 
121  Gpu::DeviceVector<IntVect> xfacelist, yfacelist, zfacelist;
122 
123  xfacelist.resize(solverChoice.advChoice.zero_xflux.size());
124  yfacelist.resize(solverChoice.advChoice.zero_yflux.size());
125  zfacelist.resize(solverChoice.advChoice.zero_zflux.size());
126 
127  if (xfacelist.size() > 0) {
128  Gpu::copy(amrex::Gpu::hostToDevice,
129  solverChoice.advChoice.zero_xflux.begin(),
130  solverChoice.advChoice.zero_xflux.end(),
131  xfacelist.begin());
132  Print() << " masking interior xfaces" << std::endl;
133  }
134  if (yfacelist.size() > 0) {
135  Gpu::copy(amrex::Gpu::hostToDevice,
136  solverChoice.advChoice.zero_yflux.begin(),
137  solverChoice.advChoice.zero_yflux.end(),
138  yfacelist.begin());
139  Print() << " masking interior yfaces" << std::endl;
140  }
141  if (zfacelist.size() > 0) {
142  Gpu::copy(amrex::Gpu::hostToDevice,
143  solverChoice.advChoice.zero_zflux.begin(),
144  solverChoice.advChoice.zero_zflux.end(),
145  zfacelist.begin());
146  Print() << " masking interior zfaces" << std::endl;
147  }
148 
149  for (MFIter mfi(phi[0]); mfi.isValid(); ++mfi) {
150  const Box& bx = mfi.validbox();
151 
152  auto phi_arr = phi[0].array(mfi);
153  auto mask_arr = mask.array(mfi);
154 
155  if (xfacelist.size() > 0) {
156  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
157  for (int iface=0; iface < xfacelist.size(); ++iface) {
158  if ((i == xfacelist[iface][0]) &&
159  (j == xfacelist[iface][1]) &&
160  (k == xfacelist[iface][2]))
161  {
162  mask_arr(i, j , k ) = 0;
163  mask_arr(i, j , k+1) = 0;
164  mask_arr(i, j+1, k ) = 0;
165  mask_arr(i, j+1, k+1) = 0;
166  }
167  }
168  });
169  }
170 
171  if (yfacelist.size() > 0) {
172  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
173  for (int iface=0; iface < yfacelist.size(); ++iface) {
174  if ((i == yfacelist[iface][0]) &&
175  (j == yfacelist[iface][1]) &&
176  (k == yfacelist[iface][2]))
177  {
178  mask_arr(i , j, k ) = 0;
179  mask_arr(i , j, k+1) = 0;
180  mask_arr(i+1, j, k ) = 0;
181  mask_arr(i+1, j, k+1) = 0;
182  }
183  }
184  });
185  }
186 
187  if (zfacelist.size() > 0) {
188  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
189  for (int iface=0; iface < zfacelist.size(); ++iface) {
190  if ((i == xfacelist[iface][0]) &&
191  (j == xfacelist[iface][1]) &&
192  (k == xfacelist[iface][2]))
193  {
194  mask_arr(i , j , k) = 0;
195  mask_arr(i , j+1, k) = 0;
196  mask_arr(i+1, j , k) = 0;
197  mask_arr(i+1, j+1, k) = 0;
198  }
199  }
200  });
201  }
202  }
203  }
204 #endif
205 
206  // ****************************************************************************
207  // Setup BCs, with solid domain boundaries being dirichlet
208  // ****************************************************************************
209  amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bc3d_lo, bc3d_hi;
210  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
211  if (geom[0].isPeriodic(dir)) {
212  bc3d_lo[dir] = LinOpBCType::Periodic;
213  bc3d_hi[dir] = LinOpBCType::Periodic;
214  } else {
215  bc3d_lo[dir] = LinOpBCType::Neumann;
216  bc3d_hi[dir] = LinOpBCType::Neumann;
217  }
218  }
219  if (havewall) {
220  Print() << " Poisson zlo BC is dirichlet" << std::endl;
221  bc3d_lo[2] = LinOpBCType::Dirichlet;
222  }
223  Print() << " bc lo : " << bc3d_lo << std::endl;
224  Print() << " bc hi : " << bc3d_hi << std::endl;
225 
226  if (!solverChoice.advChoice.have_zero_flux_faces && !havewall) {
227  Error("No solid boundaries in the computational domain");
228  }
229 
230  LPInfo info; // defaults
231 
232 /* Nodal solver cannot have hidden dimensions */
233 #if 0
234  // Allow a hidden direction if the domain is one cell wide
235  if (dom_lo.x == dom_hi.x) {
236  info.setHiddenDirection(0);
237  Print() << " domain is 2D in yz" << std::endl;
238  } else if (dom_lo.y == dom_hi.y) {
239  info.setHiddenDirection(1);
240  Print() << " domain is 2D in xz" << std::endl;
241  } else if (dom_lo.z == dom_hi.z) {
242  info.setHiddenDirection(2);
243  Print() << " domain is 2D in xy" << std::endl;
244  }
245 #endif
246 
247 #if 0
248  Vector<EBFArrayBoxFactory const*> factory_vec;
249  factory_vec.push_back(static_cast<FabFactory<FArrayBox> const*>(&EBFactory(lev));
250 #endif
251 
252  // ****************************************************************************
253  // Setup Poisson problem
254  // (A \alpha - B \nabla \cdot \beta \nabla ) \phi = f
255  //
256  // In physical space:
257  // \nabla \cdot \nabla \phi = -1
258  //
259  // In computational space:
260  // grad(phi) = T^T \nabla \phi
261  // and
262  // \nabla \cdot (h_zeta T (T^T \nabla \phi)) = -h_zeta
263  // where T = inv(J), T^T is the transpose of inv(J)
264  // ****************************************************************************
265  constexpr Real constA = zero;
266  constexpr Real constB = -one;
267 
268  MLABecLaplacian mlabec(geom_tmp, ba_tmp, dm_tmp, info);
269 
270  mlabec.setScalars(constA, constB);
271  mlabec.setACoeffs(0, zero);
272 #if 1
273  // Set beta coefficients at faces
274  Array<MultiFab, AMREX_SPACEDIM> beta;
275 
276  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
277  BoxArray ba_face = ba_tmp[0];
278  ba_face.surroundingNodes(idim); // Convert to face-centered in direction idim
279  beta[idim].define(ba_face, dm_tmp[0], 1, 0);
280  }
281 
282  auto beta0_arr = beta[0].arrays();
283  auto beta1_arr = beta[1].arrays();
284  auto beta2_arr = beta[2].arrays();
285 
286  // Note: This ignores the off-diagonal components of (h_zeta T T^T), which
287  // is equivalent to assuming that h_xi and h_eta are small.
288 
289  ParallelFor(beta[0], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
290  beta0_arr[b](i, j, k) = Compute_h_zeta_AtIface(i, j, k, dxinv, zphys_arr[b]);;
291  });
292  ParallelFor(beta[1], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
293  beta1_arr[b](i, j, k) = Compute_h_zeta_AtJface(i, j, k, dxinv, zphys_arr[b]);;
294  });
295  ParallelFor(beta[2], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
296  Real inv_h_zeta = one / Compute_h_zeta_AtKface(i, j, k, dxinv, zphys_arr[b]);
297  Real h_xi = Compute_h_xi_AtKface(i, j, k, dxinv, zphys_arr[b]);
298  Real h_eta = Compute_h_eta_AtKface(i, j, k, dxinv, zphys_arr[b]);
299  beta2_arr[b](i, j, k) = inv_h_zeta * (1 + h_xi*h_xi + h_eta*h_eta);
300  });
301 
302  mlabec.setBCoeffs(0, GetArrOfConstPtrs(beta));
303 
304  // Set RHS := -h_zeta
305  auto rhs_arr = rhs[0].arrays();
306  ParallelFor(rhs[0], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
307  rhs_arr[b](i, j, k) = -Compute_h_zeta_AtCellCenter(i, j, k, dxinv, zphys_arr[b]);
308  });
309 #else
310  mlabec.setBCoeffs(0, one);
311 #endif
312 
313  mlabec.setDomainBC(bc3d_lo, bc3d_hi);
314 
315  if (lev > 0) {
316  mlabec.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
317  }
318 
319  // If we have inhomogeneous BCs -- do this after setCoarseFineBC
320  mlabec.setLevelBC(0, nullptr);
321 
322  // ****************************************************************************
323  // Solve Poisson problem with MLMG
324  // ****************************************************************************
325  const Real reltol = solverChoice.poisson_reltol;
326  const Real abstol = solverChoice.poisson_abstol;
327  const int n_corr = solverChoice.ncorr;
328  constexpr int max_iter = 100;
329 
330  MLMG mlmg(mlabec);
331  mlmg.setMaxIter(max_iter);
332  mlmg.setVerbose(mg_verbose);
333  mlmg.setBottomVerbose(0);
334 
335  for (int icorr=0; icorr <= n_corr; ++icorr) {
336  Print()<< "Solving wall distance poisson, icorr=" << icorr << std::endl;
337 
338  mlmg.solve(GetVecOfPtrs(phi),
339  GetVecOfConstPtrs(rhs),
340  reltol, abstol);
341 
342  // ****************************************************************************
343  // Apply BCs: dirichlet (odd) on zlo, neumann (even) / periodic elsewhere
344  // ****************************************************************************
345 
346  // Overwrite with periodic fill outside domain and fine-fine fill inside
347  phi[0].FillBoundary(geom[lev].periodicity());
348 
349  if (!geom[lev].isPeriodic(0)) {
350  for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
351  {
352  Box bx = mfi.tilebox();
353  const Array4<Real>& phi_arr = phi[0].array(mfi);
354  if (bx.smallEnd(0) <= dom_lo.x) {
355  ParallelFor(makeSlab(bx,0,dom_lo.x),
356  [=] AMREX_GPU_DEVICE (int i, int j, int k)
357  {
358  phi_arr(i-1,j,k) = phi_arr(i,j,k); // even BC
359  });
360  } // lo x
361  if (bx.bigEnd(0) >= dom_hi.x) {
362  ParallelFor(makeSlab(bx,0,dom_hi.x),
363  [=] AMREX_GPU_DEVICE (int i, int j, int k)
364  {
365  phi_arr(i+1,j,k) = phi_arr(i,j,k); // even BC
366  });
367  } // hi x
368  } // mfi
369  } // not periodic in x
370 
371  if (!geom[lev].isPeriodic(1)) {
372  for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
373  {
374  Box bx = mfi.tilebox();
375  Box bx2(bx); bx2.grow(0,1);
376  const Array4<Real>& phi_arr = phi[0].array(mfi);
377  if (bx.smallEnd(1) <= dom_lo.y) {
378  ParallelFor(makeSlab(bx2,1,dom_lo.y),
379  [=] AMREX_GPU_DEVICE (int i, int j, int k)
380  {
381  phi_arr(i,j-1,k) = phi_arr(i,j,k); // even BC
382  });
383  } // lo y
384  if (bx.bigEnd(1) >= dom_hi.y) {
385  ParallelFor(makeSlab(bx2,1,dom_hi.y),
386  [=] AMREX_GPU_DEVICE (int i, int j, int k)
387  {
388  phi_arr(i,j+1,k) = phi_arr(i,j,k); // even BC
389  });
390  } // hi y
391 
392  } // mfi
393  } // not periodic in y
394 
395  for (MFIter mfi(phi[0],true); mfi.isValid(); ++mfi)
396  {
397  Box bx = mfi.tilebox();
398  Box bx3(bx); bx3.grow(0,1); bx3.grow(1,1);
399  const Array4<Real>& phi_arr = phi[0].array(mfi);
400  if (bx.smallEnd(2) <= dom_lo.z) {
401  ParallelFor(makeSlab(bx3,2,dom_lo.z),
402  [=] AMREX_GPU_DEVICE (int i, int j, int k)
403  {
404  phi_arr(i,j,k-1) = -phi_arr(i,j,k); // ODD BC
405  });
406  } // lo z
407  if (bx.bigEnd(2) >= dom_hi.z) {
408  ParallelFor(makeSlab(bx3,2,dom_hi.z),
409  [=] AMREX_GPU_DEVICE (int i, int j, int k)
410  {
411  phi_arr(i,j,k+1) = phi_arr(i,j,k); // even BC
412  });
413  } // hi z
414  } // mfi
415 
416  // ****************************************************************************
417  // Compute grad(phi) to get distances
418  // ****************************************************************************
419  auto const& phi_arr = phi[0].const_arrays();
420  //auto rhs_arr = rhs[0].arrays();
421  auto dist_arr = walldist[lev]->arrays();
422 
423  ParallelFor(*walldist[lev], [=] AMREX_GPU_DEVICE(int b, int i, int j, int k) {
424  Real dpdx{0}, dpdy{0}, dpdz{0};
425 
426  dpdx = terrpoisson_flux_x(i, j, k, phi_arr[b], zphys_arr[b], dxinv[0]);
427  dpdy = terrpoisson_flux_y(i, j, k, phi_arr[b], zphys_arr[b], dxinv[1]);
428  if (k == dom_lo.z) {
429  dpdz = terrpoisson_flux_zlo_dir(i, j, k, phi_arr[b], zphys_arr[b], dxinv[0], dxinv[1]);
430  } else {
431  // This returns 0 at the wall, hence the need for the separate calc above
432  dpdz = terrpoisson_flux_z(i, j, k, phi_arr[b], zphys_arr[b], dxinv[0], dxinv[1]);
433  }
434 
435  Real magsqr_dphi = dpdx*dpdx + dpdy*dpdy + dpdz*dpdz;
436  Real mag_dphi = std::sqrt(magsqr_dphi);
437 #if 1
438  // Tucker 2003 Eqn 2
439  dist_arr[b](i, j, k) = -mag_dphi + std::sqrt(magsqr_dphi + 2*phi_arr[b](i, j, k));
440 #else
441  // DEBUG: output phi instead
442  if (i==0 && j==0) AllPrint() << "walldist"<<IntVect(i,j,k) << " = " << dist_arr[b](i,j,k) << std::endl;
443  dist_arr[b](i, j, k) = phi_arr[b](i, j, k);
444 #endif
445  // Update RHS source term to explicitly include cross-terms
446  if (n_corr > 0) {
447  // d/dxi ( h_xi * dphi/dzeta )
448  Real phi_zeta_xlo = fourth * dxinv[2] * ( phi_arr[b](i , j, k+1) - phi_arr[b](i , j, k-1)
449  + phi_arr[b](i-1, j, k+1) - phi_arr[b](i-1, j, k-1) );
450  Real phi_zeta_xhi = fourth * dxinv[2] * ( phi_arr[b](i , j, k+1) - phi_arr[b](i , j, k-1)
451  + phi_arr[b](i+1, j, k+1) - phi_arr[b](i+1, j, k-1) );
452  Real h_xi_xlo = Compute_h_xi_AtIface(i , j, k, dxinv, zphys_arr[b]);
453  Real h_xi_xhi = Compute_h_xi_AtIface(i+1, j, k, dxinv, zphys_arr[b]);
454 
455  // d/deta ( h_eta * dphi/dzeta )
456  Real phi_zeta_ylo = fourth * dxinv[2] * ( phi_arr[b](i, j , k+1) - phi_arr[b](i, j , k-1)
457  + phi_arr[b](i, j-1, k+1) - phi_arr[b](i, j-1, k-1) );
458  Real phi_zeta_yhi = fourth * dxinv[2] * ( phi_arr[b](i, j , k+1) - phi_arr[b](i, j , k-1)
459  + phi_arr[b](i, j+1, k+1) - phi_arr[b](i, j+1, k-1) );
460  Real h_eta_ylo = Compute_h_eta_AtJface(i, j , k, dxinv, zphys_arr[b]);
461  Real h_eta_yhi = Compute_h_eta_AtJface(i, j+1, k, dxinv, zphys_arr[b]);
462 
463  // d/dzeta ( h_xi * dphi/dxi )
464  Real phi_xi_zlo = fourth * dxinv[0] * ( phi_arr[b](i+1, j, k ) - phi_arr[b](i-1, j, k )
465  + phi_arr[b](i+1, j, k-1) - phi_arr[b](i-1, j, k-1) );
466  Real phi_xi_zhi = fourth * dxinv[0] * ( phi_arr[b](i+1, j, k ) - phi_arr[b](i-1, j, k )
467  + phi_arr[b](i+1, j, k+1) - phi_arr[b](i-1, j, k+1) );
468  Real h_xi_zlo = Compute_h_xi_AtKface(i, j, k , dxinv, zphys_arr[b]);
469  Real h_xi_zhi = Compute_h_xi_AtKface(i, j, k+1, dxinv, zphys_arr[b]);
470 
471  // d/dzeta ( h_eta * dphi/deta )
472  Real phi_eta_zlo = fourth * dxinv[1] * ( phi_arr[b](i, j+1, k ) - phi_arr[b](i, j-1, k )
473  + phi_arr[b](i, j+1, k-1) - phi_arr[b](i, j-1, k-1) );
474  Real phi_eta_zhi = fourth * dxinv[1] * ( phi_arr[b](i, j+1, k ) - phi_arr[b](i, j-1, k )
475  + phi_arr[b](i, j+1, k+1) - phi_arr[b](i, j-1, k+1) );
476  Real h_eta_zlo = Compute_h_eta_AtKface(i, j, k , dxinv, zphys_arr[b]);
477  Real h_eta_zhi = Compute_h_eta_AtKface(i, j, k+1, dxinv, zphys_arr[b]);
478 
479  Real detJ = Compute_h_zeta_AtCellCenter(i, j, k, dxinv, zphys_arr[b]);
480 
481  rhs_arr[b](i, j, k) = -detJ
482  + dxinv[0] * ( h_xi_xhi * phi_zeta_xhi - h_xi_xlo * phi_zeta_xlo)
483  + dxinv[1] * ( h_eta_yhi * phi_zeta_yhi - h_eta_ylo * phi_zeta_ylo)
484  + dxinv[2] * ( h_xi_zhi * phi_xi_zhi - h_xi_zlo * phi_xi_zlo
485  + h_eta_zhi * phi_eta_zhi - h_eta_zlo * phi_eta_zlo);
486  }
487  });
488  } // corrector loop
489 }
beta
amrex::Real beta
Definition: ERF_InitCustomPert_DataAssimilation_ISV.H:10
Compute_h_xi_AtIface
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_xi_AtIface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:117
Compute_h_zeta_AtCellCenter
AMREX_FORCE_INLINE AMREX_GPU_DEVICE amrex::Real Compute_h_zeta_AtCellCenter(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:55
Compute_h_zeta_AtKface
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_zeta_AtKface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:184
Compute_h_zeta_AtIface
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_zeta_AtIface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:104
Compute_h_xi_AtKface
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_xi_AtKface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:198
Compute_h_zeta_AtJface
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_zeta_AtJface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:144
Compute_h_eta_AtJface
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_eta_AtJface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:170
Compute_h_eta_AtKface
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_eta_AtKface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:211
terrpoisson_flux_x
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE T terrpoisson_flux_x(int i, int j, int k, amrex::Array4< T const > const &sol, amrex::Array4< T const > const &zp, T dxinv) noexcept
Definition: ERF_TerrainPoisson_3D_K.H:9
terrpoisson_flux_zlo_dir
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE T terrpoisson_flux_zlo_dir(int i, int j, int k, amrex::Array4< T const > const &sol, amrex::Array4< T const > const &zp, T dxinv, T dyinv) noexcept
Definition: ERF_TerrainPoisson_3D_K.H:163
terrpoisson_flux_z
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE T terrpoisson_flux_z(int i, int j, int k, amrex::Array4< T const > const &sol, amrex::Array4< T const > const &zp, T dxinv, T dyinv) noexcept
Definition: ERF_TerrainPoisson_3D_K.H:84
terrpoisson_flux_y
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE T terrpoisson_flux_y(int i, int j, int k, amrex::Array4< T const > const &sol, amrex::Array4< T const > const &zp, T dyinv) noexcept
Definition: ERF_TerrainPoisson_3D_K.H:47
ERF::mg_verbose
static int mg_verbose
Definition: ERF.H:1331
SolverChoice::poisson_reltol
amrex::Real poisson_reltol
Definition: ERF_DataStruct.H:1289
SolverChoice::ncorr
int ncorr
Definition: ERF_DataStruct.H:1283
SolverChoice::poisson_abstol
amrex::Real poisson_abstol
Definition: ERF_DataStruct.H:1288
Here is the call graph for this function:

◆ post_timestep()

void ERF::post_timestep ( int  nstep,
double  time,
double  dt_lev 
)
312 {
313  BL_PROFILE("ERF::post_timestep()");
314 
315 #ifdef ERF_USE_PARTICLES
316  particleData.Redistribute(z_phys_nd);
317 #endif
318 
319  if (solverChoice.coupling_type == CouplingType::TwoWay)
320  {
321  int ncomp = vars_new[0][Vars::cons].nComp();
322  for (int lev = finest_level-1; lev >= 0; lev--)
323  {
324  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
325  // m is the map scale factor at cell centers
326  // Here we pre-divide (rho S) by m^2 before refluxing
327  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
328  const Box& bx = mfi.tilebox();
329  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
330  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
331  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
332  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
333  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
334  {
335  cons_arr(i,j,k,n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
336  });
337  } else {
338  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
339  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
340  {
341  cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
342  });
343  }
344  } // mfi
345 
346  // This call refluxes all "slow" cell-centered variables
347  // (i.e. not density or (rho theta) or velocities) from the lev/lev+1 interface onto lev
348  getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 2, 2, ncomp-2);
349 
350  // Here we multiply (rho S) by m^2 after refluxing
351  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
352  const Box& bx = mfi.tilebox();
353  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
354  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
355  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
356  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
357  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
358  {
359  cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
360  });
361  } else {
362  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
363  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
364  {
365  cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
366  });
367  }
368  } // mfi
369 
370  // We need to do this before anything else because refluxing changes the
371  // values of coarse cells underneath fine grids with the assumption they'll
372  // be over-written by averaging down
373  int src_comp;
374  if (solverChoice.anelastic[lev]) {
375  src_comp = 1;
376  } else {
377  src_comp = 0;
378  }
379  int num_comp = ncomp - src_comp;
380  AverageDownTo(lev,src_comp,num_comp);
381  }
382  }
383 
384  if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
385  sum_integrated_quantities(time);
386  sum_derived_quantities(time);
387  sum_energy_quantities(time);
388  }
389 
390  if (solverChoice.any_perturbation()) {
391  if (is_it_time_for_action(nstep, time, dt_lev0, pert_interval, -one)) {
392  turbPert.debug(time);
393  }
394  }
395 
396  if (profile_int > 0 && (nstep+1) % profile_int == 0) {
397  if (destag_profiles) {
398  // all variables cell-centered
399  write_1D_profiles(time);
400  } else {
401  // some variables staggered
402  write_1D_profiles_stag(time);
403  }
404  }
405 
406  if (solverChoice.rad_type != RadiationType::None)
407  {
408  if ( rad_datalog_int > 0 &&
409  (((nstep+1) % rad_datalog_int == 0) || (nstep==0)) ) {
410  if (rad[0]->hasDatalog()) {
411  rad[0]->WriteDataLog(time + start_time);
412  }
413  }
414  }
415 
416  if (output_1d_column) {
417 #ifdef ERF_USE_NETCDF
418  if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
419  {
420  int lev_column = 0;
421  for (int lev = finest_level; lev >= 0; lev--)
422  {
423  Real dx_lev = geom[lev].CellSize(0);
424  Real dy_lev = geom[lev].CellSize(1);
425  int i_lev = static_cast<int>(std::floor(column_loc_x / dx_lev));
426  int j_lev = static_cast<int>(std::floor(column_loc_y / dy_lev));
427  if (grids[lev].contains(IntVect(i_lev,j_lev,0))) lev_column = lev;
428  }
429  writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
430  }
431 #else
432  Abort("To output 1D column files ERF must be compiled with NetCDF");
433 #endif
434  }
435 
436  if (output_bndry_planes)
437  {
438  if (is_it_time_for_action(istep[0], time, dt_lev0, bndry_output_planes_interval, bndry_output_planes_per) &&
439  time >= bndry_output_planes_start_time)
440  {
441  bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
442  m_w2d->write_planes(istep[0], time+start_time, vars_new, is_moist);
443  }
444  }
445 
446  // Write plane/line sampler data
447  if (line_sampler && is_it_time_for_action(nstep+1, time, dt_lev0, line_sampling_interval, line_sampling_per)) {
448  line_sampler->get_sample_data(geom, vars_new);
449  line_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
450  }
451  if (plane_sampler && is_it_time_for_action(nstep+1, time, dt_lev0, plane_sampling_interval, plane_sampling_per)) {
452  plane_sampler->get_sample_data(geom, vars_new);
453  plane_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
454  }
455 
456  // Moving terrain
457  if ( solverChoice.terrain_type == TerrainType::MovingFittedMesh )
458  {
459  for (int lev = finest_level; lev >= 0; lev--)
460  {
461  // Copy z_phs_nd and detJ_cc at end of timestep
462  MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
463  MultiFab::Copy( *detJ_cc[lev], *detJ_cc_new[lev], 0, 0, 1, detJ_cc[lev]->nGrowVect());
464  MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
465 
466  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
467  }
468  }
469 
470  if (solverChoice.io_hurricane_eye_tracker and
471  (nstep == 0 or (nstep+1)%m_plot3d_int_1 == 0)) {
472 
473  int levc=finest_level;
474 
475  HurricaneEyeTracker(solverChoice);
476 
477  MultiFab& U_new = vars_new[levc][Vars::xvel];
478  MultiFab& V_new = vars_new[levc][Vars::yvel];
479  MultiFab& W_new = vars_new[levc][Vars::zvel];
480 
481  MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
482  average_face_to_cellcenter(mf_cc_vel,0,{AMREX_D_DECL(&U_new,&V_new,&W_new)},0);
483 
484  HurricaneMaxVelTracker(geom[levc],
485  mf_cc_vel,
486  t_new[0]);
487 
488  HurricaneMinPressureTracker(solverChoice.moisture_type,
489  geom[levc],
490  vars_new[levc][Vars::cons],
491  t_new[0]);
492 
493  std::string filename_tracker = MakeVTKFilename_TrackerCircle(nstep);
494  std::string filename_xy = MakeVTKFilename_EyeTracker_xy(nstep);
495  std::string filename_latlon = MakeFilename_EyeTracker_latlon(nstep);
496  std::string filename_maxvel = MakeFilename_EyeTracker_maxvel(nstep);
497  std::string filename_minpressure = MakeFilename_EyeTracker_minpressure(nstep);
498 
499  if (ParallelDescriptor::IOProcessor()) {
500  WriteVTKPolyline(filename_tracker, hurricane_tracker_circle);
501  WriteVTKPolyline(filename_xy, hurricane_eye_track_xy);
502  WriteLinePlot(filename_latlon, hurricane_eye_track_latlon);
503  WriteLinePlot(filename_maxvel, hurricane_maxvel_vs_time);
504  WriteLinePlot(filename_minpressure, hurricane_minpressure_vs_time);
505  }
506  }
507 
508 } // post_timestep
make_zcc
void make_zcc(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc)
Definition: ERF_TerrainMetrics.cpp:628
ERF::bndry_output_planes_per
static double bndry_output_planes_per
Definition: ERF.H:1407
ERF::MakeFilename_EyeTracker_maxvel
std::string MakeFilename_EyeTracker_maxvel(int nstep)
Definition: ERF_TrackerOutput.cpp:66
ERF::column_loc_y
static amrex::Real column_loc_y
Definition: ERF.H:1401
ERF::column_per
static double column_per
Definition: ERF.H:1399
ERF::column_file_name
static std::string column_file_name
Definition: ERF.H:1402
ERF::getAdvFluxReg
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg(int lev)
Definition: ERF.H:1542
ERF::HurricaneMaxVelTracker
void HurricaneMaxVelTracker(const amrex::Geometry &geom, const amrex::MultiFab &mf_cc_vel, const double &time)
Definition: ERF_HurricaneDiagnostics.cpp:501
ERF::HurricaneEyeTracker
void HurricaneEyeTracker(const SolverChoice &solverChoice)
Definition: ERF_HurricaneDiagnostics.cpp:475
ERF::column_loc_x
static amrex::Real column_loc_x
Definition: ERF.H:1400
ERF::MakeVTKFilename_TrackerCircle
std::string MakeVTKFilename_TrackerCircle(int nstep)
Definition: ERF_TrackerOutput.cpp:24
ERF::MakeVTKFilename_EyeTracker_xy
std::string MakeVTKFilename_EyeTracker_xy(int nstep)
Definition: ERF_TrackerOutput.cpp:38
ERF::HurricaneMinPressureTracker
void HurricaneMinPressureTracker(MoistureType moisture_type, const amrex::Geometry &geom, const amrex::MultiFab &mf_cons_var, const double &time)
Definition: ERF_HurricaneDiagnostics.cpp:556
ERF::bndry_output_planes_interval
static int bndry_output_planes_interval
Definition: ERF.H:1406
ERF::MakeFilename_EyeTracker_minpressure
std::string MakeFilename_EyeTracker_minpressure(int nstep)
Definition: ERF_TrackerOutput.cpp:80
ERF::WriteLinePlot
void WriteLinePlot(const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
Definition: ERF_Write1DProfiles.cpp:587
ERF::output_1d_column
static int output_1d_column
Definition: ERF.H:1397
ERF::WriteVTKPolyline
void WriteVTKPolyline(const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
Definition: ERF_TrackerOutput.cpp:94
ERF::MakeFilename_EyeTracker_latlon
std::string MakeFilename_EyeTracker_latlon(int nstep)
Definition: ERF_TrackerOutput.cpp:52
ERF::column_interval
static int column_interval
Definition: ERF.H:1398
SolverChoice::io_hurricane_eye_tracker
bool io_hurricane_eye_tracker
Definition: ERF_DataStruct.H:1478

Referenced by EvolveOneStep().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ post_update()

void ERF::post_update ( amrex::MultiFab &  state_mf,
double  time,
const amrex::Geometry &  geom 
)
private

◆ print_banner()

void ERF::print_banner ( MPI_Comm  comm,
std::ostream &  out 
)
static
65  : " << msg << std::endl;
66 }
67 
68 void ERF::print_banner (MPI_Comm comm, std::ostream& out)
69 {
70 #ifdef AMREX_USE_MPI
71  int irank = 0;
72  int num_ranks = 1;
73  MPI_Comm_size(comm, &num_ranks);
74  MPI_Comm_rank(comm, &irank);
75 
76  // Only root process does the printing
77  if (irank != 0) return;
78 #else
79  amrex::ignore_unused(comm);
80 #endif
81 
82  auto etime = std::chrono::system_clock::now();
83  auto etimet = std::chrono::system_clock::to_time_t(etime);
84 #ifndef _WIN32
85  char time_buf[64];
86  ctime_r(&etimet, time_buf);
87  const std::string tstamp(time_buf);
88 #else
89  char* time_buf = new char[64];
90  ctime_s(time_buf, 64, &etimet);
91  const std::string tstamp(time_buf);
92 #endif
93 
94  const char* githash1 = amrex::buildInfoGetGitHash(1);
95  const char* githash2 = amrex::buildInfoGetGitHash(2);
96 
97  // clang-format off
98  out << dbl_line
99  << " ERF (https://github.com/erf-model/ERF)"
100  << std::endl << std::endl
101  << " ERF Git SHA :: " << githash1 << std::endl
102  << " AMReX Git SHA :: " << githash2 << std::endl
103  << " AMReX version :: " << amrex::Version() << std::endl << std::endl
104  << " Exec. time :: " << tstamp
105  << " Build time :: " << amrex::buildInfoGetBuildDate() << std::endl
106  << " C++ compiler :: " << amrex::buildInfoGetComp()
107  << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
108  << " MPI :: "
109 #ifdef AMREX_USE_MPI
110  << "ON (Num. ranks = " << num_ranks << ")" << std::endl
111 #else
112  << "OFF " << std::endl
113 #endif
114  << " GPU :: "
115 #ifdef AMREX_USE_GPU
116  << "ON "
117 #if defined(AMREX_USE_CUDA)
118  << "(Backend: CUDA)"
119 #elif defined(AMREX_USE_HIP)
120  << "(Backend: HIP)"
121 #elif defined(AMREX_USE_SYCL)
122  << "(Backend: SYCL)"
123 #endif
124  << std::endl
125 #else
126  << "OFF" << std::endl
127 #endif
128  << " OpenMP :: "
129 #ifdef AMREX_USE_OMP
130  << "ON (Num. threads = " << omp_get_max_threads() << ")" << std::endl
131 #else
132  << "OFF" << std::endl
133 #endif
134  << std::endl;
135 
ERF::ERF
ERF()
Definition: ERF_Constructors.cpp:21
amrex::buildInfoGetBuildDate
const char * buildInfoGetBuildDate()
amrex::buildInfoGetComp
const char * buildInfoGetComp()
amrex::buildInfoGetCompVersion
const char * buildInfoGetCompVersion()

Referenced by main().

Here is the caller graph for this function:

◆ print_error()

void ERF::print_error ( MPI_Comm  comm,
const std::string &  msg 
)
static
43  :
44  input_file : Input file with simulation settings
45 
46 Optional:
47  param=value : Overrides for parameters during runtime
48 )doc" << std::endl;
49 }
50 
51 void ERF::print_error (MPI_Comm comm, const std::string& msg)
52 {
53 #ifdef AMREX_USE_MPI
54  int irank = 0;
55  int num_ranks = 1;
56  MPI_Comm_size(comm, &num_ranks);
57  MPI_Comm_rank(comm, &irank);
58 

Referenced by main().

Here is the caller graph for this function:

◆ print_summary()

static void ERF::print_summary ( std::ostream &  )
static

◆ print_tpls()

void ERF::print_tpls ( std::ostream &  out)
static
140  ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
141  << dash_line << std::endl;
142  // clang-format on
143 }
144 
145 void ERF::print_tpls (std::ostream& out)
146 {
147  amrex::Vector<std::string> tpls;
148 
149 #ifdef ERF_USE_NETCDF
150  tpls.push_back(std::string("NetCDF ") + NC_VERSION);
151 #endif
152 #ifdef AMREX_USE_SUNDIALS
153  tpls.push_back(std::string("SUNDIALS ") + SUNDIALS_VERSION);
154 #endif
155 
156  if (!tpls.empty()) {
157  out << " Enabled third-party libraries: ";
158  for (const auto& val : tpls) {
ERF::print_tpls
static void print_tpls(std::ostream &)
Definition: ERF_ConsoleIO.cpp:137

◆ print_usage()

void ERF::print_usage ( MPI_Comm  comm,
std::ostream &  out 
)
static
27 {
28 #ifdef AMREX_USE_MPI
29  int irank = 0;
30  int num_ranks = 1;
31  MPI_Comm_size(comm, &num_ranks);
32  MPI_Comm_rank(comm, &irank);
33 
34  // Only root process does the printing
35  if (irank != 0) return;
36 #else
37  amrex::ignore_unused(comm);
38 #endif
39 
40  out << R"doc(Usage:
41  ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

◆ project_initial_velocity()

void ERF::project_initial_velocity ( int  lev,
double  time,
double  l_dt 
)

Project the single-level velocity field to enforce the anelastic constraint Note that the level may or may not be level zero

32 {
33  BL_PROFILE("ERF::project_initial_velocity()");
34  // Impose FillBoundary on density since we use it in the conversion of velocity to momentum
35  vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
36 
37  const MultiFab* c_vfrac = nullptr;
38  if (solverChoice.terrain_type == TerrainType::EB) {
39  c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
40  }
41 
42  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
43  vars_new[lev][Vars::yvel], IntVect{0},
44  vars_new[lev][Vars::zvel], IntVect{0},
45  vars_new[lev][Vars::cons],
46  rU_new[lev], rV_new[lev], rW_new[lev],
47  Geom(lev).Domain(), domain_bcs_type, c_vfrac);
48 
49  Vector<MultiFab> tmp_mom;
50 
51  tmp_mom.push_back(MultiFab(vars_new[lev][Vars::cons],make_alias,0,1));
52  tmp_mom.push_back(MultiFab(rU_new[lev],make_alias,0,1));
53  tmp_mom.push_back(MultiFab(rV_new[lev],make_alias,0,1));
54  tmp_mom.push_back(MultiFab(rW_new[lev],make_alias,0,1));
55 
56  // If at lev > 0 we must first fill the velocities at the c/f interface -- this must
57  // be done *after* the projection at lev-1
58  if (lev > 0) {
59  int levc = lev-1;
60 
61  const MultiFab* c_vfrac_crse = nullptr;
62  if (solverChoice.terrain_type == TerrainType::EB) {
63  c_vfrac_crse = &((get_eb(levc).get_const_factory())->getVolFrac());
64  }
65 
66  MultiFab& S_new_crse = vars_new[levc][Vars::cons];
67  MultiFab& U_new_crse = vars_new[levc][Vars::xvel];
68  MultiFab& V_new_crse = vars_new[levc][Vars::yvel];
69  MultiFab& W_new_crse = vars_new[levc][Vars::zvel];
70 
71  VelocityToMomentum(U_new_crse, IntVect{0}, V_new_crse, IntVect{0}, W_new_crse, IntVect{0}, S_new_crse,
72  rU_new[levc], rV_new[levc], rW_new[levc],
73  Geom(levc).Domain(), domain_bcs_type, c_vfrac_crse);
74 
75  rU_new[levc].FillBoundary(geom[levc].periodicity());
76  FPr_u[levc].RegisterCoarseData({&rU_new[levc], &rU_new[levc]}, {time, time+l_dt});
77 
78  rV_new[levc].FillBoundary(geom[levc].periodicity());
79  FPr_v[levc].RegisterCoarseData({&rV_new[levc], &rV_new[levc]}, {time, time+l_dt});
80 
81  rW_new[levc].FillBoundary(geom[levc].periodicity());
82  FPr_w[levc].RegisterCoarseData({&rW_new[levc], &rW_new[levc]}, {time, time+l_dt});
83  }
84 
85  // Use the same time that was registered in the FillPatcher above so that the
86  // FillSet assertion (time >= crse_times[0] && time <= crse_times[1]) is satisfied
87  // when called at non-zero simulation time (restart or mid-run regrid).
88  project_momenta(lev, time, l_dt, tmp_mom);
89 
90  MomentumToVelocity(vars_new[lev][Vars::xvel],
91  vars_new[lev][Vars::yvel],
92  vars_new[lev][Vars::zvel],
93  vars_new[lev][Vars::cons],
94  rU_new[lev], rV_new[lev], rW_new[lev],
95  Geom(lev).Domain(), domain_bcs_type, c_vfrac);
96  }
ERF::project_momenta
void project_momenta(int lev, double l_time, double l_dt, amrex::Vector< amrex::MultiFab > &vars)
Definition: ERF_PoissonSolve.cpp:102
Here is the call graph for this function:

◆ project_momenta()

void ERF::project_momenta ( int  lev,
double  l_time,
double  l_dt,
amrex::Vector< amrex::MultiFab > &  vars 
)

Project the single-level momenta to enforce the anelastic constraint Note that the level may or may not be level zero

103 {
104  BL_PROFILE("ERF::project_momenta()");
105  Real l_dt = static_cast<Real>(l_dt_d);
106  //
107  // If at lev > 0 we must first fill the momenta at the c/f interface with interpolated coarse values
108  //
109  if (lev > 0) {
110  PhysBCFunctNoOp null_bc;
111  FPr_u[lev-1].FillSet(mom_mf[IntVars::xmom], l_time, null_bc, domain_bcs_type);
112  FPr_v[lev-1].FillSet(mom_mf[IntVars::ymom], l_time, null_bc, domain_bcs_type);
113  FPr_w[lev-1].FillSet(mom_mf[IntVars::zmom], l_time, null_bc, domain_bcs_type);
114  }
115 
116  // Make sure the solver only sees the levels over which we are solving
117  Vector<BoxArray> ba_tmp; ba_tmp.push_back(mom_mf[Vars::cons].boxArray());
118  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap());
119  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
120 
121  Box domain = geom[lev].Domain();
122 
123  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
124 
125  Vector<MultiFab> rhs;
126  Vector<MultiFab> phi;
127 
128  if (solverChoice.terrain_type == TerrainType::EB)
129  {
130  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
131  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1, MFInfo(), EBFactory(lev));
132  } else {
133  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
134  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
135  }
136 
137  MultiFab rhs_lev(rhs[0], make_alias, 0, 1);
138  MultiFab phi_lev(phi[0], make_alias, 0, 1);
139 
140  auto dx = geom[lev].CellSizeArray();
141  auto dxInv = geom[lev].InvCellSizeArray();
142 
143  // Inflow on an x-face -- note only the normal velocity is used in the projection
144  if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow") {
145  (*physbcs_u[lev])(vars_new[lev][Vars::xvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
146  IntVect{1,0,0},t_new[lev],BCVars::xvel_bc,false);
147  }
148 
149  // Inflow on a y-face -- note only the normal velocity is used in the projection
150  if (domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
151  (*physbcs_v[lev])(vars_new[lev][Vars::yvel],vars_new[lev][Vars::xvel],vars_new[lev][Vars::yvel],
152  IntVect{0,1,0},t_new[lev],BCVars::yvel_bc,false);
153  }
154 
155  if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow" ||
156  domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
157 
158  const MultiFab* c_vfrac = nullptr;
159  if (solverChoice.terrain_type == TerrainType::EB) {
160  c_vfrac = &((get_eb(lev).get_const_factory())->getVolFrac());
161  }
162 
163  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
164  vars_new[lev][Vars::yvel], IntVect{0},
165  vars_new[lev][Vars::zvel], IntVect{0},
166  vars_new[lev][Vars::cons],
167  mom_mf[IntVars::xmom],
168  mom_mf[IntVars::ymom],
169  mom_mf[IntVars::zmom],
170  Geom(lev).Domain(),
171  domain_bcs_type, c_vfrac);
172  }
173 
174  // If !fixed_density, we must convert (rho u) which came in
175  // to (rho0 u) which is what we will project
176  if (!solverChoice.fixed_density[lev]) {
177  ConvertForProjection(mom_mf[Vars::cons], r_hse,
178  mom_mf[IntVars::xmom],
179  mom_mf[IntVars::ymom],
180  mom_mf[IntVars::zmom],
181  Geom(lev).Domain(),
182  domain_bcs_type);
183  }
184 
185  //
186  // ****************************************************************************
187  // Now convert the rho0w MultiFab to hold Omega rather than rhow
188  // ****************************************************************************
189  //
190  if (solverChoice.mesh_type == MeshType::VariableDz)
191  {
192  for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
193  {
194  const Array4<Real const>& rho0u_arr = mom_mf[IntVars::xmom].const_array(mfi);
195  const Array4<Real const>& rho0v_arr = mom_mf[IntVars::ymom].const_array(mfi);
196  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
197 
198  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
199  const Array4<Real const>& mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
200  const Array4<Real const>& mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
201 
202  //
203  // Define Omega from (rho0 W) but store it in the same array
204  //
205  Box tbz = mfi.nodaltilebox(2);
206  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
207  if (k == 0) {
208  rho0w_arr(i,j,k) = zero;
209  } else {
210  Real rho0w = rho0w_arr(i,j,k);
211  rho0w_arr(i,j,k) = OmegaFromW(i,j,k,rho0w,
212  rho0u_arr,rho0v_arr,
213  mf_u,mf_v,z_nd,dxInv);
214  }
215  });
216  } // mfi
217  }
218 
219  // ****************************************************************************
220  // Allocate fluxes
221  // ****************************************************************************
222  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
223  fluxes.resize(1);
224  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
225  if (solverChoice.terrain_type == TerrainType::EB) {
226  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
227  } else {
228  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
229  }
230  }
231 
232  // ****************************************************************************
233  // Initialize phi to 0
234  // (It is essential that we do this in order to fill the corners; these are never
235  // used but the Saxpy requires the values to be initialized.)
236  // ****************************************************************************
237  phi_lev.setVal(0.0);
238 
239  // ****************************************************************************
240  // Break into subdomains
241  // ****************************************************************************
242 
243  std::map<int,int> index_map;
244 
245  BoxArray ba(grids[lev]);
246 
247  Vector<MultiFab> rhs_sub; rhs_sub.resize(1);
248  Vector<MultiFab> phi_sub; phi_sub.resize(1);
249  Vector<Array<MultiFab,AMREX_SPACEDIM>> fluxes_sub; fluxes_sub.resize(1);
250 
251  MultiFab ax_sub, ay_sub, az_sub, dJ_sub, znd_sub;
252  MultiFab mfmx_sub, mfmy_sub;
253 
254  Array<MultiFab,AMREX_SPACEDIM> rho0_u_sub;
255  Array<MultiFab const*, AMREX_SPACEDIM> rho0_u_const;
256 
257  // If we are going to solve with MLMG then we do not need to break this into subdomains
258  bool will_solve_with_mlmg = false;
259  if (solverChoice.mesh_type == MeshType::ConstantDz) {
260  will_solve_with_mlmg = true;
261 #ifdef ERF_USE_FFT
262  if (use_fft) {
263  bool all_boxes_ok = true;
264  for (int isub = 0; isub < subdomains[lev].size(); ++isub) {
265  Box my_region(subdomains[lev][isub].minimalBox());
266  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
267  if (!boxes_make_rectangle) {
268  all_boxes_ok = false;
269  }
270  } // isub
271  if (all_boxes_ok) {
272  will_solve_with_mlmg = false;
273  }
274  } // use_fft
275 #else
276  if (use_fft) {
277  amrex::Warning("You set use_fft=true but didn't build with USE_FFT = TRUE; defaulting to MLMG");
278  }
279 #endif
280  } // No terrain or grid stretching
281 
282  for (int isub = 0; isub < subdomains[lev].size(); ++isub)
283  {
284  BoxList bl_sub;
285  Vector<int> dm_sub;
286 
287  for (int j = 0; j < ba.size(); j++)
288  {
289  if (subdomains[lev][isub].intersects(ba[j]))
290  {
291  //
292  // Note that bl_sub.size() is effectively a counter which is
293  // incremented above
294  //
295  // if (ParallelDescriptor::MyProc() == j) {
296  // }
297  index_map[bl_sub.size()] = j;
298 
299  bl_sub.push_back(grids[lev][j]);
300  dm_sub.push_back(dmap[lev][j]);
301  } // intersects
302  } // loop over ba (j)
303 
304  BoxArray ba_sub(bl_sub);
305 
306  BoxList bl2d_sub = ba_sub.boxList();
307  for (auto& b : bl2d_sub) {
308  b.setRange(2,0);
309  }
310  BoxArray ba2d_sub(std::move(bl2d_sub));
311 
312  // Define MultiFabs that hold only the data in this particular subdomain
313  if (solverChoice.terrain_type == TerrainType::EB) {
314  if (ba_sub != ba) {
315  amrex::Print() << "EB Solves with multiple regions is not yet supported" << std::endl;
316  }
317  rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs_lev.nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
318  phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi_lev.nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
319 
320  mfmx_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_x]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
321  mfmy_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_y]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
322  dJ_sub.define(ba_sub, DistributionMapping(dm_sub), 1, detJ_cc[lev]->nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
323 
324  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
325  fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
326  IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false), EBFactory(lev));
327  }
328  rho0_u_sub[0].define(convert(ba_sub, IntVect::TheDimensionVector(0)), DistributionMapping(dm_sub), 1,
329  mom_mf[IntVars::xmom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
330  rho0_u_sub[1].define(convert(ba_sub, IntVect::TheDimensionVector(1)), DistributionMapping(dm_sub), 1,
331  mom_mf[IntVars::ymom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
332  rho0_u_sub[2].define(convert(ba_sub, IntVect::TheDimensionVector(2)), DistributionMapping(dm_sub), 1,
333  mom_mf[IntVars::zmom].nGrowVect(), MFInfo{}.SetAlloc(false), EBFactory(lev));
334  } else {
335  rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
336  phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi_lev.nGrowVect(), MFInfo{}.SetAlloc(false));
337 
338  mfmx_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_x]->nGrowVect(), MFInfo{}.SetAlloc(false));
339  mfmy_sub.define(ba2d_sub, DistributionMapping(dm_sub), 1, mapfac[lev][MapFacType::m_y]->nGrowVect(), MFInfo{}.SetAlloc(false));
340  dJ_sub.define(ba_sub, DistributionMapping(dm_sub), 1, detJ_cc[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
341 
342  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
343  fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
344  IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false));
345  }
346  rho0_u_sub[0].define(convert(ba_sub, IntVect::TheDimensionVector(0)), DistributionMapping(dm_sub), 1,
347  mom_mf[IntVars::xmom].nGrowVect(), MFInfo{}.SetAlloc(false));
348  rho0_u_sub[1].define(convert(ba_sub, IntVect::TheDimensionVector(1)), DistributionMapping(dm_sub), 1,
349  mom_mf[IntVars::ymom].nGrowVect(), MFInfo{}.SetAlloc(false));
350  rho0_u_sub[2].define(convert(ba_sub, IntVect::TheDimensionVector(2)), DistributionMapping(dm_sub), 1,
351  mom_mf[IntVars::zmom].nGrowVect(), MFInfo{}.SetAlloc(false));
352  }
353 
354  // Link the new MultiFabs to the FABs in the original MultiFabs (no copy required)
355  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
356  {
357  int orig_index = index_map[mfi.index()];
358  rhs_sub[0].setFab(mfi, FArrayBox(rhs_lev[orig_index], amrex::make_alias, 0, 1));
359  phi_sub[0].setFab(mfi, FArrayBox(phi_lev[orig_index], amrex::make_alias, 0, 1));
360 
361  mfmx_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::m_x])[orig_index], amrex::make_alias, 0, 1));
362  mfmy_sub.setFab(mfi, FArrayBox((*mapfac[lev][MapFacType::m_y])[orig_index], amrex::make_alias, 0, 1));
363 
364  fluxes_sub[0][0].setFab(mfi,FArrayBox(fluxes[0][0][orig_index], amrex::make_alias, 0, 1));
365  fluxes_sub[0][1].setFab(mfi,FArrayBox(fluxes[0][1][orig_index], amrex::make_alias, 0, 1));
366  fluxes_sub[0][2].setFab(mfi,FArrayBox(fluxes[0][2][orig_index], amrex::make_alias, 0, 1));
367 
368  rho0_u_sub[0].setFab(mfi,FArrayBox(mom_mf[IntVars::xmom][orig_index], amrex::make_alias, 0, 1));
369  rho0_u_sub[1].setFab(mfi,FArrayBox(mom_mf[IntVars::ymom][orig_index], amrex::make_alias, 0, 1));
370  rho0_u_sub[2].setFab(mfi,FArrayBox(mom_mf[IntVars::zmom][orig_index], amrex::make_alias, 0, 1));
371  }
372 
373  rho0_u_const[0] = &rho0_u_sub[0];
374  rho0_u_const[1] = &rho0_u_sub[1];
375  rho0_u_const[2] = &rho0_u_sub[2];
376 
377  if (solverChoice.mesh_type != MeshType::ConstantDz) {
378  ax_sub.define(convert(ba_sub,IntVect(1,0,0)), DistributionMapping(dm_sub), 1,
379  ax[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
380  ay_sub.define(convert(ba_sub,IntVect(0,1,0)), DistributionMapping(dm_sub), 1,
381  ay[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
382  az_sub.define(convert(ba_sub,IntVect(0,0,1)), DistributionMapping(dm_sub), 1,
383  az[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
384  znd_sub.define(convert(ba_sub,IntVect(1,1,1)), DistributionMapping(dm_sub), 1,
385  z_phys_nd[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
386 
387  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
388  int orig_index = index_map[mfi.index()];
389  ax_sub.setFab(mfi, FArrayBox((*ax[lev])[orig_index], amrex::make_alias, 0, 1));
390  ay_sub.setFab(mfi, FArrayBox((*ay[lev])[orig_index], amrex::make_alias, 0, 1));
391  az_sub.setFab(mfi, FArrayBox((*az[lev])[orig_index], amrex::make_alias, 0, 1));
392  znd_sub.setFab(mfi, FArrayBox((*z_phys_nd[lev])[orig_index], amrex::make_alias, 0, 1));
393  dJ_sub.setFab(mfi, FArrayBox((*detJ_cc[lev])[orig_index], amrex::make_alias, 0, 1));
394  }
395  }
396 
397  if (solverChoice.terrain_type == TerrainType::EB) {
398  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
399  int orig_index = index_map[mfi.index()];
400  dJ_sub.setFab(mfi, FArrayBox((*detJ_cc[lev])[orig_index], amrex::make_alias, 0, 1));
401  }
402  }
403 
404  // ****************************************************************************
405  // Compute divergence which will form RHS
406  // Note that we replace "rho0w" with the contravariant momentum, Omega
407  // ****************************************************************************
408 
409  compute_divergence(lev, rhs_sub[0], rho0_u_const, geom_tmp[0]);
410 
411  Real rhsnorm;
412 
413  // Max norm over the entire MultiFab
414  rhsnorm = rhs_sub[0].norm0();
415 
416  if (mg_verbose > 0) {
417  bool local = false;
418  Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
419  Print() << "Max/L2 norm of divergence before solve in subdomain " << isub << " at level " << lev << " : " << rhsnorm << " " <<
420  rhs_sub[0].norm2() << " and volume-weighted sum " << sum << std::endl;
421  }
422 
423  if (lev == 0 && solverChoice.use_real_bcs)
424  {
425  // We always use VariableDz if use_real_bcs is true
426  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::VariableDz);
427 
428  // Note that we always impose the projections one level at a time so this will always be a vector of length 1
429  Array<MultiFab*, AMREX_SPACEDIM> rho0_u_vec =
430  {&mom_mf[IntVars::xmom], &mom_mf[IntVars::ymom], &mom_mf[IntVars::zmom]};
431  Array<MultiFab*, AMREX_SPACEDIM> area_vec = {ax[lev].get(), ay[lev].get(), az[lev].get()};
432  //
433  // Modify ax,ay,ax to include the map factors as used in the divergence calculation
434  // We do this here so that it is seen in the call to enforceInOutSolvability
435  //
436  for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
437  {
438  Box xbx = mfi.nodaltilebox(0);
439  Box ybx = mfi.nodaltilebox(1);
440  Box zbx = mfi.nodaltilebox(2);
441  const Array4<Real >& ax_ar = ax[lev]->array(mfi);
442  const Array4<Real >& ay_ar = ay[lev]->array(mfi);
443  const Array4<Real >& az_ar = az[lev]->array(mfi);
444  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
445  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
446  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
447  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
448  ParallelFor(xbx,ybx,zbx,
449  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
450  {
451  ax_ar(i,j,k) /= mf_uy(i,j,0);
452  },
453  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
454  {
455  ay_ar(i,j,k) /= mf_vx(i,j,0);
456  },
457  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
458  {
459  az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
460  });
461  } // mfi
462 
463  if (mg_verbose > 0) {
464  Print() << "Calling enforceInOutSolvability" << std::endl;
465  }
466  enforceInOutSolvability(lev, rho0_u_vec, area_vec, geom[lev]);
467 
468  //
469  // Return ax,ay,ax to their original definition
470  //
471  for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
472  {
473  Box xbx = mfi.nodaltilebox(0);
474  Box ybx = mfi.nodaltilebox(1);
475  Box zbx = mfi.nodaltilebox(2);
476  const Array4<Real >& ax_ar = ax[lev]->array(mfi);
477  const Array4<Real >& ay_ar = ay[lev]->array(mfi);
478  const Array4<Real >& az_ar = az[lev]->array(mfi);
479  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
480  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
481  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
482  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
483  ParallelFor(xbx,ybx,zbx,
484  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
485  {
486  ax_ar(i,j,k) *= mf_uy(i,j,0);
487  },
488  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
489  {
490  ay_ar(i,j,k) *= mf_vx(i,j,0);
491  },
492  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
493  {
494  az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
495  });
496  } // mfi
497 
498  compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
499 
500  // Re-define max norm over the entire MultiFab
501  rhsnorm = rhs_lev.norm0();
502 
503  if (mg_verbose > 0)
504  {
505  bool local = false;
506  Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
507  Print() << "Max/L2 norm of divergence before solve at level " << lev << " : " << rhsnorm << " " <<
508  rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
509  }
510  } // lev 0 && use_real_bcs
511 
512  // *******************************************************************************************
513  // Enforce solvability if the problem is singular (i.e all sides Neumann or periodic)
514  // Note that solves at lev > 0 are always singular because we impose Neumann bc's on all sides
515  // *******************************************************************************************
516  bool is_singular = true;
517  if (lev == 0) {
518  if ( (domain_bc_type[0] == "Outflow" || domain_bc_type[0] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
519  if ( (domain_bc_type[1] == "Outflow" || domain_bc_type[1] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
520  if ( (domain_bc_type[3] == "Outflow" || domain_bc_type[3] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
521  if ( (domain_bc_type[4] == "Outflow" || domain_bc_type[4] == "Open") && !solverChoice.use_real_bcs ) is_singular = false;
522  if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open") ) is_singular = false;
523  } else {
524  Box my_region(subdomains[lev][isub].minimalBox());
525  if ( (domain_bc_type[5] == "Outflow" || domain_bc_type[5] == "Open") && (my_region.bigEnd(2) == domain.bigEnd(2)) ) is_singular = false;
526  }
527 
528  if (is_singular)
529  {
530  bool local = false;
531  Real sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
532 
533  Real vol;
534  if (solverChoice.mesh_type == MeshType::ConstantDz) {
535  vol = rhs_sub[0].boxArray().numPts();
536  } else {
537  vol = dJ_sub.sum();
538  }
539 
540  sum /= (vol * dx[0] * dx[1] * dx[2]);
541 
542  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
543  {
544  rhs_sub[0][mfi.index()].template minus<RunOn::Device>(sum);
545  }
546  if (mg_verbose > 0) {
547  amrex::Print() << " Subtracting " << sum << " from rhs in subdomain " << isub << std::endl;
548 
549  sum = volWgtSumMF(lev,rhs_sub[0],0,dJ_sub,mfmx_sub,mfmy_sub,false,local);
550  Print() << "Sum after subtraction " << sum << " in subdomain " << isub << std::endl;
551  }
552 
553  } // if is_singular
554 
555  rhsnorm = rhs_sub[0].norm0();
556 
557  // ****************************************************************************
558  // No need to build the solver if RHS == 0
559  // ****************************************************************************
560  if (rhsnorm <= solverChoice.poisson_abstol) return;
561 
562  double start_step = ParallelDescriptor::second();
563 
564  if (mg_verbose > 0) {
565  amrex::Print() << " Solving in subdomain " << isub << " of " << subdomains[lev].size() << " bins at level " << lev << std::endl;
566  }
567 
568  if (solverChoice.mesh_type == MeshType::VariableDz) {
569  //
570  // Modify ax,ay,ax to include the map factors as used in the divergence calculation
571  // We do this here to set the coefficients used in the stencil -- the extra factor
572  // of the mapfac comes from the gradient
573  //
574  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi)
575  {
576  Box xbx = mfi.nodaltilebox(0);
577  Box ybx = mfi.nodaltilebox(1);
578  Box zbx = mfi.nodaltilebox(2);
579  const Array4<Real >& ax_ar = ax_sub.array(mfi);
580  const Array4<Real >& ay_ar = ay_sub.array(mfi);
581  const Array4<Real >& az_ar = az_sub.array(mfi);
582  const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
583  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
584  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
585  const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
586  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
587  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
588  ParallelFor(xbx,ybx,zbx,
589  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
590  {
591  ax_ar(i,j,k) *= (mf_ux(i,j,0) / mf_uy(i,j,0));
592  },
593  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
594  {
595  ay_ar(i,j,k) *= (mf_vy(i,j,0) / mf_vx(i,j,0));
596  },
597  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
598  {
599  az_ar(i,j,k) /= (mf_mx(i,j,0)*mf_my(i,j,0));
600  });
601  } // mfi
602  }
603 
604  if (solverChoice.terrain_type != TerrainType::EB) {
605 
606 #ifdef ERF_USE_FFT
607  Box my_region(subdomains[lev][isub].minimalBox());
608 #endif
609 
610  // ****************************************************************************
611  // No terrain or grid stretching
612  // ****************************************************************************
613  if (solverChoice.mesh_type == MeshType::ConstantDz) {
614  if (will_solve_with_mlmg) {
615  solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub, geom[lev], ref_ratio, domain_bc_type,
616  mg_verbose, solverChoice.poisson_reltol, solverChoice.poisson_abstol);
617  } else {
618 #ifdef ERF_USE_FFT
619  solve_with_fft(lev, isub, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
620 #endif
621  }
622  } // No terrain or grid stretching
623  // ****************************************************************************
624  // Grid stretching (flat terrain)
625  // ****************************************************************************
626  else if (solverChoice.mesh_type == MeshType::StretchedDz) {
627 #ifndef ERF_USE_FFT
628  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT solver");
629 #else
630  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
631  if (!boxes_make_rectangle) {
632  amrex::Abort("FFT won't work unless the union of boxes is rectangular");
633  } else {
634  if (!use_fft) {
635  amrex::Warning("Using FFT even though you didn't set use_fft to true; it's the best choice");
636  }
637  solve_with_fft(lev, isub, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
638  }
639 #endif
640  } // grid stretching
641 
642  // ****************************************************************************
643  // General terrain
644  // ****************************************************************************
645  else if (solverChoice.mesh_type == MeshType::VariableDz) {
646 #ifdef ERF_USE_FFT
647  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][isub].numPts());
648  if (!boxes_make_rectangle) {
649  amrex::Abort("FFT preconditioner for GMRES won't work unless the union of boxes is rectangular");
650  } else {
651  solve_with_gmres(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0], ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
652  }
653 #else
654  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT preconditioner for GMRES");
655 #endif
656 
657  //
658  // Restore ax,ay,ax to their original definitions
659  //
660  for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
661  {
662  Box xbx = mfi.nodaltilebox(0);
663  Box ybx = mfi.nodaltilebox(1);
664  Box zbx = mfi.nodaltilebox(2);
665  const Array4<Real >& ax_ar = ax_sub.array(mfi);
666  const Array4<Real >& ay_ar = ay_sub.array(mfi);
667  const Array4<Real >& az_ar = az_sub.array(mfi);
668  const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
669  const Array4<Real const>& mf_uy = mapfac[lev][MapFacType::u_y]->const_array(mfi);
670  const Array4<Real const>& mf_vx = mapfac[lev][MapFacType::v_x]->const_array(mfi);
671  const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
672  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
673  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
674  ParallelFor(xbx,ybx,zbx,
675  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
676  {
677  ax_ar(i,j,k) *= (mf_uy(i,j,0) / mf_ux(i,j,0));
678  },
679  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
680  {
681  ay_ar(i,j,k) *= (mf_vx(i,j,0) / mf_vy(i,j,0));
682  },
683  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
684  {
685  az_ar(i,j,k) *= (mf_mx(i,j,0)*mf_my(i,j,0));
686  });
687  } // mfi
688 
689  } // MeshType::VariableDz
690 
691  // ****************************************************************************
692  // Print time in solve
693  // ****************************************************************************
694  double end_step = ParallelDescriptor::second();
695  if (mg_verbose > 0) {
696  amrex::Print() << "Time in solve " << end_step - start_step << std::endl;
697  }
698 
699  } // not EB
700  } // loop over subdomains (i)
701 
702  // ****************************************************************************
703  // When using multigrid we can solve for all of the level at once, even if there
704  // are disjoint regions
705  // ****************************************************************************
706  if (solverChoice.terrain_type == TerrainType::EB) {
707  double start_step_eb = ParallelDescriptor::second();
708  solve_with_EB_mlmg(lev, rhs_sub, phi_sub, fluxes_sub,
709  *(get_eb(lev).get_const_factory()),
710  *(get_eb(lev).get_u_const_factory()),
711  *(get_eb(lev).get_v_const_factory()),
712  *(get_eb(lev).get_w_const_factory()),
713  geom[lev], ref_ratio, domain_bc_type,
714  mg_verbose, solverChoice.poisson_reltol, solverChoice.poisson_abstol);
715  double end_step_eb = ParallelDescriptor::second();
716  if (mg_verbose > 0) {
717  amrex::Print() << "Time in solve " << end_step_eb - start_step_eb << std::endl;
718  }
719  }
720 
721  // ****************************************************************************
722  // Subtract dt grad(phi) from the momenta (rho0u, rho0v, Omega)
723  // ****************************************************************************
724  MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0);
725  MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0);
726  MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0);
727 
728  // ****************************************************************************
729  // Define gradp from fluxes -- note that fluxes is dt * change in Gp
730  // (weighted by map factor!)
731  // ****************************************************************************
732  MultiFab::Saxpy(gradp[lev][GpVars::gpx],-one/l_dt,fluxes[0][0],0,0,1,0);
733  MultiFab::Saxpy(gradp[lev][GpVars::gpy],-one/l_dt,fluxes[0][1],0,0,1,0);
734  MultiFab::Saxpy(gradp[lev][GpVars::gpz],-one/l_dt,fluxes[0][2],0,0,1,0);
735 
736  gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
737  gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
738  gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
739 
740  //
741  // This call is only to verify the divergence after the solve
742  // It is important we do this before computing the rho0w_arr from Omega back to rho0w
743  //
744  // ****************************************************************************
745  // THIS IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE
746  // ****************************************************************************
747  //
748  if (mg_verbose > 0)
749  {
750  rho0_u_const[0] = &mom_mf[IntVars::xmom];
751  rho0_u_const[1] = &mom_mf[IntVars::ymom];
752  rho0_u_const[2] = &mom_mf[IntVars::zmom];
753 
754  compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
755 
756  bool local = false;
757  Real sum = volWgtSumMF(lev,rhs_lev,0,*detJ_cc[lev],*mapfac[lev][MapFacType::m_x],*mapfac[lev][MapFacType::m_y],false,local);
758 
759  if (mg_verbose > 0) {
760  Print() << "Max/L2 norm of divergence after solve at level " << lev << " : " << rhs_lev.norm0() << " " <<
761  rhs_lev.norm2() << " and volume-weighted sum " << sum << std::endl;
762  }
763 
764 #if 0
765  // FOR DEBUGGING ONLY
766  for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
767  {
768  const Array4<Real const>& rhs_arr = rhs_lev.const_array(mfi);
769  Box bx = mfi.validbox();
770  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
771  if (std::abs(rhs_arr(i,j,k)) > Real(1.e-10)) {
772  amrex::AllPrint() << "RHS after solve at " <<
773  IntVect(i,j,k) << " " << rhs_arr(i,j,k) << std::endl;
774  }
775  });
776  } // mfi
777 #endif
778 
779  } // mg_verbose
780 
781  //
782  // ****************************************************************************
783  // Now convert the rho0w MultiFab back to holding (rho0w) rather than Omega
784  // ****************************************************************************
785  //
786  if (solverChoice.mesh_type == MeshType::VariableDz)
787  {
788  for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi)
789  {
790  Box tbz = mfi.nodaltilebox(2);
791  const Array4<Real >& rho0u_arr = mom_mf[IntVars::xmom].array(mfi);
792  const Array4<Real >& rho0v_arr = mom_mf[IntVars::ymom].array(mfi);
793  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
794  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
795  const Array4<Real const>& mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
796  const Array4<Real const>& mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
797  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
798  Real omega = rho0w_arr(i,j,k);
799  rho0w_arr(i,j,k) = WFromOmega(i,j,k,omega,
800  rho0u_arr,rho0v_arr,
801  mf_u,mf_v,z_nd,dxInv);
802  });
803  } // mfi
804  }
805 
806  // If !fixed_density, we must convert (rho0 u) back
807  // to (rho0 u) which is what we will pass back out
808  if (!solverChoice.fixed_density[lev]) {
809  ConvertForProjection(r_hse, mom_mf[Vars::cons],
810  mom_mf[IntVars::xmom],
811  mom_mf[IntVars::ymom],
812  mom_mf[IntVars::zmom],
813  Geom(lev).Domain(),
814  domain_bcs_type);
815  }
816 
817  // ****************************************************************************
818  // Update pressure variable with phi -- note that phi is dt * change in pressure
819  // ****************************************************************************
820  MultiFab::Saxpy(pp_inc[lev], one/l_dt, phi_lev,0,0,1,1);
821 }
ConvertForProjection
void ConvertForProjection(const MultiFab &den_div, const MultiFab &den_mlt, MultiFab &xmom, MultiFab &ymom, MultiFab &zmom, const Box &domain, const Vector< BCRec > &domain_bcs_type_h)
Definition: ERF_ConvertForProjection.cpp:25
enforceInOutSolvability
void enforceInOutSolvability(int, Array< MultiFab *, AMREX_SPACEDIM > &vels_vec, Array< MultiFab *, AMREX_SPACEDIM > &area_vec, const Geometry &geom)
Definition: ERF_ConvertForProjection.cpp:326
solve_with_mlmg
void solve_with_mlmg(int lev, Vector< amrex::MultiFab > &rhs, Vector< MultiFab > &p, Vector< amrex::Array< MultiFab, AMREX_SPACEDIM >> &fluxes, const Geometry &geom, const amrex::Vector< amrex::IntVect > &ref_ratio, Array< std::string, 2 *AMREX_SPACEDIM > l_domain_bc_type, int mg_verbose, Real reltol, Real abstol)
solve_with_EB_mlmg
void solve_with_EB_mlmg(int lev, Vector< amrex::MultiFab > &rhs, Vector< MultiFab > &p, Vector< amrex::Array< MultiFab, AMREX_SPACEDIM >> &fluxes, EBFArrayBoxFactory const &ebfact, eb_aux_ const &ebfact_u, eb_aux_ const &ebfact_v, eb_aux_ const &ebfact_w, const Geometry &geom, const amrex::Vector< amrex::IntVect > &ref_ratio, Array< std::string, 2 *AMREX_SPACEDIM > l_domain_bc_type, int mg_verbose, Real reltol, Real abstol)
OmegaFromW
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real OmegaFromW(int &i, int &j, int &k, amrex::Real w, const amrex::Array4< const amrex::Real > &u_arr, const amrex::Array4< const amrex::Real > &v_arr, const amrex::Array4< const amrex::Real > &mf_u, const amrex::Array4< const amrex::Real > &mf_v, const amrex::Array4< const amrex::Real > &z_nd, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &dxInv)
Definition: ERF_TerrainMetrics.H:414
WFromOmega
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real WFromOmega(int &i, int &j, int &k, amrex::Real omega, const amrex::Array4< const amrex::Real > &u_arr, const amrex::Array4< const amrex::Real > &v_arr, const amrex::Array4< const amrex::Real > &mf_u, const amrex::Array4< const amrex::Real > &mf_v, const amrex::Array4< const amrex::Real > &z_nd, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &dxInv)
Definition: ERF_TerrainMetrics.H:464
ERF::use_fft
static bool use_fft
Definition: ERF.H:1332
ERF::solve_with_gmres
void solve_with_gmres(int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &az_sub, amrex::MultiFab &, amrex::MultiFab &znd_sub)
Definition: ERF_SolveWithGMRES.cpp:12
ERF::compute_divergence
void compute_divergence(int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
Definition: ERF_ComputeDivergence.cpp:10
ERF::volWgtSumMF
amrex::Real volWgtSumMF(int lev, const amrex::MultiFab &mf, int comp, const amrex::MultiFab &dJ, const amrex::MultiFab &mfx, const amrex::MultiFab &mfy, bool finemask, bool local=true)
Definition: ERF_VolWgtSum.cpp:20
MicVar_Morr::omega
@ omega
Definition: ERF_Morrison.H:54
Here is the call graph for this function:

◆ project_velocity_tb()

void ERF::project_velocity_tb ( int  lev,
double  dt,
amrex::Vector< amrex::MultiFab > &  vars 
)

Project the single-level velocity field to enforce incompressibility with a thin body

23 {
24  BL_PROFILE("ERF::project_velocity_tb()");
25  Real l_dt = static_cast<Real>(l_dt_d);
26  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
27 
28  // Make sure the solver only sees the levels over which we are solving
29  Vector<BoxArray> ba_tmp; ba_tmp.push_back(vmf[Vars::cons].boxArray());
30  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(vmf[Vars::cons].DistributionMap());
31  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
32 
33  // Use the default settings
34  LPInfo info;
35  std::unique_ptr<MLPoisson> p_mlpoisson;
36 #if 0
37  if (overset_imask[0]) {
38  // Add overset mask around thin body
39  p_mlpoisson = std::make_unique<MLPoisson>(geom, grids, dmap, GetVecOfConstPtrs(overset_imask), info);
40  }
41  else
42 #endif
43  {
44  // Use the default settings
45  p_mlpoisson = std::make_unique<MLPoisson>(geom_tmp, ba_tmp, dm_tmp, info);
46  }
47 
48  auto bclo = get_lo_projection_bc(geom[lev],domain_bc_type);
49  auto bchi = get_hi_projection_bc(geom[lev],domain_bc_type);
50 
51  bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true;
52  p_mlpoisson->setDomainBC(bclo, bchi);
53 
54  if (lev > 0) {
55  p_mlpoisson->setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
56  }
57 
58  p_mlpoisson->setLevelBC(0, nullptr);
59 
60  Vector<MultiFab> rhs;
61  Vector<MultiFab> phi;
62  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
63  Vector<Array<MultiFab,AMREX_SPACEDIM> > deltaf; // f^* - f^{n-1}
64  Vector<Array<MultiFab,AMREX_SPACEDIM> > u_plus_dtdf; // u + dt*deltaf
65 
66  // Used to pass array of const MFs to ComputeDivergence
67  Array<MultiFab const*, AMREX_SPACEDIM> u;
68 
69  rhs.resize(1);
70  phi.resize(1);
71  fluxes.resize(1);
72  deltaf.resize(1);
73  u_plus_dtdf.resize(1);
74 
75  rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
76  phi[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
77  rhs[0].setVal(0.0);
78  phi[0].setVal(0.0);
79 
80  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
81  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
82  u_plus_dtdf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
83 
84  deltaf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
85  deltaf[0][idim].setVal(0.0); // start with f^* == f^{n-1}
86  }
87 
88 #if 0
89  // DEBUG
90  u[0] = &(vmf[Vars::xvel]);
91  u[1] = &(vmf[Vars::yvel]);
92  u[2] = &(vmf[Vars::zvel]);
93  computeDivergence(rhs[0], u, geom[0]);
94  Print() << "Max norm of divergence before solve at level 0 : " << rhs[0].norm0() << std::endl;
95 #endif
96 
97  for (int itp = 0; itp < solverChoice.ncorr; ++itp)
98  {
99  // Calculate u + dt*deltaf
100  for (int idim = 0; idim < 3; ++idim) {
101  MultiFab::Copy(u_plus_dtdf[0][idim], deltaf[0][idim], 0, 0, 1, 0);
102  u_plus_dtdf[0][0].mult(-l_dt,0,1,0);
103  }
104  MultiFab::Add(u_plus_dtdf[0][0], vmf[Vars::xvel], 0, 0, 1, 0);
105  MultiFab::Add(u_plus_dtdf[0][1], vmf[Vars::yvel], 0, 0, 1, 0);
106  MultiFab::Add(u_plus_dtdf[0][2], vmf[Vars::zvel], 0, 0, 1, 0);
107 
108  u[0] = &(u_plus_dtdf[0][0]);
109  u[1] = &(u_plus_dtdf[0][1]);
110  u[2] = &(u_plus_dtdf[0][2]);
111  computeDivergence(rhs[0], u, geom_tmp[0]);
112 
113 #if 0
114  // DEBUG
115  if (itp==0) {
116  for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
117  {
118  const Box& bx = mfi.tilebox();
119  const Array4<Real const>& divU = rhs[0].const_array(mfi);
120  const Array4<Real const>& uarr = vmf[Vars::xvel].const_array(mfi);
121  const Array4<Real const>& varr = vmf[Vars::yvel].const_array(mfi);
122  const Array4<Real const>& warr = vmf[Vars::zvel].const_array(mfi);
123  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
124  {
125  if ((i>=120) && (i<=139) && (j==0) && ((k>=127)&&(k<=128))) {
126  amrex::AllPrint() << "before project div"<<IntVect(i,j,k)<<" = "<< divU(i,j,k)
127  << " u: " << uarr(i,j,k) << " " << uarr(i+1,j,k)
128  << " v: " << varr(i,j,k) << " " << varr(i,j+1,k)
129  << " w: " << warr(i,j,k) << " " << warr(i,j,k+1)
130  << std::endl;
131  }
132  });
133  }
134  }
135 #endif
136 
137  // If all Neumann BCs, adjust RHS to make sure we can converge
138  if (need_adjust_rhs) {
139  bool local = false;
140  Real offset = volWgtSumMF(lev,rhs[0],0,*detJ_cc[lev],*mapfac[lev][MapFacType::m_x],*mapfac[lev][MapFacType::m_y],false,local);
141  // amrex::Print() << "Poisson solvability offset = " << offset << std::endl;
142  rhs[0].plus(-offset, 0, 1);
143  }
144 
145  // Initialize phi to 0
146  phi[0].setVal(0.0);
147 
148  MLMG mlmg(*p_mlpoisson);
149  int max_iter = 100;
150  mlmg.setMaxIter(max_iter);
151 
152  mlmg.setVerbose(mg_verbose);
153  //mlmg.setBottomVerbose(mg_verbose);
154 
155  // solve for dt*p
156  mlmg.solve(GetVecOfPtrs(phi),
157  GetVecOfConstPtrs(rhs),
158  solverChoice.poisson_reltol,
159  solverChoice.poisson_abstol);
160 
161  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
162 
163  // Calculate new intermediate body force with updated gradp
164  if (thin_xforce[lev]) {
165  MultiFab::Copy( deltaf[0][0], fluxes[0][0], 0, 0, 1, 0);
166  ApplyInvertedMask(deltaf[0][0], *xflux_imask[0]);
167  }
168  if (thin_yforce[lev]) {
169  MultiFab::Copy( deltaf[0][1], fluxes[0][1], 0, 0, 1, 0);
170  ApplyInvertedMask(deltaf[0][1], *yflux_imask[0]);
171  }
172  if (thin_zforce[lev]) {
173  MultiFab::Copy( deltaf[0][2], fluxes[0][2], 0, 0, 1, 0);
174  ApplyInvertedMask(deltaf[0][2], *zflux_imask[0]);
175  }
176 
177  // DEBUG
178  // for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
179  // {
180  // const Box& bx = mfi.tilebox();
181  // const Array4<Real const>& dfz_arr = deltaf[0][2].const_array(mfi);
182  // ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
183  // {
184  // if ((i>=120) && (i<=139) && (j==0) && (k==128)) {
185  // amrex::AllPrint()
186  // << " piter" << itp
187  // << " dfz"<<IntVect(i,j,k)<<" = "<< dfz_arr(i,j,k)
188  // << std::endl;
189  // }
190  // });
191  // }
192 
193  // Update pressure variable with phi -- note that phi is change in pressure, not the full pressure
194  MultiFab::Saxpy(pp_inc[lev], one, phi[0],0,0,1,0);
195 
196  // Subtract grad(phi) from the velocity components
197  Real beta = one;
198  MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
199  MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
200  MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
201  if (thin_xforce[lev]) {
202  ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
203  }
204  if (thin_yforce[lev]) {
205  ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
206  }
207  if (thin_zforce[lev]) {
208  ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
209  }
210  } // itp: pressure-force iterations
211 
212  // ****************************************************************************
213  // Define gradp from fluxes -- note that fluxes is dt * change in Gp
214  // ****************************************************************************
215  MultiFab::Saxpy(gradp[lev][GpVars::gpx],-one/l_dt,fluxes[0][0],0,0,1,0);
216  MultiFab::Saxpy(gradp[lev][GpVars::gpy],-one/l_dt,fluxes[0][1],0,0,1,0);
217  MultiFab::Saxpy(gradp[lev][GpVars::gpz],-one/l_dt,fluxes[0][2],0,0,1,0);
218 
219  gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
220  gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
221  gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
222 
223  // Subtract grad(phi) from the velocity components
224 // Real beta = one;
225 // for (int ilev = lev_min; ilev <= lev_max; ++ilev) {
226 // MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
227 // MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
228 // MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
229 // if (thin_xforce[lev]) {
230 // ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
231 // }
232 // if (thin_yforce[lev]) {
233 // ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
234 // }
235 // if (thin_zforce[lev]) {
236 // ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
237 // }
238 // }
239 
240 #if 0
241  // Confirm that the velocity is now divergence free
242  u[0] = &(vmf[Vars::xvel]);
243  u[1] = &(vmf[Vars::yvel]);
244  u[2] = &(vmf[Vars::zvel]);
245  computeDivergence(rhs[0], u, geom_tmp[0]);
246  Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
247 
248 #endif
249 }
projection_has_dirichlet
bool projection_has_dirichlet(Array< LinOpBCType, AMREX_SPACEDIM > bcs)
Definition: ERF_PoissonSolve_tb.cpp:10
offset
AMREX_FORCE_INLINE IntVect offset(const int face_dir, const int normal)
Definition: ERF_ReadBndryPlanes.cpp:28
get_lo_projection_bc
Array< LinOpBCType, AMREX_SPACEDIM > get_lo_projection_bc(Geometry const &lev_geom, Array< std::string, 2 *AMREX_SPACEDIM > l_domain_bc_type)
Definition: ERF_SolverUtils.H:13
get_hi_projection_bc
Array< LinOpBCType, AMREX_SPACEDIM > get_hi_projection_bc(Geometry const &lev_geom, Array< std::string, 2 *AMREX_SPACEDIM > l_domain_bc_type)
Definition: ERF_SolverUtils.H:34
ApplyInvertedMask
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyInvertedMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:467
Here is the call graph for this function:

◆ read_box_for_refinement()

void ERF::read_box_for_refinement ( std::string &  ref_prefix,
int &  lev_for_box,
amrex::RealBox &  real_box 
)
7 {
8  ParmParse ppr(ref_prefix);
9 
10  int num_real_lo = ppr.countval("in_box_lo");
11  int num_real_hi = ppr.countval("in_box_hi");
12 
13  int num_indx_lo = ppr.countval("in_box_lo_indices");
14  int num_indx_hi = ppr.countval("in_box_hi_indices");
15 
16  int num_indx_lo_crse = ppr.countval("in_box_lo_indices_crse");
17  int num_indx_hi_crse = ppr.countval("in_box_hi_indices_crse");
18 
19  AMREX_ALWAYS_ASSERT( (num_real_lo == num_real_hi) && (num_real_lo == 0 || num_real_lo >= 2) );
20  AMREX_ALWAYS_ASSERT( (num_indx_lo == num_indx_hi) && (num_indx_lo == 0 || num_indx_lo >= 2) );
21  AMREX_ALWAYS_ASSERT( (num_indx_lo_crse == num_indx_hi_crse) && (num_indx_lo_crse == 0 || num_indx_lo_crse >= 2) );
22 
23  // Problem low and high (in real not index space) are the same at all levels
24  const Real* plo = geom[0].ProbLo();
25  const Real* phi = geom[0].ProbHi();
26  if ( !((num_real_lo >= AMREX_SPACEDIM-1 && num_indx_lo == 0 && num_indx_lo_crse == 0) ||
27  (num_indx_lo >= AMREX_SPACEDIM-1 && num_real_lo == 0 && num_indx_lo_crse == 0) ||
28  (num_indx_lo == 0 && num_real_lo == 0 && num_indx_lo_crse == 0) ||
29  (num_indx_lo_crse >= AMREX_SPACEDIM-1 && num_real_lo == 0 && num_indx_lo == 0)
30  ) )
31  {
32  amrex::Abort("Must only specify box for refinement using real OR index space with fine/coarse grid indices");
33  }
34 
35  // Clear stale data
36  lev_for_box = max_level;
37  if (num_real_lo > 0) {
38  ppr.query("max_level",lev_for_box);
39  } else if (num_indx_lo > 0 || num_indx_lo_crse > 0) {
40  ppr.get("max_level",lev_for_box);
41  }
42  num_boxes_at_level[lev_for_box] = 0;
43  boxes_at_level[lev_for_box].clear();
44 
45  if (num_real_lo > 0) {
46 
47  std::vector<Real> rbox_lo(3), rbox_hi(3);
48  if (lev_for_box > 0 && lev_for_box <= max_level)
49  {
50  if (n_error_buf[0] != IntVect::TheZeroVector()) {
51  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
52  }
53 
54  ppr.getarr("in_box_lo",rbox_lo,0,num_real_lo);
55  ppr.getarr("in_box_hi",rbox_hi,0,num_real_hi);
56 
57  if (rbox_lo[0] < plo[0]) rbox_lo[0] = plo[0];
58  if (rbox_lo[1] < plo[1]) rbox_lo[1] = plo[1];
59  if (rbox_hi[0] > phi[0]) rbox_hi[0] = phi[0];
60  if (rbox_hi[1] > phi[1]) rbox_hi[1] = phi[1];
61  if (num_real_lo < AMREX_SPACEDIM) {
62  rbox_lo[2] = plo[2];
63  rbox_hi[2] = phi[2];
64  }
65 
66  const Box& domain = geom[lev_for_box].Domain();
67 
68  real_box = RealBox(&(rbox_lo[0]),&(rbox_hi[0]));
69 
70  Print() << "Realbox read in and intersected laterally with domain is " << real_box << std::endl;
71 
72  num_boxes_at_level[lev_for_box] += 1;
73 
74  int ilo, jlo, klo;
75  int ihi, jhi, khi;
76  const auto* dx = geom[lev_for_box].CellSize();
77  ilo = static_cast<int>((rbox_lo[0] - plo[0])/dx[0]);
78  jlo = static_cast<int>((rbox_lo[1] - plo[1])/dx[1]);
79  ihi = static_cast<int>((rbox_hi[0] - plo[0])/dx[0]-1);
80  jhi = static_cast<int>((rbox_hi[1] - plo[1])/dx[1]-1);
81  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
82  // Search for k indices corresponding to nominal grid
83  // AGL heights
84  klo = domain.smallEnd(2) - 1;
85  khi = domain.smallEnd(2) - 1;
86 
87  if (rbox_lo[2] <= zlevels_stag[lev_for_box][domain.smallEnd(2)])
88  {
89  klo = domain.smallEnd(2);
90  }
91  else
92  {
93  for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
94  if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
95  klo = k-1;
96  break;
97  }
98  }
99  }
100  AMREX_ASSERT(klo >= domain.smallEnd(2));
101 
102  if (rbox_hi[2] >= zlevels_stag[lev_for_box][domain.bigEnd(2)+1])
103  {
104  khi = domain.bigEnd(2);
105  }
106  else
107  {
108  for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
109  if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
110  khi = k-1;
111  break;
112  }
113  }
114  }
115  AMREX_ASSERT((khi <= domain.bigEnd(2)) && (khi > klo));
116 
117  // Need to update real_box because tagging is based on
118  // the initial _un_deformed grid
119  real_box = RealBox(plo[0]+ ilo *dx[0], plo[1]+ jlo *dx[1], plo[2]+ klo *dx[2],
120  plo[0]+(ihi+1)*dx[0], plo[1]+(jhi+1)*dx[1], plo[2]+(khi+1)*dx[2]);
121  } else {
122  klo = static_cast<int>((rbox_lo[2] - plo[2])/dx[2]);
123  khi = static_cast<int>((rbox_hi[2] - plo[2])/dx[2]-1);
124  }
125 
126  // Snap box indices to ref_ratio alignment (round lo down, hi up)
127  {
128  const auto& rr = ref_ratio[lev_for_box-1];
129  auto snap_lo = [](int idx, int r) { return idx - (idx % r + r) % r; };
130  auto snap_hi = [](int idx_p1, int r) { // idx_p1 = ihi+1
131  int rem = idx_p1 % r;
132  return (rem == 0) ? idx_p1 - 1 : idx_p1 + (r - rem) - 1;
133  };
134  int ilo_old = ilo, jlo_old = jlo, klo_old = klo;
135  int ihi_old = ihi, jhi_old = jhi, khi_old = khi;
136  ilo = snap_lo(ilo, rr[0]);
137  jlo = snap_lo(jlo, rr[1]);
138  klo = snap_lo(klo, rr[2]);
139  ihi = snap_hi(ihi+1, rr[0]);
140  jhi = snap_hi(jhi+1, rr[1]);
141  khi = snap_hi(khi+1, rr[2]);
142  if (ilo != ilo_old || ihi != ihi_old ||
143  jlo != jlo_old || jhi != jhi_old ||
144  klo != klo_old || khi != khi_old) {
145  amrex::Print() << "Refinement box indices snapped to ref_ratio alignment:\n"
146  << " ilo: " << ilo_old << " -> " << ilo
147  << " ihi: " << ihi_old << " -> " << ihi
148  << " jlo: " << jlo_old << " -> " << jlo
149  << " jhi: " << jhi_old << " -> " << jhi
150  << " klo: " << klo_old << " -> " << klo
151  << " khi: " << khi_old << " -> " << khi << "\n";
152  }
153  }
154 
155  Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
156 
157  bool using_pbl = (solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYJ ||
158  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNN25 ||
159  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNNEDMF ||
160  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::YSU ||
161  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MRF);
162 
163  if ( using_pbl && ( (rbox_lo[2] > plo[2]) || (rbox_hi[2] < phi[2]) ) ) {
164  amrex::Print() << "PBL models need refinement boxes that go from the bottom to the top of the domain for calculation of PBLH" << std::endl;
165  amrex::Print() << "Please set in_box_lo to geometry.prob_lo in z and in_box_hi to geometry.prob_hi in z and try again" << std::endl;
166  amrex::Abort();
167  }
168 
169  boxes_at_level[lev_for_box].push_back(bx);
170  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
171  } // (lev_for_box > 0 && lev_for_box <= max_level)
172 
173  if (solverChoice.init_type == InitType::WRFInput) {
174  amrex::Print() << "Size of num_boxes " << num_boxes_at_level.size() << std::endl;
175  amrex::Print() << "Size of num_files " << num_files_at_level.size() << std::endl;
176  amrex::Print() << "Consider lev_for_box = " << lev_for_box << std::endl;
177  amrex::Print() << "Number of boxes at level " << num_boxes_at_level[lev_for_box] << std::endl;
178  amrex::Print() << "Number of available files " << num_files_at_level[lev_for_box] << std::endl;
179  if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
180  amrex::Print() << "Will need to rely on refinement criteria from inputs file" << std::endl;
181  }
182  }
183 
184  } else if (num_indx_lo > 0) {
185 
186  std::vector<int> box_lo(3), box_hi(3);
187  if (lev_for_box > 0 && lev_for_box <= max_level)
188  {
189  if (n_error_buf[0] != IntVect::TheZeroVector()) {
190  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
191  }
192 
193  ppr.getarr("in_box_lo_indices",box_lo,0,num_indx_lo);
194  ppr.getarr("in_box_hi_indices",box_hi,0,num_indx_hi);
195 
196  if (num_indx_lo < AMREX_SPACEDIM) {
197  box_lo[2] = geom[lev_for_box].Domain().smallEnd(2);
198  box_hi[2] = geom[lev_for_box].Domain().bigEnd(2);
199  }
200 
201  Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
202  const Box& domain = geom[lev_for_box].Domain();
203 
204  if (!domain.contains(bx)) {
205  amrex::Print() << "\n";
206  amrex::Print() << "Box specified is " << bx << std::endl;
207  amrex::Print() << "But domain at level is " << domain << std::endl;
208  amrex::Error("Specified box doesn't fit in the domain");
209  }
210 
211  const auto* dx = geom[lev_for_box].CellSize();
212  real_box = RealBox(plo[0]+ box_lo[0] *dx[0], plo[1]+ box_lo[1] *dx[1], plo[2]+ box_lo[2] *dx[2],
213  plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
214 
215  Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
216  num_boxes_at_level[lev_for_box] += 1;
217 
218  // Snap box indices to ref_ratio alignment (round lo down, hi up)
219  {
220  const auto& rr = ref_ratio[lev_for_box-1];
221  auto snap_lo_fn = [](int idx, int r) { return idx - (idx % r + r) % r; };
222  auto snap_hi_fn = [](int idx_p1, int r) {
223  int rem = idx_p1 % r;
224  return (rem == 0) ? idx_p1 - 1 : idx_p1 + (r - rem) - 1;
225  };
226  int lo_old[3] = {box_lo[0], box_lo[1], box_lo[2]};
227  int hi_old[3] = {box_hi[0], box_hi[1], box_hi[2]};
228  box_lo[0] = snap_lo_fn(box_lo[0], rr[0]);
229  box_lo[1] = snap_lo_fn(box_lo[1], rr[1]);
230  box_lo[2] = snap_lo_fn(box_lo[2], rr[2]);
231  box_hi[0] = snap_hi_fn(box_hi[0]+1, rr[0]);
232  box_hi[1] = snap_hi_fn(box_hi[1]+1, rr[1]);
233  box_hi[2] = snap_hi_fn(box_hi[2]+1, rr[2]);
234  if (box_lo[0] != lo_old[0] || box_hi[0] != hi_old[0] ||
235  box_lo[1] != lo_old[1] || box_hi[1] != hi_old[1] ||
236  box_lo[2] != lo_old[2] || box_hi[2] != hi_old[2]) {
237  amrex::Print() << "Refinement box indices snapped to ref_ratio alignment:\n"
238  << " ilo: " << lo_old[0] << " -> " << box_lo[0]
239  << " ihi: " << hi_old[0] << " -> " << box_hi[0]
240  << " jlo: " << lo_old[1] << " -> " << box_lo[1]
241  << " jhi: " << hi_old[1] << " -> " << box_hi[1]
242  << " klo: " << lo_old[2] << " -> " << box_lo[2]
243  << " khi: " << hi_old[2] << " -> " << box_hi[2] << "\n";
244  }
245  bx = Box(IntVect(box_lo[0],box_lo[1],box_lo[2]),
246  IntVect(box_hi[0],box_hi[1],box_hi[2]));
247  }
248 
249  bool using_pbl = (solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYJ ||
250  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNN25 ||
251  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNNEDMF ||
252  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::YSU ||
253  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MRF);
254 
255  if ( using_pbl && ( (box_lo[2] > 0) || (box_hi[2] < domain.bigEnd(2)) ) ) {
256  amrex::Print() << "PBL models need refinement boxes that go from the bottom to the top of the domain for calculation of PBLH" << std::endl;
257  amrex::Print() << "Please set in_box_lo_indices to 0 in z and in_box_hi_indices to amr.n_cell-1 in z and try again" << std::endl;
258  amrex::Abort();
259  }
260 
261  boxes_at_level[lev_for_box].push_back(bx);
262  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
263  } // lev
264 
265  if (solverChoice.init_type == InitType::WRFInput) {
266  if ( (num_files_at_level[lev_for_box] > 0) &&
267  (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) ) {
268  amrex::Error("Number of boxes doesn't match number of input files");
269 
270  }
271  }
272 
273  } else if (num_indx_lo_crse > 0) {
274 
275  std::vector<int> box_lo(3), box_hi(3);
276  if (lev_for_box > 0 && lev_for_box <= max_level)
277  {
278  if (n_error_buf[0] != IntVect::TheZeroVector()) {
279  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
280  }
281 
282  ppr.getarr("in_box_lo_indices_crse",box_lo,0,num_indx_lo_crse);
283  ppr.getarr("in_box_hi_indices_crse",box_hi,0,num_indx_hi_crse);
284 
285  if (num_indx_lo_crse < AMREX_SPACEDIM) {
286  box_lo[2] = geom[lev_for_box-1].Domain().smallEnd(2);
287  box_hi[2] = geom[lev_for_box-1].Domain().bigEnd(2);
288  }
289 
290  Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
291 
292  if (!geom[lev_for_box-1].Domain().contains(bx)) {
293  amrex::Print() << "\n";
294  amrex::Print() << "(Coarse) Box specified is " << bx << std::endl;
295  amrex::Print() << "But (coarse) domain at level is " << geom[lev_for_box-1].Domain() << std::endl;
296  amrex::Error("Specified box doesn't fit in the domain");
297  }
298 
299  bx.refine(ref_ratio[lev_for_box-1]);
300 
301  const auto* dx = geom[lev_for_box-1].CellSize();
302 
303  real_box = RealBox(plo[0]+ box_lo[0] *dx[0], plo[1]+ box_lo[1] *dx[1], plo[2]+ box_lo[2] *dx[2],
304  plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
305 
306  Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
307  num_boxes_at_level[lev_for_box] += 1;
308  bool using_pbl = (solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYJ ||
309  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNN25 ||
310  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MYNNEDMF ||
311  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::YSU ||
312  solverChoice.turbChoice[lev_for_box].pbl_type == PBLType::MRF);
313 
314  const Box& domain = geom[lev_for_box].Domain();
315  if ( using_pbl && ( (box_lo[2] > 0) || (box_hi[2] < domain.bigEnd(2)) ) ) {
316  amrex::Print() << "PBL models need refinement boxes that go from the bottom to the top of the domain for calculation of PBLH" << std::endl;
317  amrex::Print() << "Please set in_box_lo_indices_crse to 0 in z and in_box_hi_indices_crse to amr.n_cell-1 in z and try again" << std::endl;
318  amrex::Abort();
319  }
320 
321  boxes_at_level[lev_for_box].push_back(bx);
322  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
323  } // lev
324 
325  if (solverChoice.init_type == InitType::WRFInput) {
326  if ( (num_files_at_level[lev_for_box] > 0) &&
327  (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) ) {
328  amrex::Error("Number of boxes doesn't match number of input files");
329 
330  }
331  }
332  }
333 }
ERF::num_files_at_level
amrex::Vector< int > num_files_at_level
Definition: ERF.H:912
module_mp_morr_two_moment::rr
real(c_double), private rr
Definition: ERF_module_mp_morr_two_moment.F90:224
Here is the call graph for this function:

◆ ReadCheckpointFile()

void ERF::ReadCheckpointFile ( )

ERF function for reading data from a checkpoint file during restart.

521 {
522  Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
523 
524  const auto provenance_result =
525  erf_provenance::read_job_info_file(restart_chkfile + "/job_info");
526  if (provenance_result.valid() &&
527  provenance_result.record.artifact.artifact_type == erf_provenance::ArtifactType::Checkpoint) {
528  execution_provenance = erf_provenance::make_restart_provenance(
529  execution_provenance, provenance_result.record, restart_chkfile);
530  } else {
531  // Provenance is auxiliary metadata. A missing or invalid record must not make
532  // a valid physical checkpoint unreadable.
533  if (ParallelDescriptor::IOProcessor() && !provenance_warning_emitted) {
534  provenance_warning_emitted = true;
535  const std::string reason = provenance_result.valid()
536  ? "the provenance record has artifact_type=" +
537  std::string(erf_provenance::artifact_type_token(
538  provenance_result.record.artifact.artifact_type)) +
539  ", not checkpoint"
540  : provenance_result.diagnostic;
541  Warning("Cannot recover provenance from native checkpoint '" +
542  restart_chkfile + "': " + reason +
543  ". ERF will continue with incomplete provenance.");
544  }
545  const auto failure_status = provenance_result.valid()
546  ? erf_provenance::ProvenanceReadStatus::ArtifactTypeMismatch
547  : provenance_result.status;
548  execution_provenance = erf_provenance::make_incomplete_restart_provenance(
549  execution_provenance, failure_status, restart_chkfile);
550  }
551 
552  // Header
553  std::string File(restart_chkfile + "/Header");
554 
555  VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
556 
557  Vector<char> fileCharPtr;
558  ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
559  std::string fileCharPtrString(fileCharPtr.dataPtr());
560  std::istringstream is(fileCharPtrString, std::istringstream::in);
561 
562  std::string line, word;
563 
564  int chk_ncomp_cons, chk_ncomp;
565 
566  // read in title line
567  std::getline(is, line);
568 
569  // read in finest_level
570  is >> finest_level;
571  GotoNextLine(is);
572 
573  // read the number of components
574  // for each variable we store
575 
576  // conservative, cell-centered vars
577  is >> chk_ncomp_cons;
578  GotoNextLine(is);
579 
580  // x-velocity on faces
581  is >> chk_ncomp;
582  GotoNextLine(is);
583  AMREX_ASSERT(chk_ncomp == 1);
584 
585  // y-velocity on faces
586  is >> chk_ncomp;
587  GotoNextLine(is);
588  AMREX_ASSERT(chk_ncomp == 1);
589 
590  // z-velocity on faces
591  is >> chk_ncomp;
592  GotoNextLine(is);
593  AMREX_ASSERT(chk_ncomp == 1);
594 
595  // read in array of istep
596  std::getline(is, line);
597  {
598  std::istringstream lis(line);
599  int i = 0;
600  while (lis >> word) {
601  istep[i++] = std::stoi(word);
602  }
603  }
604 
605  // read in array of dt
606  std::getline(is, line);
607  {
608  std::istringstream lis(line);
609  int i = 0;
610  while (lis >> word) {
611  dt[i++] = std::stod(word);
612  }
613  }
614 
615  // read in array of t_new
616  std::getline(is, line);
617  {
618  std::istringstream lis(line);
619  int i = 0;
620  while (lis >> word) {
621  t_new[i++] = std::stod(word);
622  }
623  }
624 
625  for (int lev = 0; lev <= finest_level; ++lev) {
626  // read in level 'lev' BoxArray from Header
627  BoxArray ba;
628  ba.readFrom(is);
629  GotoNextLine(is);
630 
631  // create a distribution mapping
632  DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
633 
634  MakeNewLevelFromScratch (lev, static_cast<Real>(t_new[lev]), ba, dm);
635  }
636 
637  // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
638  // NOTE: Data is written over ncomp, so check that we match the header file
639  int ncomp_cons = vars_new[0][Vars::cons].nComp();
640 
641  // NOTE: QKE was removed so this is for backward compatibility
642  AMREX_ASSERT((chk_ncomp_cons==ncomp_cons) || ((chk_ncomp_cons-1)==ncomp_cons));
643  //
644  // See if we have a written separate file that tells how many components and how many ghost cells
645  // we have of the base state
646  //
647  // If we can't find the file, then set the number of components to the original number = 3
648  //
649  int ncomp_base_to_read = 3;
650  IntVect ng_base = IntVect{1};
651  {
652  std::string BaseStateFile(restart_chkfile + "/num_base_state_comps");
653 
654  if (amrex::FileExists(BaseStateFile))
655  {
656  Vector<char> BaseStatefileCharPtr;
657  ParallelDescriptor::ReadAndBcastFile(BaseStateFile, BaseStatefileCharPtr);
658  std::string BaseStatefileCharPtrString(BaseStatefileCharPtr.dataPtr());
659 
660  // We set this to the default value of 3 but allow it be larger if th0 and qv0 were written
661  std::istringstream isb(BaseStatefileCharPtrString, std::istringstream::in);
662  isb >> ncomp_base_to_read;
663  isb >> ng_base;
664  }
665  }
666 
667  // read in the MultiFab data
668  for (int lev = 0; lev <= finest_level; ++lev)
669  {
670  // NOTE: For backward compatibility (chk file has QKE)
671  if ((chk_ncomp_cons-1)==ncomp_cons) {
672  MultiFab cons(grids[lev],dmap[lev],chk_ncomp_cons,0);
673  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
674 
675  // Copy up to RhoKE_comp
676  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,(RhoKE_comp+1),0);
677 
678  // Only if we have a PBL model do we need to copy QKE is src to KE in dst
679  if ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
680  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ) {
681  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+1),RhoKE_comp,1,0);
682  vars_new[lev][Vars::cons].mult(myhalf,RhoKE_comp,1,0);
683  }
684 
685  // Copy other components
686  int ncomp_remainder = ncomp_cons - (RhoKE_comp + 1);
687  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+2),(RhoKE_comp+1),ncomp_remainder,0);
688 
689  vars_new[lev][Vars::cons].setBndry(bogus_large_value);
690  } else {
691  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
692  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
693  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
694  vars_new[lev][Vars::cons].setBndry(bogus_large_value);
695  }
696 
697  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
698  VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
699  MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
700  vars_new[lev][Vars::xvel].setBndry(bogus_large_value);
701 
702  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
703  VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
704  MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
705  vars_new[lev][Vars::yvel].setBndry(bogus_large_value);
706 
707  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
708  VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
709  MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
710  vars_new[lev][Vars::zvel].setBndry(bogus_large_value);
711 
712  if (solverChoice.anelastic[lev] == 1) {
713  MultiFab ppinc(grids[lev],dmap[lev],1,0);
714  VisMF::Read(ppinc, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PP_Inc"));
715  MultiFab::Copy(pp_inc[lev],ppinc,0,0,1,0);
716  pp_inc[lev].FillBoundary(geom[lev].periodicity());
717 
718  MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
719  VisMF::Read(gpx, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpx"));
720  MultiFab::Copy(gradp[lev][GpVars::gpx],gpx,0,0,1,0);
721  gradp[lev][GpVars::gpx].FillBoundary(geom[lev].periodicity());
722 
723  MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
724  VisMF::Read(gpy, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpy"));
725  MultiFab::Copy(gradp[lev][GpVars::gpy],gpy,0,0,1,0);
726  gradp[lev][GpVars::gpy].FillBoundary(geom[lev].periodicity());
727 
728  MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
729  VisMF::Read(gpz, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpz"));
730  MultiFab::Copy(gradp[lev][GpVars::gpz],gpz,0,0,1,0);
731  gradp[lev][GpVars::gpz].FillBoundary(geom[lev].periodicity());
732  }
733 
734  // Note that we read the ghost cells of the base state (unlike above)
735 
736  // The original base state only had 3 components and 1 ghost cell -- we read this
737  // here to be consistent with the old style
738  MultiFab base(grids[lev],dmap[lev],ncomp_base_to_read,ng_base);
739  VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
740 
741  MultiFab::Copy(base_state[lev],base,0,0,ncomp_base_to_read,ng_base);
742 
743  // Create theta0 from p0, rh0
744  if (ncomp_base_to_read < 4) {
745  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
746  {
747  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
748  const Box& bx = mfi.tilebox();
749  Array4<Real> const& fab = base_state[lev].array(mfi);
750  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
751  {
752  fab(i,j,k,BaseState::th0_comp) = getRhoThetagivenP(fab(i,j,k,BaseState::p0_comp))
753  / fab(i,j,k,BaseState::r0_comp);
754  });
755  }
756  }
757  // Default theta0 to 0
758  if (ncomp_base_to_read < 5) {
759  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
760  {
761  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
762  const Box& bx = mfi.tilebox();
763  Array4<Real> const& fab = base_state[lev].array(mfi);
764  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
765  {
766  fab(i,j,k,BaseState::qv0_comp) = zero;
767  });
768  }
769  }
770  base_state[lev].FillBoundary(geom[lev].periodicity());
771 
772  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
773  // Note that we also read the ghost cells of z_phys_nd
774  IntVect ng = z_phys_nd[lev]->nGrowVect();
775  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
776  VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
777  MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
778  update_terrain_arrays(lev);
779 
780  // Compute the min dz and pass to the micro model
781  Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
782  micro->Set_dzmin(lev, dzmin);
783 
784 #if 0
785  if ( (solverChoice.init_type != InitType::WRFInput) && (solverChoice.init_type != InitType::Metgrid) ) {
786  check_mesh_type(lev);
787  }
788 #endif
789  }
790 
791  // Read in the moisture model restart variables
792  std::vector<int> qmoist_indices;
793  std::vector<std::string> qmoist_names;
794  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
795  int qmoist_nvar = static_cast<int>(qmoist_indices.size());
796  for (int var = 0; var < qmoist_nvar; var++) {
797  const int ncomp = 1;
798  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
799  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
800  VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", qmoist_names[var]));
801  MultiFab::Copy(*(qmoist[lev][qmoist_indices[var]]),moist_vars,0,0,ncomp,ng_moist);
802  }
803 
804 #if defined(ERF_USE_WINDFARM)
805  if(solverChoice.windfarm_type == WindFarmType::Fitch or
806  solverChoice.windfarm_type == WindFarmType::EWP or
807  solverChoice.windfarm_type == WindFarmType::SimpleAD){
808  IntVect ng = Nturb[lev].nGrowVect();
809  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
810  VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
811  MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
812  }
813 #endif
814 
815  // Read the LSM data
816  if (solverChoice.lsm_type != LandSurfaceType::None) {
817  amrex::Print() << "Reading LSM variables" << std::endl;
818  for (int ivar(0); ivar<lsm_data[lev].size(); ++ivar) {
819  BoxArray ba = lsm_data[lev][ivar]->boxArray();
820  DistributionMapping dm = lsm_data[lev][ivar]->DistributionMap();
821  IntVect ng = lsm_data[lev][ivar]->nGrowVect();
822  int nvar = lsm_data[lev][ivar]->nComp();
823  MultiFab lsm_vars(ba,dm,nvar,ng);
824  VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmData" + std::to_string(ivar)));
825  MultiFab::Copy(*(lsm_data[lev][ivar]),lsm_vars,0,0,nvar,ng);
826  }
827  for (int iflux(0); iflux<lsm_flux[lev].size(); ++iflux) {
828  BoxArray ba = lsm_flux[lev][iflux]->boxArray();
829  DistributionMapping dm = lsm_flux[lev][iflux]->DistributionMap();
830  IntVect ng = lsm_flux[lev][iflux]->nGrowVect();
831  int nvar = lsm_flux[lev][iflux]->nComp();
832  MultiFab lsm_vars(ba,dm,nvar,ng);
833  VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmFlux" + std::to_string(iflux)));
834  MultiFab::Copy(*(lsm_flux[lev][iflux]),lsm_vars,0,0,nvar,ng);
835  }
836  }
837 
838  // Read the radiation heating rates
839  std::string RadFileName(restart_chkfile + "/Level_0/Qrad_H");
840  if ((solverChoice.rad_type != RadiationType::None) && amrex::FileExists(RadFileName)) {
841  amrex::Print() << "Reading radiation heating rates" << std::endl;
842  int nrad = qheating_rates[lev]->nComp();
843  MultiFab mf_rad(grids[lev],dmap[lev],nrad,0);
844  VisMF::Read(mf_rad, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qrad"));
845  MultiFab::Copy(*qheating_rates[lev],mf_rad,0,0,nrad,0);
846  }
847 
848  IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
849  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
850 
851  std::string MapFacMFileName(restart_chkfile + "/Level_0/MapFactor_mx_H");
852  if (amrex::FileExists(MapFacMFileName)) {
853  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_mx"));
854  } else {
855  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
856  }
857  MultiFab::Copy(*mapfac[lev][MapFacType::m_x],mf_m,0,0,1,ng);
858 
859 #if 0
860  if (MapFacType::m_x != MapFacType::m_y) {
861  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_my"));
862  MultiFab::Copy(*mapfac[lev][MapFacType::m_y],mf_m,0,0,1,ng);
863  }
864 #endif
865 
866  ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
867  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
868 
869  std::string MapFacUFileName(restart_chkfile + "/Level_0/MapFactor_ux_H");
870  if (amrex::FileExists(MapFacUFileName)) {
871  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_ux"));
872  } else {
873  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
874  }
875  MultiFab::Copy(*mapfac[lev][MapFacType::u_x],mf_u,0,0,1,ng);
876 
877 #if 0
878  if (MapFacType::u_x != MapFacType::u_y) {
879  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_uy"));
880  MultiFab::Copy(*mapfac[lev][MapFacType::u_y],mf_u,0,0,1,ng);
881  }
882 #endif
883 
884  ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
885  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
886 
887  std::string MapFacVFileName(restart_chkfile + "/Level_0/MapFactor_vx_H");
888  if (amrex::FileExists(MapFacVFileName)) {
889  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vx"));
890  } else {
891  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
892  }
893  MultiFab::Copy(*mapfac[lev][MapFacType::v_x],mf_v,0,0,1,ng);
894 
895 #if 0
896  if (MapFacType::v_x != MapFacType::v_y) {
897  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vy"));
898  MultiFab::Copy(*mapfac[lev][MapFacType::v_y],mf_v,0,0,1,ng);
899  }
900 #endif
901 
902 
903  // NOTE: We read MOST data in ReadCheckpointFileMOST (see below)!
904 
905  // See if we wrote out SST data
906  std::string FirstSSTFileName(restart_chkfile + "/Level_0/SST_0_H");
907  if (amrex::FileExists(FirstSSTFileName))
908  {
909  amrex::Print() << "Reading SST data" << std::endl;
910  int ntimes = 1;
911  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
912  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
913  sst_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
914  for (int nt(0); nt<ntimes; ++nt) {
915  VisMF::Read(sst_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
916  "SST_" + std::to_string(nt)));
917  MultiFab::Copy(*sst_lev[lev][nt],sst_at_t,0,0,1,ng);
918  }
919  }
920 
921  // See if we wrote out TSK data
922  std::string FirstTSKFileName(restart_chkfile + "/Level_0/TSK_0_H");
923  if (amrex::FileExists(FirstTSKFileName))
924  {
925  amrex::Print() << "Reading TSK data" << std::endl;
926  int ntimes = 1;
927  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
928  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
929  tsk_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
930  for (int nt(0); nt<ntimes; ++nt) {
931  VisMF::Read(tsk_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
932  "TSK_" + std::to_string(nt)));
933  MultiFab::Copy(*tsk_lev[lev][nt],tsk_at_t,0,0,1,ng);
934  }
935  }
936 
937  std::string LMaskFileName(restart_chkfile + "/Level_0/LMASK_0_H");
938  if (amrex::FileExists(LMaskFileName))
939  {
940  amrex::Print() << "Reading LMASK data" << std::endl;
941  int ntimes = 1;
942  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
943  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
944  for (int nt(0); nt<ntimes; ++nt) {
945  VisMF::Read(lmask_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
946  "LMASK_" + std::to_string(nt)));
947  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
948  const Box& bx = mfi.growntilebox();
949  Array4<int> const& dst_arr = lmask_lev[lev][nt]->array(mfi);
950  Array4<Real> const& src_arr = lmask_at_t.array(mfi);
951  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
952  {
953  dst_arr(i,j,k) = int(src_arr(i,j,k));
954  });
955  }
956  }
957  } else {
958  lmask_lev[lev][0]->setVal(solverChoice.is_land[lev]);
959  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
960  }
961 
962  IntVect ngv = ng; ngv[2] = 0;
963 
964  // Read lat/lon if it exists
965  std::string LatFileName(restart_chkfile + "/Level_0/LAT_H");
966  if (amrex::FileExists(LatFileName)) {
967  amrex::Print() << "Reading Lat/Lon variables" << std::endl;
968  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
969  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
970  VisMF::Read(lat, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LAT"));
971  VisMF::Read(lon, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LON"));
972  lat_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
973  lon_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
974  MultiFab::Copy(*lat_m[lev],lat,0,0,1,ngv);
975  MultiFab::Copy(*lon_m[lev],lon,0,0,1,ngv);
976  }
977 
978 #ifdef ERF_USE_NETCDF
979  // Read sinPhi and cosPhi if it exists
980  std::string VarCorFileName(restart_chkfile + "/Level_0/SinPhi_H");
981  if (amrex::FileExists(VarCorFileName)) {
982  amrex::Print() << "Reading Coriolis factors" << std::endl;
983  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
984  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
985  VisMF::Read(sphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SinPhi"));
986  VisMF::Read(cphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "CosPhi"));
987  sinPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
988  cosPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
989  MultiFab::Copy(*sinPhi_m[lev],sphi,0,0,1,ngv);
990  MultiFab::Copy(*cosPhi_m[lev],cphi,0,0,1,ngv);
991  }
992 
993  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
994  if (lev == 0) {
995  amrex::Print() << "Reading C1H/C2H/RDNW/MUB/PHB variables at level " << lev << std::endl;
996  MultiFab tmp1d(ba1d[0],dmap[0],1,0);
997 
998  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C1H"));
999  MultiFab::Copy(*wrf_C1H,tmp1d,0,0,1,0);
1000 
1001  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C2H"));
1002  MultiFab::Copy(*wrf_C2H,tmp1d,0,0,1,0);
1003 
1004  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "RDNW"));
1005  MultiFab::Copy(*wrf_RDNW,tmp1d,0,0,1,0);
1006 
1007  MultiFab tmp2d(ba2d[0],dmap[0],1,wrf_MUB->nGrowVect());
1008 
1009  VisMF::Read(tmp2d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MUB"));
1010  MultiFab::Copy(*wrf_MUB,tmp2d,0,0,1,wrf_MUB->nGrowVect());
1011 
1012  MultiFab tmp3d(convert(grids[0],IntVect(0,0,1)),dmap[0],1,wrf_PHB->nGrowVect());
1013 
1014  VisMF::Read(tmp3d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PHB"));
1015  MultiFab::Copy(*wrf_PHB,tmp3d,0,0,1,ng);
1016  }
1017  }
1018 #endif
1019  } // for lev
1020 
1021  // Restore the LSM scalar step counter (e.g. NoahMP itimestep) so the
1022  // substepping schedule survives restart. lsm.Init() during
1023  // MakeNewLevelFromScratch has already reset itimestep to 0 (NoahmpIOVarInit);
1024  // override it here from the checkpoint. Older checkpoints without this file
1025  // restart with the legacy reset-to-zero behavior.
1026  if (solverChoice.lsm_type != LandSurfaceType::None) {
1027  std::string LsmStepFileName(restart_chkfile + "/lsm_step");
1028  if (amrex::FileExists(LsmStepFileName)) {
1029  Vector<char> LsmStepCharPtr;
1030  ParallelDescriptor::ReadAndBcastFile(LsmStepFileName, LsmStepCharPtr);
1031  std::string LsmStepStr(LsmStepCharPtr.dataPtr());
1032  std::istringstream lsm_is(LsmStepStr, std::istringstream::in);
1033  int step_val = 0;
1034  for (int lev = 0; lev <= finest_level; ++lev) {
1035  if (lsm_is >> step_val) {
1036  lsm.Set_LSM_Step(lev, step_val);
1037  amrex::Print() << "Restored LSM step counter at level " << lev
1038  << " to " << step_val << std::endl;
1039  }
1040  }
1041  } else {
1042  amrex::Print() << "Warning: legacy checkpoint without lsm_step file; "
1043  << "LSM substep schedule will reset (may break bitwise reproducibility)."
1044  << std::endl;
1045  }
1046 
1047  // Restore the full LSM prognostic state (e.g. NoahMP soil/snow/canopy)
1048  // from chk*/noahmp_restart. lsm.Init() during MakeNewLevelFromScratch
1049  // has already cold-initialized this state from wrfinput/tables; this
1050  // overwrites it with the checkpoint, and NoahMP's per-step In-transfer
1051  // pulls it into the physics state on the first Advance (issue #3255).
1052  // Legacy checkpoints without this directory fall back to cold-init.
1053  std::string LsmRestartDir(restart_chkfile + "/noahmp_restart");
1054  if (amrex::FileExists(LsmRestartDir + "/Level_0.nc")) {
1055  for (int lev = 0; lev <= finest_level; ++lev) {
1056  lsm.Read_Lsm_Restart(lev, LsmRestartDir);
1057  }
1058  amrex::Print() << "Restored full NoahMP prognostic state from "
1059  << LsmRestartDir << std::endl;
1060  } else {
1061  amrex::Print() << "Warning: legacy checkpoint without noahmp_restart; "
1062  << "NoahMP prognostic state cold-initialized from wrfinput "
1063  << "(land trajectory will differ from cold start)."
1064  << std::endl;
1065  }
1066  }
1067 
1068 #ifdef ERF_USE_PARTICLES
1069  restartTracers((ParGDBBase*)GetParGDB(),restart_chkfile);
1070  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
1071  dynamic_cast<LagrangianMicrophysics&>(*micro).restartParticles((ParGDBBase*)GetParGDB(),restart_chkfile);
1072  }
1073 #endif
1074 
1075 #if 0
1076 #ifdef ERF_USE_NETCDF
1077  // Read bdy_data files
1078  if ( ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
1079  solverChoice.use_real_bcs )
1080  {
1081  int ioproc = ParallelDescriptor::IOProcessorNumber(); // I/O rank
1082  int num_time;
1083  int num_var;
1084  Vector<Box> bx_v;
1085  if (ParallelDescriptor::IOProcessor()) {
1086  // Open header file and read from it
1087  std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
1088  bdy_h_file >> num_time;
1089  bdy_h_file >> num_var;
1090  bdy_h_file >> start_bdy_time;
1091  bdy_h_file >> bdy_time_interval;
1092  bdy_h_file >> real_width;
1093  bx_v.resize(4*num_var);
1094  for (int ivar(0); ivar<num_var; ++ivar) {
1095  bdy_h_file >> bx_v[4*ivar ];
1096  bdy_h_file >> bx_v[4*ivar+1];
1097  bdy_h_file >> bx_v[4*ivar+2];
1098  bdy_h_file >> bx_v[4*ivar+3];
1099  }
1100 
1101  // IO size the FABs
1102  bdy_data_xlo.resize(num_time);
1103  bdy_data_xhi.resize(num_time);
1104  bdy_data_ylo.resize(num_time);
1105  bdy_data_yhi.resize(num_time);
1106  for (int itime(0); itime<num_time; ++itime) {
1107  bdy_data_xlo[itime].resize(num_var);
1108  bdy_data_xhi[itime].resize(num_var);
1109  bdy_data_ylo[itime].resize(num_var);
1110  bdy_data_yhi[itime].resize(num_var);
1111  for (int ivar(0); ivar<num_var; ++ivar) {
1112  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
1113  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
1114  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
1115  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
1116  }
1117  }
1118 
1119  // Open data file and read from it
1120  std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
1121  for (int itime(0); itime<num_time; ++itime) {
1122  for (int ivar(0); ivar<num_var; ++ivar) {
1123  bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
1124  bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
1125  bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
1126  bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
1127  }
1128  }
1129  } // IO
1130 
1131  // Broadcast the data
1132  ParallelDescriptor::Barrier();
1133  ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
1134  ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
1135  ParallelDescriptor::Bcast(&real_width,1,ioproc);
1136  ParallelDescriptor::Bcast(&num_time,1,ioproc);
1137  ParallelDescriptor::Bcast(&num_var,1,ioproc);
1138 
1139  // Everyone size their boxes
1140  bx_v.resize(4*num_var);
1141 
1142  ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
1143 
1144  // Everyone but IO size their FABs
1145  if (!ParallelDescriptor::IOProcessor()) {
1146  bdy_data_xlo.resize(num_time);
1147  bdy_data_xhi.resize(num_time);
1148  bdy_data_ylo.resize(num_time);
1149  bdy_data_yhi.resize(num_time);
1150  for (int itime(0); itime<num_time; ++itime) {
1151  bdy_data_xlo[itime].resize(num_var);
1152  bdy_data_xhi[itime].resize(num_var);
1153  bdy_data_ylo[itime].resize(num_var);
1154  bdy_data_yhi[itime].resize(num_var);
1155  for (int ivar(0); ivar<num_var; ++ivar) {
1156  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
1157  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
1158  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
1159  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
1160  }
1161  }
1162  }
1163 
1164  for (int itime(0); itime<num_time; ++itime) {
1165  for (int ivar(0); ivar<num_var; ++ivar) {
1166  ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
1167  ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
1168  ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
1169  ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
1170  }
1171  }
1172  } // init_type == WRFInput or Metgrid
1173 #endif
1174 #endif
1175 
1176 #ifdef ERF_USE_NETCDF
1177  // Load boundary data from erfbdy during restart for metgrid or wrfinput.
1178  if (((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) &&
1179  solverChoice.use_real_bcs) {
1180 
1181  // Check for erfbdy file.
1182  std::string erfbdy_header = erfbdy_file + "/Header";
1183 
1184  // For now we disable this for InitType::WRFInput because it failed the WPS_Test_restart regression test
1185  use_erfbdy = ( (solverChoice.init_type == InitType::Metgrid) && FileSystem::Exists(erfbdy_header) );
1186 
1187  if (solverChoice.init_type == InitType::Metgrid) {
1188  if (!use_erfbdy) {
1189  Abort("Restart with init_type=metgrid requires erfbdy file: " + erfbdy_file);
1190  }
1191  }
1192 
1193  // Load from erfbdy if it exists.
1194  if (use_erfbdy) {
1195  Print() << "Restart: Loading boundary data from erfbdy file: " << erfbdy_file << std::endl;
1196 
1197  int ntimes_erfbdy;
1198  Vector<double> bdy_times;
1199  bdy_time_interval = read_times_from_erfbdy(erfbdy_file,
1200  ntimes_erfbdy, nvars_erfbdy, real_width,
1201  bdy_times, start_bdy_time, final_bdy_time);
1202 
1203  Print() << "Restart: erfbdy file contains " << ntimes_erfbdy << " times" << std::endl;
1204 
1205  bdy_data_xlo.resize(ntimes_erfbdy);
1206  bdy_data_xhi.resize(ntimes_erfbdy);
1207  bdy_data_ylo.resize(ntimes_erfbdy);
1208  bdy_data_yhi.resize(ntimes_erfbdy);
1209 
1210  // Determine which times we need based on current simulation time.
1211  double time_since_start_bdy = t_new[0] + start_time - start_bdy_time;
1212  int n_time_old = std::min(static_cast<int>(time_since_start_bdy / bdy_time_interval), ntimes_erfbdy-1);
1213  int n_time_new = n_time_old + 1;
1214 
1215  // Read the necessary times into memory.
1216  for (int itime = n_time_old; itime <= std::min(n_time_new + 1, ntimes_erfbdy - 1); ++itime) {
1217  read_from_erfbdy(itime, erfbdy_file,
1218  bdy_data_xlo, bdy_data_xhi,
1219  bdy_data_ylo, bdy_data_yhi,
1220  nvars_erfbdy, real_width);
1221  Print() << "Restart: Loaded erfbdy time index " << itime << std::endl;
1222  }
1223  }
1224  }
1225 #endif
1226 }
read_times_from_erfbdy
double read_times_from_erfbdy(const std::string &bdy_file_name, int &ntimes, int &nvars, int &real_width, Vector< double > &bdy_times, double &start_bdy_time, double &final_bdy_time)
Definition: ERF_ReadFromERFBdy.cpp:10
read_from_erfbdy
void read_from_erfbdy(int itime, const std::string &bdy_file_name, Vector< Vector< FArrayBox >> &bdy_data_xlo, Vector< Vector< FArrayBox >> &bdy_data_xhi, Vector< Vector< FArrayBox >> &bdy_data_ylo, Vector< Vector< FArrayBox >> &bdy_data_yhi, int nvars, int)
Definition: ERF_ReadFromERFBdy.cpp:65
ERF::erfbdy_file
std::string erfbdy_file
Definition: ERF.H:1382
ERF::nvars_erfbdy
int nvars_erfbdy
Definition: ERF.H:1383
ERF::use_erfbdy
bool use_erfbdy
Definition: ERF.H:1381
ERF::GotoNextLine
static void GotoNextLine(std::istream &is)
Definition: ERF_Checkpoint.cpp:25
ERF::MakeNewLevelFromScratch
void MakeNewLevelFromScratch(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:25
LandSurface::Read_Lsm_Restart
void Read_Lsm_Restart(const int &lev, const std::string &dir)
Definition: ERF_LandSurface.H:207
LandSurface::Set_LSM_Step
void Set_LSM_Step(const int &lev, int step)
Definition: ERF_LandSurface.H:198
erf_provenance::read_job_info_file
ProvenanceReadResult read_job_info_file(const std::string &path)
Definition: ERF_Provenance.cpp:482
erf_provenance::make_restart_provenance
ExecutionProvenance make_restart_provenance(const ExecutionProvenance &current_invocation, const ProvenanceRecord &parent_checkpoint, const std::string &checkpoint_path)
Definition: ERF_Provenance.cpp:519
erf_provenance::make_incomplete_restart_provenance
ExecutionProvenance make_incomplete_restart_provenance(const ExecutionProvenance &current_invocation, ProvenanceReadStatus failure_status, const std::string &checkpoint_path)
Definition: ERF_Provenance.cpp:543
erf_provenance::artifact_type_token
const char * artifact_type_token(ArtifactType type) noexcept
Definition: ERF_Provenance.cpp:334
erf_provenance::ProvenanceReadStatus::ArtifactTypeMismatch
@ ArtifactTypeMismatch
erf_provenance::ArtifactType::Checkpoint
@ Checkpoint
Here is the call graph for this function:

◆ ReadCheckpointFileSurfaceLayer()

void ERF::ReadCheckpointFileSurfaceLayer ( )

ERF function for reading additional data for MOST from a checkpoint file during restart.

This is called after the ABLMost object is instantiated.

1273 {
1274  for (int lev = 0; lev <= finest_level; ++lev)
1275  {
1276  amrex::Print() << "Reading MOST variables" << std::endl;
1277 
1278  IntVect ng(1,1,0);
1279 
1280  auto read_most_var = [&] (const std::string& name, MultiFab* dst) {
1281  const std::string mf_name = MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", name);
1282  if (amrex::FileExists(mf_name + "_H")) {
1283  MultiFab m_var;
1284  VisMF::Read(m_var, mf_name);
1285  dst->ParallelCopy(m_var, 0, 0, 1, ng, ng, geom[lev].periodicity());
1286  }
1287  };
1288 
1289  // U*
1290  read_most_var("Ustar", m_SurfaceLayer->get_u_star(lev));
1291 
1292  // W*
1293  read_most_var("Wstar", m_SurfaceLayer->get_w_star(lev));
1294 
1295  // T*
1296  read_most_var("Tstar", m_SurfaceLayer->get_t_star(lev));
1297 
1298  // Q*
1299  read_most_var("Qstar", m_SurfaceLayer->get_q_star(lev));
1300 
1301  // Olen
1302  read_most_var("Olen", m_SurfaceLayer->get_olen(lev));
1303 
1304  // Qsurf
1305  read_most_var("Qsurf", m_SurfaceLayer->get_q_surf(lev));
1306 
1307  // PBLH
1308  read_most_var("PBLH", m_SurfaceLayer->get_pblh(lev));
1309 
1310  // Z0
1311  read_most_var("Z0", m_SurfaceLayer->get_z0(lev));
1312  }
1313 }
name
std::string name
Definition: ERF_Plotfile2DCatalog.cpp:97

◆ ReadParameters()

void ERF::ReadParameters ( )
private
1946 {
1947  std::string prob_name = "Undefined";
1948  ParmParse pp_pn("erf");
1949  pp_pn.queryAdd("prob_name", prob_name);
1950  Print() << "Problem name (from inputs file) is: "
1951  << " \"" << prob_name << "\" " << std::endl;
1952 
1953  ParmParse pp(pp_prefix);
1954  ParmParse pp_amr("amr");
1955  {
1956  pp.query("regrid_level_0_on_restart", regrid_level_0_on_restart);
1957  pp.query("regrid_int", regrid_int);
1958  pp.query("check_file", check_file);
1959 
1960  // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
1961  // for those or "erf.restart" / "erf.m_check_int". The amr.* values are
1962  // queried after the erf.* values and therefore take precedence if both
1963  // are specified.
1964  pp.query("check_int", m_check_int);
1965  pp.query("check_per", m_check_per);
1966  pp_amr.query("check_int", m_check_int);
1967  pp_amr.query("check_per", m_check_per);
1968 
1969  pp.query("restart", restart_chkfile);
1970  pp_amr.query("restart", restart_chkfile);
1971 
1972  // Verbosity
1973  pp.query("v", verbose);
1974  pp.query("mg_v", mg_verbose);
1975  pp.query("use_fft", use_fft);
1976 #ifndef ERF_USE_FFT
1977  if (use_fft) {
1978  Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
1979  }
1980 #endif
1981 
1982  // Check for NaNs?
1983  pp.query("check_for_nans", check_for_nans);
1984 
1985  // Frequency of diagnostic output
1986  pp.query("sum_interval", sum_interval);
1987  pp.query("sum_period" , sum_per);
1988 
1989  pp.query("pert_interval", pert_interval);
1990 
1991  // Time step controls
1992  pp.query("cfl", cfl);
1993  pp.query("substepping_cfl", sub_cfl);
1994  pp.query("init_shrink", init_shrink);
1995  pp.query("change_max", change_max);
1996  pp.query("dt_max_initial", dt_max_initial);
1997  pp.query("dt_max", dt_max);
1998 
1999  fixed_dt.resize(max_level+1,-one);
2000  fixed_fast_dt.resize(max_level+1,-one);
2001 
2002  pp.query("fixed_dt", fixed_dt[0]);
2003  pp.query("fixed_fast_dt", fixed_fast_dt[0]);
2004 
2005  int nlevs_max = max_level + 1;
2006  istep.resize(nlevs_max, 0);
2007  nsubsteps.resize(nlevs_max, 1);
2008  // This is the default
2009  for (int lev = 1; lev <= max_level; ++lev) {
2010  nsubsteps[lev] = MaxRefRatio(lev-1);
2011  }
2012 
2013  if (max_level > 0) {
2014  ParmParse pp_erf("erf");
2015  int count = pp_erf.countval("dt_ref_ratio");
2016  if (count > 0) {
2017  Vector<int> nsub;
2018  nsub.resize(nlevs_max, 0);
2019  if (count == 1) {
2020  pp_erf.queryarr("dt_ref_ratio", nsub, 0, 1);
2021  for (int lev = 1; lev <= max_level; ++lev) {
2022  nsubsteps[lev] = nsub[0];
2023  }
2024  } else {
2025  pp_erf.queryarr("dt_ref_ratio", nsub, 0, max_level);
2026  for (int lev = 1; lev <= max_level; ++lev) {
2027  nsubsteps[lev] = nsub[lev-1];
2028  }
2029  }
2030  }
2031  }
2032 
2033  // Make sure we do this after we have defined nsubsteps above
2034  for (int lev = 1; lev <= max_level; lev++)
2035  {
2036  fixed_dt[lev] = fixed_dt[lev-1] / static_cast<Real>(nsubsteps[lev]);
2037  fixed_fast_dt[lev] = fixed_fast_dt[lev-1] / static_cast<Real>(nsubsteps[lev]);
2038  }
2039 
2040  pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
2041 
2042  // We use this to keep track of how many boxes we read in from WRF initialization
2043  num_files_at_level.resize(max_level+1,0);
2044 
2045  // We use this to keep track of how many boxes are specified thru the refinement indicators
2046  num_boxes_at_level.resize(max_level+1,0);
2047  boxes_at_level.resize(max_level+1);
2048 
2049  // We always have exactly one file at level 0
2050  num_boxes_at_level[0] = 1;
2051  boxes_at_level[0].resize(1);
2052  boxes_at_level[0][0] = geom[0].Domain();
2053 
2054 #ifdef ERF_USE_NETCDF
2055  nc_init_file.resize(max_level+1);
2056  have_read_nc_init_file.resize(max_level+1);
2057 
2058  // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
2059  // but we always have exactly one file at level 0
2060  for (int lev = 0; lev <= max_level; lev++) {
2061  const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
2062  if (pp.contains(nc_file_names.c_str())) {
2063  int num_files = pp.countval(nc_file_names.c_str());
2064  num_files_at_level[lev] = num_files;
2065  nc_init_file[lev].resize(num_files);
2066  have_read_nc_init_file[lev].resize(num_files);
2067  pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
2068  for (int j = 0; j < num_files; j++) {
2069  Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
2070  have_read_nc_init_file[lev][j] = 0;
2071  } // j
2072  } // if pp.contains
2073  } // lev
2074 
2075  // NetCDF wrfbdy lateral boundary file
2076  if (pp.query("nc_bdy_file", nc_bdy_file)) {
2077  Print() << "Reading NC bdy file name " << nc_bdy_file << std::endl;
2078  }
2079 
2080  // NetCDF wrflow lateral boundary file
2081  if (pp.query("nc_low_file", nc_low_file)) {
2082  Print() << "Reading NC low file name " << nc_low_file << std::endl;
2083  }
2084 
2085 #endif
2086 
2087  // Options for vertical interpolation of met_em*.nc data.
2088  pp.query("metgrid_debug_quiescent", metgrid_debug_quiescent);
2089  pp.query("metgrid_debug_isothermal", metgrid_debug_isothermal);
2090  pp.query("metgrid_debug_dry", metgrid_debug_dry);
2091  pp.query("metgrid_debug_psfc", metgrid_debug_psfc);
2092  pp.query("metgrid_debug_msf", metgrid_debug_msf);
2093  pp.query("metgrid_interp_theta", metgrid_interp_theta);
2094  pp.query("metgrid_basic_linear", metgrid_basic_linear);
2095  pp.query("metgrid_use_below_sfc", metgrid_use_below_sfc);
2096  pp.query("metgrid_use_sfc", metgrid_use_sfc);
2097  pp.query("metgrid_retain_sfc", metgrid_retain_sfc);
2098  pp.query("metgrid_proximity", metgrid_proximity);
2099  pp.query("metgrid_order", metgrid_order);
2100  pp.query("metgrid_force_sfc_k", metgrid_force_sfc_k);
2101 
2102  // Options for boundary file.
2103  pp.query("write_erfbdy", write_erfbdy);
2104  pp.query("erfbdy_file", erfbdy_file);
2105 
2106  // Set default to FullState for now ... later we will try Perturbation
2107  interpolation_type = StateInterpType::FullState;
2108  pp.query_enum_case_insensitive("interpolation_type" ,interpolation_type);
2109 
2110  PlotFileType plotfile3d_type_temp = PlotFileType::None;
2111  pp.query_enum_case_insensitive("plotfile_type" ,plotfile3d_type_temp);
2112  pp.query_enum_case_insensitive("plotfile_type_1",plotfile3d_type_1);
2113  pp.query_enum_case_insensitive("plotfile_type_2",plotfile3d_type_2);
2114 
2115  PlotFileType plotfile2d_type_temp = PlotFileType::None;
2116  pp.query_enum_case_insensitive("plotfile2d_type" ,plotfile2d_type_temp);
2117  pp.query_enum_case_insensitive("plotfile2d_type_1",plotfile2d_type_1);
2118  pp.query_enum_case_insensitive("plotfile2d_type_2",plotfile2d_type_2);
2119  //
2120  // This option is for backward consistency -- if only plotfile_type is set,
2121  // then it will be used for both 1 and 2 if and only if they are not set
2122  //
2123  // Default is native amrex if no type is specified
2124  //
2125  if (plotfile3d_type_temp == PlotFileType::None) {
2126  if (plotfile3d_type_1 == PlotFileType::None) {
2127  plotfile3d_type_1 = PlotFileType::Amrex;
2128  }
2129  if (plotfile3d_type_2 == PlotFileType::None) {
2130  plotfile3d_type_2 = PlotFileType::Amrex;
2131  }
2132  } else {
2133  if (plotfile3d_type_1 == PlotFileType::None) {
2134  plotfile3d_type_1 = plotfile3d_type_temp;
2135  } else {
2136  Abort("You must set either plotfile_type or plotfile_type_1, not both");
2137  }
2138  if (plotfile3d_type_2 == PlotFileType::None) {
2139  plotfile3d_type_2 = plotfile3d_type_temp;
2140  } else {
2141  Abort("You must set either plotfile_type or plotfile_type_2, not both");
2142  }
2143  }
2144  if (plotfile2d_type_temp == PlotFileType::None) {
2145  if (plotfile2d_type_1 == PlotFileType::None) {
2146  plotfile2d_type_1 = PlotFileType::Amrex;
2147  }
2148  if (plotfile2d_type_2 == PlotFileType::None) {
2149  plotfile2d_type_2 = PlotFileType::Amrex;
2150  }
2151  } else {
2152  if (plotfile2d_type_1 == PlotFileType::None) {
2153  plotfile2d_type_1 = plotfile2d_type_temp;
2154  } else {
2155  Abort("You must set either plotfile2d_type or plotfile2d_type_1, not both");
2156  }
2157  if (plotfile2d_type_2 == PlotFileType::None) {
2158  plotfile2d_type_2 = plotfile2d_type_temp;
2159  } else {
2160  Abort("You must set either plotfile2d_type or plotfile2d_type_2, not both");
2161  }
2162  }
2163 #ifndef ERF_USE_NETCDF
2164  if (plotfile3d_type_1 == PlotFileType::Netcdf ||
2165  plotfile3d_type_2 == PlotFileType::Netcdf ||
2166  plotfile2d_type_1 == PlotFileType::Netcdf ||
2167  plotfile2d_type_2 == PlotFileType::Netcdf) {
2168  Abort("Plotfile type = Netcdf is not allowed without USE_NETCDF = TRUE");
2169  }
2170 #endif
2171 
2172  pp.query("plot_file_1" , plot3d_file_1);
2173  pp.query("plot_file_2" , plot3d_file_2);
2174  pp.query("plot2d_file_1", plot2d_file_1);
2175  pp.query("plot2d_file_2", plot2d_file_2);
2176 
2177  pp.query("plot_int_1" , m_plot3d_int_1);
2178  pp.query("plot_int_2" , m_plot3d_int_2);
2179  pp.query("plot_per_1" , m_plot3d_per_1);
2180  pp.query("plot_per_2" , m_plot3d_per_2);
2181 
2182  pp.query("plot2d_int_1" , m_plot2d_int_1);
2183  pp.query("plot2d_int_2" , m_plot2d_int_2);
2184  pp.query("plot2d_per_1", m_plot2d_per_1);
2185  pp.query("plot2d_per_2", m_plot2d_per_2);
2186 
2187  pp.query("subvol_file", subvol_file);
2188 
2189  // Should we use format like plt1970-01-01_00:00:Real(00.000000) (if true) or plt00001 (if false)
2190  pp.query("use_real_time_in_pltname", use_real_time_in_pltname);
2191 
2192  // If use_real_time_in_pltname is false, how many digits should we use for the timestep?
2193  pp.query("file_name_digits", file_name_digits);
2194 
2195  // Default if subvol_int not specified
2196  m_subvol_int.resize(1); m_subvol_int[0] = -1;
2197  m_subvol_per.resize(1); m_subvol_per[0] = -one;
2198  last_subvol_step.resize(1);
2199  last_subvol_time.resize(1);
2200 
2201  int nsi = pp.countval("subvol_int");
2202  int nsr = pp.countval("subvol_per");
2203 
2204  // We must specify only subvol_int OR subvol_per
2205  AMREX_ALWAYS_ASSERT (!(nsi > 0 && nsr > 0));
2206 
2207  int nsub = -1;
2208  if (nsi > 0 || nsr > 0) {
2209  ParmParse pp_sv("erf.subvol");
2210  int n1 = pp_sv.countval("origin"); int n2 = pp_sv.countval("nxnynz"); int n3 = pp_sv.countval("dxdydz");
2211  if (n1 != n2 || n1 != n3 || n2 != n3) {
2212  Abort("WriteSubvolume: must have same number of entries in origin, nxnynz, and dxdydz.");
2213  }
2214  if ( n1%AMREX_SPACEDIM != 0) {
2215  Abort("WriteSubvolume: origin, nxnynz, and dxdydz must have multiples of AMReX_SPACEDIM");
2216  }
2217  nsub = n1/AMREX_SPACEDIM;
2218  m_subvol_int.resize(nsub);
2219  last_subvol_step.resize(nsub);
2220  last_subvol_time.resize(nsub);
2221  m_subvol_int.resize(nsub);
2222  m_subvol_per.resize(nsub);
2223  }
2224 
2225  if (nsi > 0) {
2226  for (int i = 1; i < nsub; i++) m_subvol_per[i] = -one;
2227  if ( nsi == 1) {
2228  m_subvol_int[0] = -1;
2229  pp.get("subvol_int" , m_subvol_int[0]);
2230  } else if ( nsi == nsub) {
2231  pp.getarr("subvol_int" , m_subvol_int);
2232  } else {
2233  Abort("There must either be a single value of subvol_int or one for every subdomain");
2234  }
2235  }
2236 
2237  if (nsr > 0) {
2238  for (int i = 1; i < nsub; i++) m_subvol_int[i] = -static_cast<int>(one);
2239  if ( nsr == 1) {
2240  m_subvol_per[0] = -one;
2241  pp.get("subvol_per" , m_subvol_per[0]);
2242  } else if ( nsr == nsub) {
2243  pp.getarr("subvol_per" , m_subvol_per);
2244  } else {
2245  Abort("There must either be a single value of subvol_per or one for every subdomain");
2246  }
2247  }
2248 
2249  setSubVolVariables("subvol_sampling_vars",subvol3d_var_names);
2250 
2251  pp.query("expand_plotvars_to_unif_rr",m_expand_plotvars_to_unif_rr);
2252 
2253  pp.query("plot_face_vels",m_plot_face_vels);
2254 
2255  if ( (m_plot3d_int_1 > 0 && m_plot3d_per_1 > 0) ||
2256  (m_plot3d_int_2 > 0 && m_plot3d_per_2 > zero) ) {
2257  Abort("Must choose only one of plot_int or plot_per");
2258  }
2259  if ( (m_plot2d_int_1 > 0 && m_plot2d_per_1 > 0) ||
2260  (m_plot2d_int_2 > 0 && m_plot2d_per_2 > zero) ) {
2261  Abort("Must choose only one of plot_int or plot_per");
2262  }
2263 
2264  pp.query("profile_int", profile_int);
2265  pp.query("destag_profiles", destag_profiles);
2266 
2267  pp.query("plot_lsm", plot_lsm);
2268 #ifdef ERF_USE_RRTMGP
2269  pp.query("plot_rad", plot_rad);
2270 #endif
2271  pp.query("profile_rad_int", rad_datalog_int);
2272 
2273  pp.query("output_1d_column", output_1d_column);
2274  pp.query("column_per", column_per);
2275  pp.query("column_interval", column_interval);
2276  pp.query("column_loc_x", column_loc_x);
2277  pp.query("column_loc_y", column_loc_y);
2278  pp.query("column_file_name", column_file_name);
2279 
2280  // Sampler output frequency
2281  pp.query("line_sampling_per", line_sampling_per);
2282  pp.query("line_sampling_interval", line_sampling_interval);
2283  pp.query("plane_sampling_per", plane_sampling_per);
2284  pp.query("plane_sampling_interval", plane_sampling_interval);
2285 
2286  // Specify information about outputting planes of data
2287  pp.query("output_bndry_planes", output_bndry_planes);
2288  pp.query("bndry_output_planes_interval", bndry_output_planes_interval);
2289  pp.query("bndry_output_planes_per", bndry_output_planes_per);
2290  pp.query("bndry_output_start_time", bndry_output_planes_start_time);
2291 
2292  // Specify whether ingest boundary planes of data
2293  pp.query("input_bndry_planes", input_bndry_planes);
2294 
2295  // Query the total width for wrfbdy interior ghost cells
2296  pp.query("real_width", real_width);
2297 
2298  // If using real boundaries, do we extrapolate w (or set to 0)
2299  pp.query("real_extrap_w", real_extrap_w);
2300 
2301  // Query the set and total widths for crse-fine interior ghost cells
2302  pp.query("cf_width", cf_width);
2303  pp.query("cf_set_width", cf_set_width);
2304 
2305  // AmrMesh iterate on grids?
2306  bool iterate(true);
2307  pp_amr.query("iterate_grids",iterate);
2308  if (!iterate) SetIterateToFalse();
2309  }
2310 
2311 #ifdef ERF_USE_PARTICLES
2312  readTracersParams();
2313 #endif
2314 
2315  solverChoice.init_params(max_level,pp_prefix);
2316 
2317  // Set a default value for write_erfbdy following these rules.
2318  // Prioritize write_erfbdy provided by user.
2319  // write_erfbdy must be false for restarts.
2320  // write_erfbdy defaults to true for clean starts of the metgrid or wrfinput pathways.
2321  {
2322  ParmParse pp_erfbdy(pp_prefix);
2323  bool is_restart = !restart_chkfile.empty();
2324  if (is_restart) {
2325  if (write_erfbdy) {
2326  Abort("Cannot set erf.write_erfbdy = true during restart. erfbdy should only be written during initial runs.");
2327  }
2328  } else {
2329  if (!pp_erfbdy.contains("write_erfbdy")) {
2330  if ((solverChoice.init_type == InitType::Metgrid) || (solverChoice.init_type == InitType::WRFInput)) {
2331  write_erfbdy = true;
2332  }
2333  }
2334  }
2335  }
2336 
2337  {
2338  ParmParse pp_no_prefix; // Traditionally, max_step and stop_time do not have prefix.
2339  pp_no_prefix.query("max_step", max_step);
2340  if (max_step < 0) {
2341  max_step = std::numeric_limits<int>::max();
2342  }
2343 
2344  std::string start_datetime, stop_datetime;
2345  if (pp_no_prefix.query("start_datetime", start_datetime)) {
2346  if (start_datetime.length() == 16) { // YYYY-MM-DD HH:MM
2347  start_datetime += ":00"; // add seconds
2348  }
2349  if (start_datetime.length() != 19) {
2350  Print() << "Got start_datetime = \"" << start_datetime
2351  << "\", format should be " << datetime_format << std::endl;
2352  exit(0);
2353  }
2354  start_time = static_cast<double>(getEpochTime(start_datetime, datetime_format));
2355 
2356 #ifdef ERF_USE_NETCDF
2357  if (solverChoice.init_type == InitType::WRFInput) {
2358  // This is the start time as written in the wrfinput file
2359  double start_time_from_wrfinput = read_start_time_from_wrfinput(0, nc_init_file[0][0]);
2360  if (start_time != start_time_from_wrfinput) {
2361  amrex::Print() << "start_datetime from inputs file = " << start_time <<
2362  " does not match SIMULATION START DATE from wrfinput = " <<
2363  start_time_from_wrfinput << std::endl;
2364  amrex::Abort();
2365  }
2366  } else if (solverChoice.init_type == InitType::Metgrid) {
2367  // This is the start time as written in the metgrid file
2368  double start_time_from_metgrid = read_start_time_from_metgrid(0, nc_init_file[0][0]);
2369  if (start_time != start_time_from_metgrid) {
2370  amrex::Print() << "start_datetime from inputs file = " << start_time <<
2371  " does not match SIMULATION START DATE from metgrid = " <<
2372  start_time_from_metgrid << std::endl;
2373  amrex::Abort();
2374  }
2375  }
2376 #endif
2377  Print() << "Start datetime : " << start_datetime << std::endl;
2378 
2379  use_datetime = true;
2380 
2381  } else {
2382 
2383 #ifdef ERF_USE_NETCDF
2384  if (solverChoice.init_type == InitType::WRFInput) {
2385  // This is the start time as written in the wrfinput file
2386  double start_time_from_wrfinput = read_start_time_from_wrfinput(0, nc_init_file[0][0]);
2387  start_time = start_time_from_wrfinput;
2388 
2389  use_datetime = true;
2390 
2391  if (pp_no_prefix.query("start_time", start_time)) {
2392  amrex::Print() << "start_time should not be set from inputs file; we are reading SIMULATION START DATE from wrfinput" << std::endl;
2393  amrex::Abort();
2394  }
2395  } else if (solverChoice.init_type == InitType::Metgrid) {
2396  // This is the start time as written in the metgrid file
2397  double start_time_from_metgrid = read_start_time_from_metgrid(0, nc_init_file[0][0]);
2398  start_time = start_time_from_metgrid;
2399 
2400  use_datetime = true;
2401 
2402  if (pp_no_prefix.query("start_time", start_time)) {
2403  amrex::Print() << "start_time should not be set from inputs file; we are reading SIMULATION START DATE from metgrid" << std::endl;
2404  amrex::Abort();
2405  }
2406  }
2407 #endif
2408  }
2409 
2410  if (pp_no_prefix.query("stop_datetime", stop_datetime)) {
2411  if (stop_datetime.length() == 16) { // YYYY-MM-DD HH:MM
2412  stop_datetime += ":00"; // add seconds
2413  }
2414  if (stop_datetime.length() != 19) {
2415  Print() << "Got stop_datetime = \"" << stop_datetime
2416  << "\", format should be " << datetime_format << std::endl;
2417  exit(0);
2418  }
2419 
2420  stop_time = static_cast<double>(getEpochTime(stop_datetime, datetime_format));
2421  Print() << "Stop datetime : " << start_datetime << std::endl;
2422 
2423  } else {
2424 
2425  if (pp_no_prefix.query("stop_time", stop_time)) {
2426  Print() << "Maximum simulation length based on stop_time: " << stop_time << " s (elapsed) " << std::endl;
2427  amrex::Print() <<" Adding stop time " << stop_time << " to start_time " << start_time << std::endl;
2428  stop_time += start_time;
2429  }
2430  }
2431  }
2432 
2433 #ifndef ERF_USE_NETCDF
2434  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(( (solverChoice.init_type != InitType::WRFInput) &&
2435  (solverChoice.init_type != InitType::Metgrid ) &&
2436  (solverChoice.init_type != InitType::NCFile ) ),
2437  "init_type cannot be 'WRFInput', 'Metgrid' or 'NCFile' if we don't build with netcdf!");
2438 #endif
2439 
2440  // Query the canopy model file name
2441  std::string forestfile;
2442  solverChoice.do_forest_drag = pp.query("forest_file", forestfile);
2443  if (solverChoice.do_forest_drag) {
2444  for (int lev = 0; lev <= max_level; ++lev) {
2445  m_forest_drag[lev] = std::make_unique<ForestDrag>(forestfile);
2446  }
2447  }
2448 
2449  // If init from WRFInput or Metgrid make sure a valid file name is present at level zero
2450  // We allow for the possibility that finer levels may use native refinement rather than reading from a file
2451  if ((solverChoice.init_type == InitType::WRFInput) ||
2452  (solverChoice.init_type == InitType::Metgrid) ||
2453  (solverChoice.init_type == InitType::NCFile) ) {
2454  int num_files = static_cast<int>(nc_init_file[0].size());
2455  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(num_files>0, "A file name must be present at level 0 for init type WRFInput, Metgrid or NCFile.");
2456  for (int j = 0; j < num_files; j++) {
2457  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!nc_init_file[0][j].empty(), "Valid file name must be present at level 0 for init type WRFInput, Metgrid or NCFile.");
2458  } //j
2459  } // InitType
2460 
2461  // What type of land surface model to use
2462  // NOTE: Must be checked after init_params
2463  if (solverChoice.lsm_type == LandSurfaceType::SLM) {
2464  lsm.SetModel<SLM>();
2465  Print() << "SLM land surface model!\n";
2466  } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
2467  lsm.SetModel<MM5>();
2468  Print() << "MM5 land surface model!\n";
2469 #ifdef ERF_USE_NOAHMP
2470  } else if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
2471  lsm.SetModel<NOAHMP>();
2472  Print() << "Noah-MP land surface model!\n";
2473 #endif
2474  } else if (solverChoice.lsm_type == LandSurfaceType::OceanSurf) {
2475  lsm.SetModel<OceanSurf>();
2476  Print() << "OceanSurf land surface model!\n";
2477  } else if (solverChoice.lsm_type == LandSurfaceType::None) {
2478  lsm.SetModel<NullSurf>();
2479  Print() << "Null land surface model!\n";
2480  } else {
2481  Abort("Dont know this LandSurfaceType!") ;
2482  }
2483 
2484  if (verbose > 0) {
2485  solverChoice.display(max_level,pp_prefix);
2486  }
2487 
2488  ParameterSanityChecks();
2489 }
getEpochTime
AMREX_GPU_HOST AMREX_FORCE_INLINE std::time_t getEpochTime(const std::string &dateTime, const std::string &dateTimeFormat)
Definition: ERF_EpochTime.H:34
ERF::metgrid_basic_linear
bool metgrid_basic_linear
Definition: ERF.H:1371
ERF::metgrid_debug_msf
bool metgrid_debug_msf
Definition: ERF.H:1369
ERF::plot2d_file_2
std::string plot2d_file_2
Definition: ERF.H:1198
ERF::plot3d_file_1
std::string plot3d_file_1
Definition: ERF.H:1195
ERF::plot_rad
bool plot_rad
Definition: ERF.H:1007
ERF::m_plot_face_vels
bool m_plot_face_vels
Definition: ERF.H:1213
ERF::plot3d_file_2
std::string plot3d_file_2
Definition: ERF.H:1196
ERF::regrid_int
int regrid_int
Definition: ERF.H:1188
ERF::write_erfbdy
bool write_erfbdy
Definition: ERF.H:1380
ERF::metgrid_retain_sfc
bool metgrid_retain_sfc
Definition: ERF.H:1374
ERF::file_name_digits
int file_name_digits
Definition: ERF.H:1347
ERF::metgrid_use_sfc
bool metgrid_use_sfc
Definition: ERF.H:1373
ERF::metgrid_debug_quiescent
bool metgrid_debug_quiescent
Definition: ERF.H:1365
ERF::metgrid_interp_theta
bool metgrid_interp_theta
Definition: ERF.H:1370
ERF::regrid_level_0_on_restart
bool regrid_level_0_on_restart
Definition: ERF.H:1192
ERF::metgrid_force_sfc_k
int metgrid_force_sfc_k
Definition: ERF.H:1377
ERF::setSubVolVariables
void setSubVolVariables(const std::string &pp_subvol_var_names, amrex::Vector< std::string > &subvol_var_names)
Definition: ERF_WriteSubvolume.cpp:9
ERF::real_extrap_w
bool real_extrap_w
Definition: ERF.H:1359
ERF::metgrid_use_below_sfc
bool metgrid_use_below_sfc
Definition: ERF.H:1372
ERF::subvol_file
std::string subvol_file
Definition: ERF.H:1199
ERF::metgrid_proximity
amrex::Real metgrid_proximity
Definition: ERF.H:1375
ERF::plot2d_file_1
std::string plot2d_file_1
Definition: ERF.H:1197
ERF::metgrid_debug_dry
bool metgrid_debug_dry
Definition: ERF.H:1367
ERF::metgrid_debug_isothermal
bool metgrid_debug_isothermal
Definition: ERF.H:1366
ERF::use_real_time_in_pltname
bool use_real_time_in_pltname
Definition: ERF.H:1348
ERF::metgrid_debug_psfc
bool metgrid_debug_psfc
Definition: ERF.H:1368
ERF::ParameterSanityChecks
void ParameterSanityChecks()
Definition: ERF.cpp:2493
ERF::m_expand_plotvars_to_unif_rr
bool m_expand_plotvars_to_unif_rr
Definition: ERF.H:1200
ERF::check_file
std::string check_file
Definition: ERF.H:1222
ERF::metgrid_order
int metgrid_order
Definition: ERF.H:1376
ERF::plot_lsm
bool plot_lsm
Definition: ERF.H:1215
LandSurface::SetModel
void SetModel()
Definition: ERF_LandSurface.H:31
MM5
Definition: ERF_MM5.H:26
NOAHMP
Definition: ERF_NOAHMP.H:56
NullSurf
Definition: ERF_NullSurf.H:10
OceanSurf
Definition: ERF_OceanSurf.H:19
SLM
Definition: ERF_SLM.H:26
SolverChoice::display
void display(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:1021
SolverChoice::init_params
void init_params(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:143

Referenced by ERF_shared().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ ReadStormTrackerRestart()

void ERF::ReadStormTrackerRestart ( )
316 {
317  hurricane_eye_track_xy.clear();
318  hurricane_eye_track_latlon.clear();
319  hurricane_maxvel_vs_time.clear();
320  hurricane_minpressure_vs_time.clear();
321 
322  const fs::path base_dir("Output_StormTracker");
323 
324  // Nothing to do for a fresh run.
325  if (!fs::exists(base_dir)) {
326  return;
327  }
328  //
329  // Return the alphabetically last file in a directory.
330  // Since the filenames are zero-padded, this is also the newest output.
331  //
332  auto last_file = [](const fs::path& dir) -> fs::path
333  {
334  std::vector<fs::path> files;
335 
336  for (const auto& entry : fs::directory_iterator(dir)) {
337  if (entry.is_regular_file()) {
338  files.push_back(entry.path());
339  }
340  }
341 
342  if (files.empty()) {
343  return fs::path{};
344  }
345 
346  std::sort(files.begin(), files.end());
347 
348  return files.back();
349  };
350 
351  //==========================================================
352  // Read lat/lon file
353  //==========================================================
354 
355  {
356  fs::path file = last_file(base_dir / "latlon");
357 
358  if (!file.empty())
359  {
360  std::ifstream ifs(file);
361 
362  if (!ifs.is_open()) {
363  Abort("Could not open " + file.string());
364  }
365 
366  std::string line;
367 
368  // Skip the header line.
369  std::getline(ifs, line);
370 
371  Real lat, lon;
372 
373  while (ifs >> lat >> lon)
374  {
375  hurricane_eye_track_latlon.push_back({lat, lon});
376  }
377  }
378  }
379 
380  //==========================================================
381  // Read maxvel tracker file
382  //==========================================================
383 
384  {
385  fs::path file = last_file(base_dir / "maxvel");
386 
387  if (!file.empty())
388  {
389  std::ifstream ifs(file);
390 
391  if (!ifs.is_open()) {
392  Abort("Could not open " + file.string());
393  }
394 
395  std::string line;
396 
397  // Skip the header line.
398  std::getline(ifs, line);
399 
400  amrex::Real val1, val2;
401 
402  while (ifs >> val1 >> val2)
403  {
404  hurricane_maxvel_vs_time.push_back({val1, val2});
405  }
406  }
407  }
408 
409  //==========================================================
410  // Read minpressure tracker file
411  //==========================================================
412 
413  {
414  fs::path file = last_file(base_dir / "minpressure");
415 
416  if (!file.empty())
417  {
418  std::ifstream ifs(file);
419 
420  if (!ifs.is_open()) {
421  Abort("Could not open " + file.string());
422  }
423 
424  std::string line;
425 
426  // Skip the header line.
427  std::getline(ifs, line);
428 
429  amrex::Real val1, val2;
430 
431  while (ifs >> val1 >> val2)
432  {
433  hurricane_minpressure_vs_time.push_back({val1, val2});
434  }
435  }
436  }
437 
438  //==========================================================
439  // Read XY VTK file
440  //==========================================================
441 
442  {
443  fs::path file = last_file(base_dir / "xy");
444 
445  if (!file.empty())
446  {
447  std::ifstream ifs(file);
448  std::string line;
449 
450  // Skip the first four header lines.
451  for (int i = 0; i < 4; ++i) {
452  std::getline(ifs, line);
453  }
454 
455 
456  std::getline(ifs, line);
457  std::istringstream iss(line);
458  std::string keyword, datatype;
459  int npoints;
460 
461  iss >> keyword >> npoints >> datatype;
462  hurricane_eye_track_xy.reserve(npoints);
463 
464  for (int i = 0; i < npoints; ++i)
465  {
466  Real x, y, z;
467  ifs >> x >> y >> z;
468  hurricane_eye_track_xy.push_back({x, y});
469  }
470  }
471  }
472 }

◆ ReadVelsOnlyFromCheckpointFile()

void ERF::ReadVelsOnlyFromCheckpointFile ( int  lev_to_fill,
std::string &  chkfile_for_vels 
)

ERF function for reading data from a checkpoint file during restart.

1233 {
1234  Print() << "Reading vels only from native checkpoint " << chkfile_for_vels << " at level " << lev_to_fill << "\n";
1235 
1236  // Header
1237  std::string File(chkfile_for_vels + "/Header");
1238 
1239  VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
1240 
1241  Vector<char> fileCharPtr;
1242  ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
1243  std::string fileCharPtrString(fileCharPtr.dataPtr());
1244  std::istringstream is(fileCharPtrString, std::istringstream::in);
1245 
1246  AMREX_ALWAYS_ASSERT(lev_to_fill >= 0 && lev_to_fill <= finest_level);
1247 
1248  int lev = lev_to_fill;
1249 
1250  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
1251  VisMF::Read(xvel, MultiFabFileFullPrefix(lev, chkfile_for_vels, "Level_", "XFace"));
1252  MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
1253  vars_new[lev][Vars::xvel].setBndry(bogus_large_value);
1254 
1255  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
1256  VisMF::Read(yvel, MultiFabFileFullPrefix(lev, chkfile_for_vels, "Level_", "YFace"));
1257  MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
1258  vars_new[lev][Vars::yvel].setBndry(bogus_large_value);
1259 
1260  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
1261  VisMF::Read(zvel, MultiFabFileFullPrefix(lev, chkfile_for_vels, "Level_", "ZFace"));
1262  MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
1263  vars_new[lev][Vars::zvel].setBndry(bogus_large_value);
1264 }
Here is the call graph for this function:

◆ refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )
private

Function to define the refinement criteria based on user input

473 {
474  if (max_level > 0)
475  {
476  ParmParse pp(pp_prefix);
477  Vector<std::string> refinement_indicators;
478  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
479 
480  for (int i=0; i<refinement_indicators.size(); ++i)
481  {
482  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
483 
484  ParmParse ppr(ref_prefix);
485  RealBox real_box;
486  int lev_for_box;
487 
488  read_box_for_refinement(ref_prefix, lev_for_box, real_box);
489 
490  AMRErrorTagInfo info;
491 
492  if (real_box.ok()) {
493  info.SetRealBox(real_box);
494  }
495 
496  if (ppr.countval("start_time") > 0) {
497  double ref_min_time; ppr.get("start_time",ref_min_time);
498  info.SetMinTime(static_cast<Real>(ref_min_time));
499  }
500 
501  if (ppr.countval("end_time") > 0) {
502  double ref_max_time; ppr.get("end_time",ref_max_time);
503  info.SetMaxTime(static_cast<Real>(ref_max_time));
504  }
505 
506  if (ppr.countval("max_level") > 0) {
507  int ref_max_level; ppr.get("max_level",ref_max_level);
508  info.SetMaxLevel(ref_max_level);
509  }
510 
511  // Read field_name once and validate moisture-field requests against
512  // the active moisture model so an unsupported field aborts at setup
513  // rather than after the first regrid.
514  std::string field;
515  if (ppr.countval("field_name") > 0) {
516  ppr.get("field_name", field);
517  auto is_moist_field = [](const std::string& f) {
518  return f == "qv" || f == "qc" || f == "qi" ||
519  f == "qr" || f == "qs" || f == "qg" || f == "qt";
520  };
521  if (is_moist_field(field)) {
522  const auto& mi = solverChoice.moisture_indices;
523  int comp = -1;
524  if (field == "qv") { comp = mi.qv; }
525  else if (field == "qc") { comp = mi.qc; }
526  else if (field == "qi") { comp = mi.qi; }
527  else if (field == "qr") { comp = mi.qr; }
528  else if (field == "qs") { comp = mi.qs; }
529  else if (field == "qg") { comp = mi.qg; }
530  else if (field == "qt") {
531  comp = (mi.qc >= 0 || mi.qi >= 0 || mi.qr >= 0 ||
532  mi.qs >= 0 || mi.qg >= 0) ? 0 : -1;
533  }
534  if (comp < 0) {
535  amrex::Abort("Refinement field_name '" + field +
536  "' is not available for the configured moisture model");
537  }
538  }
539  }
540 
541  if (ppr.countval("value_greater")) {
542  int num_val = ppr.countval("value_greater");
543  Vector<Real> value(num_val);
544  ppr.getarr("value_greater",value,0,num_val);
545  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
546  }
547  else if (ppr.countval("value_less"))
548  {
549  int num_val = ppr.countval("value_less");
550  Vector<Real> value(num_val);
551  ppr.getarr("value_less",value,0,num_val);
552  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
553  }
554  else if (ppr.countval("adjacent_difference_greater"))
555  {
556  int num_val = ppr.countval("adjacent_difference_greater");
557  Vector<Real> value(num_val);
558  ppr.getarr("adjacent_difference_greater",value,0,num_val);
559  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
560  }
561  else if (real_box.ok())
562  {
563  ref_tags.push_back(AMRErrorTag(info));
564  }
565  else if ( (lev_for_box > 0) && (refinement_indicators[i] != "storm_tracker") )
566  {
567  Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
568  }
569  } // loop over criteria
570  } // if max_level > 0
571 }
ERF::read_box_for_refinement
void read_box_for_refinement(std::string &ref_prefix, int &lev_for_box, amrex::RealBox &real_box)
Definition: ERF_RefineBox.cpp:6

Referenced by ERF_shared().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ remake_zphys()

void ERF::remake_zphys ( int  lev,
std::unique_ptr< amrex::MultiFab > &  temp_zphys_nd 
)
817 {
818  if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
819  {
820  if (lev == 0) {
821  temp_zphys_nd->ParallelCopy(*z_phys_nd[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect(), z_phys_nd[lev]->nGrowVect());
822  } else {
823  //
824  // First interpolate from coarser level
825  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
826  // have been pre-filled - this includes ghost cells both inside and outside
827  // the domain
828  //
829  InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
830  IntVect(0,0,0), // do NOT fill ghost cells outside the domain
831  *z_phys_nd[lev-1], 0, 0, 1,
832  geom[lev-1], geom[lev],
833  refRatio(lev-1), &node_bilinear_interp,
834  domain_bcs_type, BCVars::cons_bc);
835 
836  // This recomputes the fine values using the bottom terrain at the fine resolution,
837  // and also fills values of z_phys_nd outside the domain
838  make_terrain_fitted_coords(lev,geom[lev],*temp_zphys_nd,zlevels_stag[lev],phys_bc_type);
839 
840  } // lev > 0
841 
842  std::swap(temp_zphys_nd, z_phys_nd[lev]);
843 
844  } else {
845  if (lev == 0) {
846  temp_zphys_nd->ParallelCopy(*z_phys_nd[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect(), z_phys_nd[lev]->nGrowVect());
847  } else {
848  //
849  // First interpolate from coarser level
850  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
851  // have been pre-filled - this includes ghost cells both inside and outside
852  // the domain
853  //
854  InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
855  z_phys_nd[lev]->nGrowVect(), // DO fill ghost cells outside the domain
856  *z_phys_nd[lev-1], 0, 0, 1,
857  geom[lev-1], geom[lev],
858  refRatio(lev-1), &node_bilinear_interp,
859  domain_bcs_type, BCVars::cons_bc);
860 
861  } // lev > 0
862 
863  std::swap(temp_zphys_nd, z_phys_nd[lev]);
864  }
865 
866  if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
867  solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
868  //
869  // This assumes we have already remade the EBGeometry
870  //
871  terrain_blanking[lev]->setVal(one);
872  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, z_phys_nd[lev]->nGrowVect());
873  }
874 
875  // Compute the min dz and pass to the micro model
876  Real dzmin = get_dzmin_terrain(*z_phys_nd[lev]);
877  micro->Set_dzmin(lev, dzmin);
878 }
Here is the call graph for this function:

◆ RemakeLevel()

void ERF::RemakeLevel ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
563 {
564  //
565  // Note that "time" here is elapsed time
566  //
567  if (verbose) {
568  amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
569  }
570 
571  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::MovingFittedMesh);
572 
573  BoxArray ba_old(vars_new[lev][Vars::cons].boxArray());
574  DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
575 
576  if (verbose) {
577  amrex::Print() <<" OLD BA AT LEVEL " << lev << " " << ba_old << std::endl;
578  }
579 
580  //
581  // Re-define subdomain at this level within the domain such that
582  // 1) all boxes in a given subdomain are "connected"
583  // 2) no boxes in a subdomain touch any boxes in any other subdomain
584  //
585  subdomains[lev].clear();
586  make_subdomains(ba.simplified_list(), subdomains[lev]);
587 
588  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
589  IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
590 
591  int ngrow_vels = ComputeGhostCells(solverChoice);
592 
593  Vector<MultiFab> temp_lev_new(Vars::NumTypes);
594  Vector<MultiFab> temp_lev_old(Vars::NumTypes);
595  MultiFab temp_base_state;
596 
597  std::unique_ptr<MultiFab> temp_zphys_nd;
598 
599  //********************************************************************************************
600  // This allocates all kinds of things, including but not limited to: solution arrays,
601  // terrain arrays and metrics, and base state.
602  // *******************************************************************************************
603  init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old, temp_base_state, temp_zphys_nd);
604 
605  // ********************************************************************************************
606  // Build the data structures for terrain-related quantities
607  // ********************************************************************************************
608  if ( solverChoice.terrain_type == TerrainType::EB ||
609  solverChoice.terrain_type == TerrainType::ImmersedForcing ||
610  solverChoice.buildings_type == BuildingsType::ImmersedForcing)
611  {
612  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
613  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
614  if (solverChoice.terrain_type == TerrainType::EB) {
615  eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
616  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing ||
617  solverChoice.buildings_type == BuildingsType::ImmersedForcing) {
618  eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
619  }
620  }
621  remake_zphys(lev, temp_zphys_nd);
622  update_terrain_arrays(lev);
623 
624  // ********************************************************************************************
625  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
626  // Note that this shouldn't be necessary because the fine grid is created by interpolation
627  // from the coarse ... but just in case ...
628  // ********************************************************************************************
629  if ( (SolverChoice::mesh_type != MeshType::ConstantDz) && (solverChoice.coupling_type == CouplingType::TwoWay) ) {
630  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
631  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
632  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
633  }
634  }
635 
636  // ********************************************************************************************
637  // Build the data structures for canopy model (depends upon z_phys)
638  // ********************************************************************************************
639  if (solverChoice.do_forest_drag) {
640  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
641  }
642 
643  // *****************************************************************************************************
644  // Create the physbcs objects (after initializing the terrain but before calling FillCoarsePatch
645  // *****************************************************************************************************
646  make_physbcs(lev);
647 
648  // ********************************************************************************************
649  // Update the base state at this level by interpolation from coarser level AND copy
650  // from previous (pre-regrid) base_state array
651  // ********************************************************************************************
652  if (lev > 0) {
653  Interpolater* mapper = &cell_cons_interp;
654 
655  Vector<MultiFab*> fmf = {&base_state[lev ], &base_state[lev ]};
656  Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
657  Vector<Real> ftime = {time, time};
658  Vector<Real> ctime = {time, time};
659 
660  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
661  FillPatchTwoLevels(temp_base_state, temp_base_state.nGrowVect(), IntVect(0,0,0),
662  time, cmf, ctime, fmf, ftime,
663  0, 0, temp_base_state.nComp(), geom[lev-1], geom[lev],
664  refRatio(lev-1), mapper, domain_bcs_type,
665  BaseBCVars::rho0_bc_comp);
666 
667  // Impose bc's outside the domain
668  (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
669 
670  // *************************************************************************************************
671  // This will fill the temporary MultiFabs with data from vars_new
672  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
673  // NOTE: we must create the new base state before calling FillPatch because we will
674  // interpolate perturbational quantities
675  // *************************************************************************************************
676  FillPatchFineLevel(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
677  &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
678  {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
679  base_state[lev], temp_base_state, false);
680  } else {
681  temp_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
682  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
683  temp_lev_new[Vars::cons].ParallelCopy(vars_new[lev][Vars::cons],0,0,ncomp_cons,ngrow_state,ngrow_state);
684  temp_lev_new[Vars::xvel].ParallelCopy(vars_new[lev][Vars::xvel],0,0, 1,ngrow_vels,ngrow_vels);
685  temp_lev_new[Vars::yvel].ParallelCopy(vars_new[lev][Vars::yvel],0,0, 1,ngrow_vels,ngrow_vels);
686 
687  temp_lev_new[Vars::zvel].setVal(0.);
688  temp_lev_new[Vars::zvel].ParallelCopy(vars_new[lev][Vars::zvel],0,0, 1,
689  IntVect(ngrow_vels,ngrow_vels,0),IntVect(ngrow_vels,ngrow_vels,0));
690  }
691 
692  // Now swap the pointers since we needed both old and new in the FillPatch
693  std::swap(temp_base_state, base_state[lev]);
694 
695  // ********************************************************************************************
696  // Copy from new into old just in case
697  // ********************************************************************************************
698  MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
699  MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0, 1,ngrow_vels);
700  MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0, 1,ngrow_vels);
701  MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0, 1,IntVect(ngrow_vels,ngrow_vels,0));
702 
703  // ********************************************************************************************
704  // Now swap the pointers
705  // ********************************************************************************************
706  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
707  std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
708  std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
709  }
710 
711  //
712  // Note that t_new = time here is elapsed time
713  //
714  t_new[lev] = time;
715  t_old[lev] = time - bogus_large_value;
716 
717  // ********************************************************************************************
718  // Build the data structures for calculating diffusive/turbulent terms
719  // ********************************************************************************************
720  update_diffusive_arrays(lev, ba, dm);
721 
722  //********************************************************************************************
723  // Microphysics
724  // *******************************************************************************************
725  int q_size = micro->Get_Qmoist_Size(lev);
726  qmoist[lev].resize(q_size);
727  micro->Define(lev, solverChoice);
728  if (solverChoice.moisture_type != MoistureType::None)
729  {
730  micro->Init(lev, vars_new[lev][Vars::cons],
731  grids[lev], Geom(lev), zero,
732  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
733  }
734  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
735  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
736  }
737 
738  //********************************************************************************************
739  // Radiation
740  // *******************************************************************************************
741  if (solverChoice.rad_type != RadiationType::None)
742  {
743  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
744  }
745 
746  // ********************************************************************************************
747  // Initialize the integrator class
748  // ********************************************************************************************
749  initialize_integrator(lev, vars_new[lev][Vars::cons], vars_new[lev][Vars::xvel]);
750 
751  // We need to re-define the FillPatcher if the grids have changed
752  if (lev > 0 && cf_width >= 0) {
753  bool ba_changed = (ba != ba_old);
754  bool dm_changed = (dm != dm_old);
755  if (ba_changed || dm_changed) {
756  Define_ERFFillPatchers(lev);
757  }
758  }
759 
760  // These calls are done in AmrCore::regrid if this is a regrid at lev > 0
761  // For a level 0 regrid we must explicitly do them here
762  if (lev == 0) {
763  // Define grids[lev] to be ba
764  SetBoxArray(lev, ba);
765 
766  // Define dmap[lev] to be dm
767  SetDistributionMap(lev, dm);
768  }
769 
770  // ********************************************************************************************
771  // Initialize the 2D data structures
772  // ********************************************************************************************
773  // NOTE: 2D MFs must be filled before SurfaceLayer is defined since SL class uses sst/tsk
774  // Clear the 2D arrays
775  if (sst_lev[lev][0]) {
776  for (int n = 0; n < sst_lev[lev].size(); n++) {
777  sst_lev[lev][n].reset();
778  }
779  }
780  if (tsk_lev[lev][0]) {
781  for (int n = 0; n < tsk_lev[lev].size(); n++) {
782  tsk_lev[lev][n].reset();
783  }
784  }
785  if (lat_m[lev]) {
786  lat_m[lev].reset();
787  }
788  if (lon_m[lev]) {
789  lon_m[lev].reset();
790  }
791  if (sinPhi_m[lev]) {
792  sinPhi_m[lev].reset();
793  }
794  if (cosPhi_m[lev]) {
795  cosPhi_m[lev].reset();
796  }
797 
798  //
799  // Interpolate the 2D arrays at the lower boundary. We assume that since we created
800  // them by interpolation it is ok just to recreate them by interpolation.
801  // Note that ba2d is constructed already in init_stuff, but we have not yet defined dmap[lev]
802  // so we must explicitly pass dm.
803  Interp2DArrays(lev,ba2d[lev],dm);
804 
805  // ********************************************************************************************
806  // Update the SurfaceLayer arrays at this level
807  // ********************************************************************************************
808  if (m_SurfaceLayer != nullptr) {
809  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
810  int nlevs = finest_level+1;
811  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
812  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
813  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
814  mfv_old, Theta_prim[lev], Qv_prim[lev],
815  Qr_prim[lev], z_phys_nd[lev],
816  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
817  lsm_data[lev], lsm_data_name, lsm_flux[lev], lsm_flux_name,
818  sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
819  }
820  }
821 
822  // ********************************************************************************************
823  // Set up the Rayleigh damping vectors at this (new) level
824  // ********************************************************************************************
825  if (solverChoice.dampingChoice.rayleigh_damp_U ||solverChoice.dampingChoice.rayleigh_damp_V ||
826  solverChoice.dampingChoice.rayleigh_damp_W ||solverChoice.dampingChoice.rayleigh_damp_T)
827  {
828  initRayleigh_at_level(lev);
829  }
830 
831  // Particle redistribute handled in timeStep() after regrid() completes.
832  // Calling it here causes stale-grid crashes.
833 }
ERF::remake_zphys
void remake_zphys(int lev, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:816
Here is the call graph for this function:

◆ restart()

void ERF::restart ( )
1748 {
1749  auto dRestartTime0 = amrex::second();
1750 
1751  ReadCheckpointFile();
1752 
1753  // Force regrid on level 0 if more procs than boxes are requested
1754  regrid_level_0_on_restart = ( regrid_level_0_on_restart ||
1755  grids[0].size() < ParallelDescriptor::NProcs() );
1756 
1757  if (regrid_level_0_on_restart) {
1758  //
1759  // Coarsening before we split the grids ensures that each resulting
1760  // grid will have an even number of cells in each direction.
1761  //
1762  BoxArray new_ba(amrex::coarsen(Geom(0).Domain(),2));
1763  //
1764  // Now split up into list of grids within max_grid_size[0] limit.
1765  //
1766  new_ba.maxSize(max_grid_size[0]/2);
1767  //
1768  // Now refine these boxes back to level zero
1769  //
1770  new_ba.refine(2);
1771 
1772  if (refine_grid_layout) {
1773  ChopGrids(0, new_ba, ParallelDescriptor::NProcs());
1774  }
1775 
1776  if (new_ba != grids[0]) {
1777  DistributionMapping new_dm(new_ba);
1778  RemakeLevel(0,static_cast<Real>(t_new[0]),new_ba,new_dm);
1779  }
1780  }
1781 
1782 #ifdef ERF_USE_PARTICLES
1783  // We call this here without knowing whether the particles have already been initialized or not
1784  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,t_new[0]);
1785 #endif
1786 
1787  double cur_time = t_new[0];
1788  if (m_check_per > zero) {last_check_file_time = cur_time;}
1789  if (m_plot2d_per_1 > zero) {last_plot2d_file_time_1 = std::floor(cur_time/m_plot2d_per_1) * m_plot2d_per_1;}
1790  if (m_plot2d_per_2 > zero) {last_plot2d_file_time_2 = std::floor(cur_time/m_plot2d_per_2) * m_plot2d_per_2;}
1791  if (m_plot3d_per_1 > zero) {last_plot3d_file_time_1 = std::floor(cur_time/m_plot3d_per_1) * m_plot3d_per_1;}
1792  if (m_plot3d_per_2 > zero) {last_plot3d_file_time_2 = std::floor(cur_time/m_plot3d_per_2) * m_plot3d_per_2;}
1793 
1794  if (m_check_int > zero) {last_check_file_step = istep[0];}
1795  if (m_plot2d_int_1 > zero) {last_plot2d_file_step_1 = istep[0];}
1796  if (m_plot2d_int_2 > zero) {last_plot2d_file_step_2 = istep[0];}
1797  if (m_plot3d_int_1 > zero) {last_plot3d_file_step_1 = istep[0];}
1798  if (m_plot3d_int_2 > zero) {last_plot3d_file_step_2 = istep[0];}
1799 
1800  if (verbose > 0)
1801  {
1802  auto dRestartTime = amrex::second() - dRestartTime0;
1803  ParallelDescriptor::ReduceRealMax(dRestartTime,ParallelDescriptor::IOProcessorNumber());
1804  amrex::Print() << "Restart time = " << dRestartTime << " seconds." << '\n';
1805  }
1806 }
ERF::RemakeLevel
void RemakeLevel(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:562
ERF::ReadCheckpointFile
void ReadCheckpointFile()
Definition: ERF_Checkpoint.cpp:520

◆ sample_lines()

void ERF::sample_lines ( int  lev,
double  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters
levCurrent level
timeCurrent time
cellIntVect containing the x,y-dimension indices to sample along z
mfMultiFab from which we sample the data
570 {
571  int ifile = 0;
572 
573  const int ncomp = mf.nComp(); // cell-centered state vars
574 
575  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
576  average_face_to_cellcenter(mf_vels, 0,
577  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
578 
579  //
580  // Sample the data at a line (in direction "dir") in space
581  // In this case we sample in the vertical direction so dir = 2
582  // The "k" value of "cell" is ignored
583  //
584  int dir = 2;
585  MultiFab my_line = get_line_data(mf, dir, cell);
586  MultiFab my_line_vels = get_line_data(mf_vels, dir, cell);
587  MultiFab my_line_tau11 = get_line_data(*Tau[lev][TauType::tau11], dir, cell);
588  MultiFab my_line_tau12 = get_line_data(*Tau[lev][TauType::tau12], dir, cell);
589  MultiFab my_line_tau13 = get_line_data(*Tau[lev][TauType::tau13], dir, cell);
590  MultiFab my_line_tau22 = get_line_data(*Tau[lev][TauType::tau22], dir, cell);
591  MultiFab my_line_tau23 = get_line_data(*Tau[lev][TauType::tau23], dir, cell);
592  MultiFab my_line_tau33 = get_line_data(*Tau[lev][TauType::tau33], dir, cell);
593 
594  for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
595  {
596  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
597 
598  std::ostream& sample_log = SampleLineLog(ifile);
599  if (sample_log.good()) {
600  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
601  const auto& my_line_arr = my_line[0].const_array();
602  const auto& my_line_vels_arr = my_line_vels[0].const_array();
603  const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
604  const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
605  const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
606  const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
607  const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
608  const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
609  const Box& my_box = my_line[0].box();
610  const int klo = my_box.smallEnd(2);
611  const int khi = my_box.bigEnd(2);
612  int i = cell[0];
613  int j = cell[1];
614  for (int n = 0; n < ncomp; n++) {
615  for (int k = klo; k <= khi; k++) {
616  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
617  }
618  }
619  for (int n = 0; n < AMREX_SPACEDIM; n++) {
620  for (int k = klo; k <= khi; k++) {
621  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
622  }
623  }
624  for (int k = klo; k <= khi; k++) {
625  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
626  }
627  for (int k = klo; k <= khi; k++) {
628  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
629  }
630  for (int k = klo; k <= khi; k++) {
631  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
632  }
633  for (int k = klo; k <= khi; k++) {
634  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
635  }
636  for (int k = klo; k <= khi; k++) {
637  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
638  }
639  for (int k = klo; k <= khi; k++) {
640  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
641  }
642  sample_log << std::endl;
643  } // if good
644  } // mfi
645 }
ERF::datwidth
const int datwidth
Definition: ERF.H:1148
ERF::SampleLineLog
AMREX_FORCE_INLINE std::ostream & SampleLineLog(int i)
Definition: ERF.H:1592
ERF::datprecision
const int datprecision
Definition: ERF.H:1149

◆ sample_points()

void ERF::sample_points ( int  lev,
double  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling MultiFab data at a specified cell index.

Parameters
levLevel for the associated MultiFab data
timeCurrent time
cellIntVect containing the indexes for the cell where we want to sample
mfMultiFab from which we wish to sample data
534 {
535  int ifile = 0;
536 
537  //
538  // Sample the data at a single point in space
539  //
540  int ncomp = mf.nComp();
541  Vector<Real> my_point = get_cell_data(mf, cell);
542 
543  if (!my_point.empty()) {
544 
545  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
546 
547  std::ostream& sample_log = SamplePointLog(ifile);
548  if (sample_log.good()) {
549  sample_log << std::setw(datwidth) << time;
550  for (int i = 0; i < ncomp; ++i)
551  {
552  sample_log << std::setw(datwidth) << my_point[i];
553  }
554  sample_log << std::endl;
555  } // if good
556  } // only write from processor that holds the cell
557 }
ERF::SamplePointLog
AMREX_FORCE_INLINE std::ostream & SamplePointLog(int i)
Definition: ERF.H:1578

◆ SampleLine()

amrex::IntVect& ERF::SampleLine ( int  i)
inlineprivate
1619  {
1620  return sampleline[i];
1621  }

◆ SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int  i)
inlineprivate
1593  {
1594  return *samplelinelog[i];
1595  }

◆ SampleLineLogName()

std::string ERF::SampleLineLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith samplelinelog file.

1752 { return samplelinelogname[i]; }

◆ SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int  i)
inlineprivate
1606  {
1607  return samplepoint[i];
1608  }

◆ SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int  i)
inlineprivate
1579  {
1580  return *sampleptlog[i];
1581  }

◆ SamplePointLogName()

std::string ERF::SamplePointLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith sampleptlog file.

1749 { return sampleptlogname[i]; }

◆ SetDriverAtmosToOceanStateContract()

void ERF::SetDriverAtmosToOceanStateContract ( bool  use_state_contract)
129 {
130  m_driver_atm2ocn_uses_state_contract = use_state_contract;
131 }

◆ setPlotVariables()

void ERF::setPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
15 {
16  ParmParse pp(pp_prefix);
17 
18  if (pp.contains(pp_plot_var_names.c_str()))
19  {
20  std::string nm;
21 
22  int nPltVars = pp.countval(pp_plot_var_names.c_str());
23 
24  for (int i = 0; i < nPltVars; i++)
25  {
26  pp.get(pp_plot_var_names.c_str(), nm, i);
27 
28  // Add the named variable to our list of plot variables
29  // if it is not already in the list
30  if (!containerHasElement(plot_var_names, nm)) {
31  plot_var_names.push_back(nm);
32  }
33  }
34  } else {
35  //
36  // The default is to add none of the variables to the list
37  //
38  plot_var_names.clear();
39  }
40 
41  // Get state variables in the same order as we define them,
42  // since they may be in any order in the input list
43  Vector<std::string> tmp_plot_names;
44 
45  for (int i = 0; i < cons_names.size(); ++i) {
46  if ( containerHasElement(plot_var_names, cons_names[i]) ) {
47  if (solverChoice.moisture_type == MoistureType::None) {
48  if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
49  cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
50  {
51  tmp_plot_names.push_back(cons_names[i]);
52  }
53  } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
54  if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
55  {
56  tmp_plot_names.push_back(cons_names[i]);
57  }
58  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
59  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
60  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
61  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
62  cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
63  {
64  tmp_plot_names.push_back(cons_names[i]);
65  }
66  } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
67  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow rhoQ1, rhoQ2, rhoQ4
68  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
69  {
70  tmp_plot_names.push_back(cons_names[i]);
71  }
72  } else
73  {
74  // For moisture_type SAM, Morrison and WSM6 we have all six variables
75  tmp_plot_names.push_back(cons_names[i]);
76  }
77  }
78  }
79 
80  // check for velocity since it's not in cons_names
81  // if we are asked for any velocity component, we will need them all
82  if (containerHasElement(plot_var_names, "x_velocity") ||
83  containerHasElement(plot_var_names, "y_velocity") ||
84  containerHasElement(plot_var_names, "z_velocity")) {
85  tmp_plot_names.push_back("x_velocity");
86  tmp_plot_names.push_back("y_velocity");
87  tmp_plot_names.push_back("z_velocity");
88  }
89 
90  //
91  // If the model we are running doesn't have the variable listed in the inputs file,
92  // just ignore it rather than aborting
93  //
94  for (int i = 0; i < derived_names.size(); ++i) {
95  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
96  bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing || solverChoice.buildings_type == BuildingsType::ImmersedForcing ) ||
97  (derived_names[i] != "terrain_IB_mask") );
98  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
99  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
100  (derived_names[i] != "detJ") );
101  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
102  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
103  (derived_names[i] != "z_phys") );
104 #ifndef ERF_USE_WINDFARM
105  ok_to_add &= (derived_names[i] != "SMark0" && derived_names[i] != "SMark1");
106 #endif
107  if (ok_to_add)
108  {
109  if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
110  if (derived_names[i] != "qv" && derived_names[i] != "qc" && derived_names[i] != "qrain" &&
111  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
112  derived_names[i] != "qt" && derived_names[i] != "qn" && derived_names[i] != "qp" &&
113  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
114  {
115  tmp_plot_names.push_back(derived_names[i]);
116  }
117  } else if ( (solverChoice.moisture_type == MoistureType::Kessler ) ||
118  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
119  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow qv, qc, qrain
120  if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
121  derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
122  {
123  tmp_plot_names.push_back(derived_names[i]);
124  }
125  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
126  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
127  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ||
128  (solverChoice.moisture_type == MoistureType::MoistNoCondensation) ) { // allow qv, qc
129  if (derived_names[i] != "qrain" && derived_names[i] != "qi" && derived_names[i] != "qsnow" &&
130  derived_names[i] != "qgraup" && derived_names[i] != "qp" &&
131  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
132  {
133  tmp_plot_names.push_back(derived_names[i]);
134  }
135  } else
136  {
137  // For moisture_type SAM and Morrison we have all moist quantities
138  tmp_plot_names.push_back(derived_names[i]);
139  }
140  } // use_terrain?
141  } // hasElement
142  }
143 
144 #ifdef ERF_USE_WINDFARM
145  for (int i = 0; i < derived_names.size(); ++i) {
146  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
147  if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP) {
148  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0") {
149  tmp_plot_names.push_back(derived_names[i]);
150  }
151  }
152  if( solverChoice.windfarm_type == WindFarmType::SimpleAD or
153  solverChoice.windfarm_type == WindFarmType::GeneralAD ) {
154  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0" or derived_names[i] == "SMark1") {
155  tmp_plot_names.push_back(derived_names[i]);
156  }
157  }
158  }
159  }
160 #endif
161 
162 #ifdef ERF_USE_PARTICLES
163  const auto& particles_namelist( particleData.getNamesUnalloc() );
164  for (auto it = particles_namelist.cbegin(); it != particles_namelist.cend(); ++it) {
165  std::string tmp( (*it)+"_count" );
166  if (containerHasElement(plot_var_names, tmp) ) {
167  tmp_plot_names.push_back(tmp);
168  }
169  }
170 #endif
171 
172  plot_var_names = tmp_plot_names;
173 }
ERF::derived_names
const amrex::Vector< std::string > derived_names
Definition: ERF.H:1241
ERF::cons_names
const amrex::Vector< std::string > cons_names
Definition: ERF.H:1232

Referenced by ERF_shared().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ setPlotVariables2D()

void ERF::setPlotVariables2D ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
110 {
111  ParmParse pp(pp_prefix);
112 
113  if (!pp.contains(pp_plot_var_names.c_str())) {
114  //
115  // The default is to add none of the variables to the list
116  //
117  plot_var_names.clear();
118  return;
119  }
120 
121  Vector<std::string> requested_plot_names;
122  std::string nm;
123  const int nPltVars = pp.countval(pp_plot_var_names.c_str());
124  for (int i = 0; i < nPltVars; ++i) {
125  pp.get(pp_plot_var_names.c_str(), nm, i);
126  requested_plot_names.push_back(nm);
127  }
128 
129  const bool has_surface_layer =
130  phys_bc_type[Orientation(Direction::z, Orientation::low)] == ERF_BC::surface_layer;
131  const auto active_lsm_names = lsm.Get_DataNames();
132  const auto available_names = plotfile2d::available_diagnostic_names(solverChoice,
133  has_surface_layer,
134  active_lsm_names);
135 
136  // Keep the canonical built-in 2D ordering so the plotfile component layout
137  // stays stable even if the input request order changes.
138  const auto selection = plotfile2d::select_requested_plot_variables(requested_plot_names,
139  available_names);
140  plot_var_names = selection.accepted;
141 
142  // Unknown 2D names are skipped rather than aborting because the 2D plot
143  // list is intentionally user-configurable and may include names that are not
144  // compiled into a given build. The warning is still explicit so the user
145  // can correct the input deck.
146  warn_for_unavailable_2d_plot_vars(plotfile2d::format_plot2d_parameter_name(pp_prefix, pp_plot_var_names),
147  selection.unavailable, available_names);
148 }
LandSurface::Get_DataNames
amrex::Vector< std::string > Get_DataNames()
Definition: ERF_LandSurface.H:136
plotfile2d::format_plot2d_parameter_name
std::string format_plot2d_parameter_name(const std::string &pp_prefix, const std::string &parameter_name)
Definition: ERF_Plotfile2DUtils.cpp:75
plotfile2d::select_requested_plot_variables
PlotVariableSelection select_requested_plot_variables(const amrex::Vector< std::string > &requested, const amrex::Vector< std::string > &available)
Definition: ERF_Plotfile2DUtils.cpp:31
plotfile2d::available_diagnostic_names
amrex::Vector< std::string > available_diagnostic_names(const SolverChoice &solver_choice)
Definition: ERF_Plotfile2DWaterPath.cpp:139
Here is the call graph for this function:

◆ setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool  restarting)
private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
84 {
85  // If we are restarting then we haven't read the input_sounding file yet
86  // so we need to read it here
87  // TODO: should we store this information in the checkpoint file instead?
88  bool is_moist = (solverChoice.moisture_type != MoistureType::None);
89  if (restarting) {
90  input_sounding_data.resize_arrays();
91  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
92  input_sounding_data.read_from_file(geom[0], zlevels_stag[0], n, is_moist);
93  }
94  }
95 
96  const Real* z_inp_sound = input_sounding_data.z_inp_sound[0].dataPtr();
97  const Real* U_inp_sound = input_sounding_data.U_inp_sound[0].dataPtr();
98  const Real* V_inp_sound = input_sounding_data.V_inp_sound[0].dataPtr();
99  const Real* theta_inp_sound = input_sounding_data.theta_inp_sound[0].dataPtr();
100  const int inp_sound_size = input_sounding_data.size(0);
101 
102  int refine_fac{1};
103  for (int lev = 0; lev <= finest_level; lev++)
104  {
105  const int klo = geom[lev].Domain().smallEnd(2);
106  const int khi = geom[lev].Domain().bigEnd(2);
107  const int Nz = khi - klo + 1;
108 
109  Vector<Real> zcc(Nz);
110  Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
111  zlevels_stag[0].begin()+khi/refine_fac+2);
112  expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
113 #if 0
114  amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
115  for (int k = 0; k < zlevels_sub.size(); k++) { amrex::AllPrint() << zlevels_sub[k] << " "; }
116  amrex::AllPrint() << " --> ";
117  for (int k = 0; k < Nz; k++) { amrex::AllPrint() << zcc[k] << " "; }
118  amrex::AllPrint() << std::endl;
119 #endif
120 
121  for (int k = 0; k < Nz; k++)
122  {
123  h_rayleigh_ptrs[lev][Rayleigh::ubar][k] = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
124  h_rayleigh_ptrs[lev][Rayleigh::vbar][k] = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
125  h_rayleigh_ptrs[lev][Rayleigh::wbar][k] = zero;
126  h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
127  }
128 
129  // Copy from host version to device version
130  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
131  d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
132  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
133  d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
134  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
135  d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
136  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
137  d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
138 
139  if (lev < finest_level) {
140  refine_fac *= ref_ratio[lev][2];
141  }
142  }
143 }
expand_and_interpolate_1d
AMREX_FORCE_INLINE void expand_and_interpolate_1d(amrex::Vector< amrex::Real > &znew, const amrex::Vector< amrex::Real > &zorig, int refine_fac, bool destag=false)
Definition: ERF_Interpolation_1D.H:89
InputSoundingData::theta_inp_sound
amrex::Vector< amrex::Vector< amrex::Real > > theta_inp_sound
Definition: ERF_InputSoundingData.H:415
InputSoundingData::z_inp_sound
amrex::Vector< amrex::Vector< amrex::Real > > z_inp_sound
Definition: ERF_InputSoundingData.H:415
InputSoundingData::U_inp_sound
amrex::Vector< amrex::Vector< amrex::Real > > U_inp_sound
Definition: ERF_InputSoundingData.H:415
InputSoundingData::V_inp_sound
amrex::Vector< amrex::Vector< amrex::Real > > V_inp_sound
Definition: ERF_InputSoundingData.H:415
InputSoundingData::size
int size(int itime) const
Definition: ERF_InputSoundingData.H:390
Here is the call graph for this function:

◆ setRecordDataInfo()

void ERF::setRecordDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1649  {
1650  if (amrex::ParallelDescriptor::IOProcessor())
1651  {
1652  datalog[i] = std::make_unique<std::fstream>();
1653  datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1654  if (!datalog[i]->good()) {
1655  amrex::FileOpenFailed(filename);
1656  }
1657  }
1658  amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
1659  }

◆ setRecordDerDataInfo()

void ERF::setRecordDerDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1662  {
1663  if (amrex::ParallelDescriptor::IOProcessor())
1664  {
1665  der_datalog[i] = std::make_unique<std::fstream>();
1666  der_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1667  if (!der_datalog[i]->good()) {
1668  amrex::FileOpenFailed(filename);
1669  }
1670  }
1671  amrex::ParallelDescriptor::Barrier("ERF::setRecordDerDataInfo");
1672  }

◆ setRecordEnergyDataInfo()

void ERF::setRecordEnergyDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1675  {
1676  if (amrex::ParallelDescriptor::IOProcessor())
1677  {
1678  tot_e_datalog[i] = std::make_unique<std::fstream>();
1679  tot_e_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1680  if (!tot_e_datalog[i]->good()) {
1681  amrex::FileOpenFailed(filename);
1682  }
1683  }
1684  amrex::ParallelDescriptor::Barrier("ERF::setRecordEnergyDataInfo");
1685  }

◆ setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1705  {
1706  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1707  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1708  {
1709  const amrex::Box& bx = mfi.validbox();
1710  if (bx.contains(cell)) {
1711  samplelinelog[i] = std::make_unique<std::fstream>();
1712  samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1713  if (!samplelinelog[i]->good()) {
1714  amrex::FileOpenFailed(filename);
1715  }
1716  }
1717  }
1718  amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
1719  }

◆ setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1688  {
1689  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1690  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1691  {
1692  const amrex::Box& bx = mfi.validbox();
1693  if (bx.contains(cell)) {
1694  sampleptlog[i] = std::make_unique<std::fstream>();
1695  sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1696  if (!sampleptlog[i]->good()) {
1697  amrex::FileOpenFailed(filename);
1698  }
1699  }
1700  }
1701  amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
1702  }

◆ setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool  restarting)
private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
66 {
67  // If we are restarting then we haven't read the input_sponge file yet
68  // so we need to read it here
69  // TODO: should we store this information in the checkpoint file instead?
70  if (restarting) {
71  input_sponge_data.read_from_file(geom[0], zlevels_stag[0]);
72  }
73 
74  const Real* z_inp_sponge = input_sponge_data.z_inp_sponge.dataPtr();
75  const Real* U_inp_sponge = input_sponge_data.U_inp_sponge.dataPtr();
76  const Real* V_inp_sponge = input_sponge_data.V_inp_sponge.dataPtr();
77  const int inp_sponge_size = input_sponge_data.size();
78 
79  for (int lev = 0; lev <= finest_level; lev++)
80  {
81  const int khi = geom[lev].Domain().bigEnd()[2];
82  Vector<Real> zcc(khi+1);
83 
84  if (z_phys_cc[lev]) {
85  // use_terrain=1
86  // calculate the damping strength based on the max height at each k
87  reduce_to_max_per_height(zcc, z_phys_cc[lev]);
88  } else {
89  const auto *const prob_lo = geom[lev].ProbLo();
90  const auto *const dx = geom[lev].CellSize();
91  for (int k = 0; k <= khi; k++)
92  {
93  zcc[k] = prob_lo[2] + (k+myhalf) * dx[2];
94  }
95  }
96 
97  for (int k = 0; k <= khi; k++)
98  {
99  h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
100  h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
101  }
102 
103  // Copy from host version to device version
104  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
105  d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
106  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
107  d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
108  }
109 }
reduce_to_max_per_height
AMREX_FORCE_INLINE void reduce_to_max_per_height(amrex::Vector< amrex::Real > &v, std::unique_ptr< amrex::MultiFab > &mf)
Definition: ERF_ParFunctions.H:8
InputSpongeData::V_inp_sponge
amrex::Vector< amrex::Real > V_inp_sponge
Definition: ERF_InputSpongeData.H:111
InputSpongeData::z_inp_sponge
amrex::Vector< amrex::Real > z_inp_sponge
Definition: ERF_InputSpongeData.H:111
InputSpongeData::U_inp_sponge
amrex::Vector< amrex::Real > U_inp_sponge
Definition: ERF_InputSpongeData.H:111
InputSpongeData::size
int size() const
Definition: ERF_InputSpongeData.H:99
Here is the call graph for this function:

◆ setSubVolVariables()

void ERF::setSubVolVariables ( const std::string &  pp_subvol_var_names,
amrex::Vector< std::string > &  subvol_var_names 
)
private
11 {
12  ParmParse pp(pp_prefix);
13 
14  std::string nm;
15 
16  int nSubVolVars = pp.countval(pp_subvol_var_names.c_str());
17 
18  // We pre-populate the list with velocities, but allow these to be over-written
19  // by user input
20  if (nSubVolVars == 0)
21  {
22  subvol_var_names.push_back("x_velocity");
23  subvol_var_names.push_back("y_velocity");
24  subvol_var_names.push_back("z_velocity");
25 
26  } else {
27  for (int i = 0; i < nSubVolVars; i++)
28  {
29  pp.get(pp_subvol_var_names.c_str(), nm, i);
30 
31  // Add the named variable to our list of subvol variables
32  // if it is not already in the list
33  if (!containerHasElement(subvol_var_names, nm)) {
34  subvol_var_names.push_back(nm);
35  }
36  }
37  }
38 
39  // Get state variables in the same order as we define them,
40  // since they may be in any order in the input list
41  Vector<std::string> tmp_plot_names;
42 
43  for (int i = 0; i < cons_names.size(); ++i) {
44  if ( containerHasElement(subvol_var_names, cons_names[i]) ) {
45  if (solverChoice.moisture_type == MoistureType::None) {
46  if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
47  cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
48  {
49  tmp_plot_names.push_back(cons_names[i]);
50  }
51  } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
52  if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
53  {
54  tmp_plot_names.push_back(cons_names[i]);
55  }
56  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
57  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
58  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
59  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
60  cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
61  {
62  tmp_plot_names.push_back(cons_names[i]);
63  }
64  } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
65  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow rhoQ1, rhoQ2, rhoQ4
66  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
67  {
68  tmp_plot_names.push_back(cons_names[i]);
69  }
70  } else
71  {
72  // For moisture_type SAM and Morrison we have all six variables
73  tmp_plot_names.push_back(cons_names[i]);
74  }
75  }
76  }
77 
78  // Check for velocity since it's not in cons_names
79  if (containerHasElement(subvol_var_names, "x_velocity")) {
80  tmp_plot_names.push_back("x_velocity");
81  }
82  if (containerHasElement(subvol_var_names, "y_velocity")) {
83  tmp_plot_names.push_back("y_velocity");
84  }
85  if (containerHasElement(subvol_var_names, "z_velocity")) {
86  tmp_plot_names.push_back("z_velocity");
87  }
88 
89  //
90  // If the model we are running doesn't have the variable listed in the inputs file,
91  // just ignore it rather than aborting
92  //
93  for (int i = 0; i < derived_subvol_names.size(); ++i) {
94  if ( containerHasElement(subvol_var_names, derived_names[i]) ) {
95  bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing) ||
96  (derived_names[i] != "terrain_IB_mask") );
97  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
98  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
99  (derived_names[i] != "detJ") );
100  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
101  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
102  (derived_names[i] != "z_phys") );
103  if (ok_to_add)
104  {
105  if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
106  if (derived_names[i] != "qv" && derived_names[i] != "qc" && derived_names[i] != "qrain" &&
107  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
108  derived_names[i] != "qt" && derived_names[i] != "qn" && derived_names[i] != "qp" &&
109  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
110  {
111  tmp_plot_names.push_back(derived_names[i]);
112  }
113  } else if ( (solverChoice.moisture_type == MoistureType::Kessler ) ||
114  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
115  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow qv, qc, qrain
116  if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
117  derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
118  {
119  tmp_plot_names.push_back(derived_names[i]);
120  }
121  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
122  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
123  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
124  if (derived_names[i] != "qrain" &&
125  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
126  derived_names[i] != "qp" &&
127  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
128  {
129  tmp_plot_names.push_back(derived_names[i]);
130  }
131  } else
132  {
133  // For moisture_type SAM and Morrison we have all moist quantities
134  tmp_plot_names.push_back(derived_names[i]);
135  }
136  } // use_terrain?
137  } // hasElement
138  }
139 
140  subvol_var_names = tmp_plot_names;
141 }
ERF::derived_subvol_names
const amrex::Vector< std::string > derived_subvol_names
Definition: ERF.H:1292
Here is the call graph for this function:

◆ solve_with_gmres()

void ERF::solve_with_gmres ( int  lev,
const amrex::Box &  subdomain,
amrex::MultiFab &  rhs,
amrex::MultiFab &  p,
amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &  fluxes,
amrex::MultiFab &  ax_sub,
amrex::MultiFab &  ay_sub,
amrex::MultiFab &  az_sub,
amrex::MultiFab &  ,
amrex::MultiFab &  znd_sub 
)

Solve the Poisson equation using FFT-preconditioned GMRES

16 {
17 #ifdef ERF_USE_FFT
18  BL_PROFILE("ERF::solve_with_gmres()");
19 
20  Real reltol = solverChoice.poisson_reltol;
21  Real abstol = solverChoice.poisson_abstol;
22 
23  auto const dom_lo = lbound(Geom(lev).Domain());
24  auto const dom_hi = ubound(Geom(lev).Domain());
25 
26  auto const sub_lo = lbound(subdomain);
27  auto const sub_hi = ubound(subdomain);
28 
29  auto dx = Geom(lev).CellSizeArray();
30 
31  Geometry my_geom;
32 
33  Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
34  if (Geom(lev).isPeriodic(0) && sub_lo.x == dom_lo.x && sub_hi.x == dom_hi.x) { is_per[0] = 1;}
35  if (Geom(lev).isPeriodic(1) && sub_lo.y == dom_lo.y && sub_hi.y == dom_hi.y) { is_per[1] = 1;}
36 
37  int coord_sys = 0;
38 
39  // If subdomain == domain then we pass Geom(lev) to the FFT solver
40  if (subdomain == Geom(lev).Domain()) {
41  my_geom.define(Geom(lev).Domain(), Geom(lev).ProbDomain(), coord_sys, is_per);
42  } else {
43  // else we create a new geometry based only on the subdomain
44  // The information in my_geom used by the FFT routines is:
45  // 1) my_geom.Domain()
46  // 2) my_geom.CellSize()
47  // 3) my_geom.isAllPeriodic() / my_geom.periodicity()
48  RealBox rb( sub_lo.x *dx[0], sub_lo.y *dx[1], sub_lo.z *dx[2],
49  (sub_hi.x+1)*dx[0], (sub_hi.y+1)*dx[1], (sub_hi.z+1)*dx[2]);
50  my_geom.define(subdomain, rb, coord_sys, is_per);
51  }
52 
53  amrex::GMRES<MultiFab, TerrainPoisson> gmsolver;
54 
55  TerrainPoisson tp(my_geom, rhs.boxArray(), rhs.DistributionMap(), domain_bc_type,
56  stretched_dz_d[lev], ax_sub, ay_sub, az_sub, dJ_sub, &znd_sub,
57  solverChoice.use_real_bcs);
58 
59  gmsolver.define(tp);
60 
61  gmsolver.setVerbose(mg_verbose);
62 
63  gmsolver.setRestartLength(50);
64 
65  tp.usePrecond(true);
66 
67  gmsolver.solve(phi, rhs, reltol, abstol);
68 
69  tp.getFluxes(phi, fluxes);
70 
71  for (MFIter mfi(phi); mfi.isValid(); ++mfi)
72  {
73  Box xbx = mfi.nodaltilebox(0);
74  Box ybx = mfi.nodaltilebox(1);
75  const Array4<Real >& fx_ar = fluxes[0].array(mfi);
76  const Array4<Real >& fy_ar = fluxes[1].array(mfi);
77  const Array4<Real const>& mf_ux = mapfac[lev][MapFacType::u_x]->const_array(mfi);
78  const Array4<Real const>& mf_vy = mapfac[lev][MapFacType::v_y]->const_array(mfi);
79  ParallelFor(xbx,ybx,
80  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
81  {
82  fx_ar(i,j,k) *= mf_ux(i,j,0);
83  },
84  [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
85  {
86  fy_ar(i,j,k) *= mf_vy(i,j,0);
87  });
88  } // mfi
89 #else
90  amrex::ignore_unused(lev, rhs, phi, fluxes, ax_sub, ay_sub, az_sub, dJ_sub, znd_sub);
91 #endif
92 
93  // ****************************************************************************
94  // Impose bc's on pprime
95  // ****************************************************************************
96  ImposeBCsOnPhi(lev, phi, subdomain);
97 }
ERF::ImposeBCsOnPhi
void ImposeBCsOnPhi(int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
Definition: ERF_ImposeBCsOnPhi.cpp:12
Here is the call graph for this function:

◆ sum_derived_quantities()

void ERF::sum_derived_quantities ( double  time)
179 {
180  if (verbose <= 0 || NumDerDataLogs() <= 0) return;
181 
182  int lev = 0;
183 
184  AMREX_ALWAYS_ASSERT(lev == 0);
185 
186  auto& mfx0 = *mapfac[0][MapFacType::m_x];
187  auto& mfy0 = *mapfac[0][MapFacType::m_x];
188  auto& dJ0 = *detJ_cc[0];
189 
190  // ************************************************************************
191  // WARNING: we are not filling ghost cells other than periodic outside the domain
192  // ************************************************************************
193 
194  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
195  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
196 
197  // Average all three components of velocity (on faces) to the cell center
198  average_face_to_cellcenter(mf_cc_vel,0,
199  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
200  &vars_new[lev][Vars::yvel],
201  &vars_new[lev][Vars::zvel]});
202  mf_cc_vel.FillBoundary(geom[lev].periodicity());
203 
204  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
205  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
206  }
207 
208  MultiFab r_wted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
209  MultiFab unwted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
210  MultiFab enstrophysq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
211  MultiFab theta_mf(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
212 
213 #ifdef _OPENMP
214 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
215 #endif
216  for (MFIter mfi(unwted_magvelsq, TilingIfNotGPU()); mfi.isValid(); ++mfi)
217  {
218  const Box& bx = mfi.tilebox();
219  auto& src_fab = mf_cc_vel[mfi];
220 
221  auto& dest1_fab = unwted_magvelsq[mfi];
222  // NOTE: we send in src_fab where we should
223  derived::erf_dermagvelsq(bx, dest1_fab, 0, 1, src_fab, (*z_phys_cc[lev])[mfi], Geom(lev),
224  static_cast<Real>(t_new[0]), nullptr, lev);
225 
226  auto& dest2_fab = enstrophysq[mfi];
227  derived::erf_derenstrophysq(bx, dest2_fab, 0, 1, src_fab, (*z_phys_cc[lev])[mfi], Geom(lev),
228  static_cast<Real>(t_new[0]), nullptr, lev);
229  }
230 
231  // Copy the MF holding 1/2(u^2 + v^2 + w^2) into the MF that will hold 1/2 rho (u^2 + v^2 + w^2)d
232  MultiFab::Copy(r_wted_magvelsq, unwted_magvelsq, 0, 0, 1, 0);
233 
234  // Multiply the MF holding 1/2(u^2 + v^2 + w^2) by rho to get 1/2 rho (u^2 + v^2 + w^2)
235  MultiFab::Multiply(r_wted_magvelsq, vars_new[lev][Vars::cons], 0, 0, 1, 0);
236 
237  // Copy the MF holding (rho theta) into "theta_mf"
238  MultiFab::Copy(theta_mf, vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, 0);
239 
240  // Divide (rho theta) by rho to get theta in the MF "theta_mf"
241  MultiFab::Divide(theta_mf, vars_new[lev][Vars::cons], Rho_comp, 0, 1, 0);
242 
243  Real unwted_avg = volWgtSumMF(lev, unwted_magvelsq, 0, dJ0, mfx0, mfy0, false);
244  Real r_wted_avg = volWgtSumMF(lev, r_wted_magvelsq, 0, dJ0, mfx0, mfy0, false);
245  Real enstrsq_avg = volWgtSumMF(lev, enstrophysq, 0, dJ0, mfx0, mfy0, false);
246  Real theta_avg = volWgtSumMF(lev, theta_mf, 0, dJ0, mfx0, mfy0, false);
247 
248  // Get volume including terrain (consistent with volWgtSumMF routine)
249  MultiFab volume(grids[lev], dmap[lev], 1, 0);
250  auto const& dx = geom[lev].CellSizeArray();
251  Real cell_vol = dx[0]*dx[1]*dx[2];
252  volume.setVal(cell_vol);
253  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
254  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
255  }
256 #ifdef _OPENMP
257 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
258 #endif
259  for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
260  {
261  const Box& tbx = mfi.tilebox();
262  auto dst = volume.array(mfi);
263  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
264  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
265  ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
266  {
267  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
268  });
269  }
270  Real vol = volume.sum();
271 
272  unwted_avg /= vol;
273  r_wted_avg /= vol;
274  enstrsq_avg /= vol;
275  theta_avg /= vol;
276 
277  const int nfoo = 4;
278  Real foo[nfoo] = {unwted_avg,r_wted_avg,enstrsq_avg,theta_avg};
279  Real zero_d = zero;
280 #ifdef AMREX_LAZY
281  Lazy::QueueReduction([=]() mutable {
282 #endif
283  ParallelDescriptor::ReduceRealSum(
284  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
285 
286  if (ParallelDescriptor::IOProcessor()) {
287  int i = 0;
288  unwted_avg = foo[i++];
289  r_wted_avg = foo[i++];
290  enstrsq_avg = foo[i++];
291  theta_avg = foo[i++];
292 
293  std::ostream& data_log_der = DerDataLog(0);
294 
295  if (time == zero_d) {
296  data_log_der << std::setw(datwidth) << " time";
297  data_log_der << std::setw(datwidth) << " ke_den";
298  data_log_der << std::setw(datwidth) << " velsq";
299  data_log_der << std::setw(datwidth) << " enstrophy";
300  data_log_der << std::setw(datwidth) << " int_energy";
301  data_log_der << std::endl;
302  }
303  data_log_der << std::setw(datwidth) << std::setprecision(timeprecision) << time;
304  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << unwted_avg;
305  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << r_wted_avg;
306  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << enstrsq_avg;
307  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << theta_avg;
308  data_log_der << std::endl;
309 
310  } // if IOProcessor
311 #ifdef AMREX_LAZY
312  }
313 #endif
314 }
ERF::DerDataLog
AMREX_FORCE_INLINE std::ostream & DerDataLog(int i)
Definition: ERF.H:1556
ERF::timeprecision
const int timeprecision
Definition: ERF.H:1150
ERF::NumDerDataLogs
AMREX_FORCE_INLINE int NumDerDataLogs() noexcept
Definition: ERF.H:1570
derived::erf_dermagvelsq
void erf_dermagvelsq(const Box &bx, FArrayBox &derfab, int dcomp, int ncomp, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:507
derived::erf_derenstrophysq
void erf_derenstrophysq(const Box &bx, FArrayBox &derfab, int dcomp, int ncomp, const FArrayBox &datfab, const FArrayBox &zcc_fab, const Geometry &geomdata, Real, const int *, const int)
Definition: ERF_Derive.cpp:443
Here is the call graph for this function:

◆ sum_energy_quantities()

void ERF::sum_energy_quantities ( double  time)
318 {
319  if ( (verbose <= 0) || (tot_e_datalog.size() < 1) ) { return; }
320 
321  int lev = 0;
322 
323  auto& mfx0 = *mapfac[0][MapFacType::m_x];
324  auto& mfy0 = *mapfac[0][MapFacType::m_x];
325  auto& dJ0 = *detJ_cc[0];
326 
327  AMREX_ALWAYS_ASSERT(lev == 0);
328 
329  bool local = true;
330 
331  // ************************************************************************
332  // WARNING: we are not filling ghost cells other than periodic outside the domain
333  // ************************************************************************
334 
335  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
336  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
337 
338  // Average all three components of velocity (on faces) to the cell center
339  average_face_to_cellcenter(mf_cc_vel,0,
340  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
341  &vars_new[lev][Vars::yvel],
342  &vars_new[lev][Vars::zvel]});
343  mf_cc_vel.FillBoundary(geom[lev].periodicity());
344 
345  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
346  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
347  }
348 
349  MultiFab tot_mass (grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
350  MultiFab tot_energy(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
351 
352  auto const& dx = geom[lev].CellSizeArray();
353  bool is_moist = (solverChoice.moisture_type != MoistureType::None);
354 
355 #ifdef _OPENMP
356 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
357 #endif
358  for (MFIter mfi(tot_mass, TilingIfNotGPU()); mfi.isValid(); ++mfi)
359  {
360  const Box& bx = mfi.tilebox();
361 
362  const Array4<Real>& cc_vel_arr = mf_cc_vel.array(mfi);
363  const Array4<Real>& tot_mass_arr = tot_mass.array(mfi);
364  const Array4<Real>& tot_energy_arr = tot_energy.array(mfi);
365  const Array4<const Real>& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
366  const Array4<const Real>& z_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) :
367  Array4<const Real>{};
368  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
369  {
370  Real Qv = (is_moist) ? cons_arr(i,j,k,RhoQ1_comp) : zero;
371  Real Qc = (is_moist) ? cons_arr(i,j,k,RhoQ2_comp) : zero;
372  Real Qt = Qv + Qc;
373  Real Rhod = cons_arr(i,j,k,Rho_comp);
374  Real Rhot = Rhod * (one + Qt);
375  Real Temp = getTgivenRandRTh(Rhod, cons_arr(i,j,k,RhoTheta_comp), Qv);
376  Real TKE = myhalf * ( cc_vel_arr(i,j,k,0)*cc_vel_arr(i,j,k,0)
377  + cc_vel_arr(i,j,k,1)*cc_vel_arr(i,j,k,1)
378  + cc_vel_arr(i,j,k,2)*cc_vel_arr(i,j,k,2) );
379  Real zval = (z_arr) ? z_arr(i,j,k) : Real(k)*dx[2];
380 
381  Real Cv = Cp_d - R_d;
382  Real Cvv = Cp_v - R_v;
383  Real Cpv = Cp_v;
384 
385  tot_mass_arr(i,j,k) = Rhot;
386  tot_energy_arr(i,j,k) = Rhod * ( (Cv + Cvv*Qv + Cpv*Qc)*Temp - L_v*Qc
387  + (one + Qt)*TKE + (one + Qt)*CONST_GRAV*zval );
388 
389  });
390 
391  }
392 
393  Real tot_mass_avg = volWgtSumMF(lev, tot_mass , 0, dJ0, mfx0, mfy0, false, local);
394  Real tot_energy_avg = volWgtSumMF(lev, tot_energy, 0, dJ0, mfx0, mfy0, false, local);
395 
396  // Get volume including terrain (consistent with volWgtSumMF routine)
397  MultiFab volume(grids[lev], dmap[lev], 1, 0);
398  Real cell_vol = dx[0]*dx[1]*dx[2];
399  volume.setVal(cell_vol);
400  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
401  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
402  }
403 #ifdef _OPENMP
404 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
405 #endif
406  for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
407  {
408  const Box& tbx = mfi.tilebox();
409  auto dst = volume.array(mfi);
410  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
411  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
412  ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
413  {
414  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
415  });
416  }
417  Real vol = volume.sum();
418 
419  // Divide by the volume
420  tot_mass_avg /= vol;
421  tot_energy_avg /= vol;
422 
423  const int nfoo = 2;
424  Real foo[nfoo] = {tot_mass_avg,tot_energy_avg};
425  Real zero_d = zero;
426 #ifdef AMREX_LAZY
427  Lazy::QueueReduction([=]() mutable {
428 #endif
429  ParallelDescriptor::ReduceRealSum(
430  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
431 
432  if (ParallelDescriptor::IOProcessor()) {
433  int i = 0;
434  tot_mass_avg = foo[i++];
435  tot_energy_avg = foo[i++];
436 
437  std::ostream& data_log_energy = *tot_e_datalog[0];
438 
439  if (time == zero_d) {
440  data_log_energy << std::setw(datwidth) << " time";
441  data_log_energy << std::setw(datwidth) << " tot_mass";
442  data_log_energy << std::setw(datwidth) << " tot_energy";
443  data_log_energy << std::endl;
444  }
445  data_log_energy << std::setw(datwidth) << std::setprecision(timeprecision) << time;
446  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_mass_avg;
447  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_energy_avg;
448  data_log_energy << std::endl;
449 
450  } // if IOProcessor
451 #ifdef AMREX_LAZY
452  }
453 #endif
454 }
R_v
constexpr amrex::Real R_v
Definition: ERF_Constants.H:48
Cp_d
constexpr amrex::Real Cp_d
Definition: ERF_Constants.H:49
CONST_GRAV
constexpr amrex::Real CONST_GRAV
Definition: ERF_Constants.H:64
Cp_v
constexpr amrex::Real Cp_v
Definition: ERF_Constants.H:50
R_d
constexpr amrex::Real R_d
Definition: ERF_Constants.H:47
L_v
constexpr amrex::Real L_v
Definition: ERF_Constants.H:59
Here is the call graph for this function:

◆ sum_integrated_quantities()

void ERF::sum_integrated_quantities ( double  time)

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters
timeCurrent time
16 {
17  BL_PROFILE("ERF::sum_integrated_quantities()");
18 
19  if (verbose <= 0)
20  return;
21 
22  // Single level sum
23  Real mass_sl;
24 
25  // Multilevel sums
26  Real mass_ml = zero;
27  Real rhth_ml = zero;
28  Real scal_ml = zero;
29  Real mois_ml = zero;
30 
31  bool local = true;
32 
33  auto& mfx0 = *mapfac[0][MapFacType::m_x];
34  auto& mfy0 = *mapfac[0][MapFacType::m_x];
35  auto& dJ0 = *detJ_cc[0];
36 
37  mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,dJ0,mfx0,mfy0,false,local);
38 
39  for (int lev = 0; lev <= finest_level; lev++) {
40  auto& mfx = *mapfac[lev][MapFacType::m_x];
41  auto& mfy = *mapfac[lev][MapFacType::m_x];
42  auto& dJ = *detJ_cc[lev];
43  mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,dJ,mfx,mfy,true);
44  }
45 
46  Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,dJ0,mfx0,mfy0,false);
47  Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,dJ0,mfx0,mfy0,false);
48  Real mois_sl = zero;
49  if (solverChoice.moisture_type != MoistureType::None) {
50  int n_qstate_into_total = micro->Get_Qstate_Moist_Size() - micro->Get_Qstate_Moist_NumConc_Size();
51  for (int qoff(0); qoff<n_qstate_into_total; ++qoff) {
52  mois_sl += volWgtSumMF(0,vars_new[0][Vars::cons],RhoQ1_comp+qoff,dJ0,mfx0,mfy0,false);
53  }
54  }
55 
56  for (int lev = 0; lev <= finest_level; lev++) {
57  auto& mfx = *mapfac[lev][MapFacType::m_x];
58  auto& mfy = *mapfac[lev][MapFacType::m_x];
59  auto& dJ = *detJ_cc[lev];
60  rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,dJ,mfx,mfy,true);
61  scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,dJ,mfx,mfy,true);
62  if (solverChoice.moisture_type != MoistureType::None) {
63  int n_qstate_into_total = micro->Get_Qstate_Moist_Size() - micro->Get_Qstate_Moist_NumConc_Size();
64  for (int qoff(0); qoff<n_qstate_into_total; ++qoff) {
65  mois_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoQ1_comp+qoff,dJ,mfx,mfy,false);
66  }
67  }
68  }
69 
70  Gpu::HostVector<Real> h_avg_ustar; h_avg_ustar.resize(1);
71  Gpu::HostVector<Real> h_avg_tstar; h_avg_tstar.resize(1);
72  Gpu::HostVector<Real> h_avg_olen; h_avg_olen.resize(1);
73  if ((m_SurfaceLayer != nullptr) && (NumDataLogs() > 0)) {
74  Box domain = geom[0].Domain();
75  int zdir = 2;
76  h_avg_ustar = sumToLine(*m_SurfaceLayer->get_u_star(0),0,1,domain,zdir);
77  h_avg_tstar = sumToLine(*m_SurfaceLayer->get_t_star(0),0,1,domain,zdir);
78  h_avg_olen = sumToLine(*m_SurfaceLayer->get_olen(0) ,0,1,domain,zdir);
79 
80  // Divide by the total number of cells we are averaging over
81  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
82  h_avg_ustar[0] /= area_z;
83  h_avg_tstar[0] /= area_z;
84  h_avg_olen[0] /= area_z;
85 
86  } else {
87  h_avg_ustar[0] = zero;
88  h_avg_tstar[0] = zero;
89  h_avg_olen[0] = zero;
90  }
91 
92  const int nfoo = 8;
93  Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mois_sl,mass_ml,rhth_ml,scal_ml,mois_ml};
94  Real zero_d = zero;
95 #ifdef AMREX_LAZY
96  Lazy::QueueReduction([=]() mutable {
97 #endif
98  ParallelDescriptor::ReduceRealSum(
99  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
100 
101  if (ParallelDescriptor::IOProcessor()) {
102  int i = 0;
103  mass_sl = foo[i++];
104  rhth_sl = foo[i++];
105  scal_sl = foo[i++];
106  mois_sl = foo[i++];
107  mass_ml = foo[i++];
108  rhth_ml = foo[i++];
109  scal_ml = foo[i++];
110  mois_ml = foo[i++];
111 
112  Print() << '\n';
113  Print() << "TIME= " << std::setw(datwidth) << std::setprecision(timeprecision) << std::left << time << '\n';
114  if (finest_level == 0) {
115 #if 1
116  Print() << " MASS = " << mass_sl << '\n';
117 #else
118  Print() << " PERT MASS = " << mass_sl << '\n';
119 #endif
120  Print() << " RHO THETA = " << rhth_sl << '\n';
121  if (solverChoice.transport_scalar) { Print() << " RHO SCALAR = " << scal_sl << '\n'; }
122  if (solverChoice.moisture_type != MoistureType::None) { Print() << " RHO QTOTAL = " << mois_sl << '\n'; }
123  } else {
124 #if 1
125  Print() << " MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
126 #else
127  Print() << " PERT MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
128 #endif
129  Print() << " RHO THETA SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
130  if (solverChoice.transport_scalar) { Print() << " RHO SCALAR SL/ML = " << scal_sl << " " << scal_ml << '\n'; }
131  if (solverChoice.moisture_type != MoistureType::None) { Print() << " RHO QTOTAL SL/ML = " << mois_sl << " " << mois_ml << '\n'; }
132  }
133 
134  // The first data log only holds scalars
135  if (NumDataLogs() > 0)
136  {
137  int n_d = 0;
138  std::ostream& data_log1 = DataLog(n_d);
139  if (data_log1.good()) {
140  if (time == zero_d) {
141  data_log1 << std::setw(datwidth) << " time";
142  data_log1 << std::setw(datwidth) << " u_star";
143  data_log1 << std::setw(datwidth) << " t_star";
144  data_log1 << std::setw(datwidth) << " olen";
145  data_log1 << std::endl;
146  } // time = 0
147 
148  // Write the quantities at this time
149  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time;
150  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_ustar[0];
151  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_tstar[0];
152  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_olen[0];
153  data_log1 << std::endl;
154  } // if good
155  } // loop over i
156  } // if IOProcessor
157 #ifdef AMREX_LAZY
158  });
159 #endif
160 
161  // This is just an alias for convenience
162  int lev = 0;
163  if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
164  for (int i = 0; i < NumSamplePoints(); ++i)
165  {
166  sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
167  }
168  }
169  if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
170  for (int i = 0; i < NumSampleLines(); ++i)
171  {
172  sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
173  }
174  }
175 }
ERF::NumSampleLineLogs
AMREX_FORCE_INLINE int NumSampleLineLogs() noexcept
Definition: ERF.H:1599
ERF::NumSamplePointLogs
AMREX_FORCE_INLINE int NumSamplePointLogs() noexcept
Definition: ERF.H:1585
ERF::SampleLine
amrex::IntVect & SampleLine(int i)
Definition: ERF.H:1618
ERF::sample_points
void sample_points(int lev, double time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:533
ERF::NumSamplePoints
AMREX_FORCE_INLINE int NumSamplePoints() noexcept
Definition: ERF.H:1612
ERF::sample_lines
void sample_lines(int lev, double time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:569
ERF::NumSampleLines
AMREX_FORCE_INLINE int NumSampleLines() noexcept
Definition: ERF.H:1625
ERF::SamplePoint
amrex::IntVect & SamplePoint(int i)
Definition: ERF.H:1605
ERF::DataLog
AMREX_FORCE_INLINE std::ostream & DataLog(int i)
Definition: ERF.H:1549
ERF::NumDataLogs
AMREX_FORCE_INLINE int NumDataLogs() noexcept
Definition: ERF.H:1563
SolverChoice::transport_scalar
bool transport_scalar
Definition: ERF_DataStruct.H:1364

◆ SurfaceDataInterpolation()

void ERF::SurfaceDataInterpolation ( const int  nlevs,
const double  time,
amrex::Vector< std::unique_ptr< amrex::MultiFab >> &  z_phys_nd,
bool  regrid_forces_file_read 
)
146 {
147 
148  static amrex::Vector<double> next_read_forecast_time;
149  static amrex::Vector<double> last_read_forecast_time;
150 
151  const int nlevs = a_z_phys_nd.size();
152 
153  Real hindcast_data_interval = solverChoice.hindcast_data_interval_in_hrs*Real(3600.0);
154 
155  // Initialize static vectors once
156  if (next_read_forecast_time.empty()) {
157  next_read_forecast_time.resize(nlevs, -one);
158  last_read_forecast_time.resize(nlevs, -one);
159  Print() << "Initializing the time vector values here by " << lev << std::endl;
160  }
161 
162  if (next_read_forecast_time[lev] < zero) {
163  int next_multiple = static_cast<int>(time / hindcast_data_interval);
164  next_read_forecast_time[lev] = next_multiple * hindcast_data_interval;
165  last_read_forecast_time[lev] = next_read_forecast_time[lev];
166  }
167 
168  if (time >= next_read_forecast_time[lev] or regrid_forces_file_read) {
169 
170  Print() << "Data reading happening at level " << lev << std::endl;
171 
172  std::string folder = solverChoice.hindcast_surface_data_dir;
173 
174  // Check if folder exists and is a directory
175  if (!fs::exists(folder) || !fs::is_directory(folder)) {
176  throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
177  }
178 
179  std::vector<std::string> bin_files;
180 
181  for (const auto& entry : fs::directory_iterator(folder)) {
182  if (!entry.is_regular_file()) continue;
183 
184  std::string fname = entry.path().filename().string();
185  if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
186  bin_files.push_back(entry.path().string());
187  }
188  }
189  std::sort(bin_files.begin(), bin_files.end());
190 
191  // Check if no .bin files were found
192  if (bin_files.empty()) {
193  throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
194  }
195 
196  std::string filename1, filename2;
197 
198  int idx1 = static_cast<int>(time / hindcast_data_interval);
199  int idx2 = static_cast<int>(time / hindcast_data_interval)+1;
200  Print() << "Reading surface data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
201 
202  if (idx2 >= static_cast<int>(bin_files.size())) {
203  throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
204  }
205 
206  filename1 = bin_files[idx1];
207  filename2 = bin_files[idx2];
208 
209  FillSurfaceStateMultiFabs(lev, filename1, surface_state_1);
210  FillSurfaceStateMultiFabs(lev, filename2, surface_state_2);
211 
212  // Create the time-interpolated forecast state
213  //CreateForecastStateMultiFabs(forecast_state_interp);
214  if(!regrid_forces_file_read){
215  last_read_forecast_time[lev] = next_read_forecast_time[lev];
216  next_read_forecast_time[lev] += hindcast_data_interval;
217  Print() << "Next forecast time getting updated here " << std::endl;
218  }
219  }
220 
221  double prev_read_time = last_read_forecast_time[lev];
222  Real alpha1 = static_cast<Real>(1.0 - (time - prev_read_time)/hindcast_data_interval);
223  Real alpha2 = one - alpha1;
224 
225  amrex::Print()<< "The values of alpha1 and alpha2 are " << alpha1 << " "<< alpha2 <<std::endl;
226 
227  if (alpha1 < zero || alpha1 > one ||
228  alpha2 < zero || alpha2 > one)
229  {
230  std::stringstream ss;
231  ss << "Interpolation weights for hindcast files are incorrect: "
232  << "alpha1 = " << alpha1 << ", alpha2 = " << alpha2;
233  Abort(ss.str());
234  }
235 
236  /*MultiFab& mf_surf_interp = surface_state_interp[lev];
237 
238  // Fill the time-interpolated forecast states
239  MultiFab::LinComb(surface_state_interp[lev],
240  alpha1, surface_state_1[lev], 0,
241  alpha2, surface_state_2[lev], 0,
242  0, mf_surf_interp.nComp(), mf_surf_interp.nGrow());
243 
244  std::string pltname = "plt_interp_surface";
245  Vector<std::string> varnames_plot_mf = {"ls_mask", "SST"};
246 
247  const MultiFab& src = vars_new[0][0];
248 
249  MultiFab plot_mf(src.boxArray(),
250  src.DistributionMap(),
251  2, 0);
252 
253  plot_mf.setVal(0.0);
254 
255  for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
256  const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
257  const Array4<Real> &surf_mf_arr = surface_state_1[0].array(mfi);
258 
259  const Box& bx = mfi.validbox();
260 
261  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
262  plot_mf_arr(i,j,k,0) = surf_mf_arr(i,j,0);
263  plot_mf_arr(i,j,k,1) = surf_mf_arr(i,j,1);
264  });
265  }
266 
267  WriteSingleLevelPlotfile(
268  pltname,
269  plot_mf,
270  varnames_plot_mf,
271  geom[0],
272  time,
273  0 // level
274  );*/
275 }
ERF::FillSurfaceStateMultiFabs
void FillSurfaceStateMultiFabs(const int lev, const std::string &filename, amrex::Vector< amrex::MultiFab > &surface_state)
Definition: ERF_SurfaceDataInterpolation.cpp:19
SolverChoice::hindcast_surface_data_dir
std::string hindcast_surface_data_dir
Definition: ERF_DataStruct.H:1470
SolverChoice::hindcast_data_interval_in_hrs
amrex::Real hindcast_data_interval_in_hrs
Definition: ERF_DataStruct.H:1471

◆ timeStep()

void ERF::timeStep ( int  lev,
double  time,
int  iteration 
)
private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters
[in]levlevel of refinement (coarsest level is 0)
[in]timestart time for time advance
[in]iterationtime step counter
19 {
20  //
21  // We need to FillPatch the coarse level before assessing whether to regrid
22  // We have not done the swap yet so we fill the "new" which will become the "old"
23  //
24  MultiFab& S_new = vars_new[lev][Vars::cons];
25  MultiFab& U_new = vars_new[lev][Vars::xvel];
26  MultiFab& V_new = vars_new[lev][Vars::yvel];
27  MultiFab& W_new = vars_new[lev][Vars::zvel];
28 
29 #ifdef ERF_USE_NETCDF
30  //
31  // Since we now only read in a subset of the time slices in wrfbdy and
32  // wrflowinp, we need to check whether it's time to read in more.
33  //
34  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
35  if (solverChoice.use_real_bcs && (lev==0))
36  {
37  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
38  Array<MultiFab*, AMREX_SPACEDIM> area_vec = {ax[lev].get(), ay[lev].get(), az[lev].get()};
39 
40  int ntimes = bdy_data_xlo.size();
41  double time_since_start_bdy = time + start_time - start_bdy_time;
42  int n_time_old = std::min(static_cast<int>( (time_since_start_bdy ) / bdy_time_interval), ntimes-1);
43  int n_time_new = std::min(static_cast<int>( (time_since_start_bdy+dt[lev]) / bdy_time_interval), ntimes-1);
44 
45  for (int itime = 0; itime < ntimes; itime++)
46  {
47  /*
48  if (bdy_data_xlo[itime].size() > 0) {
49  amrex::Print() << "HAVE BDY DATA AT TIME " << itime << std::endl;
50  } else {
51  amrex::Print() << " NO BDY DATA AT TIME " << itime << std::endl;
52  }
53  */
54 
55  // Note that we never release itime == 0 because it is used for the spatial interpolation at later times
56  bool clear_itime = (itime > 0 && itime < n_time_old);
57 
58  if (clear_itime && bdy_data_xlo[itime].size() > 0) {
59  bdy_data_xlo[itime].clear();
60  bdy_data_xhi[itime].clear();
61  bdy_data_ylo[itime].clear();
62  bdy_data_yhi[itime].clear();
63  //amrex::Print() << "CLEAR BDY DATA AT TIME " << itime << std::endl;
64  }
65 
66  bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
67  //if (need_itime) { amrex::Print() << "NEED BDY DATA AT TIME " << itime << std::endl; }
68 
69  // Handle erfbdy files (AMReX native format).
70  if (use_erfbdy) {
71  if (bdy_data_xlo[itime].size() == 0 && need_itime) {
72  read_from_erfbdy(itime, erfbdy_file,
73  bdy_data_xlo, bdy_data_xhi,
74  bdy_data_ylo, bdy_data_yhi,
75  nvars_erfbdy, real_width);
76  }
77  // Handle wrfbdy files (NetCDF format).
78  } else {
79  if (bdy_data_xlo[itime].size() == 0 && need_itime) {
80  bool is_anelastic = (solverChoice.anelastic[0] == 1);
81  read_and_convert_from_wrfbdy(itime,nc_bdy_file,bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
82  wrf_MUB, wrf_C1H, wrf_C2H, wrf_RDNW, wrf_PHB, z_phys_nd[lev],
83  vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel], vars_new[lev][Vars::cons],
84  r_hse, area_vec, geom[lev], use_moist, solverChoice.rebalance_wrf_input, domain_bcs_type,
85  real_width, bdy_time_interval, is_anelastic);
86  }
87  } // use_erfbdy
88  } // itime
89  } // use_real_bcs && lev == 0
90 
91  if (!nc_low_file.empty() && (lev==0))
92  {
93  int ntimes = low_data_zlo.size();
94  double time_since_start_low = time + start_time - start_low_time;
95  int n_time_old = std::min(static_cast<int>( (time_since_start_low ) / low_time_interval), ntimes-1);
96  int n_time_new = std::min(static_cast<int>( (time_since_start_low+dt[lev]) / low_time_interval), ntimes-1);
97 
98  for (int itime = 0; itime < ntimes; itime++)
99  {
100  /*
101  if (low_data_zlo[itime].size() > 0) {
102  amrex::Print() << "HAVE LOW DATA AT TIME " << itime << std::endl;
103  } else {
104  amrex::Print() << " NO LOW DATA AT TIME " << itime << std::endl;
105  }
106  */
107 
108  bool clear_itime = (itime < n_time_old);
109 
110  if (clear_itime && low_data_zlo[itime].size() > 0) {
111  low_data_zlo[itime].clear();
112  //amrex::Print() << "CLEAR LOW DATA AT TIME " << itime << std::endl;
113  }
114 
115  bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
116  //if (need_itime) { amrex::Print() << "NEED LOW DATA AT TIME " << itime << std::endl; }
117 
118  if (low_data_zlo[itime].size() == 0 && need_itime) {
119  read_from_wrflow(itime, nc_low_file, geom[lev].Domain(), low_data_zlo);
120 
121  update_sst_tsk(itime, geom[lev], ba2d[lev],
122  sst_lev[lev], tsk_lev[lev],
123  m_SurfaceLayer, low_data_zlo,
124  S_new, *mf_PSFC[lev],
125  solverChoice.rdOcp, lmask_lev[lev][0], use_moist);
126  }
127  } // itime
128  } // have nc_low_file && lev == 0
129 #endif
130 
131  //
132  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
133  //
134  if (lev == 0) {
135  FillPatchCrseLevel(lev, time, {&S_new, &U_new, &V_new, &W_new});
136  } else if (lev < finest_level) {
137  FillPatchFineLevel(lev, time, {&S_new, &U_new, &V_new, &W_new},
138  {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
139  base_state[lev], base_state[lev]);
140  }
141 
142  if (regrid_int > 0) // We may need to regrid
143  {
144  // help keep track of whether a level was already regridded
145  // from a coarser level call to regrid
146  static Vector<int> last_regrid_step(max_level+1, 0);
147 
148  // regrid changes level "lev+1" so we don't regrid on max_level
149  // also make sure we don't regrid fine levels again if
150  // it was taken care of during a coarser regrid
151  if (lev < max_level)
152  {
153  if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
154  {
155  // regrid could add newly refine levels (if finest_level < max_level)
156  // so we save the previous finest level index
157  int old_finest = finest_level;
158 
159  if (solverChoice.coupling_type == CouplingType::TwoWay &&
160  solverChoice.moisture_type != MoistureType::None &&
161  Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian &&
162  finest_level >= 1) {
163  micro->AverageDownMicroVars(finest_level);
164  for (int flev = finest_level-1; flev >= lev; --flev) {
165  AverageDownMoistStateTo(flev);
166  }
167  }
168 
169  regrid(lev, static_cast<Real>(time));
170 
171 #ifdef ERF_USE_PARTICLES
172  particleData.Redistribute(z_phys_nd);
173 #endif
174 
175  // mark that we have regridded this level already
176  for (int k = lev; k <= finest_level; ++k) {
177  last_regrid_step[k] = istep[k];
178  }
179 
180  // if there are newly created levels, set the time step
181  for (int k = old_finest+1; k <= finest_level; ++k) {
182  dt[k] = dt[k-1] / static_cast<double>(nsubsteps[k]);
183  }
184  } // if
185  } // lev
186  }
187 
188  // Update what we call "old" and "new" time
189  t_old[lev] = t_new[lev];
190  t_new[lev] += dt[lev];
191 
192  if (Verbose()) {
193  amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
194  amrex::Print() << std::setprecision(timeprecision)
195  << "ADVANCE from elapsed time = " << t_old[lev] << " to " << t_new[lev]
196  << " with dt = " << dt[lev] << std::endl;
197  }
198 
199 #ifdef ERF_USE_WW3_COUPLING
200  amrex::Print() << " About to call send_to_ww3 from ERF_Timestep" << std::endl;
201  send_to_ww3(lev);
202  amrex::Print() << " About to call read_waves from ERF_Timestep" << std::endl;
203  read_waves(lev);
204  //send_to_ww3(lev);
205  //read_waves(lev);
206  //send_to_ww3(lev);
207 #endif
208 
209  // Advance a single level for a single time step
210  Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
211 
212  ++istep[lev];
213 
214  if (Verbose()) {
215  amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
216  amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
217  }
218 
219  if (lev < finest_level)
220  {
221  // recursive call for next-finer level
222  for (int i = 1; i <= nsubsteps[lev+1]; ++i)
223  {
224  double strt_time_for_fine = time + (i-1)*dt[lev+1];
225  timeStep(lev+1, strt_time_for_fine, i);
226  }
227  }
228 
229  if ( verbose && lev == 0 && solverChoice.moisture_type != MoistureType::None) {
230  amrex::Print() << "Cloud fraction " << time << " " << cloud_fraction(time) << std::endl;
231  }
232 }
ERF::Advance
void Advance(int lev, double time, double dt_lev, int iteration, int ncycle)
Definition: ERF_Advance.cpp:20

Referenced by EvolveOneStep().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ turbPert_amplitude()

void ERF::turbPert_amplitude ( const int  lev)
private
34 {
35  // Accessing data
36  auto& lev_new = vars_new[lev];
37 
38  // Creating local data
39  int ncons = lev_new[Vars::cons].nComp();
40  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
41 
42  // Defining BoxArray type
43  auto m_ixtype = cons_data.boxArray().ixType();
44 
45 #ifdef _OPENMP
46 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
47 #endif
48  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
49  const Box &bx = mfi.validbox();
50  const auto &cons_pert_arr = cons_data.array(mfi); // Address of perturbation array
51  const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
52 
53  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cons_pert_arr, pert_cell);
54  } // mfi
55 }
Here is the call graph for this function:

◆ turbPert_amplitude_w()

void ERF::turbPert_amplitude_w ( const int  lev)
private
61 {
62  // Accessing data
63  auto& lev_new = vars_new[lev];
64 
65  // Creating local data
66  int ncomps = lev_new[Vars::zvel].nComp();
67  MultiFab wvel_data(lev_new[Vars::zvel], make_alias, 0, ncomps);
68 
69  // Defining BoxArray type
70  auto m_ixtype = wvel_data.boxArray().ixType();
71 
72 #ifdef _OPENMP
73 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
74 #endif
75  for (MFIter mfi(lev_new[Vars::zvel], TileNoZ()); mfi.isValid(); ++mfi) {
76  const Box &bx = mfi.validbox();
77  const auto &wvel_pert_arr = wvel_data.array(mfi); // Address of perturbation array
78  const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
79 
80  turbPert.apply_tpi(lev, bx, -1, m_ixtype, wvel_pert_arr, pert_cell);
81  } // mfi
82 }
Here is the call graph for this function:

◆ turbPert_update()

void ERF::turbPert_update ( const int  lev,
const double  dt 
)
private
13 {
14  // Accessing data
15  auto& lev_new = vars_new[lev];
16 
17  // Create aliases to state data to pass to calc_tpi_update
18  int ncons = lev_new[Vars::cons].nComp();
19  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
20  MultiFab xvel_data(lev_new[Vars::xvel], make_alias, 0, 1);
21  MultiFab yvel_data(lev_new[Vars::yvel], make_alias, 0, 1);
22 
23  // Computing perturbation update time
24  turbPert.calc_tpi_update(lev, local_dt, xvel_data, yvel_data, cons_data);
25 
26  Print() << "Successfully initialized turbulent perturbation update time and amplitude at level "
27  << lev << " with type: " << turbPert.pt_type[lev] << "\n";
28 }
TurbulentPerturbation::pt_type
amrex::Vector< int > pt_type
Definition: ERF_TurbPertStruct.H:690

◆ update_box_for_refinement()

void ERF::update_box_for_refinement ( std::string &  ref_prefix,
int &  lev_for_box,
amrex::RealBox &  real_box,
const double  time 
)
337 {
338  ParmParse ppr(ref_prefix);
339 
340  Vector<double> move_start_time, move_stop_time;
341  int ni = ppr.queryarr("move_start_time", move_start_time);
342  int nj = ppr.queryarr("move_stop_time" , move_stop_time);
343  if (ni != nj) {
344  amrex::Print() << "Must be same number of start times as stop times for moving grids" << std::endl;
345  amrex::Abort();
346  }
347  for (int i = 0; i < ni; i++) {
348  if (move_stop_time[i] <= move_start_time[i]) {
349  amrex::Print() << "start time for interval " << i << " is " << move_start_time[i] << std::endl;
350  amrex::Print() << "stop time for interval " << i << " is " << move_stop_time[i] << std::endl;
351  amrex::Abort("moving grid: stop time must be greater than start time");
352  }
353  }
354  for (int i = 1; i < ni; i++) {
355  if (move_start_time[i] < move_stop_time[i-1]) {
356  amrex::Print() << "start time for interval " << i << " is " << move_start_time[i] << std::endl;
357  amrex::Print() << "stop time for interval " << i-1 << " is " << move_stop_time[i-1] << std::endl;
358  amrex::Abort("moving grid: stop time must be less than start time of the next interval");
359  }
360  }
361 
362  Vector<Real> move_speed_x, move_speed_y;
363  int ni2 = ppr.queryarr("move_speed_x", move_speed_x);
364  int nj2 = ppr.queryarr("move_speed_y", move_speed_y);
365  if (ni2 != nj2 ) {
366  amrex::Print() << "Must be same number of speeds in x- and y-directions" << std::endl;
367  amrex::Abort();
368  }
369  if (ni != ni2 ) {
370  amrex::Print() << "Must be same number of speeds as time intervals" << std::endl;
371  amrex::Abort();
372  }
373 
374  Real offset_x = zero;
375  Real offset_y = zero;
376 
377  for (int i = 0; i < ni; i++) {
378  if (time > move_start_time[i]) {
379  offset_x += move_speed_x[i] * static_cast<Real>(std::min(time,move_stop_time[i]) - move_start_time[i]);
380  offset_y += move_speed_y[i] * static_cast<Real>(std::min(time,move_stop_time[i]) - move_start_time[i]);
381  }
382  }
383 
384  RealBox orig_real_box;
385  read_box_for_refinement (ref_prefix, lev_for_box, orig_real_box);
386 
387  Real xlo = orig_real_box.lo(0) + offset_x; Real ylo = orig_real_box.lo(1) + offset_y;
388  Real xhi = orig_real_box.hi(0) + offset_x; Real yhi = orig_real_box.hi(1) + offset_y;
389  Real zlo = orig_real_box.lo(2); Real zhi = orig_real_box.hi(2);
390 
391  real_box.setLo(RealVect(xlo,ylo,zlo));
392  real_box.setHi(RealVect(xhi,yhi,zhi));
393 }
MicVar_Morr::ni
@ ni
Definition: ERF_Morrison.H:47

◆ update_diffusive_arrays()

void ERF::update_diffusive_arrays ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
private
530 {
531  // ********************************************************************************************
532  // Diffusive terms
533  // ********************************************************************************************
534  bool l_use_eb = (SolverChoice::terrain_type == TerrainType::EB);
535  bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None && !l_use_eb);
536  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
537  bool l_use_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
538  l_use_kturb );
539  bool l_need_SmnSmn = solverChoice.turbChoice[lev].use_keqn;
540  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
541  bool l_rotate = ( solverChoice.use_rotate_surface_flux );
542 
543  bool l_implicit_diff = (solverChoice.vert_implicit_fac[lev][0] > 0 ||
544  solverChoice.vert_implicit_fac[lev][1] > 0 ||
545  solverChoice.vert_implicit_fac[lev][2] > 0);
546 
547  bool l_eb_surface_layer = (l_use_eb && solverChoice.ebChoice.eb_boundary_type == EBBoundaryType::SurfaceLayer);
548 
549  BoxArray ba12 = convert(ba, IntVect(1,1,0));
550  BoxArray ba13 = convert(ba, IntVect(1,0,1));
551  BoxArray ba23 = convert(ba, IntVect(0,1,1));
552 
553  Tau[lev].resize(9);
554  Tau_corr[lev].resize(3);
555 
556  // Always resize Tau_EB structure, even if not used, because other code checks nullptr
557  Tau_EB[lev].resize(2); // tau_eb13 and tau_eb23
558  for (int comp = 0; comp < 2; ++comp) {
559  Tau_EB[lev][comp].resize(3); // xface, yface, zface
560  }
561 
562  if (l_use_diff) {
563  //
564  // NOTE: We require ghost cells in the vertical when allowing grids that don't
565  // cover the entire vertical extent of the domain at this level
566  //
567  Tau[lev][TauType::tau11] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau11]->setVal(zero);
568  Tau[lev][TauType::tau22] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau22]->setVal(zero);
569  Tau[lev][TauType::tau33] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau33]->setVal(zero);
570 
571  Tau[lev][TauType::tau12] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau12]->setVal(zero);
572  Tau[lev][TauType::tau13] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau13]->setVal(zero);
573  Tau[lev][TauType::tau23] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau23]->setVal(zero);
574  if (l_use_terrain) {
575  Tau[lev][TauType::tau21] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau21]->setVal(zero);
576  Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau31]->setVal(zero);
577  Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau32]->setVal(zero);
578  } else if (l_implicit_diff) {
579  Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau31]->setVal(zero);
580  Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); Tau[lev][TauType::tau32]->setVal(zero);
581  } else {
582  Tau[lev][TauType::tau21] = nullptr;
583  Tau[lev][TauType::tau31] = nullptr;
584  Tau[lev][TauType::tau32] = nullptr;
585  }
586 
587  // EB diffusive stresses - allocate for all three staggered grids
588  if (l_eb_surface_layer) {
589  Tau_EB[lev][EBTauType::tau_eb13][EBGridType::xface] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
590  Tau_EB[lev][EBTauType::tau_eb13][EBGridType::yface] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
591  Tau_EB[lev][EBTauType::tau_eb13][EBGridType::zface] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
592  Tau_EB[lev][EBTauType::tau_eb13][EBGridType::xface]->setVal(0.);
593  Tau_EB[lev][EBTauType::tau_eb13][EBGridType::yface]->setVal(0.);
594  Tau_EB[lev][EBTauType::tau_eb13][EBGridType::zface]->setVal(0.);
595 
596  Tau_EB[lev][EBTauType::tau_eb23][EBGridType::xface] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
597  Tau_EB[lev][EBTauType::tau_eb23][EBGridType::yface] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
598  Tau_EB[lev][EBTauType::tau_eb23][EBGridType::zface] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
599  Tau_EB[lev][EBTauType::tau_eb23][EBGridType::xface]->setVal(0.);
600  Tau_EB[lev][EBTauType::tau_eb23][EBGridType::yface]->setVal(0.);
601  Tau_EB[lev][EBTauType::tau_eb23][EBGridType::zface]->setVal(0.);
602  } else {
603  for (int comp = 0; comp < 2; ++comp) {
604  for (int grid = 0; grid < 3; ++grid) {
605  Tau_EB[lev][comp][grid] = nullptr;
606  }
607  }
608  }
609 
610  if (l_implicit_diff && solverChoice.implicit_momentum_diffusion)
611  {
612  Tau_corr[lev][0] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) ); // Tau31
613  Tau_corr[lev][1] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) ); // Tau32
614  Tau_corr[lev][0]->setVal(zero);
615  Tau_corr[lev][1]->setVal(zero);
616 #ifdef ERF_IMPLICIT_W
617  Tau_corr[lev][2] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) ); // Tau33
618  Tau_corr[lev][2]->setVal(zero);
619 #else
620  Tau_corr[lev][2] = nullptr;
621 #endif
622  } else {
623  Tau_corr[lev][0] = nullptr;
624  Tau_corr[lev][1] = nullptr;
625  Tau_corr[lev][2] = nullptr;
626  }
627 
628  SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
629  SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
630  SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
631  SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) );
632  SFS_hfx1_lev[lev]->setVal(zero);
633  SFS_hfx2_lev[lev]->setVal(zero);
634  SFS_hfx3_lev[lev]->setVal(zero);
635  SFS_diss_lev[lev]->setVal(zero);
636 
637  // EB heat fluxes
638  if (l_use_eb) {
639  hfx3_EB[lev] = std::make_unique<MultiFab>( ba, dm, 1, IntVect(1,1,1) );
640  hfx3_EB[lev]->setVal(zero);
641  } else {
642  hfx3_EB[lev] = nullptr;
643  }
644 
645  if (l_use_moist) {
646  SFS_q1fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
647  SFS_q2fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
648  SFS_q1fx3_lev[lev]->setVal(zero);
649  SFS_q2fx3_lev[lev]->setVal(zero);
650  if (l_rotate) {
651  SFS_q1fx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
652  SFS_q1fx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
653  SFS_q1fx1_lev[lev]->setVal(zero);
654  SFS_q1fx2_lev[lev]->setVal(zero);
655  } else {
656  SFS_q1fx1_lev[lev] = nullptr;
657  SFS_q1fx2_lev[lev] = nullptr;
658  }
659  } else {
660  SFS_q1fx1_lev[lev] = nullptr;
661  SFS_q1fx2_lev[lev] = nullptr;
662  SFS_q1fx3_lev[lev] = nullptr;
663  SFS_q2fx3_lev[lev] = nullptr;
664  }
665  } else {
666  for (int i = 0; i < 9; i++) {
667  Tau[lev][i] = nullptr;
668  }
669  SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
670  SFS_diss_lev[lev] = nullptr;
671  }
672 
673  if (l_use_kturb) {
674  eddyDiffs_lev[lev] = std::make_unique<MultiFab>(ba, dm, EddyDiff::NumDiffs, 2);
675  eddyDiffs_lev[lev]->setVal(zero);
676  if(l_need_SmnSmn) {
677  SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
678  } else {
679  SmnSmn_lev[lev] = nullptr;
680  }
681  } else {
682  eddyDiffs_lev[lev] = nullptr;
683  SmnSmn_lev[lev] = nullptr;
684  }
685 }
tau_eb23
@ tau_eb23
Definition: ERF_EBStruct.H:16
tau_eb13
@ tau_eb13
Definition: ERF_EBStruct.H:16
EBBoundaryType::SurfaceLayer
@ SurfaceLayer
yface
@ yface
Definition: ERF_EBStruct.H:20
zface
@ zface
Definition: ERF_EBStruct.H:20
xface
@ xface
Definition: ERF_EBStruct.H:20
EddyDiff::NumDiffs
@ NumDiffs
Definition: ERF_IndexDefines.H:220
EBChoice::eb_boundary_type
EBBoundaryType eb_boundary_type
Definition: ERF_EBStruct.H:59
SolverChoice::ebChoice
EBChoice ebChoice
Definition: ERF_DataStruct.H:1252

◆ update_terrain_arrays()

void ERF::update_terrain_arrays ( int  lev)
882 {
883  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
884  SolverChoice::mesh_type == MeshType::VariableDz) {
885  make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
886  make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
887  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
888  } else { // MeshType::ConstantDz
889  if (SolverChoice::terrain_type == TerrainType::EB) {
890  const auto& ebfact = *eb[lev]->get_const_factory();
891  const MultiFab& volfrac = ebfact.getVolFrac();
892  detJ_cc[lev] = std::make_unique<MultiFab>(volfrac, amrex::make_alias, 0, volfrac.nComp());
893  }
894  }
895 }
make_areas
void make_areas(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &ax, MultiFab &ay, MultiFab &az)
Definition: ERF_TerrainMetrics.cpp:561
make_J
void make_J(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &detJ_cc)
Definition: ERF_TerrainMetrics.cpp:523
Here is the call graph for this function:

◆ volWgtColumnSum()

void ERF::volWgtColumnSum ( int  lev,
const amrex::MultiFab &  mf,
int  comp,
amrex::MultiFab &  mf_2d,
const amrex::MultiFab &  dJ 
)
84 {
85  BL_PROFILE("ERF::volWgtSumColumnMF()");
86 
87  mf_2d.setVal(0.);
88 
89  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
90  // m is the map scale factor at cell centers
91 #ifdef _OPENMP
92 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
93 #endif
94  for (MFIter mfi(mf_to_be_summed, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
95  const Box& bx = mfi.tilebox();
96  const auto dst_arr = mf_2d.array(mfi);
97  const auto src_arr = mf_to_be_summed.array(mfi);
98  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
99  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
100  {
101  amrex::HostDevice::Atomic::Add(&dst_arr(i,j,0),src_arr(i,j,k,comp));
102  });
103  } else {
104  const auto& dJ_arr = dJ.const_array(mfi);
105  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
106  {
107  amrex::HostDevice::Atomic::Add(&dst_arr(i,j,0),src_arr(i,j,k,comp)*dJ_arr(i,j,k));
108  });
109  }
110  } // mfi
111 
112  auto const& dx = geom[lev].CellSizeArray();
113 
114  mf_2d.mult(dx[2]);
115 }
Here is the call graph for this function:

◆ volWgtSumMF()

Real ERF::volWgtSumMF ( int  lev,
const amrex::MultiFab &  mf,
int  comp,
const amrex::MultiFab &  dJ,
const amrex::MultiFab &  mfx,
const amrex::MultiFab &  mfy,
bool  finemask,
bool  local = true 
)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters
levCurrent level
mf_to_be_summed: MultiFab on which we do the volume weighted sum
dJ: volume weighting due to metric terms
mfmx: map factor in x-direction at cell centers
mfmy: map factor in y-direction at cell centers
comp: Index of the component we want to sum
finemask: If a finer level is available, determines whether we mask fine data
local: Boolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)
25 {
26  BL_PROFILE("ERF::volWgtSumMF()");
27 
28  Real sum = zero;
29  MultiFab tmp(mf_to_be_summed.boxArray(), mf_to_be_summed.DistributionMap(), 1, 0);
30 
31  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
32  // m is the map scale factor at cell centers
33 #ifdef _OPENMP
34 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
35 #endif
36  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
37  const Box& bx = mfi.tilebox();
38  const auto dst_arr = tmp.array(mfi);
39  const auto src_arr = mf_to_be_summed.array(mfi);
40  const auto& mfx_arr = mfmx.const_array(mfi);
41  const auto& mfy_arr = mfmy.const_array(mfi);
42 
43  if (SolverChoice::terrain_type != TerrainType::EB) {
44  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
45  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
46  {
47  dst_arr(i,j,k,0) = src_arr(i,j,k,comp) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
48  });
49  } else {
50  const auto& dJ_arr = dJ.const_array(mfi);
51  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
52  {
53  dst_arr(i,j,k,0) = src_arr(i,j,k,comp) * dJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
54  });
55  }
56  } else {
57  const auto& dJ_arr = dJ.const_array(mfi);
58  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
59  {
60  dst_arr(i,j,k,0) = src_arr(i,j,k,comp) * dJ_arr(i,j,k);
61  });
62  }
63 
64  } // mfi
65 
66  if (lev < finest_level && finemask) {
67  MultiFab::Multiply(tmp, *fine_mask[lev+1].get(), 0, 0, 1, 0);
68  }
69 
70  // If local = true then "sum" will be the sum only over the FABs on each rank
71  // If local = false then "sum" will be the sum over the whole MultiFab, and will be broadcast to all ranks
72  sum = tmp.sum(0,local);
73 
74  auto const& dx = geom[lev].CellSizeArray();
75 
76  sum *= dx[0]*dx[1]*dx[2];
77 
78  return sum;
79 }
Here is the call graph for this function:

◆ WeatherDataInterpolation()

void ERF::WeatherDataInterpolation ( const int  nlevs,
const double  time,
amrex::Vector< std::unique_ptr< amrex::MultiFab >> &  z_phys_nd,
bool  regrid_forces_file_read 
)
351 {
352 
353  static amrex::Vector<double> next_read_forecast_time;
354  static amrex::Vector<double> last_read_forecast_time;
355 
356  const int nlevs = static_cast<int>(a_z_phys_nd.size());
357 
358  Real hindcast_data_interval = solverChoice.hindcast_data_interval_in_hrs*Real(3600.0);
359 
360  // Initialize static vectors once
361  if (next_read_forecast_time.empty()) {
362  next_read_forecast_time.resize(nlevs, -one);
363  last_read_forecast_time.resize(nlevs, -one);
364  Print() << "Initializing the time vector values here by " << lev << std::endl;
365  }
366 
367  if (next_read_forecast_time[lev] < zero) {
368  int next_multiple = static_cast<int>(time / hindcast_data_interval);
369  next_read_forecast_time[lev] = next_multiple * hindcast_data_interval;
370  last_read_forecast_time[lev] = next_read_forecast_time[lev];
371  }
372 
373  if (time >= next_read_forecast_time[lev] or regrid_forces_file_read) {
374 
375  Print() << "Data reading happening at level " << lev << std::endl;
376 
377  std::string folder = solverChoice.hindcast_boundary_data_dir;
378 
379  // Check if folder exists and is a directory
380  if (!fs::exists(folder) || !fs::is_directory(folder)) {
381  throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
382  }
383 
384  std::vector<std::string> bin_files;
385 
386  for (const auto& entry : fs::directory_iterator(folder)) {
387  if (!entry.is_regular_file()) continue;
388 
389  std::string fname = entry.path().filename().string();
390  if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
391  bin_files.push_back(entry.path().string());
392  }
393  }
394  std::sort(bin_files.begin(), bin_files.end());
395 
396  // Check if no .bin files were found
397  if (bin_files.empty()) {
398  throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
399  }
400 
401  std::string filename1, filename2;
402 
403  int idx1 = static_cast<int>(time / hindcast_data_interval);
404  int idx2 = static_cast<int>(time / hindcast_data_interval)+1;
405  Print() << "Reading weather data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
406 
407  if (idx2 >= static_cast<int>(bin_files.size())) {
408  throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
409  }
410 
411  filename1 = bin_files[idx1];
412  filename2 = bin_files[idx2];
413 
414  FillForecastStateMultiFabs(lev, filename1, a_z_phys_nd[lev], forecast_state_1);
415  FillForecastStateMultiFabs(lev, filename2, a_z_phys_nd[lev], forecast_state_2);
416 
417  // Create the time-interpolated forecast state
418  //CreateForecastStateMultiFabs(forecast_state_interp);
419  if(!regrid_forces_file_read){
420  last_read_forecast_time[lev] = next_read_forecast_time[lev];
421  next_read_forecast_time[lev] += hindcast_data_interval;
422  Print() << "Next forecast time getting updated here " << std::endl;
423  }
424  }
425 
426  double prev_read_time = last_read_forecast_time[lev];
427  Real alpha1 = static_cast<Real>(1.0 - (time - prev_read_time)/hindcast_data_interval);
428  Real alpha2 = one - alpha1;
429 
430  amrex::Print()<< "The values of alpha1 and alpha2 are " << alpha1 << " "<< alpha2 <<std::endl;
431 
432  if (alpha1 < zero || alpha1 > one ||
433  alpha2 < zero || alpha2 > one)
434  {
435  std::stringstream ss;
436  ss << "Interpolation weights for hindcast files are incorrect: "
437  << "alpha1 = " << alpha1 << ", alpha2 = " << alpha2;
438  Abort(ss.str());
439  }
440 
441  MultiFab& erf_mf_cons = forecast_state_interp[lev][Vars::cons];
442  MultiFab& erf_mf_xvel = forecast_state_interp[lev][Vars::xvel];
443  MultiFab& erf_mf_yvel = forecast_state_interp[lev][Vars::yvel];
444  //MultiFab& erf_mf_zvel = forecast_state_interp[0][Vars::zvel];
445  MultiFab& erf_mf_latlon = forecast_state_interp[lev][4];
446 
447  // Fill the time-interpolated forecast states
448  MultiFab::LinComb(forecast_state_interp[lev][Vars::cons],
449  alpha1, forecast_state_1[lev][Vars::cons], 0,
450  alpha2, forecast_state_2[lev][Vars::cons], 0,
451  0, erf_mf_cons.nComp(), forecast_state_interp[lev][Vars::cons].nGrow());
452  MultiFab::LinComb(forecast_state_interp[lev][Vars::xvel],
453  alpha1, forecast_state_1[lev][Vars::xvel], 0,
454  alpha2, forecast_state_2[lev][Vars::xvel], 0,
455  0, erf_mf_xvel.nComp(), forecast_state_interp[lev][Vars::xvel].nGrow());
456  MultiFab::LinComb(forecast_state_interp[lev][Vars::yvel],
457  alpha1, forecast_state_1[lev][Vars::yvel], 0,
458  alpha2, forecast_state_2[lev][Vars::yvel], 0,
459  0, erf_mf_yvel.nComp(), forecast_state_interp[lev][Vars::yvel].nGrow());
460  MultiFab::LinComb(forecast_state_interp[lev][4],
461  alpha1, forecast_state_1[lev][4], 0,
462  alpha2, forecast_state_2[lev][4], 0,
463  0, erf_mf_latlon.nComp(), forecast_state_interp[lev][4].nGrow());
464 
465  /*Vector<std::string> varnames_plot_mf = {
466  "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
467  }; // Customize variable names
468 
469  std::string pltname = "plt_interp";
470 
471  MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
472  10, 0);
473 
474  plot_mf.setVal(0.0);
475 
476  for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
477  const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
478  const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
479  const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
480  const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
481  const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
482  const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
483 
484  const Box& bx = mfi.validbox();
485 
486  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
487  plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
488  plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
489  plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
490  plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
491  plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
492 
493  plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/two;
494  plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/two;
495  plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/two;
496 
497  plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
498  plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
499  });
500  }
501 
502 
503  WriteSingleLevelPlotfile(
504  pltname,
505  plot_mf,
506  varnames_plot_mf,
507  geom[0],
508  time,
509  0 // level
510  );*/
511 }
ERF::FillForecastStateMultiFabs
void FillForecastStateMultiFabs(const int lev, const std::string &filename, const std::unique_ptr< amrex::MultiFab > &z_phys_nd, amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
Definition: ERF_WeatherDataInterpolation.cpp:64
SolverChoice::hindcast_boundary_data_dir
std::string hindcast_boundary_data_dir
Definition: ERF_DataStruct.H:1470

◆ Write2DPlotFile()

void ERF::Write2DPlotFile ( int  which,
PlotFileType  plotfile_type,
amrex::Vector< std::string >  plot_var_names 
)
152 {
153  const int plot_step = istep[0];
154  const std::string plotfilename = make_2d_plotfile_name(which, plot_step,
155  plot2d_file_1, plot2d_file_2,
156  file_name_digits);
157 
158  const auto output_descriptors =
159  plotfile2d::build_sampled_level_output_descriptors(pp_prefix, which,
160  plot_var_names, solverChoice);
161 
162  Vector<std::string> varnames;
163  varnames.reserve(output_descriptors.size());
164  for (const auto& descriptor : output_descriptors) {
165  varnames.push_back(descriptor.name);
166  }
167  const int ncomp_mf = static_cast<int>(varnames.size());
168 
169  if (ncomp_mf == 0) return;
170 
171  // Vector of MultiFabs for cell-centered data
172  Vector<MultiFab> mf(finest_level+1);
173  for (int lev = 0; lev <= finest_level; ++lev) {
174  mf[lev].define(ba2d[lev], dmap[lev], ncomp_mf, 0);
175  }
176 
177 
178  // **********************************************************************************************
179  // (Effectively) 2D arrays
180  // **********************************************************************************************
181  for (int lev = 0; lev <= finest_level; ++lev)
182  {
183  // Make sure getPgivenRTh and getTgivenRandRTh don't fail
184  if (check_for_nans) {
185  check_for_negative_theta(vars_new[lev][Vars::cons]);
186  }
187 
188  int mf_comp = 0;
189 
190  // Set all components to zero in case they aren't defined below
191  mf[lev].setVal(0.0);
192 
193  // Expose domain khi and klo at each level
194  int klo = geom[lev].Domain().smallEnd(2);
195  int khi = geom[lev].Domain().bigEnd(2);
196 
197  const MultiFab* pblh_source = nullptr;
198  const MultiFab* sens_flux_source = SFS_hfx3_lev[lev].get();
199  const MultiFab* laten_flux_source = SFS_q1fx3_lev[lev].get();
200  const MultiFab* shoc_ustar_source = nullptr;
201  const MultiFab* shoc_olen_source = nullptr;
202  const MultiFab* shoc_wthv_source = nullptr;
203  const ShocDriver* native_shoc = native_shoc_driver[lev].get();
204  const bool native_shoc_owns_scalar_fluxes =
205  native_shoc && native_shoc->owns_scalar_surface_fluxes();
206  const bool native_shoc_has_consumed_flux_diagnostics =
207  native_shoc && native_shoc->has_consumed_surface_flux_diagnostics();
208  if (native_shoc && native_shoc->has_native_diagnostics()) {
209  pblh_source = &native_shoc->pblh_diagnostics();
210  }
211  // Native SHOC state_update clears the host SFS arrays after consuming
212  // them. Use SHOC's preserved snapshots only for flux components whose
213  // corresponding host SFS field existed; otherwise keep the source null
214  // so the 2D writer emits the documented -999 missing value.
215  if (plotfile2d::use_native_shoc_consumed_flux_source(
216  native_shoc_owns_scalar_fluxes,
217  native_shoc_has_consumed_flux_diagnostics,
218  SFS_hfx3_lev[lev] != nullptr)) {
219  sens_flux_source = &native_shoc->consumed_sens_flux_diagnostics();
220  }
221  if (plotfile2d::use_native_shoc_consumed_flux_source(
222  native_shoc_owns_scalar_fluxes,
223  native_shoc_has_consumed_flux_diagnostics,
224  SFS_q1fx3_lev[lev] != nullptr)) {
225  laten_flux_source = &native_shoc->consumed_laten_flux_diagnostics();
226  }
227  if (native_shoc && native_shoc->has_native_diagnostics()) {
228  shoc_ustar_source = &native_shoc->shoc_ustar_diagnostics();
229  shoc_olen_source = &native_shoc->shoc_olen_diagnostics();
230  shoc_wthv_source = &native_shoc->wthv_sec_diagnostics();
231  }
232  // pblh should follow the active PBL diagnostic provider. Native SHOC
233  // diagnoses its own PBL height in state_update mode; SurfaceLayer
234  // remains the fallback for non-SHOC configurations.
235  if (!pblh_source && m_SurfaceLayer) {
236  pblh_source = m_SurfaceLayer->get_pblh(lev);
237  }
238 
239  if (containerHasElement(plot_var_names, "z_surf")) {
240 #ifdef _OPENMP
241 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
242 #endif
243  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
244  {
245  const Box& bx = mfi.tilebox();
246  const Array4<Real>& derdat = mf[lev].array(mfi);
247  const Array4<const Real>& z_phys_arr = z_phys_nd[lev]->const_array(mfi);
248  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
249  derdat(i, j, k, mf_comp) = Compute_Z_AtWFace(i, j, 0, z_phys_arr);
250  });
251  }
252  mf_comp++;
253  }
254 
255  if (containerHasElement(plot_var_names, "landmask")) {
256 #ifdef _OPENMP
257 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
258 #endif
259  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
260  {
261  const Box& bx = mfi.tilebox();
262  const Array4<Real>& derdat = mf[lev].array(mfi);
263  const Array4<const int>& lmask_arr = lmask_lev[lev][0]->const_array(mfi);
264  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
265  derdat(i, j, k, mf_comp) = static_cast<Real>(lmask_arr(i, j, 0));
266  });
267  }
268  mf_comp++;
269  }
270 
271  if (containerHasElement(plot_var_names, "mapfac")) {
272 #ifdef _OPENMP
273 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
274 #endif
275  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
276  {
277  const Box& bx = mfi.tilebox();
278  const Array4<Real>& derdat = mf[lev].array(mfi);
279  const Array4<Real>& mf_m = mapfac[lev][MapFacType::m_x]->array(mfi);
280  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
281  derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
282  });
283  }
284  mf_comp++;
285  }
286 
287  if (containerHasElement(plot_var_names, "lat_m")) {
288  if (lat_m[lev]) {
289 #ifdef _OPENMP
290 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
291 #endif
292  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
293  {
294  const Box& bx = mfi.tilebox();
295  const Array4<Real>& derdat = mf[lev].array(mfi);
296  const Array4<Real>& data = lat_m[lev]->array(mfi);
297  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
298  derdat(i, j, k, mf_comp) = data(i,j,0);
299  });
300  }
301  }
302  mf_comp++;
303  } // lat_m
304 
305  if (containerHasElement(plot_var_names, "lon_m")) {
306  if (lon_m[lev]) {
307 #ifdef _OPENMP
308 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
309 #endif
310  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
311  {
312  const Box& bx = mfi.tilebox();
313  const Array4<Real>& derdat = mf[lev].array(mfi);
314  const Array4<Real>& data = lon_m[lev]->array(mfi);
315  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
316  derdat(i, j, k, mf_comp) = data(i,j,0);
317  });
318  }
319  } else {
320  mf[lev].setVal(0.0,mf_comp,1,0);
321  }
322 
323  mf_comp++;
324 
325  } // lon_m
326 
327  ///////////////////////////////////////////////////////////////////////
328  // Surface and single-level diagnostics use the fill helpers below.
329  // Column reductions and future interpolated surfaces keep explicit
330  // assembly paths until their contracts are isolated in dedicated
331  // helpers.
332  if (containerHasElement(plot_var_names, "u_star")) {
333  plotfile2d::fill_component_from_klevel_or_value(
334  mf[lev], mf_comp, m_SurfaceLayer ? m_SurfaceLayer->get_u_star(lev) : nullptr,
335  0, -999);
336  mf_comp++;
337  } // u_star
338 
339  if (containerHasElement(plot_var_names, "w_star")) {
340  plotfile2d::fill_component_from_klevel_or_value(
341  mf[lev], mf_comp, m_SurfaceLayer ? m_SurfaceLayer->get_w_star(lev) : nullptr,
342  0, -999);
343  mf_comp++;
344  } // w_star
345 
346  if (containerHasElement(plot_var_names, "t_star")) {
347  plotfile2d::fill_component_from_klevel_or_value(
348  mf[lev], mf_comp, m_SurfaceLayer ? m_SurfaceLayer->get_t_star(lev) : nullptr,
349  0, -999);
350  mf_comp++;
351  } // t_star
352 
353  if (containerHasElement(plot_var_names, "q_star")) {
354  plotfile2d::fill_component_from_klevel_or_value(
355  mf[lev], mf_comp, m_SurfaceLayer ? m_SurfaceLayer->get_q_star(lev) : nullptr,
356  0, -999);
357  mf_comp++;
358  } // q_star
359 
360  if (containerHasElement(plot_var_names, "Olen")) {
361  plotfile2d::fill_component_from_klevel_or_value(
362  mf[lev], mf_comp, m_SurfaceLayer ? m_SurfaceLayer->get_olen(lev) : nullptr,
363  0, -999);
364  mf_comp++;
365  } // Olen
366 
367  if (containerHasElement(plot_var_names, "pblh")) {
368  plotfile2d::fill_component_from_klevel_or_value(
369  mf[lev], mf_comp, pblh_source, 0, -999);
370  mf_comp++;
371  } // pblh
372 
373  if (containerHasElement(plot_var_names, "t_surf")) {
374  plotfile2d::fill_component_from_klevel_or_value(
375  mf[lev], mf_comp, m_SurfaceLayer ? m_SurfaceLayer->get_t_surf(lev) : nullptr,
376  0, -999);
377  mf_comp++;
378  } // t_surf
379 
380  if (containerHasElement(plot_var_names, "q_surf")) {
381  plotfile2d::fill_component_from_klevel_or_value(
382  mf[lev], mf_comp, m_SurfaceLayer ? m_SurfaceLayer->get_q_surf(lev) : nullptr,
383  0, -999);
384  mf_comp++;
385  } // q_surf
386 
387  if (containerHasElement(plot_var_names, "z0")) {
388  plotfile2d::fill_component_from_klevel_or_value(
389  mf[lev], mf_comp, m_SurfaceLayer ? m_SurfaceLayer->get_z0(lev) : nullptr,
390  0, -999);
391  mf_comp++;
392  } // z0
393 
394  if (containerHasElement(plot_var_names, "OLR")) {
395  plotfile2d::fill_component_from_klevel_or_value(
396  mf[lev], mf_comp, rad_fluxes[lev].get(), khi, -999, 2);
397  mf_comp++;
398  } // OLR
399 
400  if (containerHasElement(plot_var_names, "sens_flux")) {
401  plotfile2d::fill_component_from_klevel_or_value(
402  mf[lev], mf_comp, sens_flux_source, klo, -999);
403  mf_comp++;
404  } // sens_flux
405 
406  // Keep the legacy output name "laten_flux"; it maps to the vertical
407  // water-vapor surface flux field.
408  if (containerHasElement(plot_var_names, "laten_flux")) {
409  plotfile2d::fill_component_from_klevel_or_value(
410  mf[lev], mf_comp, laten_flux_source, klo, -999);
411  mf_comp++;
412  } // laten_flux
413 
414  if (containerHasElement(plot_var_names, "surf_pres")) {
415  bool moist = (solverChoice.moisture_type != MoistureType::None);
416 #ifdef _OPENMP
417 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
418 #endif
419  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
420  {
421  const Box& bx = mfi.tilebox();
422  const auto& derdat = mf[lev].array(mfi);
423  const auto& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
424  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
425  auto rt = cons_arr(i,j,klo,RhoTheta_comp);
426  auto qv = (moist) ? cons_arr(i,j,klo,RhoQ1_comp)/cons_arr(i,j,klo,Rho_comp)
427  : zero;
428  derdat(i, j, k, mf_comp) = getPgivenRTh(rt, qv);
429  });
430  }
431  mf_comp++;
432  } // surf_pres
433 
434  const auto precip_sources =
435  micro ? micro->Get_Surface_Precip_Accumulation_Ptrs(lev)
436  : SurfacePrecipAccumulationSources{};
437  const auto selected_precipitation =
438  plotfile2d::selected_precipitation_accumulation_components(varnames,
439  precip_sources);
440  if (selected_precipitation.n > 0) {
441  plotfile2d::fill_precipitation_accumulations(mf[lev],
442  precip_sources,
443  selected_precipitation,
444  klo);
445  }
446 
447  if (containerHasElement(plot_var_names, "precip_total_accum")) { mf_comp++; }
448  if (containerHasElement(plot_var_names, "precip_rain_accum")) { mf_comp++; }
449  if (containerHasElement(plot_var_names, "precip_snow_accum")) { mf_comp++; }
450  if (containerHasElement(plot_var_names, "precip_graupel_accum")) { mf_comp++; }
451  if (containerHasElement(plot_var_names, "precip_hail_accum")) { mf_comp++; }
452  if (containerHasElement(plot_var_names, "precip_frozen_accum")) { mf_comp++; }
453 
454  if (containerHasElement(plot_var_names, "integrated_qv")) {
455  // integrated_qv remains the legacy column-reduction example.
456  // Scheme-aware condensed water paths use a dedicated helper below
457  // rather than the single-k copy helpers.
458  MultiFab mf_qv_int(mf[lev],make_alias,mf_comp,1);
459  if (solverChoice.moisture_type != MoistureType::None) {
460  volWgtColumnSum(lev, vars_new[lev][Vars::cons], RhoQ1_comp, mf_qv_int, *detJ_cc[lev]);
461  } else {
462  mf_qv_int.setVal(0.);
463  }
464  mf_comp++;
465  }
466 
467  const auto selected_water_paths =
468  plotfile2d::selected_condensed_water_path_components(varnames, solverChoice);
469  if (selected_water_paths.n > 0) {
470  // Condensed water paths are scheme-aware column reductions. Their
471  // availability comes from solverChoice.moisture_indices, and the
472  // reduction uses the same metric convention as integrated_qv.
473  plotfile2d::fill_condensed_water_paths(mf[lev],
474  vars_new[lev][Vars::cons],
475  selected_water_paths,
476  geom[lev],
477  *detJ_cc[lev]);
478  }
479 
480  if (containerHasElement(plot_var_names, "integrated_qc")) { mf_comp++; }
481  if (containerHasElement(plot_var_names, "integrated_qi")) { mf_comp++; }
482  if (containerHasElement(plot_var_names, "integrated_qr")) { mf_comp++; }
483  if (containerHasElement(plot_var_names, "integrated_qs")) { mf_comp++; }
484  if (containerHasElement(plot_var_names, "integrated_qg")) { mf_comp++; }
485 
486  if (containerHasElement(plot_var_names, "surface_diagnostic_source")) {
487  plotfile2d::fill_component_from_klevel_or_value(
488  mf[lev], mf_comp,
489  m_SurfaceLayer ? m_SurfaceLayer->get_surface_diagnostic_source(lev) : nullptr,
490  0, 0);
491  mf_comp++;
492  } // surface_diagnostic_source
493 
494  if (containerHasElement(plot_var_names, "sensible_heat_flux")) {
495  plotfile2d::fill_sensible_heat_flux_from_klevel_or_missing(
496  mf[lev], mf_comp, sens_flux_source, klo, -999);
497  mf_comp++;
498  } // sensible_heat_flux
499 
500  if (containerHasElement(plot_var_names, "latent_heat_flux")) {
501  plotfile2d::fill_latent_heat_flux_from_klevel_or_missing(
502  mf[lev], mf_comp, laten_flux_source, klo, -999);
503  mf_comp++;
504  } // latent_heat_flux
505 
506  if (containerHasElement(plot_var_names, "shoc_u_star")) {
507  plotfile2d::fill_component_from_klevel_or_value(
508  mf[lev], mf_comp, shoc_ustar_source, klo, -999);
509  mf_comp++;
510  } // shoc_u_star
511 
512  if (containerHasElement(plot_var_names, "shoc_Olen")) {
513  plotfile2d::fill_component_from_klevel_or_value(
514  mf[lev], mf_comp, shoc_olen_source, klo, -999);
515  mf_comp++;
516  } // shoc_Olen
517 
518  if (containerHasElement(plot_var_names, "shoc_wthv_sfc")) {
519  plotfile2d::fill_component_from_klevel_or_value(
520  mf[lev], mf_comp, shoc_wthv_source, klo, -999);
521  mf_comp++;
522  } // shoc_wthv_sfc
523 
524  // Land-surface provider fields use the generic LandSurface name/index
525  // interface. The catalog supplies metadata; this block only assembles
526  // stored values and translates provider sentinels.
527  int temperature_2m_comp = -1;
528  int mixing_ratio_2m_comp = -1;
529  int near_surface_source_comp = -1;
530  for (const auto& name : plot_var_names) {
531  const auto* descriptor = plotfile2d::find_diagnostic(name);
532  if (!descriptor) {
533  descriptor = plotfile2d::find_dynamic_soil_diagnostic(name);
534  }
535  if (!is_land_surface_diagnostic(descriptor)) {
536  continue;
537  }
538 
539  if (is_unified_near_surface_diagnostic(descriptor->id)) {
540  if (descriptor->id == plotfile2d::DiagnosticID::Temperature2m) {
541  temperature_2m_comp = mf_comp;
542  } else if (descriptor->id == plotfile2d::DiagnosticID::WaterVaporMixingRatio2m) {
543  mixing_ratio_2m_comp = mf_comp;
544  } else {
545  near_surface_source_comp = mf_comp;
546  }
547  ++mf_comp;
548  continue;
549  }
550 
551  plotfile2d::fill_land_surface_component_from_klevel_or_missing(
552  mf[lev], mf_comp, lsm.Get_Data_Ptr(lev, name), 0, -999);
553  ++mf_comp;
554  }
555 
556  if (temperature_2m_comp >= 0 || mixing_ratio_2m_comp >= 0 ||
557  near_surface_source_comp >= 0) {
558  near_surface_diagnostics::Sources near_surface_sources;
559  near_surface_sources.native_temperature_vegetated =
560  lsm.Get_Data_Ptr(lev, "noahmp_temperature_2m_vegetated");
561  near_surface_sources.native_temperature_bare =
562  lsm.Get_Data_Ptr(lev, "noahmp_temperature_2m_bare");
563  near_surface_sources.native_mixing_ratio_vegetated =
564  lsm.Get_Data_Ptr(lev, "noahmp_water_vapor_mixing_ratio_2m_vegetated");
565  near_surface_sources.native_mixing_ratio_bare =
566  lsm.Get_Data_Ptr(lev, "noahmp_water_vapor_mixing_ratio_2m_bare");
567  near_surface_sources.native_vegetation_fraction =
568  lsm.Get_Data_Ptr(lev, "noahmp_vegetation_fraction");
569  near_surface_sources.theta_surface =
570  m_SurfaceLayer ? m_SurfaceLayer->get_t_surf(lev) : nullptr;
571  near_surface_sources.theta_star =
572  m_SurfaceLayer ? m_SurfaceLayer->get_t_star(lev) : nullptr;
573  near_surface_sources.mixing_ratio_surface =
574  m_SurfaceLayer ? m_SurfaceLayer->get_q_surf(lev) : nullptr;
575  near_surface_sources.mixing_ratio_star =
576  m_SurfaceLayer ? m_SurfaceLayer->get_q_star(lev) : nullptr;
577  near_surface_sources.roughness_height =
578  m_SurfaceLayer ? m_SurfaceLayer->get_z0(lev) : nullptr;
579  near_surface_sources.obukhov_length =
580  m_SurfaceLayer ? m_SurfaceLayer->get_olen(lev) : nullptr;
581  near_surface_sources.source_mask =
582  m_SurfaceLayer ? m_SurfaceLayer->get_surface_diagnostic_source(lev) : nullptr;
583  near_surface_sources.land_mask =
584  (lmask_lev[lev].empty()) ? nullptr : lmask_lev[lev][0].get();
585  near_surface_sources.cons = &vars_new[lev][Vars::cons];
586  near_surface_sources.z_phys_nd = z_phys_nd[lev].get();
587  near_surface_sources.dz = geom[lev].CellSize(2);
588  near_surface_sources.klo = klo;
589  near_surface_sources.moist = solverChoice.moisture_type != MoistureType::None;
590  near_surface_sources.has_lsm =
591  near_surface_sources.native_temperature_vegetated != nullptr &&
592  near_surface_sources.native_temperature_bare != nullptr &&
593  near_surface_sources.native_mixing_ratio_vegetated != nullptr &&
594  near_surface_sources.native_mixing_ratio_bare != nullptr &&
595  near_surface_sources.native_vegetation_fraction != nullptr;
596  near_surface_diagnostics::fill(mf[lev], temperature_2m_comp,
597  mixing_ratio_2m_comp,
598  near_surface_source_comp,
599  near_surface_sources);
600  }
601 
602  const int static_output_count = static_cast<int>(plot_var_names.size());
603  for (int out_idx = static_output_count; out_idx < static_cast<int>(output_descriptors.size()); ++out_idx) {
604  const auto& descriptor = output_descriptors[out_idx];
605  // Sampled-level descriptors are dynamic. The interpolator owns
606  // their vertical sampling so the writer remains an output
607  // assembly layer.
608  plotfile2d::SampledWindSources wind_sources;
609  wind_sources.xvel = &vars_new[lev][Vars::xvel];
610  wind_sources.yvel = &vars_new[lev][Vars::yvel];
611  wind_sources.zvel = &vars_new[lev][Vars::zvel];
612  plotfile2d::fill_sampled_level_component(
613  mf[lev], mf_comp, descriptor,
614  vars_new[lev][Vars::cons],
615  z_phys_cc[lev].get(),
616  *z_phys_nd[lev],
617  z_phys_cc[lev] != nullptr,
618  solverChoice.moisture_indices,
619  klo, khi,
620  wind_sources);
621  mf_comp++;
622  }
623 
624  if (mf_comp != ncomp_mf) {
625  Abort(plotfile2d::format_2d_component_count_error(lev, mf_comp, ncomp_mf));
626  }
627  } // lev
628 
629  Vector<Geometry> my_geom = make_2d_plot_geometries(geom, finest_level);
630 
631  if (plotfile_type == PlotFileType::Amrex)
632  {
633  Print() << "Writing 2D native plotfile " << plotfilename << "\n";
634  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
635  GetVecOfConstPtrs(mf),
636  varnames, my_geom, static_cast<Real>(t_new[0]), istep, refRatio());
637  // Native AMReX 2D plotfiles write a JSON sidecar with catalog
638  // metadata for the selected output variables only.
639  plotfile2d::write_2d_metadata_json(plotfilename, output_descriptors);
640  writeJobInfo(plotfilename, erf_provenance::ArtifactType::Plotfile2D,
641  istep[0], t_new[0]);
642 
643 #ifdef ERF_USE_NETCDF
644  } else if (plotfile_type == PlotFileType::Netcdf) {
645  int lev = 0;
646  int l_which = 0;
647  const Real* p_lo = my_geom[lev].ProbLo();
648  const Real* p_hi = my_geom[lev].ProbHi();
649  const auto dx = my_geom[lev].CellSize();
650  writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
651  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],dx[2]}, {dx[0],dx[1],dx[2]},
652  my_geom[lev].Domain(), static_cast<Real>(t_new[0]),
653  static_cast<Real>(start_bdy_time), solverChoice, zlevels_stag[lev]);
654 #endif
655  } else {
656  // Here we assume the plotfile_type is PlotFileType::None
657  Print() << "Writing no 2D plotfile since plotfile_type is none" << std::endl;
658  }
659 }
writeNCPlotFile
void writeNCPlotFile(int lev, int which_subdomain, const std::string &dir, const Vector< const MultiFab * > &plotMF, const Vector< std::string > &plot_var_names, const Vector< int > &, Array< Real, AMREX_SPACEDIM > prob_lo, Array< Real, AMREX_SPACEDIM > prob_hi, Array< Real, AMREX_SPACEDIM > dx_in, const Box &subdomain, const double &time, const double &start_bdy_time, const SolverChoice &solverChoice, const Vector< Real > &zlevels_stag)
Definition: ERF_NCPlotFile.cpp:17
descriptor
DiagnosticDescriptor descriptor
Definition: ERF_Plotfile2DCatalog.cpp:100
Compute_Z_AtWFace
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_Z_AtWFace(const int &i, const int &j, const int &k, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:376
ERF::writeJobInfo
void writeJobInfo(const std::string &dir, erf_provenance::ArtifactType artifact_type, int artifact_step, double artifact_time_seconds) const
Definition: ERF_WriteJobInfo.cpp:12
ShocDriver
Definition: ERF_ShocDriver.H:55
ShocDriver::has_consumed_surface_flux_diagnostics
bool has_consumed_surface_flux_diagnostics() const
Definition: ERF_ShocDriver.H:89
ShocDriver::consumed_laten_flux_diagnostics
const amrex::MultiFab & consumed_laten_flux_diagnostics() const
Definition: ERF_ShocDriver.H:101
ShocDriver::owns_scalar_surface_fluxes
bool owns_scalar_surface_fluxes() const
Definition: ERF_ShocDriver.cpp:893
ShocDriver::has_native_diagnostics
bool has_native_diagnostics() const
Definition: ERF_ShocDriver.H:88
ShocDriver::shoc_olen_diagnostics
const amrex::MultiFab & shoc_olen_diagnostics() const
Definition: ERF_ShocDriver.H:106
ShocDriver::shoc_ustar_diagnostics
const amrex::MultiFab & shoc_ustar_diagnostics() const
Definition: ERF_ShocDriver.H:105
ShocDriver::consumed_sens_flux_diagnostics
const amrex::MultiFab & consumed_sens_flux_diagnostics() const
Definition: ERF_ShocDriver.H:97
ShocDriver::pblh_diagnostics
const amrex::MultiFab & pblh_diagnostics() const
Definition: ERF_ShocDriver.H:95
ShocDriver::wthv_sec_diagnostics
const amrex::MultiFab & wthv_sec_diagnostics() const
Definition: ERF_ShocDriver.H:113
erf_provenance::ArtifactType::Plotfile2D
@ Plotfile2D
near_surface_diagnostics::fill
void fill(MultiFab &dst, int temperature_comp, int mixing_ratio_comp, int source_comp, const Sources &sources, Real missing_value)
Definition: ERF_NearSurfaceDiagnostics.cpp:41
plotfile2d::fill_sampled_level_component
void fill_sampled_level_component(MultiFab &dst, int dst_comp, const Plotfile2DOutputDescriptor &descriptor, const MultiFab &cons, const MultiFab *z_phys_cc, const MultiFab &z_phys_nd, bool have_z_phys_cc, const MoistureComponentIndices &moisture_indices, int klo, int khi, const SampledWindSources &wind_sources)
Definition: ERF_Plotfile2DInterpolator.cpp:167
plotfile2d::write_2d_metadata_json
void write_2d_metadata_json(const std::string &plotfilename, const amrex::Vector< std::string > &varnames)
Definition: ERF_Plotfile2DMetadata.cpp:229
plotfile2d::fill_condensed_water_paths
void fill_condensed_water_paths(MultiFab &dst, const MultiFab &cons, const SelectedWaterPathComponents &selected, const Geometry &geom, const MultiFab &detJ)
Definition: ERF_Plotfile2DWaterPath.cpp:236
plotfile2d::fill_sensible_heat_flux_from_klevel_or_missing
void fill_sensible_heat_flux_from_klevel_or_missing(MultiFab &dst, int dst_comp, const MultiFab *src, int src_k, Real missing_value)
Definition: ERF_Plotfile2DFill.cpp:103
plotfile2d::build_sampled_level_output_descriptors
amrex::Vector< Plotfile2DOutputDescriptor > build_sampled_level_output_descriptors(const std::string &pp_prefix, int which, const amrex::Vector< std::string > &static_plot_vars, const SolverChoice &solver_choice)
Definition: ERF_Plotfile2DSampledLevel.cpp:488
plotfile2d::selected_condensed_water_path_components
SelectedWaterPathComponents selected_condensed_water_path_components(const amrex::Vector< std::string > &plot_var_names, const SolverChoice &solver_choice)
Definition: ERF_Plotfile2DWaterPath.cpp:211
plotfile2d::format_2d_component_count_error
std::string format_2d_component_count_error(int lev, int filled, int expected)
Definition: ERF_Plotfile2DUtils.cpp:86
plotfile2d::find_dynamic_soil_diagnostic
const DiagnosticDescriptor * find_dynamic_soil_diagnostic(const std::string &name)
Definition: ERF_Plotfile2DCatalog.cpp:208
plotfile2d::fill_land_surface_component_from_klevel_or_missing
void fill_land_surface_component_from_klevel_or_missing(MultiFab &dst, int dst_comp, const MultiFab *src, int src_k, Real missing_value)
Definition: ERF_Plotfile2DFill.cpp:75
plotfile2d::fill_component_from_klevel_or_value
void fill_component_from_klevel_or_value(MultiFab &dst, int dst_comp, const MultiFab *src, int src_k, Real missing_value, int src_comp)
Definition: ERF_Plotfile2DFill.cpp:60
plotfile2d::use_native_shoc_consumed_flux_source
AMREX_FORCE_INLINE bool use_native_shoc_consumed_flux_source(bool native_shoc_owns_scalar_fluxes, bool native_shoc_has_consumed_flux_diagnostics, bool host_flux_field_available) noexcept
Definition: ERF_Plotfile2DUtils.H:20
plotfile2d::selected_precipitation_accumulation_components
SelectedSurfacePrecipAccumulationComponents selected_precipitation_accumulation_components(const amrex::Vector< std::string > &plot_var_names, const SurfacePrecipAccumulationSources &sources)
Definition: ERF_Plotfile2DPrecip.cpp:95
plotfile2d::fill_precipitation_accumulations
void fill_precipitation_accumulations(MultiFab &dst, const SurfacePrecipAccumulationSources &sources, const SelectedSurfacePrecipAccumulationComponents &selected, const int klo)
Definition: ERF_Plotfile2DPrecip.cpp:137
plotfile2d::fill_latent_heat_flux_from_klevel_or_missing
void fill_latent_heat_flux_from_klevel_or_missing(MultiFab &dst, int dst_comp, const MultiFab *src, int src_k, Real missing_value)
Definition: ERF_Plotfile2DFill.cpp:134
plotfile2d::find_diagnostic
const DiagnosticDescriptor * find_diagnostic(const std::string &name)
Definition: ERF_Plotfile2DCatalog.cpp:165
near_surface_diagnostics::Sources
Definition: ERF_NearSurfaceDiagnostics.H:165
near_surface_diagnostics::Sources::theta_surface
const amrex::MultiFab * theta_surface
Definition: ERF_NearSurfaceDiagnostics.H:172
near_surface_diagnostics::Sources::z_phys_nd
const amrex::MultiFab * z_phys_nd
Definition: ERF_NearSurfaceDiagnostics.H:181
near_surface_diagnostics::Sources::moist
bool moist
Definition: ERF_NearSurfaceDiagnostics.H:185
near_surface_diagnostics::Sources::has_lsm
bool has_lsm
Definition: ERF_NearSurfaceDiagnostics.H:186
near_surface_diagnostics::Sources::klo
int klo
Definition: ERF_NearSurfaceDiagnostics.H:184
near_surface_diagnostics::Sources::land_mask
const amrex::iMultiFab * land_mask
Definition: ERF_NearSurfaceDiagnostics.H:179
near_surface_diagnostics::Sources::obukhov_length
const amrex::MultiFab * obukhov_length
Definition: ERF_NearSurfaceDiagnostics.H:177
near_surface_diagnostics::Sources::roughness_height
const amrex::MultiFab * roughness_height
Definition: ERF_NearSurfaceDiagnostics.H:176
near_surface_diagnostics::Sources::native_temperature_bare
const amrex::MultiFab * native_temperature_bare
Definition: ERF_NearSurfaceDiagnostics.H:167
near_surface_diagnostics::Sources::cons
const amrex::MultiFab * cons
Definition: ERF_NearSurfaceDiagnostics.H:180
near_surface_diagnostics::Sources::theta_star
const amrex::MultiFab * theta_star
Definition: ERF_NearSurfaceDiagnostics.H:173
near_surface_diagnostics::Sources::native_temperature_vegetated
const amrex::MultiFab * native_temperature_vegetated
Definition: ERF_NearSurfaceDiagnostics.H:166
near_surface_diagnostics::Sources::mixing_ratio_surface
const amrex::MultiFab * mixing_ratio_surface
Definition: ERF_NearSurfaceDiagnostics.H:174
near_surface_diagnostics::Sources::native_mixing_ratio_bare
const amrex::MultiFab * native_mixing_ratio_bare
Definition: ERF_NearSurfaceDiagnostics.H:169
near_surface_diagnostics::Sources::dz
amrex::Real dz
Definition: ERF_NearSurfaceDiagnostics.H:183
near_surface_diagnostics::Sources::source_mask
const amrex::MultiFab * source_mask
Definition: ERF_NearSurfaceDiagnostics.H:178
near_surface_diagnostics::Sources::mixing_ratio_star
const amrex::MultiFab * mixing_ratio_star
Definition: ERF_NearSurfaceDiagnostics.H:175
near_surface_diagnostics::Sources::native_mixing_ratio_vegetated
const amrex::MultiFab * native_mixing_ratio_vegetated
Definition: ERF_NearSurfaceDiagnostics.H:168
near_surface_diagnostics::Sources::native_vegetation_fraction
const amrex::MultiFab * native_vegetation_fraction
Definition: ERF_NearSurfaceDiagnostics.H:170
plotfile2d::SampledWindSources
Definition: ERF_Plotfile2DInterpolator.H:25
plotfile2d::SampledWindSources::xvel
const amrex::MultiFab * xvel
Definition: ERF_Plotfile2DInterpolator.H:26
plotfile2d::SampledWindSources::zvel
const amrex::MultiFab * zvel
Definition: ERF_Plotfile2DInterpolator.H:28
plotfile2d::SampledWindSources::yvel
const amrex::MultiFab * yvel
Definition: ERF_Plotfile2DInterpolator.H:27
Here is the call graph for this function:

◆ Write3DPlotFile()

void ERF::Write3DPlotFile ( int  which,
PlotFileType  plotfile_type,
amrex::Vector< std::string >  plot_var_names 
)
266 {
267  auto dPlotTime0 = amrex::second();
268 
269  Real tnew = static_cast<Real>(t_new[0]);
270 
271  const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
272  const int ncomp_mf = static_cast<int>(varnames.size());
273 
274  int ncomp_cons = vars_new[0][Vars::cons].nComp();
275 
276  if (ncomp_mf == 0) return;
277 
278  // Lagrangian microphysics with AMR (TwoWay): bring fine-level moisture down
279  // to coarse cells so plotfiles at level 0 show the fine-deposit cloud.
280  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian
281  && solverChoice.coupling_type == CouplingType::TwoWay
282  && finest_level >= 1) {
283  micro->AverageDownMicroVars(finest_level);
284  for (int flev = finest_level-1; flev >= 0; --flev) {
285  AverageDownMoistStateTo(flev);
286  }
287  }
288 
289  // We Fillpatch here because some of the derived quantities require derivatives
290  // which require ghost cells to be filled. We do not need to call FillPatcher
291  // because we don't need to set interior fine points.
292  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
293 
294  // Level 0 FillPatch
295  FillPatchCrseLevel(0, t_new[0], {&vars_new[0][Vars::cons], &vars_new[0][Vars::xvel],
296  &vars_new[0][Vars::yvel], &vars_new[0][Vars::zvel]});
297 
298  for (int lev = 1; lev <= finest_level; ++lev) {
299  bool fillset = false;
300  FillPatchFineLevel(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
301  &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
302  {&vars_new[lev][Vars::cons], &rU_new[lev], &rV_new[lev], &rW_new[lev]},
303  base_state[lev], base_state[lev], fillset);
304  }
305 
306  // Get qmoist pointers if using moisture
307  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
308  for (int lev = 0; lev <= finest_level; ++lev) {
309  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
310  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
311  }
312  }
313 
314  // Vector of MultiFabs for cell-centered data
315  Vector<MultiFab> mf(finest_level+1);
316  for (int lev = 0; lev <= finest_level; ++lev) {
317  mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
318  }
319 
320  // Vector of MultiFabs for nodal data
321  Vector<MultiFab> mf_nd(finest_level+1);
322  if ( SolverChoice::mesh_type != MeshType::ConstantDz) {
323  for (int lev = 0; lev <= finest_level; ++lev) {
324  BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
325  mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
326  mf_nd[lev].setVal(0.);
327  }
328  }
329 
330  // Vector of MultiFabs for face-centered velocity
331  Vector<MultiFab> mf_u(finest_level+1);
332  Vector<MultiFab> mf_v(finest_level+1);
333  Vector<MultiFab> mf_w(finest_level+1);
334  if (m_plot_face_vels) {
335  for (int lev = 0; lev <= finest_level; ++lev) {
336  BoxArray grid_stag_u(grids[lev]); grid_stag_u.surroundingNodes(0);
337  BoxArray grid_stag_v(grids[lev]); grid_stag_v.surroundingNodes(1);
338  BoxArray grid_stag_w(grids[lev]); grid_stag_w.surroundingNodes(2);
339  mf_u[lev].define(grid_stag_u, dmap[lev], 1, 0);
340  mf_v[lev].define(grid_stag_v, dmap[lev], 1, 0);
341  mf_w[lev].define(grid_stag_w, dmap[lev], 1, 0);
342  MultiFab::Copy(mf_u[lev],vars_new[lev][Vars::xvel],0,0,1,0);
343  MultiFab::Copy(mf_v[lev],vars_new[lev][Vars::yvel],0,0,1,0);
344  MultiFab::Copy(mf_w[lev],vars_new[lev][Vars::zvel],0,0,1,0);
345  }
346  }
347 
348  // Array of MultiFabs for cell-centered velocity
349  Vector<MultiFab> mf_cc_vel(finest_level+1);
350 
351  if (containerHasElement(plot_var_names, "x_velocity" ) ||
352  containerHasElement(plot_var_names, "y_velocity" ) ||
353  containerHasElement(plot_var_names, "z_velocity" ) ||
354  containerHasElement(plot_var_names, "magvel" ) ||
355  containerHasElement(plot_var_names, "helicity" ) ||
356  containerHasElement(plot_var_names, "local_helicity") ||
357  containerHasElement(plot_var_names, "vorticity_x" ) ||
358  containerHasElement(plot_var_names, "vorticity_y" ) ||
359  containerHasElement(plot_var_names, "vorticity_z" ) ) {
360 
361  for (int lev = 0; lev <= finest_level; ++lev) {
362  mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
363  mf_cc_vel[lev].setVal(bogus_large_value);
364  average_face_to_cellcenter(mf_cc_vel[lev],0,
365  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
366  &vars_new[lev][Vars::yvel],
367  &vars_new[lev][Vars::zvel]}, 1);
368  } // lev
369  } // if (vel or vort)
370 
371  // We need ghost cells if computing vorticity
372  if ( containerHasElement(plot_var_names, "vorticity_x")||
373  containerHasElement(plot_var_names, "vorticity_y") ||
374  containerHasElement(plot_var_names, "vorticity_z") )
375  {
376  amrex::Interpolater* mapper = &cell_cons_interp;
377  FillBdyCCVels(mf_cc_vel[0],geom[0]);
378  for (int lev = 1; lev <= finest_level; ++lev)
379  {
380  Real new_time = static_cast<Real>(t_new[lev]);
381  Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
382  Vector<Real> ftime = {new_time,new_time};
383  Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
384  Vector<Real> ctime = {new_time,new_time};
385 
386  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
387  FillPatchTwoLevels(mf_cc_vel[lev], mf_cc_vel[lev].nGrowVect(), IntVect(0,0,0),
388  static_cast<Real>(t_new[lev]), cmf, ctime, fmf, ftime,
389  0, 0, mf_cc_vel[lev].nComp(), geom[lev-1], geom[lev],
390  refRatio(lev-1), mapper, domain_bcs_type,
391  BaseBCVars::rho0_bc_comp);
392  FillBdyCCVels(mf_cc_vel[lev],geom[lev]);
393  } // lev
394  } // if (vort)
395 
396 
397  for (int lev = 0; lev <= finest_level; ++lev)
398  {
399  // Make sure getPgivenRTh and getTgivenRandRTh don't fail
400  if (check_for_nans) {
401  check_for_negative_theta(vars_new[lev][Vars::cons]);
402  }
403 
404  int mf_comp = 0;
405 
406  BoxArray ba(vars_new[lev][Vars::cons].boxArray());
407  DistributionMapping dm = vars_new[lev][Vars::cons].DistributionMap();
408 
409  // First, copy any of the conserved state variables into the output plotfile
410  for (int i = 0; i < cons_names.size(); ++i) {
411  if (containerHasElement(plot_var_names, cons_names[i])) {
412  MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,0);
413  mf_comp++;
414  }
415  }
416 
417  // Next, check for velocities
418  if (containerHasElement(plot_var_names, "x_velocity")) {
419  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 0, mf_comp, 1, 0);
420  mf_comp += 1;
421  }
422  if (containerHasElement(plot_var_names, "y_velocity")) {
423  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 1, mf_comp, 1, 0);
424  mf_comp += 1;
425  }
426  if (containerHasElement(plot_var_names, "z_velocity")) {
427  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 2, mf_comp, 1, 0);
428  mf_comp += 1;
429  }
430 
431  // Create multifabs for HSE and pressure fields used to derive other quantities
432  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
433  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
434  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
435  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
436 
437  MultiFab pressure;
438 
439  if (solverChoice.anelastic[lev] == 0) {
440  if (containerHasElement(plot_var_names, "pressure") ||
441  containerHasElement(plot_var_names, "pert_pres") ||
442  containerHasElement(plot_var_names, "dpdx") ||
443  containerHasElement(plot_var_names, "dpdy") ||
444  containerHasElement(plot_var_names, "dpdz") ||
445  containerHasElement(plot_var_names, "eq_pot_temp") ||
446  containerHasElement(plot_var_names, "qsat"))
447  {
448  int ng = (containerHasElement(plot_var_names, "dpdx") || containerHasElement(plot_var_names, "dpdy") ||
449  containerHasElement(plot_var_names, "dpdz")) ? 1 : 0;
450 
451  // Allocate space for pressure
452  pressure.define(ba,dm,1,ng);
453 
454  if (ng > 0) {
455  // Default to p_hse as a way of filling ghost cells at domain boundaries
456  MultiFab::Copy(pressure,p_hse,0,0,1,1);
457  }
458 #ifdef _OPENMP
459 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
460 #endif
461  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
462  {
463  const Box& gbx = mfi.growntilebox(IntVect(ng,ng,0));
464 
465  const Array4<Real >& p_arr = pressure.array(mfi);
466  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
467  const int ncomp = vars_new[lev][Vars::cons].nComp();
468 
469  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
470  {
471  Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
472  const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
473  p_arr(i, j, k) = getPgivenRTh(rhotheta,qv_for_p);
474  });
475  } // mfi
476  pressure.FillBoundary(geom[lev].periodicity());
477  } // compute compressible pressure
478  } // not anelastic
479  else {
480  if (containerHasElement(plot_var_names, "dpdx") ||
481  containerHasElement(plot_var_names, "dpdy") ||
482  containerHasElement(plot_var_names, "dpdz") ||
483  containerHasElement(plot_var_names, "eq_pot_temp") ||
484  containerHasElement(plot_var_names, "qsat"))
485  {
486  // Copy p_hse into pressure if using anelastic
487  pressure.define(ba,dm,1,0);
488  MultiFab::Copy(pressure,p_hse,0,0,1,0);
489  }
490  }
491 
492  // ***************************************************************************************
493  // Finally, check for any derived quantities and compute them, inserting
494  // them into our output multifab
495  // ***************************************************************************************
496  auto calculate_derived = [&](const std::string& der_name,
497  MultiFab& src_mf,
498  decltype(derived::erf_dernull)& der_function)
499  {
500  if (containerHasElement(plot_var_names, der_name)) {
501  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
502 #ifdef _OPENMP
503 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
504 #endif
505  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
506  {
507  const Box& bx = mfi.tilebox();
508  auto& dfab = dmf[mfi];
509  auto& sfab = src_mf[mfi];
510  auto& zfab = (*z_phys_cc[lev])[mfi];
511  der_function(bx, dfab, 0, 1, sfab, zfab, Geom(lev), tnew, nullptr, lev);
512  }
513 
514  mf_comp++;
515  }
516  }; // end calculate_derived
517  // ***************************************************************************************
518 
519  // *****************************************************************************************
520  // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
521  // defined in ERF.H
522  // *****************************************************************************************
523 
524  if (use_moisture) {
525  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dermoisttemp);
526  } else {
527  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dertemp);
528  }
529  calculate_derived("theta", vars_new[lev][Vars::cons], derived::erf_dertheta);
530  calculate_derived("KE", vars_new[lev][Vars::cons], derived::erf_derKE);
531  calculate_derived("scalar", vars_new[lev][Vars::cons], derived::erf_derscalar);
532  calculate_derived("soundspeed", vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
533 
534  if (containerHasElement(plot_var_names, "reflectivity"))
535  {
536  if (solverChoice.moisture_type == MoistureType::Morrison ||
537  solverChoice.moisture_type == MoistureType::WSM6 ||
538  solverChoice.moisture_type == MoistureType::SAM) {
539  calculate_derived("reflectivity", vars_new[lev][Vars::cons], derived::erf_derreflectivity);
540  } else {
541  mf[lev].setVal(zero, mf_comp, 1, 0);
542  mf_comp++;
543  }
544  }
545 
546  if (containerHasElement(plot_var_names, "max_reflectivity"))
547  {
548  if (solverChoice.moisture_type == MoistureType::Morrison ||
549  solverChoice.moisture_type == MoistureType::WSM6 ||
550  solverChoice.moisture_type == MoistureType::SAM) {
551  calculate_derived("max_reflectivity", vars_new[lev][Vars::cons], derived::erf_dermaxreflectivity);
552  } else {
553  mf[lev].setVal(zero, mf_comp, 1, 0);
554  mf_comp++;
555  }
556  }
557 
558  if (solverChoice.moisture_type != MoistureType::None) {
559  calculate_derived("precipitable" , vars_new[lev][Vars::cons], derived::erf_derprecipitable);
560  }
561  calculate_derived("mucape" , vars_new[lev][Vars::cons], derived::erf_dermucape);
562 
563  calculate_derived("vorticity_x", mf_cc_vel[lev], derived::erf_dervortx);
564  calculate_derived("vorticity_y", mf_cc_vel[lev], derived::erf_dervorty);
565  calculate_derived("vorticity_z", mf_cc_vel[lev], derived::erf_dervortz);
566  calculate_derived("helicity", mf_cc_vel[lev], derived::erf_derhelicity);
567  calculate_derived("local_helicity", mf_cc_vel[lev], derived::erf_derlocalhelicity);
568  calculate_derived("magvel", mf_cc_vel[lev], derived::erf_dermagvel);
569 
570  if (containerHasElement(plot_var_names, "divU"))
571  {
572  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
573  Array<MultiFab const*, AMREX_SPACEDIM> u;
574  u[0] = &(vars_new[lev][Vars::xvel]);
575  u[1] = &(vars_new[lev][Vars::yvel]);
576  u[2] = &(vars_new[lev][Vars::zvel]);
577  compute_divergence (lev, dmf, u, geom[lev]);
578  mf_comp += 1;
579  }
580 
581  if (containerHasElement(plot_var_names, "pres_hse"))
582  {
583  MultiFab::Copy(mf[lev],p_hse,0,mf_comp,1,0);
584  mf_comp += 1;
585  }
586  if (containerHasElement(plot_var_names, "dens_hse"))
587  {
588  MultiFab::Copy(mf[lev],r_hse,0,mf_comp,1,0);
589  mf_comp += 1;
590  }
591  if (containerHasElement(plot_var_names, "theta_hse"))
592  {
593  MultiFab::Copy(mf[lev],th_hse,0,mf_comp,1,0);
594  mf_comp += 1;
595  }
596  if (containerHasElement(plot_var_names, "qv_hse"))
597  {
598  MultiFab::Copy(mf[lev],qv_hse,0,mf_comp,1,0);
599  mf_comp += 1;
600  }
601 
602  if (containerHasElement(plot_var_names, "pressure"))
603  {
604  if (solverChoice.anelastic[lev] == 1) {
605  MultiFab::Copy(mf[lev], p_hse, 0, mf_comp, 1, 0);
606  } else {
607  MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
608  }
609 
610  mf_comp += 1;
611  }
612 
613  if (containerHasElement(plot_var_names, "pert_pres"))
614  {
615  if (solverChoice.anelastic[lev] == 1) {
616  MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0);
617  } else {
618  MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
619  MultiFab::Subtract(mf[lev],p_hse,0,mf_comp,1,IntVect{0});
620  }
621  mf_comp += 1;
622  }
623 
624  if (containerHasElement(plot_var_names, "pert_dens"))
625  {
626 #ifdef _OPENMP
627 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
628 #endif
629  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
630  {
631  const Box& bx = mfi.tilebox();
632  const Array4<Real>& derdat = mf[lev].array(mfi);
633  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
634  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
635  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
636  derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
637  });
638  }
639  mf_comp ++;
640  }
641 
642  if (containerHasElement(plot_var_names, "buoyancy"))
643  {
644  MultiFab qt(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
645  MultiFab b(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
646  MultiFab S_prim(mf[lev].boxArray(), mf[lev].DistributionMap(),
647  vars_new[lev][Vars::cons].nComp()-1, 1);
648 
649  qt.setVal(0.);
650  int n_qstate_into_total = micro->Get_Qstate_Moist_Size() - micro->Get_Qstate_Moist_NumConc_Size();
651  if (solverChoice.moisture_type != MoistureType::None) {
652  make_qt(vars_new[lev][Vars::cons], qt, n_qstate_into_total);
653  }
654  cons_to_prim(vars_new[lev][Vars::cons], S_prim, 0);
655 
656  b.setVal(0.); // Need to initialize to zero because buoyancy not defined on faces at top and bottom of domain
657  make_buoyancy(lev, vars_new[lev], S_prim, qt, b, geom[lev], solverChoice, base_state[lev], n_qstate_into_total,
658  get_eb(lev), solverChoice.anelastic[lev]);
659  MultiFab::Copy(mf[lev], b, 0, mf_comp, 1, 0);
660  mf_comp ++;
661  }
662 
663  if (containerHasElement(plot_var_names, "eq_pot_temp"))
664  {
665 #ifdef _OPENMP
666 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
667 #endif
668  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
669  {
670  const Box& bx = mfi.tilebox();
671  const Array4<Real>& derdat = mf[lev].array(mfi);
672  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
673  const Array4<Real const>& p_arr = pressure.const_array(mfi);
674  const int ncomp = vars_new[lev][Vars::cons].nComp();
675  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
676  Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : zero;
677  Real qc = (use_moisture && (ncomp > RhoQ2_comp)) ? S_arr(i,j,k,RhoQ2_comp)/S_arr(i,j,k,Rho_comp) : zero;
678  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
679  Real fac = Cp_d + Cp_l*(qv + qc);
680  Real pv = erf_esatw(T)*Real(100.0);
681 
682  derdat(i, j, k, mf_comp) = T*std::pow((p_arr(i,j,k) - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
683  });
684  }
685  mf_comp ++;
686  }
687 
688  if (containerHasElement(plot_var_names, "VPD"))
689  {
690 #ifdef _OPENMP
691 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
692 #endif
693  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
694  {
695  const Box& bx = mfi.tilebox();
696  const Array4<Real>& derdat = mf[lev].array(mfi);
697  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
698  const Array4<Real const>& p_arr = pressure.const_array(mfi);
699  const int ncomp = vars_new[lev][Vars::cons].nComp();
700  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
701  {
702  const Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : zero;
703 
704  const Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
705  const Real e_sat = Real(100.0) * erf_esatw_cc(T);
706 
707  const Real P = p_arr(i,j,k);
708  const Real e_act = P * qv / (Real(0.622) + qv);
709 
710  derdat(i,j,k,mf_comp) = std::max(amrex::Real(0), e_sat - e_act) * Real(0.001);
711  });
712  }
713  mf_comp ++;
714  }
715 
716 #ifdef ERF_USE_WINDFARM
717  if ( containerHasElement(plot_var_names, "num_turb") and
718  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
719  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
720  {
721  MultiFab::Copy(mf[lev],Nturb[lev],0,mf_comp,1,0);
722  mf_comp ++;
723  }
724 
725  if ( containerHasElement(plot_var_names, "SMark0") and
726  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
727  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
728  {
729  MultiFab::Copy(mf[lev],SMark[lev],0,mf_comp,1,0);
730  mf_comp ++;
731  }
732 
733  if (containerHasElement(plot_var_names, "SMark1") and
734  (solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD))
735  {
736  MultiFab::Copy(mf[lev],SMark[lev],1,mf_comp,1,0);
737  mf_comp ++;
738  }
739 #endif
740 
741  // **********************************************************************************************
742  // Allocate space if we are computing any pressure gradients
743  // **********************************************************************************************
744 
745  Vector<MultiFab> gradp_temp; gradp_temp.resize(AMREX_SPACEDIM);
746  if (containerHasElement(plot_var_names, "dpdx") ||
747  containerHasElement(plot_var_names, "dpdy") ||
748  containerHasElement(plot_var_names, "dpdz") ||
749  containerHasElement(plot_var_names, "pres_hse_x") ||
750  containerHasElement(plot_var_names, "pres_hse_y"))
751  {
752  gradp_temp[GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp_temp[GpVars::gpx].setVal(0.);
753  gradp_temp[GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp_temp[GpVars::gpy].setVal(0.);
754  gradp_temp[GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp_temp[GpVars::gpz].setVal(0.);
755  }
756 
757  // **********************************************************************************************
758  // These are based on computing gradient of full pressure
759  // **********************************************************************************************
760 
761  if (solverChoice.anelastic[lev] == 0) {
762  if ( (containerHasElement(plot_var_names, "dpdx")) ||
763  (containerHasElement(plot_var_names, "dpdy")) ||
764  (containerHasElement(plot_var_names, "dpdz")) ) {
765  compute_gradp(pressure, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
766  get_eb(lev), gradp_temp, solverChoice);
767  }
768  }
769 
770  if (containerHasElement(plot_var_names, "dpdx"))
771  {
772  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
773  {
774  const Box& bx = mfi.tilebox();
775  const Array4<Real >& derdat = mf[lev].array(mfi);
776  const Array4<Real const>& gpx_arr = (solverChoice.anelastic[lev] == 1) ?
777  gradp[lev][GpVars::gpx].array(mfi) : gradp_temp[GpVars::gpx].array(mfi);
778  const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
779  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
780  derdat(i ,j ,k, mf_comp) = myhalf * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
781  });
782  }
783  mf_comp ++;
784  } // dpdx
785  if (containerHasElement(plot_var_names, "dpdy"))
786  {
787  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
788  {
789  const Box& bx = mfi.tilebox();
790  const Array4<Real >& derdat = mf[lev].array(mfi);
791  const Array4<Real const>& gpy_arr = (solverChoice.anelastic[lev] == 1) ?
792  gradp[lev][GpVars::gpy].array(mfi) : gradp_temp[GpVars::gpy].array(mfi);
793  const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
794  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
795  derdat(i ,j ,k, mf_comp) = myhalf * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
796  });
797  }
798  mf_comp ++;
799  } // dpdy
800  if (containerHasElement(plot_var_names, "dpdz"))
801  {
802  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
803  {
804  const Box& bx = mfi.tilebox();
805  const Array4<Real >& derdat = mf[lev].array(mfi);
806  const Array4<Real const>& gpz_arr = (solverChoice.anelastic[lev] == 1) ?
807  gradp[lev][GpVars::gpz].array(mfi) : gradp_temp[GpVars::gpz].array(mfi);
808  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
809  derdat(i ,j ,k, mf_comp) = myhalf * (gpz_arr(i,j,k+1) + gpz_arr(i,j,k));
810  });
811  }
812  mf_comp ++;
813  } // dpdz
814 
815  // **********************************************************************************************
816  // These are based on computing gradient of basestate pressure
817  // **********************************************************************************************
818 
819  if ( (containerHasElement(plot_var_names, "pres_hse_x")) ||
820  (containerHasElement(plot_var_names, "pres_hse_y")) ) {
821  compute_gradp(p_hse, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), mapfac[lev],
822  get_eb(lev), gradp_temp, solverChoice);
823  }
824 
825  if (containerHasElement(plot_var_names, "pres_hse_x"))
826  {
827  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
828  {
829  const Box& bx = mfi.tilebox();
830  const Array4<Real >& derdat = mf[lev].array(mfi);
831  const Array4<Real const>& gpx_arr = gradp_temp[0].array(mfi);
832  const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
833  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
834  derdat(i ,j ,k, mf_comp) = myhalf * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
835  });
836  }
837  mf_comp += 1;
838  } // pres_hse_x
839 
840  if (containerHasElement(plot_var_names, "pres_hse_y"))
841  {
842  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
843  {
844  const Box& bx = mfi.tilebox();
845  const Array4<Real >& derdat = mf[lev].array(mfi);
846  const Array4<Real const>& gpy_arr = gradp_temp[1].array(mfi);
847  const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
848  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
849  derdat(i ,j ,k, mf_comp) = myhalf * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
850  });
851  }
852  mf_comp += 1;
853  } // pres_hse_y
854 
855  // **********************************************************************************************
856  // Metric terms
857  // **********************************************************************************************
858 
859  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
860  if (containerHasElement(plot_var_names, "z_phys"))
861  {
862  MultiFab::Copy(mf[lev],*z_phys_cc[lev],0,mf_comp,1,0);
863  mf_comp ++;
864  }
865 
866  if (containerHasElement(plot_var_names, "detJ"))
867  {
868  MultiFab::Copy(mf[lev],*detJ_cc[lev],0,mf_comp,1,0);
869  mf_comp ++;
870  }
871  } // use_terrain
872 
873  if (containerHasElement(plot_var_names, "mapfac")) {
874  amrex::Print() << "You are plotting a 3D version of mapfac; we suggest using the 2D plotfile instead" << std::endl;
875 #ifdef _OPENMP
876 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
877 #endif
878  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
879  {
880  const Box& bx = mfi.tilebox();
881  const Array4<Real>& derdat = mf[lev].array(mfi);
882  const Array4<Real>& mf_m = mapfac[lev][MapFacType::m_x]->array(mfi);
883  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
884  derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
885  });
886  }
887  mf_comp ++;
888  }
889 
890  if (containerHasElement(plot_var_names, "lat_m")) {
891  amrex::Print() << "You are plotting a 3D version of lat_m; we suggest using the 2D plotfile instead" << std::endl;
892  if (lat_m[lev]) {
893 #ifdef _OPENMP
894 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
895 #endif
896  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
897  {
898  const Box& bx = mfi.tilebox();
899  const Array4<Real>& derdat = mf[lev].array(mfi);
900  const Array4<Real>& data = lat_m[lev]->array(mfi);
901  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
902  derdat(i, j, k, mf_comp) = data(i,j,0);
903  });
904  }
905  } else {
906  mf[lev].setVal(0.0,mf_comp,1,0);
907  }
908  mf_comp++;
909  } // lat_m
910 
911  if (containerHasElement(plot_var_names, "lon_m")) {
912  amrex::Print() << "You are plotting a 3D version of lon_m; we suggest using the 2D plotfile instead" << std::endl;
913  if (lon_m[lev]) {
914 #ifdef _OPENMP
915 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
916 #endif
917  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
918  {
919  const Box& bx = mfi.tilebox();
920  const Array4<Real>& derdat = mf[lev].array(mfi);
921  const Array4<Real>& data = lon_m[lev]->array(mfi);
922  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
923  derdat(i, j, k, mf_comp) = data(i,j,0);
924  });
925  }
926  } else {
927  mf[lev].setVal(0.0,mf_comp,1,0);
928  }
929  mf_comp++;
930  } // lon_m
931 
932  if (solverChoice.time_avg_vel) {
933  if (containerHasElement(plot_var_names, "u_t_avg")) {
934 #ifdef _OPENMP
935 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
936 #endif
937  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
938  {
939  const Box& bx = mfi.tilebox();
940  const Array4<Real>& derdat = mf[lev].array(mfi);
941  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
942  const Real norm = static_cast<Real>(t_avg_cnt[lev]);
943  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
944  {
945  derdat(i ,j ,k, mf_comp) = data(i,j,k,0) / norm;
946  });
947  }
948  mf_comp ++;
949  }
950 
951  if (containerHasElement(plot_var_names, "v_t_avg")) {
952 #ifdef _OPENMP
953 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
954 #endif
955  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
956  {
957  const Box& bx = mfi.tilebox();
958  const Array4<Real>& derdat = mf[lev].array(mfi);
959  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
960  const Real norm = static_cast<Real>(t_avg_cnt[lev]);
961  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
962  {
963  derdat(i ,j ,k, mf_comp) = data(i,j,k,1) / norm;
964  });
965  }
966  mf_comp ++;
967  }
968 
969  if (containerHasElement(plot_var_names, "w_t_avg")) {
970 #ifdef _OPENMP
971 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
972 #endif
973  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
974  {
975  const Box& bx = mfi.tilebox();
976  const Array4<Real>& derdat = mf[lev].array(mfi);
977  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
978  const Real norm = static_cast<Real>(t_avg_cnt[lev]);
979  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
980  {
981  derdat(i ,j ,k, mf_comp) = data(i,j,k,2) / norm;
982  });
983  }
984  mf_comp ++;
985  }
986 
987  if (containerHasElement(plot_var_names, "umag_t_avg")) {
988 #ifdef _OPENMP
989 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
990 #endif
991  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
992  {
993  const Box& bx = mfi.tilebox();
994  const Array4<Real>& derdat = mf[lev].array(mfi);
995  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
996  const Real norm = static_cast<Real>(t_avg_cnt[lev]);
997  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
998  {
999  derdat(i ,j ,k, mf_comp) = data(i,j,k,3) / norm;
1000  });
1001  }
1002  mf_comp ++;
1003  }
1004  }
1005 
1006  const MultiFab* eta_src = nullptr;
1007  const bool have_native_shoc_diagnostics =
1008  solverChoice.turbChoice[lev].uses_native_shoc() &&
1009  native_shoc_driver[lev] &&
1010  native_shoc_driver[lev]->has_native_diagnostics();
1011  if (solverChoice.turbChoice[lev].use_kturb) {
1012  if (have_native_shoc_diagnostics) {
1013  eta_src = &native_shoc_driver[lev]->native_diagnostics();
1014  } else
1015  {
1016  eta_src = eddyDiffs_lev[lev].get();
1017  }
1018  }
1019 
1020  if (containerHasElement(plot_var_names, "nut")) {
1021  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
1022  MultiFab cmf(vars_new[lev][Vars::cons], make_alias, 0, 1); // to provide rho only
1023 #ifdef _OPENMP
1024 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1025 #endif
1026  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
1027  {
1028  const Box& bx = mfi.tilebox();
1029  auto prim = dmf[mfi].array();
1030  auto const cons = cmf[mfi].const_array();
1031  auto const diff = (eta_src) ? eta_src->const_array(mfi) :
1032  Array4<const Real>{};
1033  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1034  {
1035  const Real rho = cons(i, j, k, Rho_comp);
1036  const Real Kmv = diff(i, j, k, EddyDiff::Mom_v);
1037  prim(i,j,k) = Kmv / rho;
1038  });
1039  }
1040 
1041  mf_comp++;
1042  }
1043 
1044  const MultiFab* shoc_or_host_eddy = have_native_shoc_diagnostics
1045  ? &native_shoc_driver[lev]->native_diagnostics()
1046  : eddyDiffs_lev[lev].get();
1047 
1048  if (containerHasElement(plot_var_names, "Kmv")) {
1049  MultiFab::Copy(mf[lev],*shoc_or_host_eddy,EddyDiff::Mom_v,mf_comp,1,0);
1050  mf_comp ++;
1051  }
1052  if (containerHasElement(plot_var_names, "Kmh")) {
1053  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
1054  mf_comp ++;
1055  }
1056  if (containerHasElement(plot_var_names, "Khv")) {
1057  MultiFab::Copy(mf[lev],*shoc_or_host_eddy,EddyDiff::Theta_v,mf_comp,1,0);
1058  mf_comp ++;
1059  }
1060  if (containerHasElement(plot_var_names, "Khh")) {
1061  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
1062  mf_comp ++;
1063  }
1064  if (containerHasElement(plot_var_names, "Lturb")) {
1065  MultiFab::Copy(mf[lev],*shoc_or_host_eddy,EddyDiff::Turb_lengthscale,mf_comp,1,0);
1066  mf_comp ++;
1067  }
1068  auto copy_native_shoc_diagnostic = [&](const MultiFab* src) {
1069  if (src != nullptr) {
1070  MultiFab::Copy(mf[lev], *src, 0, mf_comp, 1, 0);
1071  } else {
1072  mf[lev].setVal(-999, mf_comp, 1, 0);
1073  }
1074  mf_comp ++;
1075  };
1076  // Native SHOC pblh is diagnosed in meters AGL and is copied through
1077  // unchanged into the plotfile diagnostic field.
1078  if (containerHasElement(plot_var_names, "pblh")) {
1079  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1080  ? &native_shoc_driver[lev]->pblh_diagnostics()
1081  : nullptr);
1082  }
1083  if (containerHasElement(plot_var_names, "shoc_cldfrac")) {
1084  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1085  ? &native_shoc_driver[lev]->shoc_cldfrac_diagnostics()
1086  : nullptr);
1087  }
1088  if (containerHasElement(plot_var_names, "shoc_ql")) {
1089  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1090  ? &native_shoc_driver[lev]->shoc_ql_diagnostics()
1091  : nullptr);
1092  }
1093  if (containerHasElement(plot_var_names, "shoc_ql2")) {
1094  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1095  ? &native_shoc_driver[lev]->shoc_ql2_diagnostics()
1096  : nullptr);
1097  }
1098  if (containerHasElement(plot_var_names, "shoc_cond")) {
1099  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1100  ? &native_shoc_driver[lev]->shoc_cond_diagnostics()
1101  : nullptr);
1102  }
1103  if (containerHasElement(plot_var_names, "wqls_sec")) {
1104  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1105  ? &native_shoc_driver[lev]->wqls_sec_diagnostics()
1106  : nullptr);
1107  }
1108  if (containerHasElement(plot_var_names, "wthv_sec")) {
1109  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1110  ? &native_shoc_driver[lev]->wthv_sec_diagnostics()
1111  : nullptr);
1112  }
1113  if (containerHasElement(plot_var_names, "w_sec")) {
1114  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1115  ? &native_shoc_driver[lev]->w_sec_diagnostics()
1116  : nullptr);
1117  }
1118  if (containerHasElement(plot_var_names, "thl_sec")) {
1119  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1120  ? &native_shoc_driver[lev]->thl_sec_diagnostics()
1121  : nullptr);
1122  }
1123  if (containerHasElement(plot_var_names, "qw_sec")) {
1124  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1125  ? &native_shoc_driver[lev]->qw_sec_diagnostics()
1126  : nullptr);
1127  }
1128  if (containerHasElement(plot_var_names, "qwthl_sec")) {
1129  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1130  ? &native_shoc_driver[lev]->qwthl_sec_diagnostics()
1131  : nullptr);
1132  }
1133  if (containerHasElement(plot_var_names, "wthl_sec")) {
1134  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1135  ? &native_shoc_driver[lev]->wthl_sec_diagnostics()
1136  : nullptr);
1137  }
1138  if (containerHasElement(plot_var_names, "wqw_sec")) {
1139  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1140  ? &native_shoc_driver[lev]->wqw_sec_diagnostics()
1141  : nullptr);
1142  }
1143  if (containerHasElement(plot_var_names, "w3")) {
1144  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1145  ? &native_shoc_driver[lev]->w3_diagnostics()
1146  : nullptr);
1147  }
1148  if (containerHasElement(plot_var_names, "brunt")) {
1149  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1150  ? &native_shoc_driver[lev]->brunt_diagnostics()
1151  : nullptr);
1152  }
1153  if (containerHasElement(plot_var_names, "isotropy")) {
1154  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1155  ? &native_shoc_driver[lev]->isotropy_diagnostics()
1156  : nullptr);
1157  }
1158  if (containerHasElement(plot_var_names, "shear_prod")) {
1159  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1160  ? &native_shoc_driver[lev]->shear_prod_diagnostics()
1161  : nullptr);
1162  }
1163  if (containerHasElement(plot_var_names, "buoy_prod")) {
1164  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1165  ? &native_shoc_driver[lev]->buoy_prod_diagnostics()
1166  : nullptr);
1167  }
1168  if (containerHasElement(plot_var_names, "diss_tke")) {
1169  copy_native_shoc_diagnostic(have_native_shoc_diagnostics
1170  ? &native_shoc_driver[lev]->diss_tke_diagnostics()
1171  : nullptr);
1172  }
1173  if (containerHasElement(plot_var_names, "walldist")) {
1174  MultiFab::Copy(mf[lev],*walldist[lev],0,mf_comp,1,0);
1175  mf_comp ++;
1176  }
1177  if (containerHasElement(plot_var_names, "diss")) {
1178  MultiFab::Copy(mf[lev],*SFS_diss_lev[lev],0,mf_comp,1,0);
1179  mf_comp ++;
1180  }
1181 
1182  // TODO: The size of the q variables can vary with different
1183  // moisture models. Therefore, certain components may
1184  // reside at different indices. For example, Kessler is
1185  // warm but precipitating. This puts qp at index three
1186  // However, SAM is cold and precipitating so qp is index Real(4.)
1187  // Need to built an external enum struct or a better pathway.
1188 
1189  // NOTE: Protect against accessing non-existent data
1190  if (use_moisture) {
1191  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
1192  int n_qstate_moist_numconc = micro->Get_Qstate_Moist_NumConc_Size();
1193 
1194  // Moist density
1195  if(containerHasElement(plot_var_names, "moist_density"))
1196  {
1197  int n_start = RhoQ1_comp; // qv
1198  int n_end = RhoQ2_comp; // qc
1199  if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
1200  MultiFab::Copy(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1201  for (int n_comp(n_start); n_comp <= n_end; ++n_comp) {
1202  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1203  }
1204  mf_comp += 1;
1205  }
1206 
1207  if(containerHasElement(plot_var_names, "qv") && (n_qstate_moist >= 1))
1208  {
1209  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ1_comp, mf_comp, 1, 0);
1210  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1211  mf_comp += 1;
1212  }
1213 
1214  if(containerHasElement(plot_var_names, "qc") && (n_qstate_moist >= 2))
1215  {
1216  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ2_comp, mf_comp, 1, 0);
1217  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1218  mf_comp += 1;
1219  }
1220 
1221  if(containerHasElement(plot_var_names, "qi") && (n_qstate_moist >= 4))
1222  {
1223  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ3_comp, mf_comp, 1, 0);
1224  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1225  mf_comp += 1;
1226  }
1227 
1228  if(containerHasElement(plot_var_names, "qrain") && (n_qstate_moist >= 3))
1229  {
1230  int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
1231  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start , mf_comp, 1, 0);
1232  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1233  mf_comp += 1;
1234  }
1235 
1236  if(containerHasElement(plot_var_names, "qsnow") && (n_qstate_moist >= 5))
1237  {
1238  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ5_comp, mf_comp, 1, 0);
1239  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1240  mf_comp += 1;
1241  }
1242 
1243  if(containerHasElement(plot_var_names, "qgraup") && (n_qstate_moist >= 6))
1244  {
1245  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ6_comp, mf_comp, 1, 0);
1246  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1247  mf_comp += 1;
1248  }
1249 
1250  if(containerHasElement(plot_var_names, "nc") && (n_qstate_moist >= 7))
1251  {
1252  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ7_comp, mf_comp, 1, 0);
1253  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1254  mf_comp += 1;
1255  }
1256 
1257  if(containerHasElement(plot_var_names, "ni") && (n_qstate_moist >= 8))
1258  {
1259  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ8_comp, mf_comp, 1, 0);
1260  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1261  mf_comp += 1;
1262  }
1263 
1264  if(containerHasElement(plot_var_names, "nr") && (n_qstate_moist >= 9))
1265  {
1266  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ9_comp, mf_comp, 1, 0);
1267  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1268  mf_comp += 1;
1269  }
1270 
1271  if(containerHasElement(plot_var_names, "ns") && (n_qstate_moist >= 10))
1272  {
1273  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ10_comp, mf_comp, 1, 0);
1274  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1275  mf_comp += 1;
1276  }
1277 
1278  if(containerHasElement(plot_var_names, "ng") && (n_qstate_moist >= 11))
1279  {
1280  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ11_comp, mf_comp, 1, 0);
1281  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1282  mf_comp += 1;
1283  }
1284 
1285  // Precipitating + non-precipitating components
1286  //--------------------------------------------------------------------------
1287  if(containerHasElement(plot_var_names, "qt"))
1288  {
1289  int n_start = RhoQ1_comp; // qv
1290  int n_end = n_start + n_qstate_moist - n_qstate_moist_numconc;
1291  MultiFab::Copy(mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1292 
1293  for (int n_comp(n_start+1); n_comp < n_end; ++n_comp) {
1294  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1295  }
1296 
1297  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1298 
1299  mf_comp += 1;
1300  }
1301 
1302  // Non-precipitating components
1303  //--------------------------------------------------------------------------
1304  if (containerHasElement(plot_var_names, "qn"))
1305  {
1306  int n_start = RhoQ1_comp; // qv
1307  int n_end = RhoQ2_comp; // qc
1308  if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
1309  MultiFab::Copy(mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1310  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1311  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1312  }
1313  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1314  mf_comp += 1;
1315  }
1316 
1317  // Precipitating components
1318  //--------------------------------------------------------------------------
1319  if(containerHasElement(plot_var_names, "qp") && (n_qstate_moist >= 3))
1320  {
1321  int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
1322  int n_end = ncomp_cons - 1 - n_qstate_moist_numconc;
1323  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1324  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1325  MultiFab::Add( mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1326  }
1327  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1328  mf_comp += 1;
1329  }
1330 
1331  if (containerHasElement(plot_var_names, "qsat"))
1332  {
1333 #ifdef _OPENMP
1334 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1335 #endif
1336  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1337  {
1338  const Box& bx = mfi.tilebox();
1339  const Array4<Real>& derdat = mf[lev].array(mfi);
1340  const Array4<Real const>& p_arr = pressure.array(mfi);
1341  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1342  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1343  {
1344  Real qv = S_arr(i,j,k,RhoQ1_comp) / S_arr(i,j,k,Rho_comp);
1345  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
1346  Real p = p_arr(i,j,k) * Real(0.01);
1347  erf_qsatw(T, p, derdat(i,j,k,mf_comp));
1348  });
1349  }
1350  mf_comp ++;
1351  }
1352 
1353  if ( (solverChoice.moisture_type == MoistureType::Kessler) ||
1354  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
1355  (solverChoice.moisture_type == MoistureType::SAM_NoIce) )
1356  {
1357  if (containerHasElement(plot_var_names, "rain_accum"))
1358  {
1359  MultiFab::Copy(mf[lev],*(qmoist[lev][0]),0,mf_comp,1,0);
1360  mf_comp += 1;
1361  }
1362  if (containerHasElement(plot_var_names, "rel_humidity")) {
1363  Print() << "Warning: plot variable \"rel_humidity\" is not available with Kessler moisture model.\n";
1364  mf[lev].setVal(0.0, mf_comp, 1, 0);
1365  mf_comp += 1;
1366  }
1367  }
1368  else if ( (solverChoice.moisture_type == MoistureType::SAM) ||
1369  (solverChoice.moisture_type == MoistureType::Morrison) ||
1370  (solverChoice.moisture_type == MoistureType::WSM6) )
1371  {
1372  if (containerHasElement(plot_var_names, "rain_accum"))
1373  {
1374  MultiFab::Copy(mf[lev],*(qmoist[lev][0]),0,mf_comp,1,0);
1375  mf_comp += 1;
1376  }
1377  if (containerHasElement(plot_var_names, "snow_accum"))
1378  {
1379  MultiFab::Copy(mf[lev],*(qmoist[lev][1]),0,mf_comp,1,0);
1380  mf_comp += 1;
1381  }
1382  if (containerHasElement(plot_var_names, "graup_accum"))
1383  {
1384  MultiFab::Copy(mf[lev],*(qmoist[lev][2]),0,mf_comp,1,0);
1385  mf_comp += 1;
1386  }
1387  if (containerHasElement(plot_var_names, "rel_humidity")) {
1388  Print() << "Warning: plot variable \"rel_humidity\" is not available with SAM moisture model.\n";
1389  mf[lev].setVal(0.0, mf_comp, 1, 0);
1390  mf_comp += 1;
1391  }
1392  }
1393  else if(solverChoice.moisture_type == MoistureType::SuperDroplets)
1394  {
1395  if (containerHasElement(plot_var_names, "rain_accum")) {
1396  MultiFab::Copy(mf[lev],*(qmoist[lev][6]),0,mf_comp,1,0);
1397  mf_comp += 1;
1398  }
1399  if (containerHasElement(plot_var_names, "rel_humidity")) {
1400  MultiFab::Copy(mf[lev],*(qmoist[lev][5]),0,mf_comp,1,0);
1401  mf_comp += 1;
1402  }
1403  if (containerHasElement(plot_var_names, "condensation_rate")) {
1404  MultiFab::Copy(mf[lev],*(qmoist[lev][3]),0,mf_comp,1,0);
1405  mf_comp += 1;
1406  }
1407  }
1408 
1409  } // if use_moisture
1410 
1411  if (containerHasElement(plot_var_names, "terrain_IB_mask"))
1412  {
1413  MultiFab* terrain_blank = terrain_blanking[lev].get();
1414  MultiFab::Copy(mf[lev],*terrain_blank,0,mf_comp,1,0);
1415  mf_comp ++;
1416  }
1417 
1418  if (containerHasElement(plot_var_names, "volfrac")) {
1419  if ( solverChoice.terrain_type == TerrainType::EB ||
1420  solverChoice.terrain_type == TerrainType::ImmersedForcing)
1421  {
1422  MultiFab::Copy(mf[lev], EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
1423  } else {
1424  mf[lev].setVal(1.0, mf_comp, 1, 0);
1425  }
1426  mf_comp += 1;
1427  }
1428 
1429 #ifdef ERF_COMPUTE_ERROR
1430  // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
1431  if (containerHasElement(plot_var_names, "xvel_err") ||
1432  containerHasElement(plot_var_names, "yvel_err") ||
1433  containerHasElement(plot_var_names, "zvel_err"))
1434  {
1435  //
1436  // Moving terrain ANALYTICAL
1437  //
1438  Real H = geom[lev].ProbHi()[2];
1439  Real Ampl = Real(0.16);
1440  Real wavelength = Real(100.);
1441  Real kp = two * PI / wavelength;
1442  Real g = CONST_GRAV;
1443  Real omega = std::sqrt(g * kp);
1444  Real omega_t = omega * t_new[lev];
1445 
1446  const auto dx = geom[lev].CellSizeArray();
1447 
1448 #ifdef _OPENMP
1449 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1450 #endif
1451  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1452  {
1453  const Box& bx = mfi.validbox();
1454  Box xbx(bx); xbx.surroundingNodes(0);
1455  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1456  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1457 
1458  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1459 
1460  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1461  {
1462  Real x = i * dx[0];
1463  Real z = fourth * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1464 
1465  Real z_base = Ampl * std::sin(kp * x - omega_t);
1466  z -= z_base;
1467 
1468  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1469 
1470  xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
1471  });
1472 
1473  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1474  {
1475  Real x = (i + myhalf) * dx[0];
1476  Real z = fourth * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1477 
1478  Real z_base = Ampl * std::sin(kp * x - omega_t);
1479  z -= z_base;
1480 
1481  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1482 
1483  zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
1484  });
1485  }
1486 
1487  MultiFab temp_mf(mf[lev].boxArray(), mf[lev].DistributionMap(), AMREX_SPACEDIM, 0);
1488  average_face_to_cellcenter(temp_mf,0,
1489  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
1490 
1491  if (containerHasElement(plot_var_names, "xvel_err")) {
1492  MultiFab::Copy(mf[lev],temp_mf,0,mf_comp,1,0);
1493  mf_comp += 1;
1494  }
1495  if (containerHasElement(plot_var_names, "yvel_err")) {
1496  MultiFab::Copy(mf[lev],temp_mf,1,mf_comp,1,0);
1497  mf_comp += 1;
1498  }
1499  if (containerHasElement(plot_var_names, "zvel_err")) {
1500  MultiFab::Copy(mf[lev],temp_mf,2,mf_comp,1,0);
1501  mf_comp += 1;
1502  }
1503 
1504  // Now restore the velocities to what they were
1505 #ifdef _OPENMP
1506 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1507 #endif
1508  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1509  {
1510  const Box& bx = mfi.validbox();
1511  Box xbx(bx); xbx.surroundingNodes(0);
1512 
1513  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1514  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1515 
1516  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1517 
1518  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1519  {
1520  Real x = i * dx[0];
1521  Real z = fourth * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1522  Real z_base = Ampl * std::sin(kp * x - omega_t);
1523 
1524  z -= z_base;
1525 
1526  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1527  xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
1528  });
1529  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1530  {
1531  Real x = (i + myhalf) * dx[0];
1532  Real z = fourth * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1533  Real z_base = Ampl * std::sin(kp * x - omega_t);
1534 
1535  z -= z_base;
1536  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1537 
1538  zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
1539  });
1540  }
1541  } // end xvel_err, yvel_err, zvel_err
1542 
1543  if (containerHasElement(plot_var_names, "pp_err"))
1544  {
1545  // Moving terrain ANALYTICAL
1546 #ifdef _OPENMP
1547 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1548 #endif
1549  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1550  {
1551  const Box& bx = mfi.tilebox();
1552  const Array4<Real>& derdat = mf[lev].array(mfi);
1553  const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
1554  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1555 
1556  const auto dx = geom[lev].CellSizeArray();
1557  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1558  const Array4<Real const>& p_arr = pressure.const_array(mfi);
1559  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
1560 
1561  Real H = geom[lev].ProbHi()[2];
1562  Real Ampl = Real(0.16);
1563  Real wavelength = Real(100.);
1564  Real kp = two * PI / wavelength;
1565  Real g = CONST_GRAV;
1566  Real omega = std::sqrt(g * kp);
1567  Real omega_t = omega * t_new[lev];
1568 
1569  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1570  {
1571  derdat(i, j, k, mf_comp) = p_arr(i,j,k) - p0_arr(i,j,k);
1572 
1573  Real rho_hse = r0_arr(i,j,k);
1574 
1575  Real x = (i + myhalf) * dx[0];
1576  Real z = Real(0.125) * ( z_nd(i,j,k ) + z_nd(i+1,j,k ) + z_nd(i,j+1,k ) + z_nd(i+1,j+1,k )
1577  +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
1578  Real z_base = Ampl * std::sin(kp * x - omega_t);
1579 
1580  z -= z_base;
1581  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1582  Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
1583  std::sin(kp * x - omega_t) * r0_arr(i,j,k);
1584 
1585  derdat(i,j,k,mf_comp) -= pprime_exact;
1586  });
1587  }
1588  mf_comp += 1;
1589  }
1590 #endif
1591 
1592  if (solverChoice.rad_type != RadiationType::None) {
1593  if (containerHasElement(plot_var_names, "qsrc_sw")) {
1594  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 0, mf_comp, 1, 0);
1595  mf_comp += 1;
1596  }
1597  if (containerHasElement(plot_var_names, "qsrc_lw")) {
1598  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 1, mf_comp, 1, 0);
1599  mf_comp += 1;
1600  }
1601  }
1602 
1603  // *****************************************************************************************
1604  // End of derived variables corresponding to "derived_names" in ERF.H
1605  //
1606  // Particles and microphysics can provide additional outputs, which are handled below.
1607  // *****************************************************************************************
1608 
1609 #ifdef ERF_USE_PARTICLES
1610  const auto& particles_namelist( particleData.getNames() );
1611 
1612  if (containerHasElement(plot_var_names, "tracer_particles_count")) {
1613  if (particles_namelist.size() == 0) {
1614  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
1615  temp_dat.setVal(0);
1616  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1617  mf_comp += 1;
1618  } else {
1619  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++) {
1620  if (containerHasElement(plot_var_names, std::string(particles_namelist[i]+"_count"))) {
1621  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
1622  temp_dat.setVal(0);
1623  if (particleData.HasSpecies(particles_namelist[i])) {
1624  particleData[particles_namelist[i]]->Increment(temp_dat, lev);
1625  }
1626  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1627  mf_comp += 1;
1628  }
1629  }
1630  }
1631  }
1632 
1633  Vector<std::string> particle_mesh_plot_names(0);
1634  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
1635 
1636  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
1637  std::string plot_var_name(particle_mesh_plot_names[i]);
1638  if (containerHasElement(plot_var_names, plot_var_name) ) {
1639  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
1640  temp_dat.setVal(0);
1641  particleData.GetMeshPlotVar(plot_var_name, temp_dat, *z_phys_nd[lev], lev);
1642  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1643  mf_comp += 1;
1644  }
1645  }
1646 #endif
1647 
1648  {
1649  Vector<std::string> microphysics_plot_names;
1650  micro->GetPlotVarNames(microphysics_plot_names);
1651  for (auto& plot_name : microphysics_plot_names) {
1652  if (containerHasElement(plot_var_names, plot_name)) {
1653  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
1654  temp_dat.setVal(0);
1655  micro->GetPlotVar(plot_name, temp_dat, lev);
1656  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1657  mf_comp += 1;
1658  }
1659  }
1660  }
1661  } // lev
1662 
1663  if (solverChoice.terrain_type == TerrainType::EB)
1664  {
1665  for (int lev = 0; lev <= finest_level; ++lev) {
1666  EB_set_covered(mf[lev], zero);
1667  }
1668  }
1669 
1670  // Fill terrain distortion MF (nu_nd)
1671  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1672  for (int lev(0); lev <= finest_level; ++lev) {
1673  MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
1674  Real dz = Geom()[lev].CellSizeArray()[2];
1675  for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1676  const Box& bx = mfi.tilebox();
1677  Array4<Real> mf_arr = mf_nd[lev].array(mfi);
1678  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1679  {
1680  mf_arr(i,j,k,2) -= k * dz;
1681  });
1682  }
1683  }
1684  }
1685 
1686  std::string plotfilename;
1687  std::string plotfilenameU;
1688  std::string plotfilenameV;
1689  std::string plotfilenameW;
1690 
1691  if (which == 1) {
1692  if (use_real_time_in_pltname) {
1693  const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
1694  plotfilename = plot3d_file_1+"_"+getTimestamp(start_time+tnew, dt_format,false);
1695  } else {
1696  plotfilename = Concatenate(plot3d_file_1, istep[0], file_name_digits);
1697  }
1698  plotfilenameU = Concatenate(plot3d_file_1+"U", istep[0], file_name_digits);
1699  plotfilenameV = Concatenate(plot3d_file_1+"V", istep[0], file_name_digits);
1700  plotfilenameW = Concatenate(plot3d_file_1+"W", istep[0], file_name_digits);
1701  } else if (which == 2) {
1702  if (use_real_time_in_pltname) {
1703  const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
1704  plotfilename = plot3d_file_2+"_"+getTimestamp(start_time+tnew, dt_format,false);
1705  } else {
1706  plotfilename = Concatenate(plot3d_file_2, istep[0], file_name_digits);
1707  }
1708  plotfilenameU = Concatenate(plot3d_file_2+"U", istep[0], file_name_digits);
1709  plotfilenameV = Concatenate(plot3d_file_2+"V", istep[0], file_name_digits);
1710  plotfilenameW = Concatenate(plot3d_file_2+"W", istep[0], file_name_digits);
1711  }
1712 
1713  // LSM writes it's own data
1714  if (which==1 && plot_lsm) {
1715  lsm.Plot_Lsm_Data(tnew, istep, refRatio());
1716  }
1717 
1718 #ifdef ERF_USE_RRTMGP
1719  /*
1720  // write additional RRTMGP data
1721  // TODO: currently single level only
1722  if (which==1 && plot_rad) {
1723  rad[0]->writePlotfile(plot_file_1, tnew, istep[0]);
1724  }
1725  */
1726 #endif
1727 
1728  // Single level
1729  if (finest_level == 0)
1730  {
1731  if (plotfile_type == PlotFileType::Amrex)
1732  {
1733  Print() << "Writing native 3D plotfile " << plotfilename << "\n";
1734  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1735  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1736  GetVecOfConstPtrs(mf),
1737  GetVecOfConstPtrs(mf_nd),
1738  varnames,
1739  Geom(), tnew, istep, refRatio());
1740  } else {
1741  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1742  GetVecOfConstPtrs(mf),
1743  varnames,
1744  Geom(), tnew, istep, refRatio());
1745  }
1746  writeJobInfo(plotfilename, erf_provenance::ArtifactType::Plotfile3D,
1747  istep[0], t_new[0]);
1748 
1749  if (m_plot_face_vels) {
1750  Print() << "Writing face velocities" << std::endl;
1751  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1752  GetVecOfConstPtrs(mf_u),
1753  {"x_velocity_stag"},
1754  Geom(), tnew, istep, refRatio());
1755  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1756  GetVecOfConstPtrs(mf_v),
1757  {"y_velocity_stag"},
1758  Geom(), tnew, istep, refRatio());
1759  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1760  GetVecOfConstPtrs(mf_w),
1761  {"z_velocity_stag"},
1762  Geom(), tnew, istep, refRatio());
1763  }
1764 
1765 #ifdef ERF_USE_PARTICLES
1766  particleData.writePlotFile(plotfilename, z_phys_nd);
1767 #endif
1768 #ifdef ERF_USE_NETCDF
1769  } else if (plotfile_type == PlotFileType::Netcdf) {
1770  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type != MeshType::VariableDz);
1771  int lev = 0;
1772  int l_which = 0;
1773  const Real* p_lo = geom[lev].ProbLo();
1774  const Real* p_hi = geom[lev].ProbHi();
1775  const auto dx = geom[lev].CellSize();
1776  writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1777  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
1778  geom[lev].Domain(), tnew, start_bdy_time, solverChoice, zlevels_stag[lev]);
1779 #endif
1780  } else {
1781  // Here we assume the plotfile_type is PlotFileType::None
1782  Print() << "Writing no 3D plotfile since plotfile_type is none" << std::endl;
1783  }
1784 
1785  } else { // Multilevel
1786 
1787  if (plotfile_type == PlotFileType::Amrex) {
1788 
1789  int lev0 = 0;
1790  int desired_ratio = std::max(std::max(ref_ratio[lev0][0],ref_ratio[lev0][1]),ref_ratio[lev0][2]);
1791  bool any_ratio_one = ( ( (ref_ratio[lev0][0] == 1) || (ref_ratio[lev0][1] == 1) ) ||
1792  (ref_ratio[lev0][2] == 1) );
1793  for (int lev = 1; lev < finest_level; lev++) {
1794  any_ratio_one = any_ratio_one ||
1795  ( ( (ref_ratio[lev][0] == 1) || (ref_ratio[lev][1] == 1) ) ||
1796  (ref_ratio[lev][2] == 1) );
1797  }
1798 
1799  if (any_ratio_one && m_expand_plotvars_to_unif_rr)
1800  {
1801  Vector<IntVect> r2(finest_level);
1802  Vector<Geometry> g2(finest_level+1);
1803  Vector<MultiFab> mf2(finest_level+1);
1804 
1805  mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
1806 
1807  // Copy level 0 as is
1808  MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
1809 
1810  // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
1811  Array<int,AMREX_SPACEDIM> periodicity =
1812  {Geom()[lev0].isPeriodic(0),Geom()[lev0].isPeriodic(1),Geom()[lev0].isPeriodic(2)};
1813  g2[lev0].define(Geom()[lev0].Domain(),&(Geom()[lev0].ProbDomain()),0,periodicity.data());
1814 
1815  r2[0] = IntVect(desired_ratio/ref_ratio[lev0][0],
1816  desired_ratio/ref_ratio[lev0][1],
1817  desired_ratio/ref_ratio[lev0][2]);
1818 
1819  for (int lev = 1; lev <= finest_level; ++lev) {
1820  if (lev > 1) {
1821  r2[lev-1][0] = r2[lev-2][0] * desired_ratio / ref_ratio[lev-1][0];
1822  r2[lev-1][1] = r2[lev-2][1] * desired_ratio / ref_ratio[lev-1][1];
1823  r2[lev-1][2] = r2[lev-2][2] * desired_ratio / ref_ratio[lev-1][2];
1824  }
1825 
1826  mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
1827 
1828  // Set the new problem domain
1829  Box d2(Geom()[lev].Domain());
1830  d2.refine(r2[lev-1]);
1831 
1832  g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
1833  }
1834 
1835  //
1836  // We need to make a temporary that is the size of ncomp_mf
1837  // in order to not get an out of bounds error
1838  // even though the values will not be used
1839  //
1840  Vector<BCRec> temp_domain_bcs_type;
1841  temp_domain_bcs_type.resize(ncomp_mf);
1842 
1843  //
1844  // Do piecewise constant interpolation of mf into mf2
1845  //
1846  for (int lev = 1; lev <= finest_level; ++lev) {
1847  Interpolater* mapper_c = &pc_interp;
1848  InterpFromCoarseLevel(mf2[lev], static_cast<Real>(t_new[lev]), mf[lev],
1849  0, 0, ncomp_mf,
1850  geom[lev], g2[lev],
1851  null_bc_for_fill, 0, null_bc_for_fill, 0,
1852  r2[lev-1], mapper_c, temp_domain_bcs_type, 0);
1853  }
1854 
1855  // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
1856  Vector<IntVect> rr(finest_level);
1857  for (int lev = 0; lev < finest_level; ++lev) {
1858  rr[lev] = IntVect(desired_ratio);
1859  }
1860 
1861  Print() << "Writing 3D plotfile " << plotfilename << "\n";
1862  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1863  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1864  GetVecOfConstPtrs(mf2),
1865  GetVecOfConstPtrs(mf_nd),
1866  varnames,
1867  g2, tnew, istep, rr);
1868  } else {
1869  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1870  GetVecOfConstPtrs(mf2), varnames,
1871  g2, tnew, istep, rr);
1872  }
1873 
1874  } else {
1875  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1876  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1877  GetVecOfConstPtrs(mf),
1878  GetVecOfConstPtrs(mf_nd),
1879  varnames,
1880  geom, tnew, istep, ref_ratio);
1881  } else {
1882  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1883  GetVecOfConstPtrs(mf), varnames,
1884  geom, tnew, istep, ref_ratio);
1885  }
1886  if (m_plot_face_vels) {
1887  Print() << "Writing face velocities" << std::endl;
1888  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1889  GetVecOfConstPtrs(mf_u),
1890  {"x_velocity_stag"},
1891  geom, tnew, istep, ref_ratio);
1892  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1893  GetVecOfConstPtrs(mf_v),
1894  {"y_velocity_stag"},
1895  geom, tnew, istep, ref_ratio);
1896  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1897  GetVecOfConstPtrs(mf_w),
1898  {"z_velocity_stag"},
1899  geom, tnew, istep, ref_ratio);
1900  }
1901  } // ref_ratio test
1902 
1903  writeJobInfo(plotfilename, erf_provenance::ArtifactType::Plotfile3D,
1904  istep[0], t_new[0]);
1905 
1906 #ifdef ERF_USE_PARTICLES
1907  particleData.writePlotFile(plotfilename, z_phys_nd);
1908 #endif
1909 
1910 #ifdef ERF_USE_NETCDF
1911  } else if (plotfile_type == PlotFileType::Netcdf) {
1912  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type != MeshType::VariableDz);
1913  for (int lev = 0; lev <= finest_level; ++lev) {
1914  for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
1915  Box bounding_region = (lev == 0) ? geom[lev].Domain() : boxes_at_level[lev][which_box];
1916  const Real* p_lo = geom[lev].ProbLo();
1917  const Real* p_hi = geom[lev].ProbHi();
1918  const auto dx = geom[lev].CellSizeArray();
1919  writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1920  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
1921  bounding_region, tnew, start_bdy_time, solverChoice, zlevels_stag[lev]);
1922  }
1923  }
1924 #endif
1925  }
1926  } // end multi-level
1927 
1928  if (verbose > 0)
1929  {
1930  auto dPlotTime = amrex::second() - dPlotTime0;
1931  ParallelDescriptor::ReduceRealMax(dPlotTime,ParallelDescriptor::IOProcessorNumber());
1932  amrex::Print() << "3DPlotfile write time = " << dPlotTime << " seconds." << '\n';
1933  }
1934 }
PI
constexpr amrex::Real PI
Definition: ERF_Constants.H:42
Cp_l
constexpr amrex::Real Cp_l
Definition: ERF_Constants.H:51
RhoQ4_comp
#define RhoQ4_comp
Definition: ERF_IndexDefines.H:45
RhoQ11_comp
#define RhoQ11_comp
Definition: ERF_IndexDefines.H:52
RhoQ9_comp
#define RhoQ9_comp
Definition: ERF_IndexDefines.H:50
RhoQ8_comp
#define RhoQ8_comp
Definition: ERF_IndexDefines.H:49
RhoQ7_comp
#define RhoQ7_comp
Definition: ERF_IndexDefines.H:48
RhoQ10_comp
#define RhoQ10_comp
Definition: ERF_IndexDefines.H:51
Ampl
Real Ampl
Definition: ERF_InitCustomPert_MovingTerrain.H:4
H
Real H
Definition: ERF_InitCustomPert_MovingTerrain.H:7
wavelength
Real wavelength
Definition: ERF_InitCustomPert_MovingTerrain.H:5
kp
Real kp
Definition: ERF_InitCustomPert_MovingTerrain.H:8
make_buoyancy
void make_buoyancy(int lev, const Vector< MultiFab > &S_data, const MultiFab &S_prim, const MultiFab &qt, MultiFab &buoyancy, const Geometry geom, const SolverChoice &solverChoice, const MultiFab &base_state, const int n_qstate, const eb_ &ebfact, const int anelastic)
Definition: ERF_MakeBuoyancy.cpp:32
erf_esatw
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real erf_esatw(amrex::Real t, bool use_empirical=false)
Definition: ERF_MicrophysicsUtils.H:123
erf_esatw_cc
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real erf_esatw_cc(amrex::Real t)
Definition: ERF_MicrophysicsUtils.H:62
null_bc_for_fill
static PhysBCFunctNoOp null_bc_for_fill
Definition: ERF_Plotfile.cpp:11
ERF::FillBdyCCVels
void FillBdyCCVels(amrex::MultiFab &mf_cc_vel, amrex::Geometry &lev_geom)
Definition: ERF_FillBdyCCVels.cpp:11
ERF::PlotFileVarNames
static amrex::Vector< std::string > PlotFileVarNames(amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:253
ERF::WriteMultiLevelPlotfileWithTerrain
void WriteMultiLevelPlotfileWithTerrain(const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, double time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
Definition: ERF_Plotfile.cpp:1937
LandSurface::Plot_Lsm_Data
void Plot_Lsm_Data(double time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &ref_ratio)
Definition: ERF_LandSurface.H:210
EddyDiff::Turb_lengthscale
@ Turb_lengthscale
Definition: ERF_IndexDefines.H:215
EddyDiff::Mom_h
@ Mom_h
Definition: ERF_IndexDefines.H:205
EddyDiff::Theta_h
@ Theta_h
Definition: ERF_IndexDefines.H:206
MetGridTmpDstVars::P
@ P
Definition: ERF_IndexDefines.H:165
derived::erf_derKE
void erf_derKE(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:342
derived::erf_dermoisttemp
void erf_dermoisttemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:266
derived::erf_dertemp
void erf_dertemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:243
derived::erf_dersoundspeed
void erf_dersoundspeed(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:209
derived::erf_derlocalhelicity
void erf_derlocalhelicity(const Box &bx, FArrayBox &derfab, int dcomp, int, const FArrayBox &datfab, const FArrayBox &, const Geometry &geomdata, Real, const int *, const int)
Definition: ERF_Derive.cpp:616
derived::erf_dervorty
void erf_dervorty(const Box &bx, FArrayBox &derfab, int dcomp, int ncomp, const FArrayBox &datfab, const FArrayBox &zcc_fab, const Geometry &geomdata, Real, const int *, const int)
Definition: ERF_Derive.cpp:386
derived::erf_dernull
void erf_dernull(const Box &, FArrayBox &, int, int, const FArrayBox &, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:189
derived::erf_dermucape
void erf_dermucape(const Box &bx, FArrayBox &derfab, int dcomp, int ncomp, const FArrayBox &datfab, const FArrayBox &zcc_fab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:746
derived::erf_derreflectivity
void erf_derreflectivity(const Box &bx, FArrayBox &derfab, int dcomp, int, const FArrayBox &datfab, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:534
derived::erf_derprecipitable
void erf_derprecipitable(const Box &bx, FArrayBox &derfab, int dcomp, int, const FArrayBox &datfab, const FArrayBox &zcc_fab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:701
derived::erf_dervortx
void erf_dervortx(const Box &bx, FArrayBox &derfab, int dcomp, int ncomp, const FArrayBox &datfab, const FArrayBox &zcc_fab, const Geometry &geomdata, Real, const int *, const int)
Definition: ERF_Derive.cpp:357
erf_provenance::ArtifactType::Plotfile3D
@ Plotfile3D
module_model_constants::g
real(c_double), parameter g
Definition: ERF_module_model_constants.F90:19
Here is the call graph for this function:

◆ write_1D_profiles()

void ERF::write_1D_profiles ( double  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters
timeCurrent time
18 {
19  BL_PROFILE("ERF::write_1D_profiles()");
20 
21  if (NumDataLogs() > 1)
22  {
23  // Define the 1d arrays we will need
24  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
25  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
26  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
27  Gpu::HostVector<Real> h_avg_wthv;
28  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
29  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
30  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
31  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
32  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
33  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
34 
35  if (NumDataLogs() > 1) {
36  derive_diag_profiles(time,
37  h_avg_u, h_avg_v, h_avg_w,
38  h_avg_rho, h_avg_th, h_avg_ksgs,
39  h_avg_Kmv, h_avg_Khv,
40  h_avg_qv, h_avg_qc, h_avg_qr,
41  h_avg_wqv, h_avg_wqc, h_avg_wqr,
42  h_avg_qi, h_avg_qs, h_avg_qg,
43  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
44  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
45  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
46  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
47  h_avg_wthv);
48  }
49 
50  if (NumDataLogs() > 3 && time > zero) {
51  derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
52  h_avg_tau22, h_avg_tau23, h_avg_tau33,
53  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
54  h_avg_sgsdiss);
55  }
56 
57  int hu_size = h_avg_u.size();
58 
59  auto const& dx = geom[0].CellSizeArray();
60  if (ParallelDescriptor::IOProcessor()) {
61  if (NumDataLogs() > 1) {
62  std::ostream& data_log1 = DataLog(1);
63  if (data_log1.good()) {
64  // Write the quantities at this time
65  for (int k = 0; k < hu_size; k++) {
66  Real z;
67  if (zlevels_stag[0].size() > 1) {
68  z = myhalf * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
69  } else {
70  z = (k + myhalf)* dx[2];
71  }
72  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
73  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
74  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
75  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
76  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
77  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
78  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
79  << std::endl;
80  } // loop over z
81  } // if good
82  } // NumDataLogs
83 
84  if (NumDataLogs() > 2) {
85  std::ostream& data_log2 = DataLog(2);
86  if (data_log2.good()) {
87  // Write the perturbational quantities at this time
88  for (int k = 0; k < hu_size; k++) {
89  Real z;
90  if (zlevels_stag[0].size() > 1) {
91  z = myhalf * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
92  } else {
93  z = (k + myhalf)* dx[2];
94  }
95  Real thv = h_avg_th[k] * (one + epsv*h_avg_qv[k] - h_avg_qc[k] - h_avg_qr[k]);
96  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
97  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
98  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " "
99  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " "
100  << h_avg_uw[k] - h_avg_u[k]*h_avg_w[k] << " "
101  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " "
102  << h_avg_vw[k] - h_avg_v[k]*h_avg_w[k] << " "
103  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " "
104  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " "
105  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " "
106  << h_avg_wth[k] - h_avg_w[k]*h_avg_th[k] << " "
107  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
108  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
109  // - <u_i u_i> * <u_j>
110  // - 2*<u_i> * <u_i u_j>
111  // + 2*<u_i>*<u_i> * <u_j>
112  << h_avg_uiuiu[k]
113  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
114  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
115  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
116  << " " // (u'_i u'_i)u'
117  << h_avg_uiuiv[k]
118  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
119  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
120  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
121  << " " // (u'_i u'_i)v'
122  << h_avg_uiuiw[k]
123  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
124  - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
125  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
126  << " " // (u'_i u'_i)w'
127  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " "
128  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " "
129  << h_avg_pw[k] - h_avg_p[k]*h_avg_w[k] << " "
130  << h_avg_wqv[k] - h_avg_qv[k]*h_avg_w[k] << " "
131  << h_avg_wqc[k] - h_avg_qc[k]*h_avg_w[k] << " "
132  << h_avg_wqr[k] - h_avg_qr[k]*h_avg_w[k] << " "
133  << h_avg_wthv[k] - h_avg_w[k]*thv
134  << std::endl;
135  } // loop over z
136  } // if good
137  } // NumDataLogs
138 
139  if (NumDataLogs() > 3 && time > zero) {
140  std::ostream& data_log3 = DataLog(3);
141  if (data_log3.good()) {
142  // Write the average stresses
143  for (int k = 0; k < hu_size; k++) {
144  Real z;
145  if (zlevels_stag[0].size() > 1) {
146  z = myhalf * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
147  } else {
148  z = (k + myhalf)* dx[2];
149  }
150  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
151  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
152  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
153  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
154  << h_avg_sgshfx[k] << " "
155  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
156  << h_avg_sgsdiss[k]
157  << std::endl;
158  } // loop over z
159  } // if good
160  } // if (NumDataLogs() > 3)
161  } // if IOProcessor
162  } // if (NumDataLogs() > 1)
163 }
ERF::derive_diag_profiles
void derive_diag_profiles(double time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles.cpp:190
ERF::derive_stress_profiles
void derive_stress_profiles(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles.cpp:488

◆ write_1D_profiles_stag()

void ERF::write_1D_profiles_stag ( double  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Quantities are output at their native grid locations. Therefore, w and associated flux quantities <(•)'w'>, tau13, and tau23 (where '•' includes u, v, p, theta, ...) will be output at staggered heights (i.e., coincident with z faces) rather than cell-center heights to avoid performing additional averaging. Unstaggered (i.e., cell-centered) quantities are output alongside staggered quantities at the lower cell faces in the log file; these quantities will have a zero value at the big end, corresponding to k=Nz+one

The structure of file should follow ERF_Write1DProfiles.cpp

Parameters
timeCurrent time
26 {
27  BL_PROFILE("ERF::write_1D_profiles()");
28 
29  if (NumDataLogs() > 1)
30  {
31  // Define the 1d arrays we will need
32  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
33  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
34  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
35  Gpu::HostVector<Real> h_avg_wthv;
36  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
37  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
38  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
39  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
40  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
41  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
42 
43  if (NumDataLogs() > 1) {
44  derive_diag_profiles_stag(time,
45  h_avg_u, h_avg_v, h_avg_w,
46  h_avg_rho, h_avg_th, h_avg_ksgs,
47  h_avg_Kmv, h_avg_Khv,
48  h_avg_qv, h_avg_qc, h_avg_qr,
49  h_avg_wqv, h_avg_wqc, h_avg_wqr,
50  h_avg_qi, h_avg_qs, h_avg_qg,
51  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
52  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
53  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
54  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
55  h_avg_wthv);
56  }
57 
58  if (NumDataLogs() > 3 && time > zero) {
59  derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
60  h_avg_tau22, h_avg_tau23, h_avg_tau33,
61  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
62  h_avg_sgsdiss);
63  }
64 
65  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
66 
67  auto const& dx = geom[0].CellSizeArray();
68  if (ParallelDescriptor::IOProcessor()) {
69  if (NumDataLogs() > 1) {
70  std::ostream& data_log1 = DataLog(1);
71  if (data_log1.good()) {
72  // Write the quantities at this time
73  for (int k = 0; k < unstag_size; k++) {
74  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
75  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
76  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
77  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
78  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
79  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
80  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
81  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
82  << std::endl;
83  } // loop over z
84  // Write top face values
85  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
86  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
87  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
88  << 0 << " " << 0 << " " << h_avg_w[unstag_size] << " "
89  << 0 << " " << 0 << " " << 0 << " " // rho, theta, ksgs
90  << 0 << " " << 0 << " " // Kmv, Khv
91  << 0 << " " << 0 << " " << 0 << " " // qv, qc, qr
92  << 0 << " " << 0 << " " << 0 // qi, qs, qg
93  << std::endl;
94  } // if good
95  } // NumDataLogs
96 
97  if (NumDataLogs() > 2) {
98  std::ostream& data_log2 = DataLog(2);
99  if (data_log2.good()) {
100  // Write the perturbational quantities at this time
101  // For surface values (k=0), assume w = uw = vw = ww = 0
102  Real w_cc = h_avg_w[1] / 2; // w at first cell center
103  Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
104  Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
105  Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
106  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
107  << std::setw(datwidth) << std::setprecision(datprecision) << 0 << " "
108  << h_avg_uu[0] - h_avg_u[0]*h_avg_u[0] << " " // u'u'
109  << h_avg_uv[0] - h_avg_u[0]*h_avg_v[0] << " " // u'v'
110  << 0 << " " // u'w'
111  << h_avg_vv[0] - h_avg_v[0]*h_avg_v[0] << " " // v'v'
112  << 0 << " " // v'w'
113  << 0 << " " // w'w'
114  << h_avg_uth[0] - h_avg_u[0]*h_avg_th[0] << " " // u'th'
115  << h_avg_vth[0] - h_avg_v[0]*h_avg_th[0] << " " // v'th'
116  << 0 << " " // w'th'
117  << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
118  << h_avg_uiuiu[0]
119  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
120  - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
121  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
122  << " " // (u'_i u'_i)u'
123  << h_avg_uiuiv[0]
124  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
125  - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
126  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
127  << " " // (u'_i u'_i)v'
128  << 0 << " " // (u'_i u'_i)w'
129  << h_avg_pu[0] - h_avg_p[0]*h_avg_u[0] << " " // p'u'
130  << h_avg_pv[0] - h_avg_p[0]*h_avg_v[0] << " " // p'v'
131  << 0 << " " // p'w'
132  << 0 << " " // qv'w'
133  << 0 << " " // qc'w'
134  << 0 << " " // qr'w'
135  << 0 // thv'w'
136  << std::endl;
137 
138  // For internal values, interpolate scalar quantities to faces
139  for (int k = 1; k < unstag_size; k++) {
140  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
141  Real uface = myhalf*(h_avg_u[k] + h_avg_u[k-1]);
142  Real vface = myhalf*(h_avg_v[k] + h_avg_v[k-1]);
143  Real thface = myhalf*(h_avg_th[k] + h_avg_th[k-1]);
144  Real pface = myhalf*(h_avg_p[k] + h_avg_p[k-1]);
145  Real qvface = myhalf*(h_avg_qv[k] + h_avg_qv[k-1]);
146  Real qcface = myhalf*(h_avg_qc[k] + h_avg_qc[k-1]);
147  Real qrface = myhalf*(h_avg_qr[k] + h_avg_qr[k-1]);
148  Real uuface = myhalf*(h_avg_uu[k] + h_avg_uu[k-1]);
149  Real vvface = myhalf*(h_avg_vv[k] + h_avg_vv[k-1]);
150  Real thvface = thface * (one + epsv*qvface - qcface - qrface);
151  w_cc = myhalf*(h_avg_w[k-1] + h_avg_w[k]);
152  uw_cc = myhalf*(h_avg_uw[k-1] + h_avg_uw[k]);
153  vw_cc = myhalf*(h_avg_vw[k-1] + h_avg_vw[k]);
154  ww_cc = myhalf*(h_avg_ww[k-1] + h_avg_ww[k]);
155  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
156  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
157  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " " // u'u'
158  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " " // u'v'
159  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
160  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " " // v'v'
161  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
162  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
163  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " " // u'th'
164  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " " // v'th'
165  << h_avg_wth[k] - h_avg_w[k]*thface << " " // w'th'
166  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
167  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
168  // - <u_i u_i> * <u_j>
169  // - 2*<u_i> * <u_i u_j>
170  // + 2*<u_i>*<u_i> * <u_j>
171  << h_avg_uiuiu[k]
172  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
173  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
174  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
175  << " " // cell-centered (u'_i u'_i)u'
176  << h_avg_uiuiv[k]
177  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
178  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
179  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
180  << " " // cell-centered (u'_i u'_i)v'
181  << h_avg_uiuiw[k]
182  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
183  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
184  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
185  << " " // face-centered (u'_i u'_i)w'
186  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " " // cell-centered p'u'
187  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " " // cell-centered p'v'
188  << h_avg_pw[k] - pface*h_avg_w[k] << " " // face-centered p'w'
189  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
190  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
191  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
192  << h_avg_wthv[k] - thvface*h_avg_w[k]
193  << std::endl;
194  } // loop over z
195 
196  // Write top face values, extrapolating scalar quantities
197  const int k = unstag_size;
198  Real uface = Real(1.5)*h_avg_u[k-1] - myhalf*h_avg_u[k-2];
199  Real vface = Real(1.5)*h_avg_v[k-1] - myhalf*h_avg_v[k-2];
200  Real thface = Real(1.5)*h_avg_th[k-1] - myhalf*h_avg_th[k-2];
201  Real pface = Real(1.5)*h_avg_p[k-1] - myhalf*h_avg_p[k-2];
202  Real qvface = Real(1.5)*h_avg_qv[k-1] - myhalf*h_avg_qv[k-2];
203  Real qcface = Real(1.5)*h_avg_qc[k-1] - myhalf*h_avg_qc[k-2];
204  Real qrface = Real(1.5)*h_avg_qr[k-1] - myhalf*h_avg_qr[k-2];
205  Real uuface = Real(1.5)*h_avg_uu[k-1] - myhalf*h_avg_uu[k-2];
206  Real vvface = Real(1.5)*h_avg_vv[k-1] - myhalf*h_avg_vv[k-2];
207  Real thvface = thface * (one + epsv*qvface - qcface - qrface);
208  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
209  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
210  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
211  << 0 << " " // u'u'
212  << 0 << " " // u'v'
213  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
214  << 0 << " " // v'v'
215  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
216  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
217  << 0 << " " // u'th'
218  << 0 << " " // v'th'
219  << h_avg_wth[k] - thface*h_avg_w[k] << " " // w'th'
220  << 0 << " " // th'th'
221  << 0 << " " // (u'_i u'_i)u'
222  << 0 << " " // (u'_i u'_i)v'
223  << h_avg_uiuiw[k]
224  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
225  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
226  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
227  << " " // (u'_i u'_i)w'
228  << 0 << " " // pu'
229  << 0 << " " // pv'
230  << h_avg_pw[k] - pface*h_avg_w[k] << " " // pw'
231  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
232  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
233  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
234  << h_avg_wthv[k] - thvface*h_avg_w[k]
235  << std::endl;
236  } // if good
237  } // NumDataLogs
238 
239  if (NumDataLogs() > 3 && time > zero) {
240  std::ostream& data_log3 = DataLog(3);
241  if (data_log3.good()) {
242  // Write the average stresses
243  for (int k = 0; k < unstag_size; k++) {
244  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
245  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
246  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
247  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
248  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
249  << h_avg_sgshfx[k] << " "
250  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
251  << h_avg_sgsdiss[k]
252  << std::endl;
253  } // loop over z
254  // Write top face values
255  Real NANval = zero;
256  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
257  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
258  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
259  << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
260  << NANval << " " << h_avg_tau23[unstag_size] << " " << NANval << " "
261  << h_avg_sgshfx[unstag_size] << " "
262  << h_avg_sgsq1fx[unstag_size] << " " << h_avg_sgsq2fx[unstag_size] << " "
263  << NANval
264  << std::endl;
265  } // if good
266  } // if (NumDataLogs() > 3)
267  } // if IOProcessor
268  } // if (NumDataLogs() > 1)
269 }
ERF::derive_diag_profiles_stag
void derive_diag_profiles_stag(double time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles_stag.cpp:296
ERF::derive_stress_profiles_stag
void derive_stress_profiles_stag(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles_stag.cpp:618

◆ WriteAtFinalTime()

void ERF::WriteAtFinalTime ( )
277 {
278  // Write plotfiles at final time
279  if ( (m_plot3d_int_1 > 0 || m_plot3d_per_1 > zero) && istep[0] > last_plot3d_file_step_1 ) {
280  Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
281  if (m_plot3d_per_1 > zero) {last_plot3d_file_time_1 += m_plot3d_per_1;}
282  }
283  if ( (m_plot3d_int_2 > 0 || m_plot3d_per_2 > zero) && istep[0] > last_plot3d_file_step_2) {
284  Write3DPlotFile(2,plotfile3d_type_1,plot3d_var_names_2);
285  if (m_plot3d_per_2 > zero) {last_plot3d_file_time_2 += m_plot3d_per_2;}
286  }
287  if ( (m_plot2d_int_1 > 0 || m_plot2d_per_1 > zero) && istep[0] > last_plot2d_file_step_1 ) {
288  Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
289  if (m_plot2d_per_1 > zero) {last_plot2d_file_time_1 += m_plot2d_per_1;}
290  }
291  if ( (m_plot2d_int_2 > 0 || m_plot2d_per_2 > zero) && istep[0] > last_plot2d_file_step_2) {
292  Write2DPlotFile(2,plotfile2d_type_1,plot2d_var_names_2);
293  if (m_plot2d_per_2 > zero) {last_plot2d_file_time_2 += m_plot2d_per_2;}
294  }
295 
296  for (int i = 0; i < m_subvol_int.size(); i++) {
297  if ( (m_subvol_int[i] > 0 || m_subvol_per[i] > zero) && istep[0] > last_subvol_step[i]) {
298  WriteSubvolume(i,subvol3d_var_names);
299  if (m_subvol_per[i] > zero) {last_subvol_time[i] += m_subvol_per[i];}
300  }
301  }
302 
303  if ( (m_check_int > 0 || m_check_per > zero) && istep[0] > last_check_file_step) {
304  WriteCheckpointFile();
305  if (m_check_per > zero) {last_check_file_time += m_check_per;}
306  }
307 }

◆ WriteAtIntermediateTime()

void ERF::WriteAtIntermediateTime ( int  nstep,
double  time 
)
234 {
235  if (writeNow(cur_time, step+1, m_plot3d_int_1, m_plot3d_per_1, dt[0], last_plot3d_file_time_1)) {
236  last_plot3d_file_step_1 = step+1;
237  Write3DPlotFile(1,plotfile3d_type_1,plot3d_var_names_1);
238  for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
239  if (m_plot3d_per_1 > zero) {last_plot3d_file_time_1 += m_plot3d_per_1;}
240  }
241  if (writeNow(cur_time, step+1, m_plot3d_int_2, m_plot3d_per_2, dt[0], last_plot3d_file_time_2)) {
242  last_plot3d_file_step_2 = step+1;
243  Write3DPlotFile(2,plotfile3d_type_2,plot3d_var_names_2);
244  for (int lev = 0; lev <= finest_level; ++lev) {lsm.Plot(lev, step+1);}
245  if (m_plot3d_per_2 > zero) {last_plot3d_file_time_2 += m_plot3d_per_2;}
246  }
247 
248  if (writeNow(cur_time, step+1, m_plot2d_int_1, m_plot2d_per_1, dt[0], last_plot2d_file_time_1)) {
249  last_plot2d_file_step_1 = step+1;
250  Write2DPlotFile(1,plotfile2d_type_1,plot2d_var_names_1);
251  if (m_plot2d_per_1 > zero) {last_plot2d_file_time_1 += m_plot2d_per_1;}
252  }
253 
254  if (writeNow(cur_time, step+1, m_plot2d_int_2, m_plot2d_per_2, dt[0], last_plot2d_file_time_2)) {
255  last_plot2d_file_step_2 = step+1;
256  Write2DPlotFile(2,plotfile2d_type_2,plot2d_var_names_2);
257  if (m_plot2d_per_2 > zero) {last_plot2d_file_time_2 += m_plot2d_per_2;}
258  }
259 
260  for (int i = 0; i < m_subvol_int.size(); i++) {
261  if (writeNow(cur_time, step+1, m_subvol_int[i], m_subvol_per[i], dt[0], last_subvol_time[i])) {
262  last_subvol_step[i] = step+1;
263  WriteSubvolume(i,subvol3d_var_names);
264  if (m_subvol_per[i] > zero) {last_subvol_time[i] += m_subvol_per[i];}
265  }
266  }
267 
268  if (writeNow(cur_time, step+1, m_check_int, m_check_per, dt[0], last_check_file_time)) {
269  last_check_file_step = step+1;
270  WriteCheckpointFile();
271  if (m_check_per > zero) {last_check_file_time += m_check_per;}
272  }
273 }
ERF::writeNow
bool writeNow(double cur_time, const int nstep, const int plot_int, const double plot_per, const double dt_0, double &last_file_time)
Definition: ERF.cpp:2765
LandSurface::Plot
void Plot(const int &lev, const int &nstep)
Definition: ERF_LandSurface.H:89

Referenced by EvolveOneStep().

Here is the caller graph for this function:

◆ writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream &  os)
static
176 {
177  std::string PrettyLine = std::string(78, '=') + "\n";
178  std::string OtherLine = std::string(78, '-') + "\n";
179  std::string SkipSpace = std::string(8, ' ');
180 
181  // build information
182  os << PrettyLine;
183  os << " ERF Build Information\n";
184  os << PrettyLine;
185 
186  os << "build date: " << buildInfoGetBuildDate() << "\n";
187  os << "build machine: " << buildInfoGetBuildMachine() << "\n";
188  os << "build dir: " << buildInfoGetBuildDir() << "\n";
189  os << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
190 
191  os << "\n";
192 
193  os << "COMP: " << buildInfoGetComp() << "\n";
194  os << "COMP version: " << buildInfoGetCompVersion() << "\n";
195 
196  os << "C++ compiler: " << buildInfoGetCXXName() << "\n";
197  os << "C++ flags: " << buildInfoGetCXXFlags() << "\n";
198 
199  os << "\n";
200 
201  os << "Link flags: " << buildInfoGetLinkFlags() << "\n";
202  os << "Libraries: " << buildInfoGetLibraries() << "\n";
203 
204  os << "\n";
205 
206  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
207  os << buildInfoGetModuleName(n) << ": "
208  << buildInfoGetModuleVal(n) << "\n";
209  }
210 
211  os << "\n";
212  const char* githash1 = buildInfoGetGitHash(1);
213  const char* githash2 = buildInfoGetGitHash(2);
214  if (strlen(githash1) > 0) {
215  os << "ERF git hash: " << githash1 << "\n";
216  }
217  if (strlen(githash2) > 0) {
218  os << "AMReX git hash: " << githash2 << "\n";
219  }
220 
221  const char* buildgithash = buildInfoGetBuildGitHash();
222  const char* buildgitname = buildInfoGetBuildGitName();
223  if (strlen(buildgithash) > 0) {
224  os << buildgitname << " git hash: " << buildgithash << "\n";
225  }
226 
227  os << "\n";
228  os << " ERF Compile time variables: \n";
229 
230  os << "\n";
231  os << " ERF Defines: \n";
232 #ifdef _OPENMP
233  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
234  << std::endl;
235 #else
236  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
237  << std::endl;
238 #endif
239 
240 #ifdef MPI_VERSION
241  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
242  << MPI_VERSION << std::endl;
243 #else
244  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
245  << "UNDEFINED" << std::endl;
246 #endif
247 
248 #ifdef MPI_SUBVERSION
249  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
250  << MPI_SUBVERSION << std::endl;
251 #else
252  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
253  << "UNDEFINED" << std::endl;
254 #endif
255 
256  os << "\n\n";
257 }

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ WriteCheckpointFile()

void ERF::WriteCheckpointFile ( ) const

ERF function for writing a checkpoint file.

36 {
37  auto dCheckTime0 = amrex::second();
38 
39  // chk00010 write a checkpoint file with this root directory
40  // chk00010/Header this contains information you need to save (e.g., finest_level, t_new, etc.) and also
41  // the BoxArrays at each level
42  // chk00010/Level_0/
43  // chk00010/Level_1/
44  // etc. these subdirectories will hold the MultiFab data at each level of refinement
45 
46  // checkpoint file name, e.g., chk00010
47  const std::string& checkpointname = Concatenate(check_file,istep[0],file_name_digits);
48 
49  Print() << "Writing native checkpoint " << checkpointname << "\n";
50 
51  const int nlevels = finest_level+1;
52 
53  // ---- prebuild a hierarchy of directories
54  // ---- dirName is built first. if dirName exists, it is renamed. then build
55  // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
56  // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
57  // ---- after all directories are built
58  // ---- ParallelDescriptor::IOProcessor() creates the directories
59  PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
60 
61  int ncomp_cons = vars_new[0][Vars::cons].nComp();
62 
63  // write Header file
64  if (ParallelDescriptor::IOProcessor()) {
65 
66  std::string HeaderFileName(checkpointname + "/Header");
67  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
68  std::ofstream HeaderFile;
69  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
70  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
71  std::ofstream::trunc |
72  std::ofstream::binary);
73  if(! HeaderFile.good()) {
74  FileOpenFailed(HeaderFileName);
75  }
76 
77  HeaderFile.precision(17);
78 
79  // write out title line
80  HeaderFile << "Checkpoint file for ERF\n";
81 
82  // write out finest_level
83  HeaderFile << finest_level << "\n";
84 
85  // write the number of components
86  // for each variable we store
87 
88  // conservative, cell-centered vars
89  HeaderFile << ncomp_cons << "\n";
90 
91  // x-velocity on faces
92  HeaderFile << 1 << "\n";
93 
94  // y-velocity on faces
95  HeaderFile << 1 << "\n";
96 
97  // z-velocity on faces
98  HeaderFile << 1 << "\n";
99 
100  // write out array of istep
101  for (int i = 0; i < istep.size(); ++i) {
102  HeaderFile << istep[i] << " ";
103  }
104  HeaderFile << "\n";
105 
106  // write out array of dt
107  for (int i = 0; i < dt.size(); ++i) {
108  HeaderFile << dt[i] << " ";
109  }
110  HeaderFile << "\n";
111 
112  // write out array of t_new
113  for (int i = 0; i < t_new.size(); ++i) {
114  HeaderFile << t_new[i] << " ";
115  }
116  HeaderFile << "\n";
117 
118  // write the BoxArray at each level
119  for (int lev = 0; lev <= finest_level; ++lev) {
120  boxArray(lev).writeOn(HeaderFile);
121  HeaderFile << '\n';
122  }
123 
124  // Write separate file that tells how many components we have of the base state
125  std::string BaseStateFileName(checkpointname + "/num_base_state_comps");
126  std::ofstream BaseStateFile;
127  BaseStateFile.open(BaseStateFileName.c_str(), std::ofstream::out |
128  std::ofstream::trunc |
129  std::ofstream::binary);
130  if(! BaseStateFile.good()) {
131  FileOpenFailed(BaseStateFileName);
132  } else {
133  // write out number of components in base state
134  BaseStateFile << BaseState::num_comps << "\n";
135  BaseStateFile << base_state[0].nGrowVect() << "\n";
136  }
137 
138  // Persist the LSM scalar step counter (e.g. NoahMP itimestep). Without
139  // this, restart resets the substep schedule, which changes the LSM->MOST
140  // flux firing times and produces a non-bitwise trajectory vs. cold start.
141  if (solverChoice.lsm_type != LandSurfaceType::None) {
142  std::string LsmStepFileName(checkpointname + "/lsm_step");
143  std::ofstream LsmStepFile;
144  LsmStepFile.open(LsmStepFileName.c_str(), std::ofstream::out |
145  std::ofstream::trunc |
146  std::ofstream::binary);
147  if(! LsmStepFile.good()) {
148  FileOpenFailed(LsmStepFileName);
149  } else {
150  for (int lev = 0; lev <= finest_level; ++lev) {
151  LsmStepFile << lsm.Get_LSM_Step(lev) << " ";
152  }
153  LsmStepFile << "\n";
154  }
155  }
156  }
157 
158  // write the MultiFab data to, e.g., chk00010/Level_0/
159  // Here we make copies of the MultiFab with no ghost cells
160  for (int lev = 0; lev <= finest_level; ++lev)
161  {
162  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
163  MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
164  VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
165 
166  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
167  MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
168  VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
169 
170  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
171  MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
172  VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
173 
174  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
175  MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
176  VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
177 
178  if (solverChoice.anelastic[lev] == 1) {
179  MultiFab ppinc(grids[lev],dmap[lev],1,0);
180  MultiFab::Copy(ppinc,pp_inc[lev],0,0,1,0);
181  VisMF::Write(ppinc, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PP_Inc"));
182 
183  MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
184  MultiFab::Copy(gpx,gradp[lev][GpVars::gpx],0,0,1,0);
185  VisMF::Write(gpx, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpx"));
186 
187  MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
188  MultiFab::Copy(gpy,gradp[lev][GpVars::gpy],0,0,1,0);
189  VisMF::Write(gpy, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpy"));
190 
191  MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
192  MultiFab::Copy(gpz,gradp[lev][GpVars::gpz],0,0,1,0);
193  VisMF::Write(gpz, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpz"));
194  }
195 
196  // Note that we write the ghost cells of the base state (unlike above)
197  IntVect ng_base = base_state[lev].nGrowVect();
198  int ncomp_base = base_state[lev].nComp();
199  MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
200  MultiFab::Copy(base,base_state[lev],0,0,ncomp_base,ng_base);
201  VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
202 
203  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
204  // Note that we also write the ghost cells of z_phys_nd
205  IntVect ng = z_phys_nd[lev]->nGrowVect();
206  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
207  MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
208  VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
209  }
210 
211  // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
212  // Write the moisture model restart variables
213  std::vector<int> qmoist_indices;
214  std::vector<std::string> qmoist_names;
215  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
216  int qmoist_nvar = static_cast<int>(qmoist_indices.size());
217  for (int var = 0; var < qmoist_nvar; var++) {
218  const int ncomp = 1;
219  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
220  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
221  MultiFab::Copy(moist_vars,*(qmoist[lev][qmoist_indices[var]]),0,0,ncomp,ng_moist);
222  VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", qmoist_names[var]));
223  }
224 
225 #if defined(ERF_USE_WINDFARM)
226  if(solverChoice.windfarm_type == WindFarmType::Fitch or
227  solverChoice.windfarm_type == WindFarmType::EWP or
228  solverChoice.windfarm_type == WindFarmType::SimpleAD){
229  IntVect ng_turb = Nturb[lev].nGrowVect();
230  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng_turb);
231  MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng_turb);
232  VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
233  }
234 #endif
235 
236  // Write the LSM data
237  if (solverChoice.lsm_type != LandSurfaceType::None) {
238  for (int ivar(0); ivar<lsm_data[lev].size(); ++ivar) {
239  BoxArray ba = lsm_data[lev][ivar]->boxArray();
240  DistributionMapping dm = lsm_data[lev][ivar]->DistributionMap();
241  IntVect ng = lsm_data[lev][ivar]->nGrowVect();
242  int nvar = lsm_data[lev][ivar]->nComp();
243  MultiFab lsm_vars(ba,dm,nvar,ng);
244  MultiFab::Copy(lsm_vars,*(lsm_data[lev][ivar]),0,0,nvar,ng);
245  VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmData" + std::to_string(ivar)));
246  }
247  for (int iflux(0); iflux<lsm_flux[lev].size(); ++iflux) {
248  BoxArray ba = lsm_flux[lev][iflux]->boxArray();
249  DistributionMapping dm = lsm_flux[lev][iflux]->DistributionMap();
250  IntVect ng = lsm_flux[lev][iflux]->nGrowVect();
251  int nvar = lsm_flux[lev][iflux]->nComp();
252  MultiFab lsm_vars(ba,dm,nvar,ng);
253  MultiFab::Copy(lsm_vars,*(lsm_flux[lev][iflux]),0,0,nvar,ng);
254  VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmFlux" + std::to_string(iflux)));
255  }
256 
257  // Write the full LSM prognostic state (e.g. NoahMP soil/snow/canopy)
258  // to chk*/noahmp_restart/Level_<lev>.nc so a restart reproduces a
259  // cold-start trajectory bitwise (issue #3255). The LSM model writes
260  // its blocks collectively; no-op for models without such state.
261  // Create the subdir on rank 0 and barrier before the collective open
262  // so no rank races ahead of the directory existing.
263  std::string LsmRestartDir(checkpointname + "/noahmp_restart");
264  if (ParallelDescriptor::IOProcessor()) {
265  amrex::UtilCreateDirectory(LsmRestartDir, 0755);
266  }
267  ParallelDescriptor::Barrier();
268  lsm.Write_Lsm_Restart(lev, LsmRestartDir);
269  }
270 
271  // Write the radiation heating rates
272  if ((solverChoice.rad_type != RadiationType::None) && (qheating_rates[lev])) {
273  int nrad = qheating_rates[lev]->nComp();
274  MultiFab mf_rad(grids[lev],dmap[lev],nrad,0);
275  MultiFab::Copy(mf_rad,*qheating_rates[lev],0,0,nrad,0);
276  VisMF::Write(mf_rad, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qrad"));
277  }
278 
279  IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
280  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
281  MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_x],0,0,1,ng);
282  VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_mx"));
283 
284 #if 0
285  if (MapFacType::m_x != MapFacType::m_y) {
286  MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_y],0,0,1,ng);
287  VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_my"));
288  }
289 #endif
290 
291  ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
292  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
293  MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_x],0,0,1,ng);
294  VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_ux"));
295 
296 #if 0
297  if (MapFacType::m_x != MapFacType::m_y) {
298  MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_y],0,0,1,ng);
299  VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_uy"));
300  }
301 #endif
302 
303  ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
304  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
305  MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_x],0,0,1,ng);
306  VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vx"));
307 
308 #if 0
309  if (MapFacType::m_x != MapFacType::m_y) {
310  MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_y],0,0,1,ng);
311  VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vy"));
312  }
313 #endif
314 
315  if (m_SurfaceLayer) {
316  amrex::Print() << "Writing SurfaceLayer variables at level " << lev << std::endl;
317  ng = IntVect(1,1,0);
318  MultiFab m_var(ba2d[lev],dmap[lev],1,ng);
319  MultiFab* src = nullptr;
320 
321  // U*
322  src = m_SurfaceLayer->get_u_star(lev);
323  MultiFab::Copy(m_var,*src,0,0,1,ng);
324  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Ustar"));
325 
326  // W*
327  src = m_SurfaceLayer->get_w_star(lev);
328  MultiFab::Copy(m_var,*src,0,0,1,ng);
329  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Wstar"));
330 
331  // T*
332  src = m_SurfaceLayer->get_t_star(lev);
333  MultiFab::Copy(m_var,*src,0,0,1,ng);
334  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Tstar"));
335 
336  // Q*
337  src = m_SurfaceLayer->get_q_star(lev);
338  MultiFab::Copy(m_var,*src,0,0,1,ng);
339  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qstar"));
340 
341  // Olen
342  src = m_SurfaceLayer->get_olen(lev);
343  MultiFab::Copy(m_var,*src,0,0,1,ng);
344  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Olen"));
345 
346  // Qsurf
347  src = m_SurfaceLayer->get_q_surf(lev);
348  MultiFab::Copy(m_var,*src,0,0,1,ng);
349  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qsurf"));
350 
351  // PBLH
352  src = m_SurfaceLayer->get_pblh(lev);
353  MultiFab::Copy(m_var,*src,0,0,1,ng);
354  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PBLH"));
355 
356  // Z0
357  src = m_SurfaceLayer->get_z0(lev);
358  MultiFab::Copy(m_var,*src,0,0,1,ng);
359  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z0"));
360  }
361 
362  if (sst_lev[lev][0]) {
363  int ntimes = 1;
364  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
365  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
366  for (int nt(0); nt<ntimes; ++nt) {
367  MultiFab::Copy(sst_at_t,*sst_lev[lev][nt],0,0,1,ng);
368  VisMF::Write(sst_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
369  "SST_" + std::to_string(nt)));
370  }
371  }
372 
373  if (tsk_lev[lev][0]) {
374  int ntimes = 1;
375  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
376  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
377  for (int nt(0); nt<ntimes; ++nt) {
378  MultiFab::Copy(tsk_at_t,*tsk_lev[lev][nt],0,0,1,ng);
379  VisMF::Write(tsk_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
380  "TSK_" + std::to_string(nt)));
381  }
382  }
383 
384  {
385  int ntimes = 1;
386  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
387  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
388  for (int nt(0); nt<ntimes; ++nt) {
389  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
390  const Box& bx = mfi.growntilebox();
391  Array4<int> const& src_arr = lmask_lev[lev][nt]->array(mfi);
392  Array4<Real> const& dst_arr = lmask_at_t.array(mfi);
393  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
394  {
395  dst_arr(i,j,k) = Real(src_arr(i,j,k));
396  });
397  }
398  VisMF::Write(lmask_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
399  "LMASK_" + std::to_string(nt)));
400  }
401  }
402 
403  IntVect ngv = ng; ngv[2] = 0;
404 
405  // Write lat/lon if it exists
406  if (lat_m[lev] && lon_m[lev]) {
407  amrex::Print() << "Writing Lat/Lon variables at level " << lev << std::endl;
408  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
409  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
410  MultiFab::Copy(lat,*lat_m[lev],0,0,1,ngv);
411  MultiFab::Copy(lon,*lon_m[lev],0,0,1,ngv);
412  VisMF::Write(lat, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LAT"));
413  VisMF::Write(lon, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LON"));
414  }
415 
416 
417 #ifdef ERF_USE_NETCDF
418  // Write sinPhi and cosPhi if it exists
419  if (cosPhi_m[lev] && sinPhi_m[lev] && solverChoice.variable_coriolis) {
420  amrex::Print() << "Writing Coriolis factors at level " << lev << std::endl;
421  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
422  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
423  MultiFab::Copy(sphi,*sinPhi_m[lev],0,0,1,ngv);
424  MultiFab::Copy(cphi,*cosPhi_m[lev],0,0,1,ngv);
425  VisMF::Write(sphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SinPhi"));
426  VisMF::Write(cphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "CosPhi"));
427  }
428 
429  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
430  if (lev == 0) {
431  amrex::Print() << "Writing C1H/C2H/RDNW/MUB/PHB variables at level " << lev << std::endl;
432  MultiFab tmp1d(ba1d[0],dmap[0],1,0);
433 
434  MultiFab::Copy(tmp1d,*wrf_C1H,0,0,1,0);
435  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C1H"));
436 
437  MultiFab::Copy(tmp1d,*wrf_C2H,0,0,1,0);
438  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C2H"));
439 
440  MultiFab::Copy(tmp1d,*wrf_RDNW,0,0,1,0);
441  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "RDNW"));
442 
443  MultiFab tmp2d(ba2d[0],dmap[0],1,wrf_MUB->nGrowVect());
444 
445  MultiFab::Copy(tmp2d,*wrf_MUB,0,0,1,wrf_MUB->nGrowVect());
446  VisMF::Write(tmp2d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MUB"));
447 
448  MultiFab tmp3d(convert(grids[0],IntVect(0,0,1)),dmap[0],1,wrf_PHB->nGrowVect());
449 
450  MultiFab::Copy(tmp3d,*wrf_PHB,0,0,1,ng);
451  VisMF::Write(tmp3d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PHB"));
452  }
453  }
454 #endif
455  } // for lev
456 
457 #ifdef ERF_USE_PARTICLES
458  particleData.Checkpoint(checkpointname);
459 #endif
460 
461 #if 0
462 #ifdef ERF_USE_NETCDF
463  // Write bdy_data files
464  if ( ParallelDescriptor::IOProcessor() &&
465  ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
466  solverChoice.use_real_bcs )
467  {
468  // Vector dimensions
469  int num_time = bdy_data_xlo.size();
470  int num_var = bdy_data_xlo[0].size();
471 
472  // Open header file and write to it
473  std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
474  bdy_h_file << std::setprecision(1) << std::fixed;
475  bdy_h_file << num_time << "\n";
476  bdy_h_file << num_var << "\n";
477  bdy_h_file << start_bdy_time << "\n";
478  bdy_h_file << bdy_time_interval << "\n";
479  bdy_h_file << real_width << "\n";
480  for (int ivar(0); ivar<num_var; ++ivar) {
481  bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
482  bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
483  bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
484  bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
485  }
486 
487  // Open data file and write to it
488  std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
489  for (int itime(0); itime<num_time; ++itime) {
490  if (bdy_data_xlo[itime].size() > 0) {
491  for (int ivar(0); ivar<num_var; ++ivar) {
492  bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
493  bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
494  bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
495  bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
496  }
497  }
498  }
499  }
500 #endif
501 #endif
502 
503  // Write job_info after checkpoint state so the provenance record describes
504  // the completed output attempt and can seed a later restart lineage.
505  writeJobInfo(checkpointname, erf_provenance::ArtifactType::Checkpoint,
506  istep[0], t_new[0]);
507 
508  if (verbose > 0)
509  {
510  auto dCheckTime = amrex::second() - dCheckTime0;
511  ParallelDescriptor::ReduceRealMax(dCheckTime,ParallelDescriptor::IOProcessorNumber());
512  amrex::Print() << "Checkpoint write time = " << dCheckTime << " seconds." << '\n';
513  }
514 }
LandSurface::Get_LSM_Step
int Get_LSM_Step(const int &lev) const
Definition: ERF_LandSurface.H:195
LandSurface::Write_Lsm_Restart
void Write_Lsm_Restart(const int &lev, const std::string &dir) const
Definition: ERF_LandSurface.H:204
SolverChoice::variable_coriolis
bool variable_coriolis
Definition: ERF_DataStruct.H:1436
Here is the call graph for this function:

◆ WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain ( std::ostream &  HeaderFile,
int  nlevels,
const amrex::Vector< amrex::BoxArray > &  bArray,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
double  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName,
const std::string &  levelPrefix,
const std::string &  mfPrefix 
) const
2035 {
2036  AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
2037  AMREX_ALWAYS_ASSERT(nlevels <= my_ref_ratio.size()+1);
2038  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
2039 
2040  HeaderFile.precision(17);
2041 
2042  // ---- this is the generic plot file type name
2043  HeaderFile << versionName << '\n';
2044 
2045  HeaderFile << varnames.size() << '\n';
2046 
2047  for (int ivar = 0; ivar < varnames.size(); ++ivar) {
2048  HeaderFile << varnames[ivar] << "\n";
2049  }
2050  HeaderFile << AMREX_SPACEDIM << '\n';
2051  HeaderFile << my_time << '\n';
2052  HeaderFile << finest_level << '\n';
2053  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
2054  HeaderFile << my_geom[0].ProbLo(i) << ' ';
2055  }
2056  HeaderFile << '\n';
2057  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
2058  HeaderFile << my_geom[0].ProbHi(i) << ' ';
2059  }
2060  HeaderFile << '\n';
2061  for (int i = 0; i < finest_level; ++i) {
2062  HeaderFile << my_ref_ratio[i][0] << ' ';
2063  }
2064  HeaderFile << '\n';
2065  for (int i = 0; i <= finest_level; ++i) {
2066  HeaderFile << my_geom[i].Domain() << ' ';
2067  }
2068  HeaderFile << '\n';
2069  for (int i = 0; i <= finest_level; ++i) {
2070  HeaderFile << level_steps[i] << ' ';
2071  }
2072  HeaderFile << '\n';
2073  for (int i = 0; i <= finest_level; ++i) {
2074  for (int k = 0; k < AMREX_SPACEDIM; ++k) {
2075  HeaderFile << my_geom[i].CellSize()[k] << ' ';
2076  }
2077  HeaderFile << '\n';
2078  }
2079  HeaderFile << (int) my_geom[0].Coord() << '\n';
2080  HeaderFile << "0\n";
2081 
2082  for (int level = 0; level <= finest_level; ++level) {
2083  HeaderFile << level << ' ' << bArray[level].size() << ' ' << my_time << '\n';
2084  HeaderFile << level_steps[level] << '\n';
2085 
2086  const IntVect& domain_lo = my_geom[level].Domain().smallEnd();
2087  for (int i = 0; i < bArray[level].size(); ++i)
2088  {
2089  // Need to shift because the RealBox ctor we call takes the
2090  // physical location of index (0,0,0). This does not affect
2091  // the usual cases where the domain index starts with zero
2092  const Box& b = shift(bArray[level][i], -domain_lo);
2093  RealBox loc = RealBox(b, my_geom[level].CellSize(), my_geom[level].ProbLo());
2094  for (int n = 0; n < AMREX_SPACEDIM; ++n) {
2095  HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
2096  }
2097  }
2098 
2099  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
2100  }
2101  HeaderFile << "1" << "\n";
2102  HeaderFile << "3" << "\n";
2103  HeaderFile << "amrexvec_nu_x" << "\n";
2104  HeaderFile << "amrexvec_nu_y" << "\n";
2105  HeaderFile << "amrexvec_nu_z" << "\n";
2106  std::string mf_nodal_prefix = "Nu_nd";
2107  for (int level = 0; level <= finest_level; ++level) {
2108  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
2109  }
2110 }
Coord
Coord
Definition: ERF_DataStruct.H:92
Here is the call graph for this function:

◆ writeJobInfo()

void ERF::writeJobInfo ( const std::string &  dir,
erf_provenance::ArtifactType  artifact_type,
int  artifact_step,
double  artifact_time_seconds 
) const
16 {
17  if (!ParallelDescriptor::IOProcessor()) {
18  return;
19  }
20 
21  // job_info file with details about the run
22  std::ofstream jobInfoFile;
23  std::string FullPathJobInfoFile = dir;
24  FullPathJobInfoFile += "/job_info";
25  jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out | std::ios::trunc);
26  if (!jobInfoFile.good()) {
27  FileOpenFailed(FullPathJobInfoFile);
28  }
29 
30  erf_provenance::ProvenanceRecord provenance;
31  provenance.execution = execution_provenance;
32  provenance.artifact.artifact_uuid = erf_provenance::generate_uuid_v4();
33  provenance.artifact.artifact_type = artifact_type;
34  provenance.artifact.artifact_step = artifact_step;
35  provenance.artifact.artifact_time_seconds = artifact_time_seconds;
36  provenance.artifact.artifact_created_utc = erf_provenance::current_utc();
37 
38  std::string PrettyLine = "==================================================="
39  "============================\n";
40  std::string OtherLine = "----------------------------------------------------"
41  "----------------------------\n";
42  std::string SkipSpace = " ";
43 
44  // job information
45  jobInfoFile << PrettyLine;
46  jobInfoFile << " ERF Job Information\n";
47  jobInfoFile << PrettyLine;
48 
49  jobInfoFile << erf_provenance::serialize_provenance_block(provenance);
50  jobInfoFile << "\n";
51 
52  jobInfoFile << "inputs file: " << inputs_name << "\n\n";
53 
54  jobInfoFile << "number of MPI processes: "
55  << ParallelDescriptor::NProcs() << "\n";
56 #ifdef _OPENMP
57  jobInfoFile << "number of threads: " << omp_get_max_threads() << "\n";
58 #endif
59 
60  jobInfoFile << "\n";
61  jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
62  << getCPUTime() / Real(3600.0);
63 
64  jobInfoFile << "\n\n";
65 
66  if (use_datetime) {
67  const std::string dt_format = "%Y-%m-%d %H:%M:%S"; // ISO 8601 standard
68  jobInfoFile << "Simulation time: " << getTimestamp(start_time+t_new[0], dt_format) << "\n";
69  jobInfoFile << "\n\n";
70  }
71 
72  // Output information
73  jobInfoFile << PrettyLine;
74  jobInfoFile << " Output Information\n";
75  jobInfoFile << PrettyLine;
76 
77  jobInfoFile << "artifact type: "
78  << erf_provenance::artifact_type_token(artifact_type) << "\n";
79  jobInfoFile << "artifact step: " << artifact_step << "\n";
80  const auto existing_precision = jobInfoFile.precision();
81  jobInfoFile << std::setprecision(17)
82  << "simulation time: " << artifact_time_seconds << "\n";
83  jobInfoFile.precision(existing_precision);
84 
85  time_t now = time(nullptr);
86 
87  // Convert now to tm struct for local timezone
88  tm* localtm = localtime(&now);
89  jobInfoFile << "output data / time: " << asctime(localtm);
90 
91  std::string currentDir = FileSystem::CurrentPath();
92  jobInfoFile << "output dir: " << currentDir << "\n";
93 
94  jobInfoFile << "\n\n";
95 
96  // build information
97  jobInfoFile << PrettyLine;
98  jobInfoFile << " Build Information\n";
99  jobInfoFile << PrettyLine;
100 
101  jobInfoFile << "build date: " << buildInfoGetBuildDate() << "\n";
102  jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
103  jobInfoFile << "build dir: " << buildInfoGetBuildDir() << "\n";
104  jobInfoFile << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
105 
106  jobInfoFile << "\n";
107 
108  jobInfoFile << "COMP: " << buildInfoGetComp() << "\n";
109  jobInfoFile << "COMP version: " << buildInfoGetCompVersion() << "\n";
110 
111  jobInfoFile << "\n";
112 
113  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
114  jobInfoFile << buildInfoGetModuleName(n) << ": "
115  << buildInfoGetModuleVal(n) << "\n";
116  }
117 
118  jobInfoFile << "\n";
119 
120  const char* githash1 = buildInfoGetGitHash(1);
121  const char* githash2 = buildInfoGetGitHash(2);
122  if (strlen(githash1) > 0) {
123  jobInfoFile << "ERF git hash: " << githash1 << "\n";
124  }
125  if (strlen(githash2) > 0) {
126  jobInfoFile << "AMReX git hash: " << githash2 << "\n";
127  }
128 
129  const char* buildgithash = buildInfoGetBuildGitHash();
130  const char* buildgitname = buildInfoGetBuildGitName();
131  if (strlen(buildgithash) > 0) {
132  jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
133  }
134 
135  jobInfoFile << "\n\n";
136 
137  // grid information
138  jobInfoFile << PrettyLine;
139  jobInfoFile << " Grid Information\n";
140  jobInfoFile << PrettyLine;
141 
142  int f_lev = finest_level;
143 
144  for (int i = 0; i <= f_lev; i++) {
145  jobInfoFile << " level: " << i << "\n";
146  jobInfoFile << " number of boxes = " << grids[i].size() << "\n";
147  jobInfoFile << " maximum zones = ";
148  for (int n = 0; n < AMREX_SPACEDIM; n++) {
149  jobInfoFile << geom[i].Domain().length(n) << " ";
150  }
151  jobInfoFile << "\n\n";
152  }
153 
154  jobInfoFile << " Boundary conditions\n";
155 
156  jobInfoFile << " -x: " << domain_bc_type[0] << "\n";
157  jobInfoFile << " +x: " << domain_bc_type[3] << "\n";
158  jobInfoFile << " -y: " << domain_bc_type[1] << "\n";
159  jobInfoFile << " +y: " << domain_bc_type[4] << "\n";
160  jobInfoFile << " -z: " << domain_bc_type[2] << "\n";
161  jobInfoFile << " +z: " << domain_bc_type[5] << "\n";
162 
163  jobInfoFile << "\n\n";
164 
165  // runtime parameters
166  jobInfoFile << PrettyLine;
167  jobInfoFile << " Inputs File Parameters\n";
168  jobInfoFile << PrettyLine;
169 
170  ParmParse::dumpTable(jobInfoFile, true);
171  jobInfoFile.close();
172 }
inputs_name
std::string inputs_name
Definition: main.cpp:15
ERF::getCPUTime
static double getCPUTime()
Definition: ERF.H:1634
erf_provenance::current_utc
std::string current_utc()
Definition: ERF_Provenance.cpp:315
erf_provenance::serialize_provenance_block
std::string serialize_provenance_block(const ProvenanceRecord &record)
Definition: ERF_Provenance.cpp:344
erf_provenance::generate_uuid_v4
std::string generate_uuid_v4()
Definition: ERF_Provenance.cpp:270
erf_provenance::ArtifactProvenance::artifact_step
int artifact_step
Definition: ERF_Provenance.H:81
erf_provenance::ArtifactProvenance::artifact_uuid
std::string artifact_uuid
Definition: ERF_Provenance.H:79
erf_provenance::ArtifactProvenance::artifact_time_seconds
double artifact_time_seconds
Definition: ERF_Provenance.H:82
erf_provenance::ArtifactProvenance::artifact_created_utc
std::string artifact_created_utc
Definition: ERF_Provenance.H:83
erf_provenance::ArtifactProvenance::artifact_type
ArtifactType artifact_type
Definition: ERF_Provenance.H:80
erf_provenance::ProvenanceRecord
Definition: ERF_Provenance.H:87
erf_provenance::ProvenanceRecord::artifact
ArtifactProvenance artifact
Definition: ERF_Provenance.H:89
erf_provenance::ProvenanceRecord::execution
ExecutionProvenance execution
Definition: ERF_Provenance.H:88
Here is the call graph for this function:

◆ WriteLinePlot()

void ERF::WriteLinePlot ( const std::string &  filename,
amrex::Vector< std::array< amrex::Real, 2 >> &  points_xy 
)
589 {
590  std::ofstream ofs(filename);
591  if (!ofs.is_open()) {
592  amrex::Print() << "Error: Could not open file " << filename << " for writing.\n";
593  return;
594  }
595 
596  ofs << std::setprecision(10) << std::scientific;
597  ofs << "# x y\n";
598 
599  for (const auto& p : points_xy) {
600  ofs << p[0] << " " << p[1] << "\n";
601  }
602 
603  ofs.close();
604 
605  amrex::Print() << "Line plot data written to " << filename << "\n";
606 }

◆ WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain ( const std::string &  plotfilename,
int  nlevels,
const amrex::Vector< const amrex::MultiFab * > &  mf,
const amrex::Vector< const amrex::MultiFab * > &  mf_nd,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
double  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName = "HyperCLaw-V1.1",
const std::string &  levelPrefix = "Level_",
const std::string &  mfPrefix = "Cell",
const amrex::Vector< std::string > &  extra_dirs = amrex::Vector<std::string>() 
) const
1949 {
1950  BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
1951 
1952  AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
1953  AMREX_ALWAYS_ASSERT(nlevels <= rr.size()+1);
1954  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
1955  AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
1956 
1957  bool callBarrier(false);
1958  PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
1959  if (!extra_dirs.empty()) {
1960  for (const auto& d : extra_dirs) {
1961  const std::string ed = plotfilename+"/"+d;
1962  PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
1963  }
1964  }
1965  ParallelDescriptor::Barrier();
1966 
1967  if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
1968  Vector<BoxArray> boxArrays(nlevels);
1969  for(int level(0); level < boxArrays.size(); ++level) {
1970  boxArrays[level] = mf[level]->boxArray();
1971  }
1972 
1973  auto f = [=,this]() {
1974  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
1975  std::string HeaderFileName(plotfilename + "/Header");
1976  std::ofstream HeaderFile;
1977  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
1978  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
1979  std::ofstream::trunc |
1980  std::ofstream::binary);
1981  if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
1982  WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
1983  my_geom, time, level_steps, rr, versionName,
1984  levelPrefix, mfPrefix);
1985  };
1986 
1987  if (AsyncOut::UseAsyncOut()) {
1988  AsyncOut::Submit(std::move(f));
1989  } else {
1990  f();
1991  }
1992  }
1993 
1994  std::string mf_nodal_prefix = "Nu_nd";
1995  for (int level = 0; level <= finest_level; ++level)
1996  {
1997  if (AsyncOut::UseAsyncOut()) {
1998  VisMF::AsyncWrite(*mf[level],
1999  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
2000  true);
2001  VisMF::AsyncWrite(*mf_nd[level],
2002  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
2003  true);
2004  } else {
2005  const MultiFab* data;
2006  std::unique_ptr<MultiFab> mf_tmp;
2007  if (mf[level]->nGrowVect() != 0) {
2008  mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
2009  mf[level]->DistributionMap(),
2010  mf[level]->nComp(), 0, MFInfo(),
2011  mf[level]->Factory());
2012  MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
2013  data = mf_tmp.get();
2014  } else {
2015  data = mf[level];
2016  }
2017  VisMF::Write(*data , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
2018  VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
2019  }
2020  }
2021 }
ERF::WriteGenericPlotfileHeaderWithTerrain
void WriteGenericPlotfileHeaderWithTerrain(std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, double time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
Definition: ERF_Plotfile.cpp:2024
Here is the call graph for this function:

◆ WriteMyEBSurface()

void ERF::WriteMyEBSurface ( )

◆ writeNow()

bool ERF::writeNow ( double  cur_time,
const int  nstep,
const int  plot_int,
const double  plot_per,
const double  dt_0,
double &  last_file_time 
)
2767 {
2768  bool write_now = false;
2769 
2770  if ( plot_int > 0) {
2771 
2772  write_now = (nstep % plot_int == 0);
2773 
2774  } else if (plot_per > zero) {
2775 
2776  amrex::Print() << "CUR NEXT PER " << cur_time << " " << next_file_time << " " << plot_per << std::endl;
2777 
2778  // Only write now if nstep newly matches the number of elapsed periods
2779  write_now = (cur_time > (next_file_time - 0.1*dt_0));
2780  }
2781 
2782  return write_now;
2783 }

◆ WriteSubvolume()

void ERF::WriteSubvolume ( int  isub,
amrex::Vector< std::string >  subvol_var_names 
)
146 {
147  ParmParse pp("erf.subvol");
148 
149  Vector<Real> origin;
150  Vector< int> ncell;
151  Vector<Real> delta;
152 
153  // **************************************************************
154  // Read in the origin, number of cells in each dir, and resolution
155  // **************************************************************
156 
157  int lev_for_sub = 0;
158  int offset = isub * AMREX_SPACEDIM;
159 
160  pp.getarr("origin",origin,offset,AMREX_SPACEDIM);
161  pp.getarr("nxnynz", ncell,offset,AMREX_SPACEDIM);
162  pp.getarr("dxdydz", delta,offset,AMREX_SPACEDIM);
163 
164  bool found = false;
165  for (int i = 0; i <= finest_level; i++) {
166  if (!found) {
167  if (almostEqual(delta[offset+0],geom[i].CellSize(0)) &&
168  almostEqual(delta[offset+1],geom[i].CellSize(1)) &&
169  almostEqual(delta[offset+2],geom[i].CellSize(2)) ) {
170 
171  amrex::Print() << "WriteSubvolume:Resolution specified matches that of level " << i << std::endl;
172  found = true;
173  lev_for_sub = i;
174  }
175  }
176  }
177 
178  if (!found) {
179  amrex::Abort("Resolution specified for subvol does not match the resolution of any of the levels.");
180  }
181 
182 
183  // **************************************************************
184  // Now that we know which level we're at, we can figure out which (i,j,k) the origin corresponds to
185  // Note we use Real(1.0001) as a fudge factor since the division of two reals --> integer will do a floor
186  // **************************************************************
187  int i0 = static_cast<int>((origin[offset+0] - geom[lev_for_sub].ProbLo(0)) * Real(1.0001) / delta[offset+0]);
188  int j0 = static_cast<int>((origin[offset+1] - geom[lev_for_sub].ProbLo(1)) * Real(1.0001) / delta[offset+1]);
189  int k0 = static_cast<int>((origin[offset+2] - geom[lev_for_sub].ProbLo(2)) * Real(1.0001) / delta[offset+2]);
190 
191  found = false;
192  if (almostEqual(geom[lev_for_sub].ProbLo(0)+i0*delta[offset+0],origin[offset+0]) &&
193  almostEqual(geom[lev_for_sub].ProbLo(1)+j0*delta[offset+1],origin[offset+1]) &&
194  almostEqual(geom[lev_for_sub].ProbLo(2)+k0*delta[offset+2],origin[offset+2]) )
195  {
196  amrex::Print() << "WriteSubvolume:Specified origin is the lower left corner of cell " << IntVect(i0,j0,k0) << std::endl;
197  found = true;
198  }
199 
200  if (!found) {
201  amrex::Abort("Origin specified does not correspond to a node at this level.");
202  }
203 
204  Box domain(geom[lev_for_sub].Domain());
205 
206  Box bx(IntVect(i0,j0,k0),IntVect(i0+ncell[offset+0]-1,j0+ncell[offset+1]-1,k0+ncell[offset+2]-1));
207  amrex::Print() << "WriteSubvolume:Box requested is " << bx << std::endl;
208 
209  if (!domain.contains(bx))
210  {
211  amrex::Abort("WriteSubvolume:Box requested is larger than the existing domain");
212  }
213 
214  Vector<int> cs(AMREX_SPACEDIM);
215  int count = pp.countval("chunk_size");
216  if (count > 0) {
217  pp.queryarr("chunk_size",cs,0,AMREX_SPACEDIM);
218  } else {
219  cs[0] = max_grid_size[0][0];
220  cs[1] = max_grid_size[0][1];
221  cs[2] = max_grid_size[0][2];
222  }
223  IntVect chunk_size(cs[0],cs[1],cs[2]);
224 
225  BoxArray ba(bx);
226  ba.maxSize(chunk_size);
227 
228  amrex::Print() << "WriteSubvolume:BoxArray is " << ba << std::endl;
229 
230  Vector<std::string> varnames;
231  varnames.insert(varnames.end(), subvol_var_names.begin(), subvol_var_names.end());
232 
233  int ncomp_mf = subvol_var_names.size();
234 
235  DistributionMapping dm(ba);
236 
237  MultiFab mf(ba, dm, ncomp_mf, 0);
238 
239  int mf_comp = 0;
240 
241  // *****************************************************************************************
242 
243  // First, copy any of the conserved state variables into the output plotfile
244  for (int i = 0; i < cons_names.size(); ++i) {
245  if (containerHasElement(subvol_var_names, cons_names[i])) {
246  mf.ParallelCopy(vars_new[lev_for_sub][Vars::cons],i,mf_comp,1,1,0);
247  mf_comp++;
248  }
249  }
250 
251  // *****************************************************************************************
252 
253  if (containerHasElement(subvol_var_names, "x_velocity") ||
254  containerHasElement(subvol_var_names, "y_velocity") ||
255  containerHasElement(subvol_var_names, "z_velocity"))
256  {
257  MultiFab mf_cc_vel(grids[lev_for_sub], dmap[lev_for_sub], AMREX_SPACEDIM, 0);
258  average_face_to_cellcenter(mf_cc_vel,0,
259  Array<const MultiFab*,3>{&vars_new[lev_for_sub][Vars::xvel],
260  &vars_new[lev_for_sub][Vars::yvel],
261  &vars_new[lev_for_sub][Vars::zvel]});
262  if (containerHasElement(subvol_var_names, "x_velocity")) {
263  mf.ParallelCopy(mf_cc_vel,0,mf_comp,1,0,0);
264  mf_comp++;
265  }
266  if (containerHasElement(subvol_var_names, "y_velocity")) {
267  mf.ParallelCopy(mf_cc_vel,1,mf_comp,1,0,0);
268  mf_comp++;
269  }
270  if (containerHasElement(subvol_var_names, "z_velocity")) {
271  mf.ParallelCopy(mf_cc_vel,2,mf_comp,1,0,0);
272  mf_comp++;
273  }
274  }
275 
276  // *****************************************************************************************
277 
278  // Finally, check for any derived quantities and compute them, inserting
279  // them into our output multifab
280  auto calculate_derived = [&](const std::string& der_name,
281  MultiFab& src_mf,
282  decltype(derived::erf_dernull)& der_function)
283  {
284  if (containerHasElement(subvol_var_names, der_name)) {
285  MultiFab dmf(src_mf.boxArray(), src_mf.DistributionMap(), 1, 0);
286 #ifdef _OPENMP
287 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
288 #endif
289  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
290  {
291  const Box& tbx = mfi.tilebox();
292  auto& dfab = dmf[mfi];
293  auto& sfab = src_mf[mfi];
294  auto& zfab = (*z_phys_cc[lev_for_sub])[mfi];
295  der_function(tbx, dfab, 0, 1, sfab, zfab, Geom(lev_for_sub),
296  static_cast<Real>(t_new[0]), nullptr, lev_for_sub);
297  }
298  mf.ParallelCopy(dmf,0,mf_comp,1,0,0);
299  mf_comp++;
300  }
301  };
302 
303  // *****************************************************************************************
304  // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
305  // defined in ERF.H
306  // *****************************************************************************************
307 
308  if (solverChoice.moisture_type != MoistureType::None) {
309  calculate_derived("temp", vars_new[lev_for_sub][Vars::cons], derived::erf_dermoisttemp);
310  } else {
311  calculate_derived("temp", vars_new[lev_for_sub][Vars::cons], derived::erf_dertemp);
312  }
313  calculate_derived("theta", vars_new[lev_for_sub][Vars::cons], derived::erf_dertheta);
314  calculate_derived("KE", vars_new[lev_for_sub][Vars::cons], derived::erf_derKE);
315  calculate_derived("scalar", vars_new[lev_for_sub][Vars::cons], derived::erf_derscalar);
316  calculate_derived("soundspeed", vars_new[lev_for_sub][Vars::cons], derived::erf_dersoundspeed);
317  if (solverChoice.moisture_type != MoistureType::None) {
318  calculate_derived("precipitable", vars_new[lev_for_sub][Vars::cons], derived::erf_derprecipitable);
319  calculate_derived("mucape", vars_new[lev_for_sub][Vars::cons], derived::erf_dermucape);
320  }
321 
322  // *****************************************************************************************
323 
324  double time = t_new[lev_for_sub];
325 
326  std::string sf = subvol_file + "_" + std::to_string(isub);
327  std::string subvol_filename;
328 
329  if (use_real_time_in_pltname) {
330  const std::string dt_format = "%Y-%m-%d_%H:%M:%S"; // ISO 8601 standard
331  subvol_filename = sf + getTimestamp(start_time+time, dt_format);
332  } else {
333  subvol_filename = Concatenate(sf + "_", istep[0], file_name_digits);
334  }
335 
336  amrex::Print() <<"Writing subvolume into " << subvol_filename << std::endl;
337  WriteSingleLevelPlotfile(subvol_filename,mf,varnames,geom[lev_for_sub],static_cast<amrex::Real>(time),istep[0]);
338 
339 }
module_mp_morr_two_moment::cs
real(c_double), private cs
Definition: ERF_module_mp_morr_two_moment.F90:203
Here is the call graph for this function:

◆ WriteVTKPolyline()

void ERF::WriteVTKPolyline ( const std::string &  filename,
amrex::Vector< std::array< amrex::Real, 2 >> &  points_xy 
)
96 {
97  std::ofstream vtkfile(filename);
98  if (!vtkfile.is_open()) {
99  std::cerr << "Error: Cannot open file " << filename << std::endl;
100  return;
101  }
102 
103  int num_points = points_xy.size();
104  if (num_points == 0) {
105  vtkfile << "# vtk DataFile Version three\n";
106  vtkfile << "Storm Track\n";
107  vtkfile << "ASCII\n";
108  vtkfile << "DATASET POLYDATA\n";
109  vtkfile << "POINTS " << num_points << " float\n";
110  vtkfile.close();
111  return;
112  }
113  if (num_points < 2) {
114  points_xy.push_back(points_xy[0]);
115  }
116  num_points = points_xy.size();
117 
118  vtkfile << "# vtk DataFile Version three\n";
119  vtkfile << "Storm Track\n";
120  vtkfile << "ASCII\n";
121  vtkfile << "DATASET POLYDATA\n";
122 
123  // Write points (Z=0 assumed)
124  vtkfile << "POINTS " << num_points << " float\n";
125  for (const auto& pt : points_xy) {
126  vtkfile << pt[0] << " " << pt[1] << 10000.0 << std::endl;
127  }
128 
129  // Write polyline connectivity
130  vtkfile << "LINES 1 " << num_points + 1 << "\n";
131  vtkfile << num_points << " ";
132  for (int i = 0; i < num_points; ++i) {
133  vtkfile << i << " ";
134  }
135  vtkfile << "\n";
136 
137  vtkfile.close();
138 }

Member Data Documentation

◆ advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg
private

Referenced by ERF_shared(), and getAdvFluxReg().

◆ avg_xmom

amrex::Vector<amrex::MultiFab> ERF::avg_xmom
private

Referenced by ERF_shared().

◆ avg_ymom

amrex::Vector<amrex::MultiFab> ERF::avg_ymom
private

Referenced by ERF_shared().

◆ avg_zmom

amrex::Vector<amrex::MultiFab> ERF::avg_zmom
private

Referenced by ERF_shared().

◆ ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax
private

Referenced by ERF_shared().

◆ ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src
private

Referenced by ERF_shared().

◆ ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay
private

Referenced by ERF_shared().

◆ ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src
private

Referenced by ERF_shared().

◆ az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az
private

Referenced by ERF_shared().

◆ az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src
private

Referenced by ERF_shared().

◆ ba1d

amrex::Vector<amrex::BoxArray> ERF::ba1d
private

Referenced by ERF_shared().

◆ ba2d

amrex::Vector<amrex::BoxArray> ERF::ba2d
private

Referenced by ERF_shared(), and PackAtmosphericStates().

◆ base_state

amrex::Vector<amrex::MultiFab> ERF::base_state
private

Referenced by ERF_shared().

◆ base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new
private

Referenced by ERF_shared().

◆ bckgnd_state

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::bckgnd_state

◆ bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1
staticprivate

◆ bndry_output_planes_per

double ERF::bndry_output_planes_per = -1.0
staticprivate

◆ bndry_output_planes_start_time

double ERF::bndry_output_planes_start_time = 0.0
staticprivate

◆ boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level
private

◆ cf_set_width

int ERF::cf_set_width {0}
private

◆ cf_width

int ERF::cf_width {0}
private

◆ cfl

Real ERF::cfl = Real(0.8)
staticprivate

◆ change_max

Real ERF::change_max = Real(1.1)
staticprivate

◆ check_file

std::string ERF::check_file {"chk"}
private

◆ check_for_nans

int ERF::check_for_nans = 0
staticprivate

◆ column_file_name

std::string ERF::column_file_name = "column_data.nc"
staticprivate

◆ column_interval

int ERF::column_interval = -1
staticprivate

◆ column_loc_x

Real ERF::column_loc_x = zero
staticprivate

◆ column_loc_y

Real ERF::column_loc_y = zero
staticprivate

◆ column_per

double ERF::column_per = -1.0
staticprivate

◆ cons_names

const amrex::Vector<std::string> ERF::cons_names
private
Initial value:
{"density", "rhotheta", "rhoKE", "rhoadv_0",
"rhoQ1", "rhoQ2", "rhoQ3",
"rhoQ4", "rhoQ5", "rhoQ6",
"rhoQ7", "rhoQ8", "rhoQ9",
"rhoQ10", "rhoQ11"}

◆ cosPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::cosPhi_m
private

Referenced by ERF_shared().

◆ d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density
private

◆ d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure
private

◆ d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc
private

◆ d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv
private

◆ d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature
private

◆ d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs
private

Referenced by ERF_shared().

◆ d_sinesq_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_ptrs
private

Referenced by ERF_shared().

◆ d_sinesq_stag_ptrs

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_sinesq_stag_ptrs
private

Referenced by ERF_shared().

◆ d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs
private

◆ d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos
private

◆ d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos
private

◆ d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid
private

◆ datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog
private

Referenced by DataLog(), NumDataLogs(), and setRecordDataInfo().

◆ datalogname

amrex::Vector<std::string> ERF::datalogname
private

Referenced by DataLogName().

◆ datetime_format

const std::string ERF::datetime_format = "%Y-%m-%d %H:%M:%S"
private

◆ datprecision

const int ERF::datprecision = 6
private

◆ datwidth

const int ERF::datwidth = 14
private

◆ der_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::der_datalog
private

Referenced by DerDataLog(), NumDerDataLogs(), and setRecordDerDataInfo().

◆ der_datalogname

amrex::Vector<std::string> ERF::der_datalogname
private

Referenced by DerDataLogName().

◆ derived_names

const amrex::Vector<std::string> ERF::derived_names
private

◆ derived_subvol_names

const amrex::Vector<std::string> ERF::derived_subvol_names {"soundspeed", "temp", "theta", "KE", "scalar"}
private

◆ destag_profiles

bool ERF::destag_profiles = true
private

◆ detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc
private

Referenced by ERF_shared().

◆ detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new
private

Referenced by ERF_shared().

◆ detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src
private

Referenced by ERF_shared().

◆ domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type
private

◆ domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type
private

◆ domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d
private

◆ dt

amrex::Vector<double> ERF::dt
private

Referenced by ERF_shared(), and EvolveOneStep().

◆ dt_max

double ERF::dt_max = 1.0e9
staticprivate

◆ dt_max_initial

double ERF::dt_max_initial = static_cast<double>(bogus_large_value)
staticprivate

Referenced by ERF_shared().

◆ dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio
private

Referenced by ERF_shared().

◆ dz_min

amrex::Vector<amrex::Real> ERF::dz_min
private

◆ eb

amrex::Vector<std::unique_ptr<eb_> > ERF::eb
private

Referenced by EBFactory(), ERF_shared(), and get_eb().

◆ eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev
private

Referenced by ERF_shared().

◆ erfbdy_file

std::string ERF::erfbdy_file {"erfbdy"}
private

◆ execution_provenance

erf_provenance::ExecutionProvenance ERF::execution_provenance
private

Referenced by ERF_shared().

◆ file_name_digits

int ERF::file_name_digits = 5
private

◆ fine_mask

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::fine_mask
private

Referenced by ERF_shared().

◆ finished_wave

bool ERF::finished_wave = false
private

◆ fixed_dt

amrex::Vector<double> ERF::fixed_dt
private

◆ fixed_fast_dt

amrex::Vector<double> ERF::fixed_fast_dt
private

◆ fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0
staticprivate

◆ forecast_state_1

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_1

Referenced by ERF_shared().

◆ forecast_state_2

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_2

Referenced by ERF_shared().

◆ forecast_state_interp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_interp

Referenced by ERF_shared().

◆ FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c
private

◆ FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u
private

◆ FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v
private

◆ FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w
private

◆ gradp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::gradp
private

Referenced by ERF_shared().

◆ h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density
private

◆ h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure
private

◆ h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc
private

◆ h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv
private

◆ h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature
private

◆ h_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs
private

Referenced by ERF_shared().

◆ h_sinesq_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_ptrs
private

Referenced by ERF_shared().

◆ h_sinesq_stag_ptrs

amrex::Vector<amrex::Vector<amrex::Real> > ERF::h_sinesq_stag_ptrs
private

Referenced by ERF_shared().

◆ h_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs
private

◆ h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos
private

◆ h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos
private

◆ h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid
private

◆ have_read_nc_init_file

Vector< Vector< int > > ERF::have_read_nc_init_file = {{0}}
staticprivate

◆ hfx3_EB

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::hfx3_EB
private

Referenced by ERF_shared().

◆ hurricane_eye_track_latlon

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_latlon

◆ hurricane_eye_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_eye_track_xy

◆ hurricane_maxvel_vs_time

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_maxvel_vs_time

◆ hurricane_minpressure_vs_time

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_minpressure_vs_time

◆ hurricane_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_track_xy

◆ hurricane_tracker_circle

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_tracker_circle

◆ Hwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave
private

Referenced by ERF_shared().

◆ Hwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave_onegrid
private

Referenced by ERF_shared().

◆ init_shrink

Real ERF::init_shrink = one
staticprivate

◆ input_bndry_planes

int ERF::input_bndry_planes = 0
staticprivate

◆ input_sounding_data

InputSoundingData ERF::input_sounding_data
private

◆ input_sponge_data

InputSpongeData ERF::input_sponge_data
private

◆ interpolation_type

StateInterpType ERF::interpolation_type
staticprivate

◆ istep

amrex::Vector<int> ERF::istep
private

Referenced by EvolveOneStep().

◆ lagged_delta_rt

amrex::Vector<amrex::MultiFab> ERF::lagged_delta_rt
private

Referenced by ERF_shared().

◆ land_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::land_type_lev
private

Referenced by ERF_shared().

◆ last_check_file_step

int ERF::last_check_file_step = -1
staticprivate

◆ last_check_file_time

double ERF::last_check_file_time = 0.0
staticprivate

◆ last_plot2d_file_step_1

int ERF::last_plot2d_file_step_1 = -1
staticprivate

◆ last_plot2d_file_step_2

int ERF::last_plot2d_file_step_2 = -1
staticprivate

◆ last_plot2d_file_time_1

double ERF::last_plot2d_file_time_1 = 0.0
staticprivate

◆ last_plot2d_file_time_2

double ERF::last_plot2d_file_time_2 = 0.0
staticprivate

◆ last_plot3d_file_step_1

int ERF::last_plot3d_file_step_1 = -1
staticprivate

◆ last_plot3d_file_step_2

int ERF::last_plot3d_file_step_2 = -1
staticprivate

◆ last_plot3d_file_time_1

double ERF::last_plot3d_file_time_1 = 0.0
staticprivate

◆ last_plot3d_file_time_2

double ERF::last_plot3d_file_time_2 = 0.0
staticprivate

◆ last_subvol_step

amrex::Vector<int> ERF::last_subvol_step
private

◆ last_subvol_time

amrex::Vector<double> ERF::last_subvol_time
private

◆ lat_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lat_m
private

Referenced by ERF_shared().

◆ line_sampler

std::unique_ptr<LineSampler> ERF::line_sampler = nullptr
private

◆ line_sampling_interval

int ERF::line_sampling_interval = -1
private

◆ line_sampling_per

amrex::Real ERF::line_sampling_per = -one
private

◆ lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev
private

Referenced by ERF_shared().

◆ lon_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lon_m
private

Referenced by ERF_shared().

◆ lsm

LandSurface ERF::lsm
private

Referenced by ApplyOceanSurfaceState(), ERF_shared(), and GetOceanToAtmosSurfaceLayout().

◆ lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data
private

Referenced by ERF_shared().

◆ lsm_data_name

amrex::Vector<std::string> ERF::lsm_data_name
private

◆ lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux
private

Referenced by ERF_shared().

◆ lsm_flux_name

amrex::Vector<std::string> ERF::lsm_flux_name
private

◆ Lwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave
private

Referenced by ERF_shared().

◆ Lwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave_onegrid
private

Referenced by ERF_shared().

◆ m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_extdir_vals
private

◆ m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_neumann_vals
private

◆ m_bc_nonreflecting

amrex::Array<bool, AMREX_SPACEDIM*2> ERF::m_bc_nonreflecting = {{false}}
private

◆ m_check_int

int ERF::m_check_int = -1
private

◆ m_check_per

double ERF::m_check_per = -1.0
private

◆ m_driver_atm2ocn_uses_state_contract

bool ERF::m_driver_atm2ocn_uses_state_contract = true

Referenced by ConfigureDriverAtmosToOceanCoupling(), and SetDriverAtmosToOceanStateContract().

◆ m_driver_has_atm2ocn_coupling

bool ERF::m_driver_has_atm2ocn_coupling = false

Referenced by ConfigureDriverAtmosToOceanCoupling().

◆ m_driver_uses_two_way_coupling

bool ERF::m_driver_uses_two_way_coupling = false

Referenced by ConfigureDriverAtmosToOceanCoupling().

◆ m_expand_plotvars_to_unif_rr

bool ERF::m_expand_plotvars_to_unif_rr = false
private

◆ m_forest_drag

amrex::Vector<std::unique_ptr<ForestDrag> > ERF::m_forest_drag
private

Referenced by ERF_shared().

◆ m_plot2d_int_1

int ERF::m_plot2d_int_1 = -1
private

◆ m_plot2d_int_2

int ERF::m_plot2d_int_2 = -1
private

◆ m_plot2d_per_1

double ERF::m_plot2d_per_1 = -1.0
private

◆ m_plot2d_per_2

double ERF::m_plot2d_per_2 = -1.0
private

◆ m_plot3d_int_1

int ERF::m_plot3d_int_1 = -1
private

◆ m_plot3d_int_2

int ERF::m_plot3d_int_2 = -1
private

◆ m_plot3d_per_1

double ERF::m_plot3d_per_1 = -1.0
private

◆ m_plot3d_per_2

double ERF::m_plot3d_per_2 = -1.0
private

◆ m_plot_face_vels

bool ERF::m_plot_face_vels = false
private

◆ m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr
private

◆ m_subvol_int

amrex::Vector<int> ERF::m_subvol_int
private

◆ m_subvol_per

amrex::Vector<double> ERF::m_subvol_per
private

◆ m_SurfaceLayer

std::unique_ptr<SurfaceLayer> ERF::m_SurfaceLayer = nullptr
private

◆ m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr
private

◆ mapfac

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::mapfac
private

Referenced by ERF_shared().

◆ max_step

int ERF::max_step = -1
private

◆ metgrid_basic_linear

bool ERF::metgrid_basic_linear {false}
private

◆ metgrid_debug_dry

bool ERF::metgrid_debug_dry {false}
private

◆ metgrid_debug_isothermal

bool ERF::metgrid_debug_isothermal {false}
private

◆ metgrid_debug_msf

bool ERF::metgrid_debug_msf {false}
private

◆ metgrid_debug_psfc

bool ERF::metgrid_debug_psfc {false}
private

◆ metgrid_debug_quiescent

bool ERF::metgrid_debug_quiescent {false}
private

◆ metgrid_force_sfc_k

int ERF::metgrid_force_sfc_k {6}
private

◆ metgrid_interp_theta

bool ERF::metgrid_interp_theta {false}
private

◆ metgrid_order

int ERF::metgrid_order {2}
private

◆ metgrid_proximity

amrex::Real ERF::metgrid_proximity {amrex::Real(500.0)}
private

◆ metgrid_retain_sfc

bool ERF::metgrid_retain_sfc {false}
private

◆ metgrid_use_below_sfc

bool ERF::metgrid_use_below_sfc {true}
private

◆ metgrid_use_sfc

bool ERF::metgrid_use_sfc {true}
private

◆ mf_PSFC

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_PSFC
private

Referenced by ERF_shared().

◆ mg_verbose

int ERF::mg_verbose = 0
staticprivate

◆ micro

std::unique_ptr<Microphysics> ERF::micro
private

◆ mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem
private

Referenced by ERF_shared().

◆ native_shoc_driver

amrex::Vector<std::unique_ptr<ShocDriver> > ERF::native_shoc_driver
private

Referenced by ERF_shared().

◆ nc_bdy_file

std::string ERF::nc_bdy_file
staticprivate

◆ nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}
staticprivate

◆ nc_low_file

std::string ERF::nc_low_file
staticprivate

◆ ng_dens_hse

int ERF::ng_dens_hse
staticprivate

◆ ng_pres_hse

int ERF::ng_pres_hse
staticprivate

◆ nsubsteps

amrex::Vector<int> ERF::nsubsteps
private

◆ num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level
private

◆ num_files_at_level

amrex::Vector<int> ERF::num_files_at_level
private

◆ nvars_erfbdy

int ERF::nvars_erfbdy {0}
private

◆ output_1d_column

int ERF::output_1d_column = 0
staticprivate

◆ output_bndry_planes

int ERF::output_bndry_planes = 0
staticprivate

◆ pert_interval

int ERF::pert_interval = -1
staticprivate

◆ phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type
private

◆ physbcs_base

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_base> > ERF::physbcs_base
private

Referenced by ERF_shared().

◆ physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons
private

Referenced by ERF_shared().

◆ physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u
private

Referenced by ERF_shared().

◆ physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v
private

Referenced by ERF_shared().

◆ physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w
private

Referenced by ERF_shared().

◆ plane_sampler

std::unique_ptr<PlaneSampler> ERF::plane_sampler = nullptr
private

◆ plane_sampling_interval

int ERF::plane_sampling_interval = -1
private

◆ plane_sampling_per

amrex::Real ERF::plane_sampling_per = -one
private

◆ plot2d_file_1

std::string ERF::plot2d_file_1 {"plt2d_1_"}
private

◆ plot2d_file_2

std::string ERF::plot2d_file_2 {"plt2d_2_"}
private

◆ plot2d_var_names_1

amrex::Vector<std::string> ERF::plot2d_var_names_1
private

◆ plot2d_var_names_2

amrex::Vector<std::string> ERF::plot2d_var_names_2
private

◆ plot3d_file_1

std::string ERF::plot3d_file_1 {"plt_1_"}
private

◆ plot3d_file_2

std::string ERF::plot3d_file_2 {"plt_2_"}
private

◆ plot3d_var_names_1

amrex::Vector<std::string> ERF::plot3d_var_names_1
private

Referenced by ERF_shared().

◆ plot3d_var_names_2

amrex::Vector<std::string> ERF::plot3d_var_names_2
private

Referenced by ERF_shared().

◆ plot_file_on_restart

bool ERF::plot_file_on_restart = true
staticprivate

◆ plot_lsm

bool ERF::plot_lsm = false
private

◆ plot_rad

bool ERF::plot_rad = false
private

◆ plotfile2d_type_1

PlotFileType ERF::plotfile2d_type_1 = PlotFileType::None
staticprivate

◆ plotfile2d_type_2

PlotFileType ERF::plotfile2d_type_2 = PlotFileType::None
staticprivate

◆ plotfile3d_type_1

PlotFileType ERF::plotfile3d_type_1 = PlotFileType::None
staticprivate

◆ plotfile3d_type_2

PlotFileType ERF::plotfile3d_type_2 = PlotFileType::None
staticprivate

◆ pp_inc

amrex::Vector<amrex::MultiFab> ERF::pp_inc
private

Referenced by ERF_shared().

◆ pp_prefix

std::string ERF::pp_prefix {"erf"}

◆ previousCPUTimeUsed

double ERF::previousCPUTimeUsed = 0.0
staticprivate

Referenced by getCPUTime().

◆ prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr
private

Referenced by ERF_shared().

◆ profile_int

int ERF::profile_int = -1
private

◆ qfx3_EB

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::qfx3_EB
private

◆ qheating_rates

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::qheating_rates
private

Referenced by ERF_shared().

◆ qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist
private

◆ Qr_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qr_prim
private

Referenced by ERF_shared().

◆ Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim
private

Referenced by ERF_shared().

◆ rad

amrex::Vector<std::unique_ptr<IRadiation> > ERF::rad
private

Referenced by ERF_shared().

◆ rad_datalog_int

int ERF::rad_datalog_int = -1
private

Referenced by ERF_shared().

◆ rad_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rad_fluxes
private

Referenced by ERF_shared(), and PackAtmosphericStates().

◆ real_extrap_w

bool ERF::real_extrap_w {true}
private

◆ real_width

int ERF::real_width {0}
private

◆ ref_tags

Vector< AMRErrorTag > ERF::ref_tags
staticprivate

◆ regrid_int

int ERF::regrid_int = -1
private

◆ regrid_level_0_on_restart

bool ERF::regrid_level_0_on_restart = false
private

◆ restart_chkfile

std::string ERF::restart_chkfile = ""
private

◆ rhoqt_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rhoqt_src
private

Referenced by ERF_shared().

◆ rhotheta_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::rhotheta_src
private

Referenced by ERF_shared().

◆ rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new
private

Referenced by ERF_shared().

◆ rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old
private

Referenced by ERF_shared().

◆ rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new
private

Referenced by ERF_shared().

◆ rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old
private

Referenced by ERF_shared().

◆ rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new
private

Referenced by ERF_shared().

◆ rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old
private

Referenced by ERF_shared().

◆ sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline
private

Referenced by NumSampleLines(), and SampleLine().

◆ samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog
private

Referenced by NumSampleLineLogs(), SampleLineLog(), and setRecordSampleLineInfo().

◆ samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname
private

Referenced by SampleLineLogName().

◆ samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint
private

Referenced by NumSamplePoints(), and SamplePoint().

◆ sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog
private

Referenced by NumSamplePointLogs(), SamplePointLog(), and setRecordSamplePointInfo().

◆ sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname
private

Referenced by SamplePointLogName().

◆ SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev
private

Referenced by ERF_shared().

◆ SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev
private

Referenced by ERF_shared().

◆ SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev
private

Referenced by ERF_shared().

◆ SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev
private

Referenced by ERF_shared(), and PackAtmosphericStates().

◆ SFS_q1fx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx1_lev
private

Referenced by ERF_shared().

◆ SFS_q1fx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx2_lev
private

Referenced by ERF_shared().

◆ SFS_q1fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx3_lev
private

Referenced by ERF_shared(), and PackAtmosphericStates().

◆ SFS_q2fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q2fx3_lev
private

Referenced by ERF_shared().

◆ sinPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sinPhi_m
private

Referenced by ERF_shared().

◆ SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev
private

Referenced by ERF_shared().

◆ soil_type_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::soil_type_lev
private

Referenced by ERF_shared().

◆ solverChoice

SolverChoice ERF::solverChoice
staticprivate

Referenced by ApplyOceanSurfaceState(), ERF_shared(), and PackAtmosphericStates().

◆ sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev
private

Referenced by ERF_shared().

◆ start_time

double ERF::start_time = 0.0
staticprivate

Referenced by EvolveOneStep().

◆ startCPUTime

double ERF::startCPUTime = 0.0
staticprivate

Referenced by getCPUTime().

◆ stop_time

double ERF::stop_time = std::numeric_limits<double>::max()
staticprivate

Referenced by EvolveOneStep().

◆ stretched_dz_d

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::stretched_dz_d
private

Referenced by ERF_shared().

◆ stretched_dz_h

amrex::Vector<amrex::Vector<amrex::Real> > ERF::stretched_dz_h
private

Referenced by ERF_shared().

◆ sub_cfl

Real ERF::sub_cfl = one
staticprivate

◆ subdomains

amrex::Vector<amrex::Vector<amrex::BoxArray> > ERF::subdomains
private

◆ subvol3d_var_names

amrex::Vector<std::string> ERF::subvol3d_var_names
private

◆ subvol_file

std::string ERF::subvol_file {"subvol"}
private

◆ sum_interval

int ERF::sum_interval = -1
staticprivate

◆ sum_per

double ERF::sum_per = -1.0
staticprivate

◆ surface_state_1

amrex::Vector<amrex::MultiFab> ERF::surface_state_1

Referenced by ERF_shared().

◆ surface_state_2

amrex::Vector<amrex::MultiFab> ERF::surface_state_2

Referenced by ERF_shared().

◆ surface_state_interp

amrex::Vector<amrex::MultiFab> ERF::surface_state_interp

Referenced by ERF_shared().

◆ t_avg_cnt

amrex::Vector<double> ERF::t_avg_cnt
private

Referenced by ERF_shared().

◆ t_new

amrex::Vector<double> ERF::t_new
private

Referenced by ERF_shared(), and EvolveOneStep().

◆ t_old

amrex::Vector<double> ERF::t_old
private

Referenced by ERF_shared().

◆ Tau

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau
private

Referenced by ERF_shared(), and PackAtmosphericStates().

◆ Tau_corr

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau_corr
private

Referenced by ERF_shared().

◆ Tau_EB

amrex::Vector<amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > > ERF::Tau_EB
private

Referenced by ERF_shared().

◆ terrain_blanking

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking
private

Referenced by ERF_shared().

◆ th_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::th_bc_data
private

Referenced by ERF_shared().

◆ Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim
private

Referenced by ERF_shared().

◆ thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce
private

Referenced by ERF_shared().

◆ thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce
private

Referenced by ERF_shared().

◆ thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce
private

Referenced by ERF_shared().

◆ timeprecision

const int ERF::timeprecision = 13
private

◆ tot_e_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::tot_e_datalog
private

Referenced by setRecordEnergyDataInfo().

◆ tot_e_datalogname

amrex::Vector<std::string> ERF::tot_e_datalogname
private

◆ tsk_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::tsk_lev
private

Referenced by ERF_shared().

◆ turbPert

TurbulentPerturbation ERF::turbPert
private

◆ urb_frac_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::urb_frac_lev
private

Referenced by ERF_shared().

◆ use_datetime

bool ERF::use_datetime = false
private

◆ use_erfbdy

bool ERF::use_erfbdy {false}
private

◆ use_fft

bool ERF::use_fft = false
staticprivate

◆ use_real_time_in_pltname

bool ERF::use_real_time_in_pltname = false
private

◆ vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new
private

Referenced by ERF_shared(), and PackAtmosphericStates().

◆ vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old
private

Referenced by ERF_shared().

◆ vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg
private

Referenced by ERF_shared().

◆ verbose

int ERF::verbose = 0
staticprivate

Referenced by PackAtmosphericStates().

◆ walldist

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::walldist
private

Referenced by ERF_shared().

◆ weather_forecast_data_1

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_1

◆ weather_forecast_data_2

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_2

◆ wrf_C1H

std::unique_ptr<amrex::MultiFab> ERF::wrf_C1H
private

◆ wrf_C2H

std::unique_ptr<amrex::MultiFab> ERF::wrf_C2H
private

◆ wrf_MUB

std::unique_ptr<amrex::MultiFab> ERF::wrf_MUB
private

◆ wrf_PHB

std::unique_ptr<amrex::MultiFab> ERF::wrf_PHB
private

◆ wrf_RDNW

std::unique_ptr<amrex::MultiFab> ERF::wrf_RDNW
private

◆ write_erfbdy

bool ERF::write_erfbdy {false}
private

◆ xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask
private

Referenced by ERF_shared().

◆ xvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::xvel_bc_data
private

Referenced by ERF_shared().

◆ yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask
private

Referenced by ERF_shared().

◆ yvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::yvel_bc_data
private

Referenced by ERF_shared().

◆ z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc
private

Referenced by ERF_shared().

◆ z_phys_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc_src
private

Referenced by ERF_shared().

◆ z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd
private

Referenced by ERF_shared().

◆ z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new
private

Referenced by ERF_shared().

◆ z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src
private

Referenced by ERF_shared().

◆ z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk
private

Referenced by ERF_shared().

◆ zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask
private

Referenced by ERF_shared().

◆ zlevels_stag

amrex::Vector<amrex::Vector<amrex::Real> > ERF::zlevels_stag
private

Referenced by ERF_shared().

◆ zmom_crse_rhs

amrex::Vector<amrex::MultiFab> ERF::zmom_crse_rhs
private

Referenced by ERF_shared().

◆ zvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::zvel_bc_data
private

Referenced by ERF_shared().

The documentation for this class was generated from the following files: