ERF
Energy Research and Forecasting: An Atmospheric Modeling Code
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
ERF Class Reference

#include <ERF.H>

Inheritance diagram for ERF:
Collaboration diagram for ERF:

Public Member Functions

 ERF ()
 
 ~ERF () override
 
void ERF_shared ()
 
 ERF (ERF &&) noexcept=delete
 
ERFoperator= (ERF &&other) noexcept=delete
 
 ERF (const ERF &other)=delete
 
ERFoperator= (const ERF &other)=delete
 
void Evolve ()
 
void ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override
 
void HurricaneTracker (int lev, const amrex::MultiFab &U_new, const amrex::MultiFab &V_new, const amrex::MultiFab &W_new, const amrex::Real velmag_threshold, const bool is_track_io, amrex::TagBoxArray *tags=nullptr)
 
std::string MakeVTKFilename (int nstep)
 
void WriteVTKPolyline (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
 
void InitData ()
 
void InitData_pre ()
 
void InitData_post ()
 
void WriteMyEBSurface ()
 
void compute_divergence (int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
 
void project_velocities (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars, amrex::MultiFab &p)
 
void project_velocities_tb (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars, amrex::MultiFab &p)
 
void poisson_wall_dist (int lev)
 
void solve_with_EB_mlmg (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
 
void solve_with_gmres (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
 
void solve_with_mlmg (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
 
void ImposeBCsOnPhi (int lev, amrex::MultiFab &phi)
 
amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > get_projection_bc (amrex::Orientation::Side side) const noexcept
 
bool projection_has_dirichlet (amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > bcs) const
 
void init_only (int lev, amrex::Real time)
 
void restart ()
 
bool writeNow (const amrex::Real cur_time, const amrex::Real dt, const int nstep, const int plot_int, const amrex::Real plot_per)
 
void post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev)
 
void sum_integrated_quantities (amrex::Real time)
 
void sum_derived_quantities (amrex::Real time)
 
void sum_energy_quantities (amrex::Real time)
 
void write_1D_profiles (amrex::Real time)
 
void write_1D_profiles_stag (amrex::Real time)
 
amrex::Real cloud_fraction (amrex::Real time)
 
void FillBdyCCVels (amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)
 
void sample_points (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void sample_lines (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void derive_diag_profiles (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_diag_profiles_stag (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
void derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
amrex::Real volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, const amrex::MultiFab &mapfac, bool finemask)
 
void MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void ClearLevel (int lev) override
 
void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
amrex::Real estTimeStep (int lev, long &dt_fast_ratio) const
 
void advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
 
void advance_microphysics (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
 
void advance_lsm (int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)
 
amrex::MultiFab & build_fine_mask (int lev)
 
void MakeHorizontalAverages ()
 
void MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)
 
void derive_upwp (amrex::Vector< amrex::Real > &h_havg)
 
void WritePlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
 
void WriteSubvolume ()
 
void WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
 
void WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
 
void erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
 
void init_from_input_sounding (int lev)
 
void input_sponge (int lev)
 
void init_from_hse (int lev)
 
void init_thin_body (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
 
void MakeEBGeometry ()
 
void make_eb_box ()
 
void make_eb_regular ()
 
void AverageDownTo (int crse_lev, int scomp, int ncomp)
 
void WriteCheckpointFile () const
 
void ReadCheckpointFile ()
 
void ReadCheckpointFileSurfaceLayer ()
 
void init_zphys (int lev, amrex::Real time)
 
void remake_zphys (int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
 
void update_terrain_arrays (int lev)
 
void writeJobInfo (const std::string &dir) const
 

Static Public Member Functions

static bool is_it_time_for_action (int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
 
static void writeBuildInfo (std::ostream &os)
 
static void print_banner (MPI_Comm, std::ostream &)
 
static void print_usage (MPI_Comm, std::ostream &)
 
static void print_error (MPI_Comm, const std::string &msg)
 
static void print_summary (std::ostream &)
 
static void print_tpls (std::ostream &)
 

Public Attributes

amrex::Vector< std::array< amrex::Real, 2 > > hurricane_track_xy
 
std::string pp_prefix {"erf"}
 

Private Member Functions

void ReadParameters ()
 
void ParameterSanityChecks ()
 
void AverageDown ()
 
void update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void Construct_ERFFillPatchers (int lev)
 
void Define_ERFFillPatchers (int lev)
 
void init1DArrays ()
 
void init_bcs ()
 
void init_custom (int lev)
 
void init_uniform (int lev)
 
void init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
 
void turbPert_update (const int lev, const amrex::Real dt)
 
void turbPert_amplitude (const int lev)
 
void initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
 
void make_physbcs (int lev)
 
void initializeMicrophysics (const int &)
 
void FillPatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
 
void FillPatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
 
void FillIntermediatePatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
 
void FillCoarsePatch (int lev, amrex::Real time)
 
void timeStep (int lev, amrex::Real time, int iteration)
 
void Advance (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
 
void initHSE ()
 Initialize HSE. More...
 
void initHSE (int lev)
 
void initRayleigh ()
 Initialize Rayleigh damping profiles. More...
 
void initSponge ()
 Initialize sponge profiles. More...
 
void setRayleighRefFromSounding (bool restarting)
 Set Rayleigh mean profiles from input sounding. More...
 
void setSpongeRefFromSounding (bool restarting)
 Set sponge mean profiles from input sounding. More...
 
void ComputeDt (int step=-1)
 
std::string PlotFileName (int lev) const
 
void setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void init_Dirichlet_bc_data (const std::string input_file)
 
void InitializeFromFile ()
 
void InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)
 
void post_update (amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
 
void fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
 
void init_geo_wind_profile (const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
 
void refinement_criteria_setup ()
 
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg (int lev)
 
AMREX_FORCE_INLINE std::ostream & DataLog (int i)
 
AMREX_FORCE_INLINE std::ostream & DerDataLog (int i)
 
AMREX_FORCE_INLINE int NumDataLogs () noexcept
 
AMREX_FORCE_INLINE int NumDerDataLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SamplePointLog (int i)
 
AMREX_FORCE_INLINE int NumSamplePointLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SampleLineLog (int i)
 
AMREX_FORCE_INLINE int NumSampleLineLogs () noexcept
 
amrex::IntVect & SamplePoint (int i)
 
AMREX_FORCE_INLINE int NumSamplePoints () noexcept
 
amrex::IntVect & SampleLine (int i)
 
AMREX_FORCE_INLINE int NumSampleLines () noexcept
 
void setRecordDataInfo (int i, const std::string &filename)
 
void setRecordDerDataInfo (int i, const std::string &filename)
 
void setRecordEnergyDataInfo (int i, const std::string &filename)
 
void setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
void setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
std::string DataLogName (int i) const noexcept
 The filename of the ith datalog file. More...
 
std::string DerDataLogName (int i) const noexcept
 
std::string SamplePointLogName (int i) const noexcept
 The filename of the ith sampleptlog file. More...
 
std::string SampleLineLogName (int i) const noexcept
 The filename of the ith samplelinelog file. More...
 
eb_ const & get_eb (int lev) const noexcept
 
amrex::EBFArrayBoxFactory const & EBFactory (int lev) const noexcept
 

Static Private Member Functions

static amrex::Vector< std::string > PlotFileVarNames (amrex::Vector< std::string > plot_var_names)
 
static void GotoNextLine (std::istream &is)
 
static AMREX_FORCE_INLINE int ComputeGhostCells (const SolverChoice &sc)
 
static amrex::Real getCPUTime ()
 
static int nghost_eb_basic ()
 
static int nghost_eb_volume ()
 
static int nghost_eb_full ()
 

Private Attributes

amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
 
InputSoundingData input_sounding_data
 
InputSpongeData input_sponge_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
 
std::unique_ptr< ProblemBaseprob = nullptr
 
amrex::Vector< int > num_boxes_at_level
 
amrex::Vector< int > num_files_at_level
 
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
 
amrex::Vector< int > istep
 
amrex::Vector< int > nsubsteps
 
amrex::Vector< amrex::Real > t_new
 
amrex::Vector< amrex::Real > t_old
 
amrex::Vector< amrex::Real > dt
 
amrex::Vector< long > dt_mri_ratio
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
 
amrex::Vector< amrex::Real > t_avg_cnt
 
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
 
amrex::Vector< amrex::MultiFab > pp_inc
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
 
amrex::Vector< amrex::MultiFab > rU_old
 
amrex::Vector< amrex::MultiFab > rU_new
 
amrex::Vector< amrex::MultiFab > rV_old
 
amrex::Vector< amrex::MultiFab > rV_new
 
amrex::Vector< amrex::MultiFab > rW_old
 
amrex::Vector< amrex::MultiFab > rW_new
 
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
 
std::unique_ptr< Microphysicsmicro
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
 
LandSurface lsm
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
 
int cf_width {0}
 
int cf_set_width {0}
 
amrex::Vector< ERFFillPatcherFPr_c
 
amrex::Vector< ERFFillPatcherFPr_u
 
amrex::Vector< ERFFillPatcherFPr_v
 
amrex::Vector< ERFFillPatcherFPr_w
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
 
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mapfac_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mapfac_u
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mapfac_v
 
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
 
amrex::Vector< amrex::MultiFab > base_state
 
amrex::Vector< amrex::MultiFab > base_state_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
 
bool finished_wave = false
 
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
 
amrex::Vector< amrex::BCRec > domain_bcs_type
 
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
 
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_maxm_bc_extdir_vals
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_maxm_bc_neumann_vals
 
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
 
int last_plot_file_step_1
 
int last_plot_file_step_2
 
int last_subvol
 
int last_check_file_step
 
int plot_file_on_restart = 1
 
const int datwidth = 14
 
const int datprecision = 6
 
const int timeprecision = 13
 
int max_step = std::numeric_limits<int>::max()
 
amrex::Real start_time = 0.0
 
amrex::Real stop_time = std::numeric_limits<amrex::Real>::max()
 
bool use_datetime = false
 
const std::string datetime_format = "%Y-%m-%d %H:%M:%S"
 
std::string restart_chkfile = ""
 
amrex::Vector< amrex::Real > fixed_dt
 
amrex::Vector< amrex::Real > fixed_fast_dt
 
int regrid_int = -1
 
bool regrid_level_0_on_restart = false
 
std::string plot_file_1 {"plt_1_"}
 
std::string plot_file_2 {"plt_2_"}
 
std::string subvol_file {"subvol"}
 
bool m_expand_plotvars_to_unif_rr = false
 
int m_plot_int_1 = -1
 
int m_plot_int_2 = -1
 
int m_subvol_int = -1
 
amrex::Real m_plot_per_1 = -1.0
 
amrex::Real m_plot_per_2 = -1.0
 
amrex::Real m_subvol_per = -1.0
 
bool m_plot_face_vels = false
 
bool plot_lsm = false
 
int profile_int = -1
 
bool destag_profiles = true
 
std::string check_file {"chk"}
 
int m_check_int = -1
 
amrex::Real m_check_per = -1.0
 
amrex::Vector< std::string > plot_var_names_1
 
amrex::Vector< std::string > plot_var_names_2
 
const amrex::Vector< std::string > cons_names
 
const amrex::Vector< std::string > derived_names
 
TurbulentPerturbation turbPert
 
int real_width {0}
 
int real_set_width {0}
 
bool metgrid_debug_quiescent {false}
 
bool metgrid_debug_isothermal {false}
 
bool metgrid_debug_dry {false}
 
bool metgrid_debug_psfc {false}
 
bool metgrid_debug_msf {false}
 
bool metgrid_interp_theta {false}
 
bool metgrid_basic_linear {false}
 
bool metgrid_use_below_sfc {true}
 
bool metgrid_use_sfc {true}
 
bool metgrid_retain_sfc {false}
 
amrex::Real metgrid_proximity {500.0}
 
int metgrid_order {2}
 
int metgrid_force_sfc_k {6}
 
amrex::Vector< amrex::BoxArray > ba1d
 
amrex::Vector< amrex::BoxArray > ba2d
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_C1H
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_C2H
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_MUB
 
amrex::Vector< amrex::Vector< amrex::Real > > h_rhotheta_src
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhotheta_src
 
amrex::Vector< amrex::Vector< amrex::Real > > h_rhoqt_src
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhoqt_src
 
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
 
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
 
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
 
amrex::Vector< amrex::Real > h_havg_density
 
amrex::Vector< amrex::Real > h_havg_temperature
 
amrex::Vector< amrex::Real > h_havg_pressure
 
amrex::Vector< amrex::Real > h_havg_qv
 
amrex::Vector< amrex::Real > h_havg_qc
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_density
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_temperature
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_pressure
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qv
 
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qc
 
std::unique_ptr< WriteBndryPlanesm_w2d = nullptr
 
std::unique_ptr< ReadBndryPlanesm_r2d = nullptr
 
std::unique_ptr< SurfaceLayerm_SurfaceLayer = nullptr
 
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
 
amrex::MultiFab fine_mask
 
amrex::Vector< amrex::Real > dz_min
 
int sampler_interval = -1
 
amrex::Real sampler_per = -1.0
 
std::unique_ptr< SampleDatadata_sampler = nullptr
 
amrex::Vector< std::unique_ptr< std::fstream > > datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
 
amrex::Vector< std::string > datalogname
 
amrex::Vector< std::string > der_datalogname
 
amrex::Vector< std::string > tot_e_datalogname
 
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
 
amrex::Vector< std::string > sampleptlogname
 
amrex::Vector< amrex::IntVect > samplepoint
 
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
 
amrex::Vector< std::string > samplelinelogname
 
amrex::Vector< amrex::IntVect > sampleline
 
amrex::Vector< std::unique_ptr< eb_ > > eb
 

Static Private Attributes

static amrex::Real cfl = 0.8
 
static amrex::Real sub_cfl = 1.0
 
static amrex::Real init_shrink = 1.0
 
static amrex::Real change_max = 1.1
 
static amrex::Real dt_max_initial = 2.0e100
 
static amrex::Real dt_max = 1e9
 
static int fixed_mri_dt_ratio = 0
 
static SolverChoice solverChoice
 
static int verbose = 0
 
static int mg_verbose = 0
 
static bool use_fft = false
 
static int sum_interval = -1
 
static int pert_interval = -1
 
static amrex::Real sum_per = -1.0
 
static PlotFileType plotfile_type_1 = PlotFileType::None
 
static PlotFileType plotfile_type_2 = PlotFileType::None
 
static StateInterpType interpolation_type
 
static std::string sponge_type
 
static amrex::Vector< amrex::Vector< std::string > > nc_init_file = {{""}}
 
static std::string nc_bdy_file
 
static std::string nc_low_file
 
static bool init_sounding_ideal = false
 
static int output_1d_column = 0
 
static int column_interval = -1
 
static amrex::Real column_per = -1.0
 
static amrex::Real column_loc_x = 0.0
 
static amrex::Real column_loc_y = 0.0
 
static std::string column_file_name = "column_data.nc"
 
static int output_bndry_planes = 0
 
static int bndry_output_planes_interval = -1
 
static amrex::Real bndry_output_planes_per = -1.0
 
static amrex::Real bndry_output_planes_start_time = 0.0
 
static int input_bndry_planes = 0
 
static int ng_dens_hse
 
static int ng_pres_hse
 
static amrex::Vector< amrex::AMRErrorTag > ref_tags
 
static amrex::Real startCPUTime = 0.0
 
static amrex::Real previousCPUTimeUsed = 0.0
 

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

◆ ERF() [1/3]

ERF::ERF ( )
93 {
94  int fix_random_seed = 0;
95  ParmParse pp("erf"); pp.query("fix_random_seed", fix_random_seed);
96  // Note that the value of 1024UL is not significant -- the point here is just to set the
97  // same seed for all MPI processes for the purpose of regression testing
98  if (fix_random_seed) {
99  Print() << "Fixing the random seed" << std::endl;
100  InitRandom(1024UL);
101  }
102 
103  ERF_shared();
104 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real pp(amrex::Real y)
Definition: ERF_MicrophysicsUtils.H:219
void ERF_shared()
Definition: ERF.cpp:107
Here is the call graph for this function:

◆ ~ERF()

ERF::~ERF ( )
overridedefault

◆ ERF() [2/3]

ERF::ERF ( ERF &&  )
deletenoexcept

◆ ERF() [3/3]

ERF::ERF ( const ERF other)
delete

Member Function Documentation

◆ Advance()

void ERF::Advance ( int  lev,
amrex::Real  time,
amrex::Real  dt_lev,
int  iteration,
int  ncycle 
)
private

Function that advances the solution at one level for a single time step – this does some preliminaries then calls erf_advance

Parameters
[in]levlevel of refinement (coarsest level is 0)
[in]timestart time for time advance
[in]dt_levtime step for this time advance
21 {
22  BL_PROFILE("ERF::Advance()");
23 
24  // We must swap the pointers so the previous step's "new" is now this step's "old"
25  std::swap(vars_old[lev], vars_new[lev]);
26 
27  MultiFab& S_old = vars_old[lev][Vars::cons];
28  MultiFab& S_new = vars_new[lev][Vars::cons];
29 
30  MultiFab& U_old = vars_old[lev][Vars::xvel];
31  MultiFab& V_old = vars_old[lev][Vars::yvel];
32  MultiFab& W_old = vars_old[lev][Vars::zvel];
33 
34  MultiFab& U_new = vars_new[lev][Vars::xvel];
35  MultiFab& V_new = vars_new[lev][Vars::yvel];
36  MultiFab& W_new = vars_new[lev][Vars::zvel];
37 
38  // We need to set these because otherwise in the first call to erf_advance we may
39  // read uninitialized data on ghost values in setting the bc's on the velocities
40  U_new.setVal(1.e34,U_new.nGrowVect());
41  V_new.setVal(1.e34,V_new.nGrowVect());
42  W_new.setVal(1.e34,W_new.nGrowVect());
43 
44  //
45  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
46  // If lev == 0 we have already FillPatched this in ERF::TimeStep
47  //
48 // if (lev == 0) {
49 // FillPatch(lev, time, {&S_old, &U_old, &V_old, &W_old});
50 // } else {
51  if (lev > 0) {
52  FillPatch(lev, time, {&S_old, &U_old, &V_old, &W_old},
53  {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]},
54  base_state[lev], base_state[lev]);
55  }
56 
57  //
58  // So we must convert the fillpatched to momenta, including the ghost values
59  //
60  VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
61  V_old, rV_old[lev].nGrowVect(),
62  W_old, rW_old[lev].nGrowVect(),
63  S_old, rU_old[lev], rV_old[lev], rW_old[lev],
64  Geom(lev).Domain(),
66 
67  // Update the inflow perturbation update time and amplitude
68  if (solverChoice.pert_type == PerturbationType::Source ||
69  solverChoice.pert_type == PerturbationType::Direct)
70  {
71  turbPert.calc_tpi_update(lev, dt_lev, U_old, V_old, S_old);
72  }
73 
74  // If PerturbationType::Direct is selected, directly add the computed perturbation
75  // on the conserved field
76  if (solverChoice.pert_type == PerturbationType::Direct)
77  {
78  auto m_ixtype = S_old.boxArray().ixType(); // Conserved term
79  for (MFIter mfi(S_old,TileNoZ()); mfi.isValid(); ++mfi) {
80  Box bx = mfi.tilebox();
81  const Array4<Real> &cell_data = S_old.array(mfi);
82  const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
83  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cell_data, pert_cell);
84  }
85  }
86 
87  // configure SurfaceLayer params if needed
88  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
89  if (m_SurfaceLayer) {
90  IntVect ng = Theta_prim[lev]->nGrowVect();
91  MultiFab::Copy( *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
92  MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp , 0, 1, ng);
93  if (solverChoice.moisture_type != MoistureType::None) {
94  ng = Qv_prim[lev]->nGrowVect();
95 
96  MultiFab::Copy( *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
97  MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp , 0, 1, ng);
98 
99  if (solverChoice.RhoQr_comp > -1) {
100  MultiFab::Copy( *Qr_prim[lev], S_old, solverChoice.RhoQr_comp, 0, 1, ng);
101  MultiFab::Divide(*Qr_prim[lev], S_old, Rho_comp , 0, 1, ng);
102  } else {
103  Qr_prim[lev]->setVal(0.0);
104  }
105  }
106  // NOTE: std::swap above causes the field ptrs to be out of date.
107  // Reassign the field ptrs for MAC avg computation.
108  m_SurfaceLayer->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim, Qr_prim);
109  m_SurfaceLayer->update_pblh(lev, vars_old, z_phys_cc[lev].get(),
113  m_SurfaceLayer->update_fluxes(lev, time);
114  }
115  }
116 
117 #if defined(ERF_USE_WINDFARM)
118  if (solverChoice.windfarm_type != WindFarmType::None) {
119  advance_windfarm(Geom(lev), dt_lev, S_old,
120  U_old, V_old, W_old, vars_windfarm[lev],
121  Nturb[lev], SMark[lev], time);
122  }
123 
124 #endif
125 
126  const BoxArray& ba = S_old.boxArray();
127  const DistributionMapping& dm = S_old.DistributionMap();
128 
129  int nvars = S_old.nComp();
130 
131  // Source array for conserved cell-centered quantities -- this will be filled
132  // in the call to make_sources in ERF_TI_slow_rhs_fun.H
133  MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0.0);
134 
135  // Source arrays for momenta -- these will be filled
136  // in the call to make_mom_sources in ERF_TI_slow_rhs_fun.H
137  BoxArray ba_x(ba); ba_x.surroundingNodes(0);
138  MultiFab xmom_source(ba_x,dm,nvars,1); xmom_source.setVal(0.0);
139 
140  BoxArray ba_y(ba); ba_y.surroundingNodes(1);
141  MultiFab ymom_source(ba_y,dm,nvars,1); ymom_source.setVal(0.0);
142 
143  BoxArray ba_z(ba); ba_z.surroundingNodes(2);
144  MultiFab zmom_source(ba_z,dm,nvars,1); zmom_source.setVal(0.0);
145 
146  // We don't need to call FillPatch on cons_mf because we have fillpatch'ed S_old above
147  MultiFab cons_mf(ba,dm,nvars,S_old.nGrowVect());
148  MultiFab::Copy(cons_mf,S_old,0,0,S_old.nComp(),S_old.nGrowVect());
149 
150  amrex::Vector<MultiFab> state_old;
151  amrex::Vector<MultiFab> state_new;
152 
153  // **************************************************************************************
154  // Here we define state_old and state_new which are to be advanced
155  // **************************************************************************************
156  // Initial solution
157  // Note that "old" and "new" here are relative to each RK stage.
158  state_old.push_back(MultiFab(cons_mf , amrex::make_alias, 0, nvars)); // cons
159  state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0, 1)); // xmom
160  state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0, 1)); // ymom
161  state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0, 1)); // zmom
162 
163  // Final solution
164  // state_new at the end of the last RK stage holds the t^{n+1} data
165  state_new.push_back(MultiFab(S_new , amrex::make_alias, 0, nvars)); // cons
166  state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0, 1)); // xmom
167  state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0, 1)); // ymom
168  state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0, 1)); // zmom
169 
170  // **************************************************************************************
171  // Update the dycore
172  // **************************************************************************************
173  advance_dycore(lev, state_old, state_new,
174  U_old, V_old, W_old,
175  U_new, V_new, W_new,
176  cc_source, xmom_source, ymom_source, zmom_source,
177  Geom(lev), dt_lev, time);
178 
179  // **************************************************************************************
180  // Update the microphysics (moisture)
181  // **************************************************************************************
182  advance_microphysics(lev, S_new, dt_lev, iteration, time);
183 
184  // **************************************************************************************
185  // Update the land surface model
186  // **************************************************************************************
187  advance_lsm(lev, S_new, U_new, V_new, dt_lev);
188 
189 #if defined(ERF_USE_RRTMGP)
190  // **************************************************************************************
191  // Update the radiation
192  // **************************************************************************************
193  advance_radiation(lev, S_new, dt_lev);
194 #endif
195 
196 #ifdef ERF_USE_PARTICLES
197  // **************************************************************************************
198  // Update the particle positions
199  // **************************************************************************************
200  evolveTracers( lev, dt_lev, vars_new, z_phys_nd );
201 #endif
202 
203  // ***********************************************************************************************
204  // Impose domain boundary conditions here so that in FillPatching the fine data we won't
205  // need to re-fill these
206  // ***********************************************************************************************
207  if (lev < finest_level) {
208  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
210  0,vars_new[lev][Vars::cons].nComp(),
211  vars_new[lev][Vars::cons].nGrowVect(),time,BCVars::cons_bc,true);
212  (*physbcs_u[lev])(vars_new[lev][Vars::xvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
213  ngvect_vels,time,BCVars::xvel_bc,true);
214  (*physbcs_v[lev])(vars_new[lev][Vars::yvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
215  ngvect_vels,time,BCVars::yvel_bc,true);
216  (*physbcs_w[lev])(vars_new[lev][Vars::zvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
217  ngvect_vels,time,BCVars::zvel_bc,true);
218  }
219 
220  // **************************************************************************************
221  // Register old and new coarse data if we are at a level less than the finest level
222  // **************************************************************************************
223  if (lev < finest_level)
224  {
225  if (cf_width > 0) {
226  // We must fill the ghost cells of these so that the parallel copy works correctly
227  state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
228  state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
229  FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
230  {time, time + dt_lev});
231  }
232 
233  if (cf_width >= 0) {
234  // We must fill the ghost cells of these so that the parallel copy works correctly
235  state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
236  state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
237  FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
238  {time, time + dt_lev});
239 
240  state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
241  state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
242  FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
243  {time, time + dt_lev});
244 
245  state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
246  state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
247  FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
248  {time, time + dt_lev});
249  }
250 
251  //
252  // Now create a MultiFab that holds (S_new - S_old) / dt from the coarse level interpolated
253  // on to the coarse/fine boundary at the fine resolution
254  //
255  Interpolater* mapper_f = &face_cons_linear_interp;
256 
257  // PhysBCFunctNoOp null_bc;
258  // MultiFab tempx(vars_new[lev+1][Vars::xvel].boxArray(),vars_new[lev+1][Vars::xvel].DistributionMap(),1,0);
259  // tempx.setVal(0.0);
260  // xmom_crse_rhs[lev+1].setVal(0.0);
261  // FPr_u[lev].FillSet(tempx , time , null_bc, domain_bcs_type);
262  // FPr_u[lev].FillSet(xmom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
263  // MultiFab::Subtract(xmom_crse_rhs[lev+1],tempx,0,0,1,IntVect{0});
264  // xmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
265 
266  // MultiFab tempy(vars_new[lev+1][Vars::yvel].boxArray(),vars_new[lev+1][Vars::yvel].DistributionMap(),1,0);
267  // tempy.setVal(0.0);
268  // ymom_crse_rhs[lev+1].setVal(0.0);
269  // FPr_v[lev].FillSet(tempy , time , null_bc, domain_bcs_type);
270  // FPr_v[lev].FillSet(ymom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
271  // MultiFab::Subtract(ymom_crse_rhs[lev+1],tempy,0,0,1,IntVect{0});
272  // ymom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
273 
274  MultiFab temp_state(zmom_crse_rhs[lev+1].boxArray(),zmom_crse_rhs[lev+1].DistributionMap(),1,0);
275  InterpFromCoarseLevel(temp_state, IntVect{0}, IntVect{0}, state_old[IntVars::zmom], 0, 0, 1,
276  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
277  InterpFromCoarseLevel(zmom_crse_rhs[lev+1], IntVect{0}, IntVect{0}, state_new[IntVars::zmom], 0, 0, 1,
278  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
279  MultiFab::Subtract(zmom_crse_rhs[lev+1],temp_state,0,0,1,IntVect{0});
280  zmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
281  }
282 
283  // ***********************************************************************************************
284  // Update the time averaged velocities if they are requested
285  // ***********************************************************************************************
287  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
288  }
289 }
@ nvars
Definition: ERF_DataStruct.H:74
#define Rho_comp
Definition: ERF_IndexDefines.H:36
#define RhoTheta_comp
Definition: ERF_IndexDefines.H:37
#define RhoQ1_comp
Definition: ERF_IndexDefines.H:42
@ surface_layer
AMREX_FORCE_INLINE amrex::IntVect TileNoZ()
Definition: ERF_TileNoZ.H:11
void Time_Avg_Vel_atCC(const Real &dt, Real &t_avg_cnt, MultiFab *vel_t_avg, MultiFab &xvel, MultiFab &yvel, MultiFab &zvel)
Definition: ERF_TimeAvgVel.cpp:9
void VelocityToMomentum(const amrex::MultiFab &xvel_in, const amrex::IntVect &xvel_ngrow, const amrex::MultiFab &yvel_in, const amrex::IntVect &yvel_ngrow, const amrex::MultiFab &zvel_in, const amrex::IntVect &zvel_ngrow, const amrex::MultiFab &cons_in, amrex::MultiFab &xmom_out, amrex::MultiFab &ymom_out, amrex::MultiFab &zmom_out, const amrex::Box &domain, const amrex::Vector< amrex::BCRec > &domain_bcs_type_h)
amrex::Vector< amrex::MultiFab > rU_new
Definition: ERF.H:762
amrex::Vector< ERFFillPatcher > FPr_u
Definition: ERF.H:807
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
Definition: ERF.H:737
amrex::Vector< ERFFillPatcher > FPr_v
Definition: ERF.H:808
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
Definition: ERF.H:749
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
Definition: ERF.H:832
static SolverChoice solverChoice
Definition: ERF.H:1026
amrex::Vector< ERFFillPatcher > FPr_c
Definition: ERF.H:806
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
Definition: ERF.H:741
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
Definition: ERF.H:752
amrex::Vector< amrex::MultiFab > base_state
Definition: ERF.H:867
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
Definition: ERF.H:757
amrex::Vector< amrex::MultiFab > rV_new
Definition: ERF.H:764
amrex::Vector< amrex::BCRec > domain_bcs_type
Definition: ERF.H:883
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
Definition: ERF.H:758
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
Definition: ERF.H:750
amrex::Vector< amrex::Real > t_avg_cnt
Definition: ERF.H:742
amrex::Vector< amrex::MultiFab > rU_old
Definition: ERF.H:761
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
Definition: ERF.H:756
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
Definition: ERF.H:751
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
Definition: ERF.H:831
amrex::Vector< amrex::MultiFab > rW_new
Definition: ERF.H:766
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
Definition: ERF.H:770
void advance_lsm(int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)
Definition: ERF_AdvanceLSM.cpp:5
TurbulentPerturbation turbPert
Definition: ERF.H:1029
amrex::Vector< amrex::MultiFab > rW_old
Definition: ERF.H:765
std::unique_ptr< SurfaceLayer > m_SurfaceLayer
Definition: ERF.H:1188
amrex::Vector< ERFFillPatcher > FPr_w
Definition: ERF.H:809
amrex::Vector< amrex::Real > dt
Definition: ERF.H:731
void advance_microphysics(int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
Definition: ERF_AdvanceMicrophysics.cpp:5
int cf_width
Definition: ERF.H:804
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
Definition: ERF.H:896
amrex::Vector< amrex::MultiFab > rV_old
Definition: ERF.H:763
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
Definition: ERF.H:738
void advance_dycore(int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
Definition: ERF_AdvanceDycore.cpp:38
void FillPatch(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
@ zvel_bc
Definition: ERF_IndexDefines.H:89
@ yvel_bc
Definition: ERF_IndexDefines.H:88
@ cons_bc
Definition: ERF_IndexDefines.H:76
@ xvel_bc
Definition: ERF_IndexDefines.H:87
@ ymom
Definition: ERF_IndexDefines.H:152
@ cons
Definition: ERF_IndexDefines.H:150
@ zmom
Definition: ERF_IndexDefines.H:153
@ xmom
Definition: ERF_IndexDefines.H:151
@ ng
Definition: ERF_Morrison.H:48
@ xvel
Definition: ERF_IndexDefines.H:141
@ cons
Definition: ERF_IndexDefines.H:140
@ zvel
Definition: ERF_IndexDefines.H:143
@ yvel
Definition: ERF_IndexDefines.H:142
int RhoQr_comp
Definition: ERF_DataStruct.H:771
int RhoQc_comp
Definition: ERF_DataStruct.H:765
int RhoQv_comp
Definition: ERF_DataStruct.H:764
MoistureType moisture_type
Definition: ERF_DataStruct.H:746
PerturbationType pert_type
Definition: ERF_DataStruct.H:733
WindFarmType windfarm_type
Definition: ERF_DataStruct.H:747
bool time_avg_vel
Definition: ERF_DataStruct.H:730
amrex::Vector< amrex::MultiFab > pb_cell
Definition: ERF_TurbPertStruct.H:581
void calc_tpi_update(const int lev, const amrex::Real dt, amrex::MultiFab &mf_xvel, amrex::MultiFab &mf_yvel, amrex::MultiFab &mf_cons)
Definition: ERF_TurbPertStruct.H:217
void apply_tpi(const int &lev, const amrex::Box &vbx, const int &comp, const amrex::IndexType &m_ixtype, const amrex::Array4< amrex::Real > &src_arr, const amrex::Array4< amrex::Real const > &pert_cell)
Definition: ERF_TurbPertStruct.H:296
Here is the call graph for this function:

◆ advance_dycore()

void ERF::advance_dycore ( int  level,
amrex::Vector< amrex::MultiFab > &  state_old,
amrex::Vector< amrex::MultiFab > &  state_new,
amrex::MultiFab &  xvel_old,
amrex::MultiFab &  yvel_old,
amrex::MultiFab &  zvel_old,
amrex::MultiFab &  xvel_new,
amrex::MultiFab &  yvel_new,
amrex::MultiFab &  zvel_new,
amrex::MultiFab &  source,
amrex::MultiFab &  xmom_src,
amrex::MultiFab &  ymom_src,
amrex::MultiFab &  zmom_src,
amrex::Geometry  fine_geom,
amrex::Real  dt,
amrex::Real  time 
)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters
[in]levellevel of refinement (coarsest level is 0)
[in]state_oldold-time conserved variables
[in]state_newnew-time conserved variables
[in]xvel_oldold-time x-component of velocity
[in]yvel_oldold-time y-component of velocity
[in]zvel_oldold-time z-component of velocity
[in]xvel_newnew-time x-component of velocity
[in]yvel_newnew-time y-component of velocity
[in]zvel_newnew-time z-component of velocity
[in]cc_srcsource term for conserved variables
[in]xmom_srcsource term for x-momenta
[in]ymom_srcsource term for y-momenta
[in]zmom_srcsource term for z-momenta
[in]fine_geomcontainer for geometry information at current level
[in]dt_advancetime step for this time advance
[in]old_timeold time for this time advance
47 {
48  BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
49 
50  const Box& domain = fine_geom.Domain();
51 
55 
56  MultiFab r_hse (base_state[level], make_alias, BaseState::r0_comp , 1);
57  MultiFab p_hse (base_state[level], make_alias, BaseState::p0_comp , 1);
58  MultiFab pi_hse(base_state[level], make_alias, BaseState::pi0_comp, 1);
59 
60  // These pointers are used in the MRI utility functions
61  MultiFab* r0 = &r_hse;
62  MultiFab* p0 = &p_hse;
63  MultiFab* pi0 = &pi_hse;
64 
65  Real* dptr_rhotheta_src = solverChoice.custom_rhotheta_forcing ? d_rhotheta_src[level].data() : nullptr;
66  Real* dptr_rhoqt_src = solverChoice.custom_moisture_forcing ? d_rhoqt_src[level].data() : nullptr;
67  Real* dptr_wbar_sub = solverChoice.custom_w_subsidence ? d_w_subsid[level].data() : nullptr;
68 
69  // Turbulent Perturbation Pointer
70  //Real* dptr_rhotheta_src = solverChoice.pert_type ? d_rhotheta_src[level].data() : nullptr;
71 
72  Vector<Real*> d_rayleigh_ptrs_at_lev;
73  d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
74  d_rayleigh_ptrs_at_lev[Rayleigh::ubar] = solverChoice.rayleigh_damp_U ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
75  d_rayleigh_ptrs_at_lev[Rayleigh::vbar] = solverChoice.rayleigh_damp_V ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
76  d_rayleigh_ptrs_at_lev[Rayleigh::wbar] = solverChoice.rayleigh_damp_W ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
77  d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.rayleigh_damp_T ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
78 
79  Vector<Real*> d_sponge_ptrs_at_lev;
80  if(sc.sponge_type=="input_sponge")
81  {
82  d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
83  d_sponge_ptrs_at_lev[Sponge::ubar_sponge] = d_sponge_ptrs[level][Sponge::ubar_sponge].data();
84  d_sponge_ptrs_at_lev[Sponge::vbar_sponge] = d_sponge_ptrs[level][Sponge::vbar_sponge].data();
85  }
86 
87  bool l_use_terrain_fitted_coords = (solverChoice.mesh_type != MeshType::ConstantDz);
88  bool l_use_kturb = tc.use_kturb;
89  bool l_use_diff = ( (dc.molec_diff_type != MolecDiffType::None) ||
90  l_use_kturb );
91  bool l_use_moisture = ( solverChoice.moisture_type != MoistureType::None );
92  bool l_implicit_substepping = ( solverChoice.substepping_type[level] == SubsteppingType::Implicit );
93 
94  const bool use_SurfLayer = (m_SurfaceLayer != nullptr);
95  const FArrayBox* z_0 = (use_SurfLayer) ? m_SurfaceLayer->get_z0(level) : nullptr;
96 
97  const BoxArray& ba = state_old[IntVars::cons].boxArray();
98  const BoxArray& ba_z = zvel_old.boxArray();
99  const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
100 
101  int num_prim = state_old[IntVars::cons].nComp() - 1;
102 
103  MultiFab S_prim (ba , dm, num_prim, state_old[IntVars::cons].nGrowVect());
104  MultiFab pi_stage (ba , dm, 1, state_old[IntVars::cons].nGrowVect());
105  MultiFab fast_coeffs(ba_z, dm, 5, 0);
106  MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
107  MultiFab* SmnSmn = SmnSmn_lev[level].get();
108 
109  // **************************************************************************************
110  // Compute strain for use in slow RHS and Smagorinsky model
111  // **************************************************************************************
112  {
113  BL_PROFILE("erf_advance_strain");
114  if (l_use_diff) {
115 
116  const BCRec* bc_ptr_h = domain_bcs_type.data();
117  const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
118 
119 #ifdef _OPENMP
120 #pragma omp parallel if (Gpu::notInLaunchRegion())
121 #endif
122  for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
123  {
124  Box bxcc = mfi.growntilebox(IntVect(1,1,0));
125  Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
126  Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
127  Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
128 
129  if (bxcc.smallEnd(2) != domain.smallEnd(2)) {
130  bxcc.growLo(2,1);
131  tbxxy.growLo(2,1);
132  tbxxz.growLo(2,1);
133  tbxyz.growLo(2,1);
134  }
135 
136  if (bxcc.bigEnd(2) != domain.bigEnd(2)) {
137  bxcc.growHi(2,1);
138  tbxxy.growHi(2,1);
139  tbxxz.growHi(2,1);
140  tbxyz.growHi(2,1);
141  }
142 
143  const Array4<const Real> & u = xvel_old.array(mfi);
144  const Array4<const Real> & v = yvel_old.array(mfi);
145  const Array4<const Real> & w = zvel_old.array(mfi);
146 
147  Array4<Real> tau11 = Tau[level][TauType::tau11].get()->array(mfi);
148  Array4<Real> tau22 = Tau[level][TauType::tau22].get()->array(mfi);
149  Array4<Real> tau33 = Tau[level][TauType::tau33].get()->array(mfi);
150  Array4<Real> tau12 = Tau[level][TauType::tau12].get()->array(mfi);
151  Array4<Real> tau13 = Tau[level][TauType::tau13].get()->array(mfi);
152  Array4<Real> tau23 = Tau[level][TauType::tau23].get()->array(mfi);
153 
154  Array4<Real> tau21 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau21].get()->array(mfi) : Array4<Real>{};
155  Array4<Real> tau31 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau31].get()->array(mfi) : Array4<Real>{};
156  Array4<Real> tau32 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau32].get()->array(mfi) : Array4<Real>{};
157  const Array4<const Real>& z_nd = z_phys_nd[level]->const_array(mfi);
158 
159  const Array4<const Real> mf_m = mapfac_m[level]->array(mfi);
160  const Array4<const Real> mf_u = mapfac_u[level]->array(mfi);
161  const Array4<const Real> mf_v = mapfac_v[level]->array(mfi);
162 
163  if (l_use_terrain_fitted_coords) {
164  ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
165  u, v, w,
166  tau11, tau22, tau33,
167  tau12, tau21,
168  tau13, tau31,
169  tau23, tau32,
170  z_nd, detJ_cc[level]->const_array(mfi), bc_ptr_h, dxInv,
171  mf_m, mf_u, mf_v);
172  } else {
173  ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
174  u, v, w,
175  tau11, tau22, tau33,
176  tau12, tau13, tau23,
177  bc_ptr_h, dxInv,
178  mf_m, mf_u, mf_v);
179  }
180  } // mfi
181  } // l_use_diff
182  } // profile
183 
184 #include "ERF_TI_utils.H"
185 
186  // Additional SFS quantities, calculated once per timestep
187  MultiFab* Hfx1 = SFS_hfx1_lev[level].get();
188  MultiFab* Hfx2 = SFS_hfx2_lev[level].get();
189  MultiFab* Hfx3 = SFS_hfx3_lev[level].get();
190  MultiFab* Q1fx1 = SFS_q1fx1_lev[level].get();
191  MultiFab* Q1fx2 = SFS_q1fx2_lev[level].get();
192  MultiFab* Q1fx3 = SFS_q1fx3_lev[level].get();
193  MultiFab* Q2fx3 = SFS_q2fx3_lev[level].get();
194  MultiFab* Diss = SFS_diss_lev[level].get();
195 
196  // *************************************************************************
197  // Calculate cell-centered eddy viscosity & diffusivities
198  //
199  // Notes -- we fill all the data in ghost cells before calling this so
200  // that we can fill the eddy viscosity in the ghost regions and
201  // not have to call a boundary filler on this data itself
202  //
203  // LES - updates both horizontal and vertical eddy viscosity components
204  // PBL - only updates vertical eddy viscosity components so horizontal
205  // components come from the LES model or are left as zero.
206  // *************************************************************************
207  if (l_use_kturb)
208  {
209  // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
210  const BCRec* bc_ptr_h = domain_bcs_type.data();
211  ComputeTurbulentViscosity(xvel_old, yvel_old,Tau[level],
212  state_old[IntVars::cons],
213  *walldist[level].get(),
214  *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
215  fine_geom, *mapfac_u[level], *mapfac_v[level],
216  z_phys_nd[level], solverChoice,
217  m_SurfaceLayer, z_0, l_use_terrain_fitted_coords,
218  l_use_moisture, level, bc_ptr_h);
219  }
220 
221  // ***********************************************************************************************
222  // Update user-defined source terms -- these are defined once per time step (not per RK stage)
223  // ***********************************************************************************************
225  prob->update_rhotheta_sources(old_time,
226  h_rhotheta_src[level], d_rhotheta_src[level],
227  fine_geom, z_phys_cc[level]);
228  }
229 
231  prob->update_rhoqt_sources(old_time,
232  h_rhoqt_src[level], d_rhoqt_src[level],
233  fine_geom, z_phys_cc[level]);
234  }
235 
237  prob->update_geostrophic_profile(old_time,
238  h_u_geos[level], d_u_geos[level],
239  h_v_geos[level], d_v_geos[level],
240  fine_geom, z_phys_cc[level]);
241  }
242 
244  prob->update_w_subsidence(old_time,
245  h_w_subsid[level], d_w_subsid[level],
246  fine_geom, z_phys_nd[level]);
247  }
248 
249  // ***********************************************************************************************
250  // Convert old velocity available on faces to old momentum on faces to be used in time integration
251  // ***********************************************************************************************
252  MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
253 
254  //
255  // This is an optimization since we won't need more than one ghost
256  // cell of momentum in the integrator if not using numerical diffusion
257  //
258  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
259  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
260  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
261 
262  VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
263  state_old[IntVars::xmom],
264  state_old[IntVars::ymom],
265  state_old[IntVars::zmom],
266  domain, domain_bcs_type);
267 
268  MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
269  MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
270  MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
271 
272  bool fast_only = false;
273  bool vel_and_mom_synced = true;
274 
275  apply_bcs(state_old, old_time,
276  state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
277  fast_only, vel_and_mom_synced);
278  cons_to_prim(state_old[IntVars::cons], state_old[IntVars::cons].nGrow());
279 
280 #include "ERF_TI_no_substep_fun.H"
281 #include "ERF_TI_substep_fun.H"
282 #include "ERF_TI_slow_rhs_fun.H"
283 
284  // ***************************************************************************************
285  // Setup the integrator and integrate for a single timestep
286  // **************************************************************************************
287  MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
288 
289  // Define rhs and 'post update' utility function that is called after calculating
290  // any state data (e.g. at RK stages or at the end of a timestep)
291  mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
292  mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
293 
294  if (solverChoice.anelastic[level]) {
295  mri_integrator.set_slow_rhs_inc(slow_rhs_fun_inc);
296  }
297 
298  mri_integrator.set_fast_rhs(fast_rhs_fun);
300  mri_integrator.set_no_substep(no_substep_fun);
301 
302  mri_integrator.advance(state_old, state_new, old_time, dt_advance);
303 
304  if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
305 }
void ComputeStrain_N(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau13, Array4< Real > &tau23, const BCRec *bc_ptr, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_m, const Array4< const Real > &mf_u, const Array4< const Real > &mf_v)
Definition: ERF_ComputeStrain_N.cpp:28
void ComputeStrain_T(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau21, Array4< Real > &tau13, Array4< Real > &tau31, Array4< Real > &tau23, Array4< Real > &tau32, const Array4< const Real > &z_nd, const Array4< const Real > &detJ, const BCRec *bc_ptr, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_m, const Array4< const Real > &mf_u, const Array4< const Real > &mf_v)
Definition: ERF_ComputeStrain_T.cpp:33
void ComputeTurbulentViscosity(const MultiFab &xvel, const MultiFab &yvel, Vector< std::unique_ptr< MultiFab >> &Tau_lev, const MultiFab &cons_in, const MultiFab &wdist, MultiFab &eddyViscosity, MultiFab &Hfx1, MultiFab &Hfx2, MultiFab &Hfx3, MultiFab &Diss, const Geometry &geom, const MultiFab &mapfac_u, const MultiFab &mapfac_v, const std::unique_ptr< MultiFab > &z_phys_nd, const SolverChoice &solverChoice, std::unique_ptr< SurfaceLayer > &SurfLayer, const amrex::FArrayBox *z_0, const bool &use_terrain_fitted_coords, const bool &use_moisture, int level, const BCRec *bc_ptr, bool vert_only)
Definition: ERF_ComputeTurbulentViscosity.cpp:520
@ tau12
Definition: ERF_DataStruct.H:21
@ tau23
Definition: ERF_DataStruct.H:21
@ tau33
Definition: ERF_DataStruct.H:21
@ tau22
Definition: ERF_DataStruct.H:21
@ tau11
Definition: ERF_DataStruct.H:21
@ tau32
Definition: ERF_DataStruct.H:21
@ tau31
Definition: ERF_DataStruct.H:21
@ tau21
Definition: ERF_DataStruct.H:21
@ tau13
Definition: ERF_DataStruct.H:21
@ ubar
Definition: ERF_DataStruct.H:74
@ wbar
Definition: ERF_DataStruct.H:74
@ vbar
Definition: ERF_DataStruct.H:74
@ thetabar
Definition: ERF_DataStruct.H:74
@ nvars_sponge
Definition: ERF_DataStruct.H:79
@ vbar_sponge
Definition: ERF_DataStruct.H:79
@ ubar_sponge
Definition: ERF_DataStruct.H:79
auto no_substep_fun
Definition: ERF_TI_no_substep_fun.H:4
auto slow_rhs_fun_inc
Definition: ERF_TI_slow_rhs_fun.H:413
auto slow_rhs_fun_pre
Definition: ERF_TI_slow_rhs_fun.H:6
auto slow_rhs_fun_post
Definition: ERF_TI_slow_rhs_fun.H:322
auto fast_rhs_fun
Definition: ERF_TI_substep_fun.H:4
auto apply_bcs
Definition: ERF_TI_utils.H:50
auto cons_to_prim
Definition: ERF_TI_utils.H:4
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
Definition: ERF.H:857
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
Definition: ERF.H:744
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhotheta_src
Definition: ERF.H:1137
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
Definition: ERF.H:824
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
Definition: ERF.H:822
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
Definition: ERF.H:1142
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mapfac_u
Definition: ERF.H:861
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mapfac_m
Definition: ERF.H:860
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
Definition: ERF.H:822
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
Definition: ERF.H:834
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
Definition: ERF.H:813
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
Definition: ERF.H:1170
amrex::Vector< amrex::Vector< amrex::Real > > h_rhoqt_src
Definition: ERF.H:1139
amrex::Vector< long > dt_mri_ratio
Definition: ERF.H:732
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
Definition: ERF.H:812
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
Definition: ERF.H:825
static int verbose
Definition: ERF.H:1061
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
Definition: ERF.H:824
std::unique_ptr< ProblemBase > prob
Definition: ERF.H:719
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
Definition: ERF.H:823
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
Definition: ERF.H:1149
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
Definition: ERF.H:1148
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhoqt_src
Definition: ERF.H:1140
amrex::Vector< amrex::Vector< amrex::Real > > h_rhotheta_src
Definition: ERF.H:1136
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
Definition: ERF.H:1145
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
Definition: ERF.H:814
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
Definition: ERF.H:1146
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
Definition: ERF.H:1143
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
Definition: ERF.H:822
static int fixed_mri_dt_ratio
Definition: ERF.H:947
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
Definition: ERF.H:1167
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
Definition: ERF.H:824
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mapfac_v
Definition: ERF.H:862
Definition: ERF_MRI.H:16
void set_slow_rhs_pre(std::function< void(T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const int)> F)
Definition: ERF_MRI.H:134
void set_no_substep(std::function< void(T &, T &, T &, amrex::Real, amrex::Real, int)> F)
Definition: ERF_MRI.H:164
void set_slow_rhs_inc(std::function< void(T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const int)> F)
Definition: ERF_MRI.H:138
void set_fast_rhs(std::function< void(int, int, int, T &, const T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const amrex::Real)> F)
Definition: ERF_MRI.H:147
void set_slow_fast_timestep_ratio(const int timestep_ratio=1)
Definition: ERF_MRI.H:154
amrex::Real advance(T &S_old, T &S_new, amrex::Real time, const amrex::Real time_step)
Definition: ERF_MRI.H:174
void set_slow_rhs_post(std::function< void(T &, T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const int)> F)
Definition: ERF_MRI.H:142
@ pi0_comp
Definition: ERF_IndexDefines.H:65
@ p0_comp
Definition: ERF_IndexDefines.H:64
@ r0_comp
Definition: ERF_IndexDefines.H:63
real(c_double), parameter p0
Definition: ERF_module_model_constants.F90:40
Definition: ERF_DiffStruct.H:19
MolecDiffType molec_diff_type
Definition: ERF_DiffStruct.H:81
bool rayleigh_damp_T
Definition: ERF_DataStruct.H:690
static MeshType mesh_type
Definition: ERF_DataStruct.H:655
bool rayleigh_damp_V
Definition: ERF_DataStruct.H:688
DiffChoice diffChoice
Definition: ERF_DataStruct.H:664
bool custom_rhotheta_forcing
Definition: ERF_DataStruct.H:716
bool custom_w_subsidence
Definition: ERF_DataStruct.H:718
bool rayleigh_damp_U
Definition: ERF_DataStruct.H:687
bool custom_geostrophic_profile
Definition: ERF_DataStruct.H:719
amrex::Vector< SubsteppingType > substepping_type
Definition: ERF_DataStruct.H:670
bool use_num_diff
Definition: ERF_DataStruct.H:736
bool custom_moisture_forcing
Definition: ERF_DataStruct.H:717
amrex::Vector< TurbChoice > turbChoice
Definition: ERF_DataStruct.H:666
amrex::Vector< int > anelastic
Definition: ERF_DataStruct.H:671
bool rayleigh_damp_W
Definition: ERF_DataStruct.H:689
SpongeChoice spongeChoice
Definition: ERF_DataStruct.H:665
Definition: ERF_SpongeStruct.H:15
std::string sponge_type
Definition: ERF_SpongeStruct.H:58
Definition: ERF_TurbStruct.H:31
bool use_kturb
Definition: ERF_TurbStruct.H:282
Here is the call graph for this function:

◆ advance_lsm()

void ERF::advance_lsm ( int  lev,
amrex::MultiFab &  cons_in,
amrex::MultiFab &  xvel_in,
amrex::MultiFab &  yvel_in,
const amrex::Real &  dt_advance 
)
10 {
11  if (solverChoice.lsm_type != LandSurfaceType::None) {
12  if (solverChoice.lsm_type == LandSurfaceType::NOAH) {
13  lsm.Advance(lev, cons_in, xvel_in, yvel_in, SFS_hfx3_lev[lev].get(), SFS_q1fx3_lev[lev].get(), dt_advance, istep[lev]);
14  } else {
15  lsm.Advance(lev, dt_advance);
16  }
17  }
18 }
LandSurface lsm
Definition: ERF.H:788
amrex::Vector< int > istep
Definition: ERF.H:725
void Advance(const int &lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, amrex::MultiFab *hfx3_out, amrex::MultiFab *qfx3_out, const amrex::Real &dt_advance, const int &nstep)
Definition: ERF_LandSurface.H:52
LandSurfaceType lsm_type
Definition: ERF_DataStruct.H:749

◆ advance_microphysics()

void ERF::advance_microphysics ( int  lev,
amrex::MultiFab &  cons_in,
const amrex::Real &  dt_advance,
const int &  iteration,
const amrex::Real &  time 
)
10 {
11  if (solverChoice.moisture_type != MoistureType::None) {
12  micro->Update_Micro_Vars_Lev(lev, cons);
13  micro->Advance(lev, dt_advance, iteration, time, solverChoice, vars_new, z_phys_nd, phys_bc_type);
14  micro->Update_State_Vars_Lev(lev, cons);
15  }
16 }
std::unique_ptr< Microphysics > micro
Definition: ERF.H:772

◆ appendPlotVariables()

void ERF::appendPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
174 {
175  ParmParse pp(pp_prefix);
176 
177  Vector<std::string> plot_var_names(0);
178  if (pp.contains(pp_plot_var_names.c_str())) {
179  std::string nm;
180  int nPltVars = pp.countval(pp_plot_var_names.c_str());
181  for (int i = 0; i < nPltVars; i++) {
182  pp.get(pp_plot_var_names.c_str(), nm, i);
183  // Add the named variable to our list of plot variables
184  // if it is not already in the list
185  if (!containerHasElement(plot_var_names, nm)) {
186  plot_var_names.push_back(nm);
187  }
188  }
189  }
190 
191  Vector<std::string> tmp_plot_names(0);
192 #ifdef ERF_USE_PARTICLES
193  Vector<std::string> particle_mesh_plot_names;
194  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
195  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
196  std::string tmp(particle_mesh_plot_names[i]);
197  if (containerHasElement(plot_var_names, tmp) ) {
198  tmp_plot_names.push_back(tmp);
199  }
200  }
201 #endif
202 
203  for (int i = 0; i < tmp_plot_names.size(); i++) {
204  a_plot_var_names.push_back( tmp_plot_names[i] );
205  }
206 
207  // Finally, check to see if we found all the requested variables
208  for (const auto& plot_name : plot_var_names) {
209  if (!containerHasElement(a_plot_var_names, plot_name)) {
210  if (amrex::ParallelDescriptor::IOProcessor()) {
211  Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
212  }
213  }
214  }
215 }
bool containerHasElement(const V &iterable, const T &query)
Definition: ERF_Container.H:5
std::string pp_prefix
Definition: ERF.H:462
Here is the call graph for this function:

◆ AverageDown()

void ERF::AverageDown ( )
private
17 {
18  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
19 
20  int src_comp, num_comp;
21  for (int lev = finest_level-1; lev >= 0; --lev)
22  {
23  // If anelastic we don't average down rho because rho == rho0.
24  if (solverChoice.anelastic[lev]) {
25  src_comp = 1;
26  } else {
27  src_comp = 0;
28  }
29  num_comp = vars_new[0][Vars::cons].nComp() - src_comp;
30  AverageDownTo(lev,src_comp,num_comp);
31  }
32 }
void AverageDownTo(int crse_lev, int scomp, int ncomp)
Definition: ERF_AverageDown.cpp:36
CouplingType coupling_type
Definition: ERF_DataStruct.H:745

◆ AverageDownTo()

void ERF::AverageDownTo ( int  crse_lev,
int  scomp,
int  ncomp 
)
37 {
38  if (solverChoice.anelastic[crse_lev]) {
39  AMREX_ALWAYS_ASSERT(scomp == 1);
40  } else {
41  AMREX_ALWAYS_ASSERT(scomp == 0);
42  }
43 
44  AMREX_ALWAYS_ASSERT(ncomp == vars_new[crse_lev][Vars::cons].nComp() - scomp);
45  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
46 
47  // ******************************************************************************************
48  // First do cell-centered quantities
49  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
50  // m is the map scale factor at cell centers
51  // Here we pre-divide (rho S) by m^2 before average down
52  // ******************************************************************************************
53  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
54  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
55  const Box& bx = mfi.tilebox();
56  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
57  const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
58  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
59  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
60  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
61  {
62  cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
63  });
64  } else {
65  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
66  {
67  cons_arr(i,j,k,scomp+n) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
68  });
69  }
70  } // mfi
71  } // lev
72 
73  int fine_lev = crse_lev+1;
74 
75  if (interpolation_type == StateInterpType::Perturbational) {
76  // Make the fine rho and (rho theta) be perturbational
77  MultiFab::Divide(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
78  Rho_comp,RhoTheta_comp,1,IntVect{0});
79  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
80  BaseState::r0_comp,Rho_comp,1,IntVect{0});
81  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
82  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
83 
84  // Make the crse rho and (rho theta) be perturbational
85  MultiFab::Divide(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
86  Rho_comp,RhoTheta_comp,1,IntVect{0});
87  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
88  BaseState::r0_comp,Rho_comp,1,IntVect{0});
89  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
90  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
91  }
92 
93  average_down(vars_new[crse_lev+1][Vars::cons],vars_new[crse_lev ][Vars::cons],
94  scomp, ncomp, refRatio(crse_lev));
95 
96  if (interpolation_type == StateInterpType::Perturbational) {
97  // Restore the fine data to what it was
98  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
99  BaseState::r0_comp,Rho_comp,1,IntVect{0});
100  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
101  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
102  MultiFab::Multiply(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
103  Rho_comp,RhoTheta_comp,1,IntVect{0});
104 
105  // Make the crse data be full values not perturbational
106  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
107  BaseState::r0_comp,Rho_comp,1,IntVect{0});
108  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
109  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
110  MultiFab::Multiply(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
111  Rho_comp,RhoTheta_comp,1,IntVect{0});
112  }
113 
114  vars_new[crse_lev][Vars::cons].FillBoundary(geom[crse_lev].periodicity());
115 
116  // Here we multiply (rho S) by m^2 after average down
117  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
118  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
119  const Box& bx = mfi.tilebox();
120  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
121  const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
122  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
123  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
124  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
125  {
126  cons_arr(i,j,k,scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)) / detJ_arr(i,j,k);
127  });
128  } else {
129  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
130  {
131  cons_arr(i,j,k,scomp+n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
132  });
133  }
134  } // mfi
135  } // lev
136 
137  // ******************************************************************************************
138  // Now average down momenta.
139  // Note that vars_new holds velocities not momenta, but we want to do conservative
140  // averaging so we first convert to momentum, then average down, then convert
141  // back to velocities -- only on the valid region
142  // ******************************************************************************************
143  for (int lev = crse_lev; lev <= crse_lev+1; lev++)
144  {
145  // FillBoundary for density so we can go back and forth between velocity and momentum
146  vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
147 
148  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
149  vars_new[lev][Vars::yvel], IntVect(0,0,0),
150  vars_new[lev][Vars::zvel], IntVect(0,0,0),
151  vars_new[lev][Vars::cons],
152  rU_new[lev],
153  rV_new[lev],
154  rW_new[lev],
155  Geom(lev).Domain(),
157  }
158 
159  average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
160  average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
161  average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
162 
163  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
165  vars_new[lev][Vars::yvel],
166  vars_new[lev][Vars::zvel],
167  vars_new[lev][Vars::cons],
168  rU_new[lev],
169  rV_new[lev],
170  rW_new[lev],
171  Geom(lev).Domain(),
173  }
174 }
void MomentumToVelocity(MultiFab &xvel, MultiFab &yvel, MultiFab &zvel, const MultiFab &density, const MultiFab &xmom_in, const MultiFab &ymom_in, const MultiFab &zmom_in, const Box &domain, const Vector< BCRec > &domain_bcs_type_h)
Definition: ERF_MomentumToVelocity.cpp:25
static StateInterpType interpolation_type
Definition: ERF.H:1074
@ th0_comp
Definition: ERF_IndexDefines.H:66
Here is the call graph for this function:

◆ build_fine_mask()

MultiFab & ERF::build_fine_mask ( int  level)

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters
levelFine level index which masks underlying coarser data
649 {
650  // Mask for zeroing covered cells
651  AMREX_ASSERT(level > 0);
652 
653  const BoxArray& cba = grids[level-1];
654  const DistributionMapping& cdm = dmap[level-1];
655 
656  // TODO -- we should make a vector of these a member of ERF class
657  fine_mask.define(cba, cdm, 1, 0, MFInfo());
658  fine_mask.setVal(1.0);
659 
660  BoxArray fba = grids[level];
661  iMultiFab ifine_mask = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
662 
663  const auto fma = fine_mask.arrays();
664  const auto ifma = ifine_mask.arrays();
665  ParallelFor(fine_mask, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
666  {
667  fma[bno](i,j,k) = ifma[bno](i,j,k);
668  });
669 
670  return fine_mask;
671 }
amrex::MultiFab fine_mask
Definition: ERF.H:1200

◆ ClearLevel()

void ERF::ClearLevel ( int  lev)
override
560 {
561  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
562  vars_new[lev][var_idx].clear();
563  vars_old[lev][var_idx].clear();
564  }
565 
566  base_state[lev].clear();
567 
568  rU_new[lev].clear();
569  rU_old[lev].clear();
570  rV_new[lev].clear();
571  rV_old[lev].clear();
572  rW_new[lev].clear();
573  rW_old[lev].clear();
574 
575  if (lev > 0) {
576  zmom_crse_rhs[lev].clear();
577  }
578 
579  if (solverChoice.anelastic[lev] == 1) {
580  pp_inc[lev].clear();
581  }
582 
583  // Clears the integrator memory
584  mri_integrator_mem[lev].reset();
585 
586  // Clears the physical boundary condition routines
587  physbcs_cons[lev].reset();
588  physbcs_u[lev].reset();
589  physbcs_v[lev].reset();
590  physbcs_w[lev].reset();
591  physbcs_base[lev].reset();
592 
593  // Clears the flux register array
594  advflux_reg[lev]->reset();
595 }
amrex::Vector< amrex::MultiFab > pp_inc
Definition: ERF.H:746
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
Definition: ERF.H:878
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
Definition: ERF.H:753
@ NumTypes
Definition: ERF_IndexDefines.H:144

◆ cloud_fraction()

Real ERF::cloud_fraction ( amrex::Real  time)
396 {
397  BL_PROFILE("ERF::cloud_fraction()");
398 
399  int lev = 0;
400  // This holds all of qc
401  MultiFab qc(vars_new[lev][Vars::cons],make_alias,RhoQ2_comp,1);
402 
403  int direction = 2; // z-direction
404  Box const& domain = geom[lev].Domain();
405 
406  auto const& qc_arr = qc.const_arrays();
407 
408  // qc_2d is an BaseFab<int> holding the max value over the column
409  auto qc_2d = ReduceToPlane<ReduceOpMax,int>(direction, domain, qc,
410  [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) -> int
411  {
412  if (qc_arr[box_no](i,j,k) > 0) {
413  return 1;
414  } else {
415  return 0;
416  }
417  });
418 
419  auto* p = qc_2d.dataPtr();
420 
421  Long numpts = qc_2d.numPts();
422 
423  AMREX_ASSERT(numpts < Long(std::numeric_limits<int>::max));
424 
425 #if 1
426  if (ParallelDescriptor::UseGpuAwareMpi()) {
427  ParallelDescriptor::ReduceIntMax(p,static_cast<int>(numpts));
428  } else {
429  Gpu::PinnedVector<int> hv(numpts);
430  Gpu::copyAsync(Gpu::deviceToHost, p, p+numpts, hv.data());
431  Gpu::streamSynchronize();
432  ParallelDescriptor::ReduceIntMax(hv.data(),static_cast<int>(numpts));
433  Gpu::copyAsync(Gpu::hostToDevice, hv.data(), hv.data()+numpts, p);
434  }
435 
436  // Sum over component 0
437  Long num_cloudy = qc_2d.template sum<RunOn::Device>(0);
438 
439 #else
440  //
441  // We need this if we allow domain decomposition in the vertical
442  // but for now we leave it commented out
443  //
444  Long num_cloudy = Reduce::Sum<Long>(numpts,
445  [=] AMREX_GPU_DEVICE (Long i) -> Long {
446  if (p[i] == 1) {
447  return 1;
448  } else {
449  return 0;
450  }
451  });
452  ParallelDescriptor::ReduceLongSum(num_cloudy);
453 #endif
454 
455  Real num_total = qc_2d.box().d_numPts();
456 
457  Real cloud_frac = num_cloudy / num_total;
458 
459  return cloud_frac;
460 }
#define RhoQ2_comp
Definition: ERF_IndexDefines.H:43
@ qc
Definition: ERF_SatAdj.H:36

◆ compute_divergence()

void ERF::compute_divergence ( int  lev,
amrex::MultiFab &  rhs,
amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM >  rho0_u_const,
amrex::Geometry const &  geom_at_lev 
)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level 0.

11 {
12  BL_PROFILE("ERF::compute_divergence()");
13 
14  auto dxInv = geom_at_lev.InvCellSizeArray();
15 
16  // ****************************************************************************
17  // Compute divergence which will form RHS
18  // Note that we replace "rho0w" with the contravariant momentum, Omega
19  // ****************************************************************************
20  if (solverChoice.terrain_type == TerrainType::EB)
21  {
22  bool already_on_centroids = true;
23  EB_computeDivergence(rhs, rho0_u_const, geom_at_lev, already_on_centroids);
24  }
25  else if (SolverChoice::mesh_type == MeshType::ConstantDz)
26  {
27  computeDivergence(rhs, rho0_u_const, geom_at_lev);
28  }
29  else
30  {
31  for ( MFIter mfi(rhs,TilingIfNotGPU()); mfi.isValid(); ++mfi)
32  {
33  Box bx = mfi.tilebox();
34  const Array4<Real const>& rho0u_arr = rho0_u_const[0]->const_array(mfi);
35  const Array4<Real const>& rho0v_arr = rho0_u_const[1]->const_array(mfi);
36  const Array4<Real const>& rho0w_arr = rho0_u_const[2]->const_array(mfi);
37  const Array4<Real >& rhs_arr = rhs.array(mfi);
38 
39  const Array4<Real const>& mf_m = mapfac_m[lev]->const_array(mfi);
40 
41  if (SolverChoice::mesh_type == MeshType::StretchedDz) {
42  Real* stretched_dz_d_ptr = stretched_dz_d[lev].data();
43  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
44  Real dz = stretched_dz_d_ptr[k];
45  Real mfsq = mf_m(i,j,0) * mf_m(i,j,0);
46  rhs_arr(i,j,k) = mfsq * ( (rho0u_arr(i+1,j ,k ) - rho0u_arr(i,j,k)) * dxInv[0]
47  +(rho0v_arr(i ,j+1,k ) - rho0v_arr(i,j,k)) * dxInv[1]
48  +(rho0w_arr(i ,j ,k+1) - rho0w_arr(i,j,k)) / dz );
49  });
50  } else {
51 
52  //
53  // Note we compute the divergence using "rho0w" == Omega
54  //
55  const Array4<Real const>& ax_arr = ax[lev]->const_array(mfi);
56  const Array4<Real const>& ay_arr = ay[lev]->const_array(mfi);
57  const Array4<Real const>& dJ_arr = detJ_cc[lev]->const_array(mfi);
58  //
59  // az == 1 for terrain-fitted coordinates
60  //
61  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
62  {
63  Real mfsq = mf_m(i,j,0) * mf_m(i,j,0);
64  rhs_arr(i,j,k) = mfsq * ( (ax_arr(i+1,j,k)*rho0u_arr(i+1,j,k) - ax_arr(i,j,k)*rho0u_arr(i,j,k)) * dxInv[0]
65  +(ay_arr(i,j+1,k)*rho0v_arr(i,j+1,k) - ay_arr(i,j,k)*rho0v_arr(i,j,k)) * dxInv[1]
66  +( rho0w_arr(i,j,k+1) - rho0w_arr(i,j,k)) * dxInv[2] ) / dJ_arr(i,j,k);
67  });
68  }
69  } // mfi
70  }
71 }
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
Definition: ERF.H:865
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
Definition: ERF.H:835
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
Definition: ERF.H:836
static TerrainType terrain_type
Definition: ERF_DataStruct.H:649

◆ ComputeDt()

void ERF::ComputeDt ( int  step = -1)
private

Function that calls estTimeStep for each level

12 {
13  Vector<Real> dt_tmp(finest_level+1);
14 
15  for (int lev = 0; lev <= finest_level; ++lev)
16  {
17  dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
18  }
19 
20  ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
21 
22  Real dt_0 = dt_tmp[0];
23  int n_factor = 1;
24  for (int lev = 0; lev <= finest_level; ++lev) {
25  dt_tmp[lev] = amrex::min(dt_tmp[lev], change_max*dt[lev]);
26  n_factor *= nsubsteps[lev];
27  dt_0 = amrex::min(dt_0, n_factor*dt_tmp[lev]);
28  if (step == 0){
29  dt_0 *= init_shrink;
30  if (verbose && init_shrink != 1.0) {
31  Print() << "Timestep 0: shrink initial dt at level " << lev << " by " << init_shrink << std::endl;
32  }
33  }
34  }
35  // Limit dt's by the value of stop_time.
36  const Real eps = 1.e-3*dt_0;
37  if (t_new[0] + dt_0 > stop_time - eps) {
38  dt_0 = stop_time - t_new[0];
39  }
40 
41  dt[0] = dt_0;
42  for (int lev = 1; lev <= finest_level; ++lev) {
43  dt[lev] = dt[lev-1] / nsubsteps[lev];
44  }
45 }
amrex::Real stop_time
Definition: ERF.H:928
amrex::Vector< amrex::Real > t_new
Definition: ERF.H:729
amrex::Real estTimeStep(int lev, long &dt_fast_ratio) const
Definition: ERF_ComputeTimestep.cpp:54
amrex::Vector< int > nsubsteps
Definition: ERF.H:726
static amrex::Real init_shrink
Definition: ERF.H:939
static amrex::Real change_max
Definition: ERF.H:940

◆ ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells ( const SolverChoice sc)
inlinestaticprivate
1207  {
1208  int ngrow = 0;
1209 
1210  if (sc.use_num_diff)
1211  {
1212  ngrow = 3;
1213  } else {
1214  if (
1221  { ngrow = 3; }
1222  else if (
1229  { ngrow = 3; }
1230  else if (
1239  { ngrow = 3; }
1240  else if (
1249  { ngrow = 4; }
1250  else
1251  {
1252  if (sc.terrain_type == TerrainType::EB){
1253  ngrow = 3;
1254  } else {
1255  ngrow = 2;
1256  }
1257  }
1258  }
1259 
1260  return ngrow;
1261  }
@ Centered_6th
AdvType moistscal_horiz_adv_type
Definition: ERF_AdvStruct.H:343
AdvType dycore_vert_adv_type
Definition: ERF_AdvStruct.H:340
AdvType moistscal_vert_adv_type
Definition: ERF_AdvStruct.H:344
AdvType dryscal_horiz_adv_type
Definition: ERF_AdvStruct.H:341
AdvType dycore_horiz_adv_type
Definition: ERF_AdvStruct.H:339
AdvType dryscal_vert_adv_type
Definition: ERF_AdvStruct.H:342
AdvChoice advChoice
Definition: ERF_DataStruct.H:663

◆ Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int  lev)
private
2077 {
2078  auto& fine_new = vars_new[lev];
2079  auto& crse_new = vars_new[lev-1];
2080  auto& ba_fine = fine_new[Vars::cons].boxArray();
2081  auto& ba_crse = crse_new[Vars::cons].boxArray();
2082  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2083  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2084 
2085  int ncomp = vars_new[lev][Vars::cons].nComp();
2086 
2087  FPr_c.emplace_back(ba_fine, dm_fine, geom[lev] ,
2088  ba_crse, dm_crse, geom[lev-1],
2089  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2090  FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2091  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2092  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2093  FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2094  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2095  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2096  FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2097  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2098  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2099 }
int cf_set_width
Definition: ERF.H:805

◆ DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int  i)
inlineprivate
1272  {
1273  return *datalog[i];
1274  }
amrex::Vector< std::unique_ptr< std::fstream > > datalog
Definition: ERF.H:1448

◆ DataLogName()

std::string ERF::DataLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith datalog file.

1464 { return datalogname[i]; }
amrex::Vector< std::string > datalogname
Definition: ERF.H:1451

◆ Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int  lev)
private
2103 {
2104  auto& fine_new = vars_new[lev];
2105  auto& crse_new = vars_new[lev-1];
2106  auto& ba_fine = fine_new[Vars::cons].boxArray();
2107  auto& ba_crse = crse_new[Vars::cons].boxArray();
2108  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2109  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2110 
2111  int ncomp = fine_new[Vars::cons].nComp();
2112 
2113  FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev] ,
2114  ba_crse, dm_crse, geom[lev-1],
2115  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2116  FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2117  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2118  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2119  FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2120  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2121  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2122  FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2123  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2124  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2125 }

◆ DerDataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DerDataLog ( int  i)
inlineprivate
1279  {
1280  return *der_datalog[i];
1281  }
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
Definition: ERF.H:1449

◆ DerDataLogName()

std::string ERF::DerDataLogName ( int  i) const
inlineprivatenoexcept
1465 { return der_datalogname[i]; }
amrex::Vector< std::string > der_datalogname
Definition: ERF.H:1452

◆ derive_diag_profiles()

void ERF::derive_diag_profiles ( amrex::Real  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
205 {
206  // We assume that this is always called at level 0
207  int lev = 0;
208 
209  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
210  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
211  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, uw, vv, vw, ww, uth, vth, wth,
212  // 0 1 2 3 4 5 6 7 8 9 10 11 12 13
213  // thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr,
214  // 14 15 16 17 18 19 20 21 22 23 24 25 26 27
215  // qi, qs, qg, wthv
216  // 28 29 30 31
217  MultiFab mf_out(grids[lev], dmap[lev], 32, 0);
218 
219  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
220 
221  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
222  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
223  MultiFab w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
224 
225  average_face_to_cellcenter(mf_vels,0,
226  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
227 
228  int zdir = 2;
229  auto domain = geom[0].Domain();
230 
231  // Sum in the horizontal plane
232  h_avg_u = sumToLine(mf_vels ,0,1,domain,zdir);
233  h_avg_v = sumToLine(mf_vels ,1,1,domain,zdir);
234  h_avg_w = sumToLine(mf_vels ,2,1,domain,zdir);
235 
236  int hu_size = h_avg_u.size();
237 
238  // Divide by the total number of cells we are averaging over
239  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
240  for (int k = 0; k < hu_size; ++k) {
241  h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
242  }
243 
244  Gpu::DeviceVector<Real> d_avg_u(hu_size, Real(0.0));
245  Gpu::DeviceVector<Real> d_avg_v(hu_size, Real(0.0));
246  Gpu::DeviceVector<Real> d_avg_w(hu_size, Real(0.0));
247 
248 #if 0
249  auto* avg_u_ptr = d_avg_u.data();
250  auto* avg_v_ptr = d_avg_v.data();
251  auto* avg_w_ptr = d_avg_w.data();
252 #endif
253 
254  Gpu::copy(Gpu::hostToDevice, h_avg_u.begin(), h_avg_u.end(), d_avg_u.begin());
255  Gpu::copy(Gpu::hostToDevice, h_avg_v.begin(), h_avg_v.end(), d_avg_v.begin());
256  Gpu::copy(Gpu::hostToDevice, h_avg_w.begin(), h_avg_w.end(), d_avg_w.begin());
257 
258  int nvars = vars_new[lev][Vars::cons].nComp();
259  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
260 
261  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
262 
263  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
264 
265  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
266  {
267  const Box& bx = mfi.tilebox();
268  const Array4<Real>& fab_arr = mf_out.array(mfi);
269  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
270  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
271  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
272  const Array4<Real>& cons_arr = mf_cons.array(mfi);
273  const Array4<Real>& p0_arr = p_hse.array(mfi);
274  const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
275  Array4<const Real>{};
276 
277  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
278  {
279  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
280  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
281  fab_arr(i, j, k, 1) = theta;
282  Real ksgs = 0.0;
283  if (l_use_KE) {
284  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
285  }
286  fab_arr(i, j, k, 2) = ksgs;
287 #if 1
288  if (l_use_kturb) {
289  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
290  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
291  } else {
292  fab_arr(i, j, k, 3) = 0.0;
293  fab_arr(i, j, k, 4) = 0.0;
294  }
295 #else
296  // Here we hijack the "Kturb" variable name to print out the resolved kinetic energy
297  Real upert = u_cc_arr(i,j,k) - avg_u_ptr[k];
298  Real vpert = v_cc_arr(i,j,k) - avg_v_ptr[k];
299  Real wpert = w_cc_arr(i,j,k) - avg_w_ptr[k];
300  fab_arr(i, j, k, 3) = 0.5 * (upert*upert + vpert*vpert + wpert*wpert);
301 #endif
302  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
303  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
304  fab_arr(i, j, k, 7) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k); // u*w
305  fab_arr(i, j, k, 8) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
306  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k); // v*w
307  fab_arr(i, j, k,10) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k); // w*w
308  fab_arr(i, j, k,11) = u_cc_arr(i,j,k) * theta; // u*th
309  fab_arr(i, j, k,12) = v_cc_arr(i,j,k) * theta; // v*th
310  fab_arr(i, j, k,13) = w_cc_arr(i,j,k) * theta; // w*th
311  fab_arr(i, j, k,14) = theta * theta; // th*th
312 
313  // if the number of fields is changed above, then be sure to update
314  // the following def!
315  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,8) + fab_arr(i,j,k,10);
316  fab_arr(i, j, k,15) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
317  fab_arr(i, j, k,16) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
318  fab_arr(i, j, k,17) = uiui * w_cc_arr(i,j,k); // (ui*ui)*w
319 
320  if (!use_moisture) {
321  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
322  p -= p0_arr(i,j,k);
323  fab_arr(i, j, k,18) = p; // p
324  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
325  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
326  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
327  fab_arr(i, j, k,22) = 0.; // qv
328  fab_arr(i, j, k,23) = 0.; // qc
329  fab_arr(i, j, k,24) = 0.; // qr
330  fab_arr(i, j, k,25) = 0.; // w*qv
331  fab_arr(i, j, k,26) = 0.; // w*qc
332  fab_arr(i, j, k,27) = 0.; // w*qr
333  fab_arr(i, j, k,28) = 0.; // qi
334  fab_arr(i, j, k,29) = 0.; // qs
335  fab_arr(i, j, k,30) = 0.; // qg
336  fab_arr(i, j, k,31) = 0.; // w*thv
337  }
338  });
339  } // mfi
340 
341  if (use_moisture)
342  {
343  int n_qstate = micro->Get_Qstate_Size();
344 
345  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
346  {
347  const Box& bx = mfi.tilebox();
348  const Array4<Real>& fab_arr = mf_out.array(mfi);
349  const Array4<Real>& cons_arr = mf_cons.array(mfi);
350  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
351  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
352  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
353  const Array4<Real>& p0_arr = p_hse.array(mfi);
354 
355  int rhoqr_comp = solverChoice.RhoQr_comp;
356 
357  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
358  {
359  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
360  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
361  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
362  Real(0.0);
363  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
364 
365  p -= p0_arr(i,j,k);
366  fab_arr(i, j, k,18) = p; // p
367  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
368  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
369  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
370  fab_arr(i, j, k,22) = qv; // qv
371  fab_arr(i, j, k,23) = qc; // qc
372  fab_arr(i, j, k,24) = qr; // qr
373  fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * qv; // w*qv
374  fab_arr(i, j, k,26) = w_cc_arr(i,j,k) * qc; // w*qc
375  fab_arr(i, j, k,27) = w_cc_arr(i,j,k) * qr; // w*qr
376  if (n_qstate > 3) {
377  fab_arr(i, j, k,28) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
378  fab_arr(i, j, k,29) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
379  fab_arr(i, j, k,30) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
380  } else {
381  fab_arr(i, j, k,28) = 0.0; // qi
382  fab_arr(i, j, k,29) = 0.0; // qs
383  fab_arr(i, j, k,30) = 0.0; // qg
384  }
385  Real ql = qc + qr;
386  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
387  Real thv = theta * (1 + 0.61*qv - ql);
388  fab_arr(i, j, k,31) = w_cc_arr(i,j,k) * thv; // w*thv
389  });
390  } // mfi
391  } // use_moisture
392 
393  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
394  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
395  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
396  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
397  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
398  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
399  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
400  h_avg_uw = sumToLine(mf_out, 7,1,domain,zdir);
401  h_avg_vv = sumToLine(mf_out, 8,1,domain,zdir);
402  h_avg_vw = sumToLine(mf_out, 9,1,domain,zdir);
403  h_avg_ww = sumToLine(mf_out,10,1,domain,zdir);
404  h_avg_uth = sumToLine(mf_out,11,1,domain,zdir);
405  h_avg_vth = sumToLine(mf_out,12,1,domain,zdir);
406  h_avg_wth = sumToLine(mf_out,13,1,domain,zdir);
407  h_avg_thth = sumToLine(mf_out,14,1,domain,zdir);
408  h_avg_uiuiu = sumToLine(mf_out,15,1,domain,zdir);
409  h_avg_uiuiv = sumToLine(mf_out,16,1,domain,zdir);
410  h_avg_uiuiw = sumToLine(mf_out,17,1,domain,zdir);
411  h_avg_p = sumToLine(mf_out,18,1,domain,zdir);
412  h_avg_pu = sumToLine(mf_out,19,1,domain,zdir);
413  h_avg_pv = sumToLine(mf_out,20,1,domain,zdir);
414  h_avg_pw = sumToLine(mf_out,21,1,domain,zdir);
415  h_avg_qv = sumToLine(mf_out,22,1,domain,zdir);
416  h_avg_qc = sumToLine(mf_out,23,1,domain,zdir);
417  h_avg_qr = sumToLine(mf_out,24,1,domain,zdir);
418  h_avg_wqv = sumToLine(mf_out,25,1,domain,zdir);
419  h_avg_wqc = sumToLine(mf_out,26,1,domain,zdir);
420  h_avg_wqr = sumToLine(mf_out,27,1,domain,zdir);
421  h_avg_qi = sumToLine(mf_out,28,1,domain,zdir);
422  h_avg_qs = sumToLine(mf_out,29,1,domain,zdir);
423  h_avg_qg = sumToLine(mf_out,30,1,domain,zdir);
424  h_avg_wthv = sumToLine(mf_out,31,1,domain,zdir);
425 
426  // Divide by the total number of cells we are averaging over
427  int h_avg_u_size = static_cast<int>(h_avg_u.size());
428  for (int k = 0; k < h_avg_u_size; ++k) {
429  h_avg_rho[k] /= area_z;
430  h_avg_ksgs[k] /= area_z;
431  h_avg_Kmv[k] /= area_z;
432  h_avg_Khv[k] /= area_z;
433  h_avg_th[k] /= area_z;
434  h_avg_thth[k] /= area_z;
435  h_avg_uu[k] /= area_z;
436  h_avg_uv[k] /= area_z;
437  h_avg_uw[k] /= area_z;
438  h_avg_vv[k] /= area_z;
439  h_avg_vw[k] /= area_z;
440  h_avg_ww[k] /= area_z;
441  h_avg_uth[k] /= area_z;
442  h_avg_vth[k] /= area_z;
443  h_avg_wth[k] /= area_z;
444  h_avg_uiuiu[k] /= area_z;
445  h_avg_uiuiv[k] /= area_z;
446  h_avg_uiuiw[k] /= area_z;
447  h_avg_p[k] /= area_z;
448  h_avg_pu[k] /= area_z;
449  h_avg_pv[k] /= area_z;
450  h_avg_pw[k] /= area_z;
451  h_avg_qv[k] /= area_z;
452  h_avg_qc[k] /= area_z;
453  h_avg_qr[k] /= area_z;
454  h_avg_wqv[k] /= area_z;
455  h_avg_wqc[k] /= area_z;
456  h_avg_wqr[k] /= area_z;
457  h_avg_qi[k] /= area_z;
458  h_avg_qs[k] /= area_z;
459  h_avg_qg[k] /= area_z;
460  h_avg_wthv[k] /= area_z;
461  }
462 
463 #if 0
464  // Here we print the integrated total kinetic energy as computed in the 1D profile above
465  Real sum = 0.;
466  Real dz = geom[0].ProbHi(2) / static_cast<Real>(h_avg_u_size);
467  for (int k = 0; k < h_avg_u_size; ++k) {
468  sum += h_avg_kturb[k] * h_avg_rho[k] * dz;
469  }
470  amrex::Print() << "ITKE " << time << " " << sum << " using " << h_avg_u_size << " " << dz << std::endl;
471 #endif
472 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getPgivenRTh(const amrex::Real rhotheta, const amrex::Real qv=0.)
Definition: ERF_EOS.H:84
#define RhoQ3_comp
Definition: ERF_IndexDefines.H:44
#define RhoQ6_comp
Definition: ERF_IndexDefines.H:47
#define RhoQ5_comp
Definition: ERF_IndexDefines.H:46
#define RhoKE_comp
Definition: ERF_IndexDefines.H:38
@ Theta_v
Definition: ERF_IndexDefines.H:168
@ Mom_v
Definition: ERF_IndexDefines.H:167
@ theta
Definition: ERF_MM5.H:20
@ qv
Definition: ERF_Kessler.H:28
Here is the call graph for this function:

◆ derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag ( amrex::Real  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
311 {
312  // We assume that this is always called at level 0
313  int lev = 0;
314 
315  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
316  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
317  // Note: "uiui" == u_i*u_i = u*u + v*v + w*w
318  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, vv, uth, vth,
319  // indices: 0 1 2 3 4 5 6 7 8 9
320  // thth, uiuiu, uiuiv, p, pu, pv, qv, qc, qr, qi, qs, qg
321  // 10 11 12 13 14 15 16 17 18 19 20 21
322  MultiFab mf_out(grids[lev], dmap[lev], 22, 0);
323 
324  // This will hold uw, vw, ww, wth, uiuiw, pw, wqv, wqc, wqr, wthv
325  // indices: 0 1 2 3 4 5 6 7 8 9
326  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 10, 0);
327 
328  // This is only used to average u and v; w is not averaged to cell centers
329  MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
330 
331  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
332  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
333  MultiFab w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers (staggered)
334 
335  int zdir = 2;
336  auto domain = geom[0].Domain();
337  Box stag_domain = domain;
338  stag_domain.convert(IntVect(0,0,1));
339 
340  int nvars = vars_new[lev][Vars::cons].nComp();
341  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
342 
343  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
344 
345  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
346 
347  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
348  {
349  const Box& bx = mfi.tilebox();
350  const Array4<Real>& fab_arr = mf_out.array(mfi);
351  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
352  const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
353  const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
354  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
355  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
356  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
357  const Array4<Real>& cons_arr = mf_cons.array(mfi);
358  const Array4<Real>& p0_arr = p_hse.array(mfi);
359  const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
360  Array4<const Real>{};
361 
362  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
363  {
364  u_cc_arr(i,j,k) = 0.5 * (u_arr(i,j,k) + u_arr(i+1,j ,k));
365  v_cc_arr(i,j,k) = 0.5 * (v_arr(i,j,k) + v_arr(i ,j+1,k));
366 
367  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
368  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
369  fab_arr(i, j, k, 1) = theta;
370  Real ksgs = 0.0;
371  if (l_use_KE) {
372  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
373  }
374  fab_arr(i, j, k, 2) = ksgs;
375  if (l_use_kturb) {
376  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
377  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
378  } else {
379  fab_arr(i, j, k, 3) = 0.0;
380  fab_arr(i, j, k, 4) = 0.0;
381  }
382  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
383  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
384  fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
385  fab_arr(i, j, k, 8) = u_cc_arr(i,j,k) * theta; // u*th
386  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * theta; // v*th
387  fab_arr(i, j, k,10) = theta * theta; // th*th
388 
389  Real wcc = 0.5 * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
390 
391  // if the number of fields is changed above, then be sure to update
392  // the following def!
393  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,7) + wcc*wcc;
394  fab_arr(i, j, k,11) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
395  fab_arr(i, j, k,12) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
396 
397  if (!use_moisture) {
398  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
399  p -= p0_arr(i,j,k);
400  fab_arr(i, j, k,13) = p; // p
401  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
402  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
403  fab_arr(i, j, k,16) = 0.; // qv
404  fab_arr(i, j, k,17) = 0.; // qc
405  fab_arr(i, j, k,18) = 0.; // qr
406  fab_arr(i, j, k,19) = 0.; // qi
407  fab_arr(i, j, k,20) = 0.; // qs
408  fab_arr(i, j, k,21) = 0.; // qg
409  }
410  });
411 
412  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
413  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
414  {
415  // average to z faces (first to cell centers, then in z)
416  Real uface = 0.25 * ( u_arr(i ,j,k) + u_arr(i ,j,k-1)
417  + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
418  Real vface = 0.25 * ( v_arr(i,j ,k) + v_arr(i,j ,k-1)
419  + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
420  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
421  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
422  Real thface = 0.5*(theta0 + theta1);
423  fab_arr_stag(i,j,k,0) = uface * w_fc_arr(i,j,k); // u*w
424  fab_arr_stag(i,j,k,1) = vface * w_fc_arr(i,j,k); // v*w
425  fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
426  fab_arr_stag(i,j,k,3) = thface * w_fc_arr(i,j,k); // th*w
427  Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
428  fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
429  if (!use_moisture) {
430  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp)) - p0_arr(i,j,k );
431  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
432  Real pface = 0.5 * (p0 + p1);
433  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
434  fab_arr_stag(i,j,k,6) = 0.; // w*qv
435  fab_arr_stag(i,j,k,7) = 0.; // w*qc
436  fab_arr_stag(i,j,k,8) = 0.; // w*qr
437  fab_arr_stag(i,j,k,9) = 0.; // w*thv
438  }
439  });
440 
441  } // mfi
442 
443  if (use_moisture)
444  {
445  int n_qstate = micro->Get_Qstate_Size();
446 
447  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
448  {
449  const Box& bx = mfi.tilebox();
450  const Array4<Real>& fab_arr = mf_out.array(mfi);
451  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
452  const Array4<Real>& cons_arr = mf_cons.array(mfi);
453  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
454  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
455  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
456  const Array4<Real>& p0_arr = p_hse.array(mfi);
457 
458  int rhoqr_comp = solverChoice.RhoQr_comp;
459 
460  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
461  {
462  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
463  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
464  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
465  Real(0.0);
466  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
467 
468  p -= p0_arr(i,j,k);
469  fab_arr(i, j, k,13) = p; // p
470  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
471  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
472  fab_arr(i, j, k,16) = qv; // qv
473  fab_arr(i, j, k,17) = qc; // qc
474  fab_arr(i, j, k,18) = qr; // qr
475  if (n_qstate > 3) { // SAM model
476  fab_arr(i, j, k,19) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
477  fab_arr(i, j, k,20) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
478  fab_arr(i, j, k,21) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
479  } else {
480  fab_arr(i, j, k,19) = 0.0; // qi
481  fab_arr(i, j, k,20) = 0.0; // qs
482  fab_arr(i, j, k,21) = 0.0; // qg
483  }
484  });
485 
486  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
487  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
488  {
489  Real qv0 = cons_arr(i,j,k ,RhoQ1_comp) / cons_arr(i,j,k ,Rho_comp);
490  Real qv1 = cons_arr(i,j,k-1,RhoQ1_comp) / cons_arr(i,j,k-1,Rho_comp);
491  Real qc0 = cons_arr(i,j,k ,RhoQ2_comp) / cons_arr(i,j,k ,Rho_comp);
492  Real qc1 = cons_arr(i,j,k-1,RhoQ2_comp) / cons_arr(i,j,k-1,Rho_comp);
493  Real qr0 = (rhoqr_comp > -1) ? cons_arr(i,j,k ,RhoQ3_comp) / cons_arr(i,j,k ,Rho_comp) :
494  Real(0.0);
495  Real qr1 = (rhoqr_comp > -1) ? cons_arr(i,j,k-1,RhoQ3_comp) / cons_arr(i,j,k-1,Rho_comp) :
496  Real(0.0);
497  Real qvface = 0.5 * (qv0 + qv1);
498  Real qcface = 0.5 * (qc0 + qc1);
499  Real qrface = 0.5 * (qr0 + qr1);
500 
501  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp), qv0) - p0_arr(i,j,k );
502  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv1) - p0_arr(i,j,k-1);
503  Real pface = 0.5 * (p0 + p1);
504 
505  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
506  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
507  Real thface = 0.5*(theta0 + theta1);
508  Real ql = qcface + qrface;
509  Real thv = thface * (1 + 0.61*qvface - ql);
510 
511  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
512  fab_arr_stag(i,j,k,6) = qvface * w_fc_arr(i,j,k); // w*qv
513  fab_arr_stag(i,j,k,7) = qcface * w_fc_arr(i,j,k); // w*qc
514  fab_arr_stag(i,j,k,8) = qrface * w_fc_arr(i,j,k); // w*qr
515  fab_arr_stag(i,j,k,9) = thv * w_fc_arr(i,j,k); // w*thv
516  });
517  } // mfi
518  } // use_moisture
519 
520  // Sum in the horizontal plane
521  h_avg_u = sumToLine(u_cc,0,1, domain,zdir);
522  h_avg_v = sumToLine(v_cc,0,1, domain,zdir);
523  h_avg_w = sumToLine(w_fc,0,1,stag_domain,zdir);
524 
525  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
526  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
527  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
528  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
529  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
530  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
531  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
532  h_avg_vv = sumToLine(mf_out, 7,1,domain,zdir);
533  h_avg_uth = sumToLine(mf_out, 8,1,domain,zdir);
534  h_avg_vth = sumToLine(mf_out, 9,1,domain,zdir);
535  h_avg_thth = sumToLine(mf_out,10,1,domain,zdir);
536  h_avg_uiuiu = sumToLine(mf_out,11,1,domain,zdir);
537  h_avg_uiuiv = sumToLine(mf_out,12,1,domain,zdir);
538  h_avg_p = sumToLine(mf_out,13,1,domain,zdir);
539  h_avg_pu = sumToLine(mf_out,14,1,domain,zdir);
540  h_avg_pv = sumToLine(mf_out,15,1,domain,zdir);
541  h_avg_qv = sumToLine(mf_out,16,1,domain,zdir);
542  h_avg_qc = sumToLine(mf_out,17,1,domain,zdir);
543  h_avg_qr = sumToLine(mf_out,18,1,domain,zdir);
544  h_avg_qi = sumToLine(mf_out,19,1,domain,zdir);
545  h_avg_qs = sumToLine(mf_out,20,1,domain,zdir);
546  h_avg_qg = sumToLine(mf_out,21,1,domain,zdir);
547 
548  h_avg_uw = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
549  h_avg_vw = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
550  h_avg_ww = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
551  h_avg_wth = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
552  h_avg_uiuiw = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
553  h_avg_pw = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
554  h_avg_wqv = sumToLine(mf_out_stag,6,1,stag_domain,zdir);
555  h_avg_wqc = sumToLine(mf_out_stag,7,1,stag_domain,zdir);
556  h_avg_wqr = sumToLine(mf_out_stag,8,1,stag_domain,zdir);
557  h_avg_wthv = sumToLine(mf_out_stag,9,1,stag_domain,zdir);
558 
559  // Divide by the total number of cells we are averaging over
560  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
561  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
562  for (int k = 0; k < unstag_size; ++k) {
563  h_avg_u[k] /= area_z;
564  h_avg_v[k] /= area_z;
565  h_avg_rho[k] /= area_z;
566  h_avg_ksgs[k] /= area_z;
567  h_avg_Kmv[k] /= area_z;
568  h_avg_Khv[k] /= area_z;
569  h_avg_th[k] /= area_z;
570  h_avg_thth[k] /= area_z;
571  h_avg_uu[k] /= area_z;
572  h_avg_uv[k] /= area_z;
573  h_avg_vv[k] /= area_z;
574  h_avg_uth[k] /= area_z;
575  h_avg_vth[k] /= area_z;
576  h_avg_uiuiu[k] /= area_z;
577  h_avg_uiuiv[k] /= area_z;
578  h_avg_p[k] /= area_z;
579  h_avg_pu[k] /= area_z;
580  h_avg_pv[k] /= area_z;
581  h_avg_qv[k] /= area_z;
582  h_avg_qc[k] /= area_z;
583  h_avg_qr[k] /= area_z;
584  h_avg_qi[k] /= area_z;
585  h_avg_qs[k] /= area_z;
586  h_avg_qg[k] /= area_z;
587  }
588 
589  for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
590  h_avg_w[k] /= area_z;
591  h_avg_uw[k] /= area_z;
592  h_avg_vw[k] /= area_z;
593  h_avg_ww[k] /= area_z;
594  h_avg_wth[k] /= area_z;
595  h_avg_uiuiw[k] /= area_z;
596  h_avg_pw[k] /= area_z;
597  h_avg_wqv[k] /= area_z;
598  h_avg_wqc[k] /= area_z;
599  h_avg_wqr[k] /= area_z;
600  h_avg_wthv[k] /= area_z;
601  }
602 }
Here is the call graph for this function:

◆ derive_stress_profiles()

void ERF::derive_stress_profiles ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
480 {
481  int lev = 0;
482 
483  // This will hold the stress tensor components
484  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
485 
486  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
487 
488  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
489 
490  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
491  {
492  const Box& bx = mfi.tilebox();
493  const Array4<Real>& fab_arr = mf_out.array(mfi);
494 
495  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
496 
497  // NOTE: These are from the last RK stage...
498  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
499  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
500  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
501  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
502  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
503  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
504 
505  // These should be re-calculated during ERF_slow_rhs_post
506  // -- just vertical SFS kinematic heat flux for now
507  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
508  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
509  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
510  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
511  Array4<const Real>{};
512  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
513  Array4<const Real>{};
514  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
515 
516  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
517  {
518  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
519  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
520  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
521  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
522  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
523  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
524  fab_arr(i, j, k, 2) = ( tau13_arr(i,j,k ) + tau13_arr(i+1,j,k )
525  + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) )
526  / ( rho_arr(i,j,k ) + rho_arr(i+1,j,k )
527  + rho_arr(i,j,k+1) + rho_arr(i+1,j,k+1) );
528  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
529  fab_arr(i, j, k, 4) = ( tau23_arr(i,j,k ) + tau23_arr(i,j+1,k )
530  + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) )
531  / ( rho_arr(i,j,k ) + rho_arr(i,j+1,k )
532  + rho_arr(i,j,k+1) + rho_arr(i,j+1,k+1) );
533  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
534  fab_arr(i, j, k, 6) = 0.5 * ( hfx3_arr(i,j,k) + hfx3_arr(i,j,k+1) ) / rho_arr(i,j,k);
535  fab_arr(i, j, k, 7) = (l_use_moist) ? 0.5 * ( q1fx3_arr(i,j,k) + q1fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
536  fab_arr(i, j, k, 8) = (l_use_moist) ? 0.5 * ( q2fx3_arr(i,j,k) + q2fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
537  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
538  });
539  }
540 
541  int zdir = 2;
542  auto domain = geom[0].Domain();
543 
544  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
545  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
546  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
547  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
548  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
549  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
550  h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
551  h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
552  h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
553  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
554 
555  int ht_size = h_avg_tau11.size();
556 
557  // Divide by the total number of cells we are averaging over
558  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
559  for (int k = 0; k < ht_size; ++k) {
560  h_avg_tau11[k] /= area_z;
561  h_avg_tau12[k] /= area_z;
562  h_avg_tau13[k] /= area_z;
563  h_avg_tau22[k] /= area_z;
564  h_avg_tau23[k] /= area_z;
565  h_avg_tau33[k] /= area_z;
566  h_avg_hfx3[k] /= area_z;
567  h_avg_q1fx3[k] /= area_z;
568  h_avg_q2fx3[k] /= area_z;
569  h_avg_diss[k] /= area_z;
570  }
571 }

◆ derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
610 {
611  int lev = 0;
612 
613  // This will hold the stress tensor components
614  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
615 
616  // This will hold Tau13 and Tau23
617  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 5, 0);
618 
619  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
620 
621  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
622 
623  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
624  {
625  const Box& bx = mfi.tilebox();
626  const Array4<Real>& fab_arr = mf_out.array(mfi);
627  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
628 
629  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
630 
631  // NOTE: These are from the last RK stage...
632  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
633  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
634  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
635  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
636  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
637  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
638 
639  // These should be re-calculated during ERF_slow_rhs_post
640  // -- just vertical SFS kinematic heat flux for now
641  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
642  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
643  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
644  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
645  Array4<const Real>{};
646  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
647  Array4<const Real>{};
648  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
649 
650  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
651  {
652  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
653  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
654  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
655  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
656  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
657  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
658  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
659  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
660  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
661  });
662 
663  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
664  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
665  {
666  Real rho_face = 0.5 * (rho_arr(i,j,k-1) + rho_arr(i,j,k));
667  // average from edge to face center
668  fab_arr_stag(i,j,k,0) = 0.5*(tau13_arr(i,j,k) + tau13_arr(i+1,j ,k)) / rho_face;
669  fab_arr_stag(i,j,k,1) = 0.5*(tau23_arr(i,j,k) + tau23_arr(i ,j+1,k)) / rho_face;
670 
671  fab_arr_stag(i,j,k,2) = hfx3_arr(i,j,k) / rho_face;
672  fab_arr_stag(i,j,k,3) = (l_use_moist) ? q1fx3_arr(i,j,k) / rho_face : 0.0;
673  fab_arr_stag(i,j,k,4) = (l_use_moist) ? q2fx3_arr(i,j,k) / rho_face : 0.0;
674  });
675  }
676 
677  int zdir = 2;
678  auto domain = geom[0].Domain();
679  Box stag_domain = domain;
680  stag_domain.convert(IntVect(0,0,1));
681 
682  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
683  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
684 // h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
685  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
686 // h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
687  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
688 // h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
689 // h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
690 // h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
691  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
692 
693  h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
694  h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
695  h_avg_hfx3 = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
696  h_avg_q1fx3 = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
697  h_avg_q2fx3 = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
698 
699  int ht_size = h_avg_tau11.size(); // _un_staggered
700 
701  // Divide by the total number of cells we are averaging over
702  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
703  for (int k = 0; k < ht_size; ++k) {
704  h_avg_tau11[k] /= area_z;
705  h_avg_tau12[k] /= area_z;
706  h_avg_tau13[k] /= area_z;
707  h_avg_tau22[k] /= area_z;
708  h_avg_tau23[k] /= area_z;
709  h_avg_tau33[k] /= area_z;
710  h_avg_hfx3[k] /= area_z;
711  h_avg_q1fx3[k] /= area_z;
712  h_avg_q2fx3[k] /= area_z;
713  h_avg_diss[k] /= area_z;
714  }
715  // staggered heights
716  h_avg_tau13[ht_size] /= area_z;
717  h_avg_tau23[ht_size] /= area_z;
718  h_avg_hfx3[ht_size] /= area_z;
719  h_avg_q1fx3[ht_size] /= area_z;
720  h_avg_q2fx3[ht_size] /= area_z;
721 }

◆ derive_upwp()

void ERF::derive_upwp ( amrex::Vector< amrex::Real > &  h_havg)

◆ EBFactory()

amrex::EBFArrayBoxFactory const& ERF::EBFactory ( int  lev) const
inlineprivatenoexcept
1482  {
1483  return *(eb[lev]->get_const_factory());
1484  }
amrex::Vector< std::unique_ptr< eb_ > > eb
Definition: ERF.H:1474

Referenced by WriteMyEBSurface().

Here is the caller graph for this function:

◆ erf_enforce_hse()

void ERF::erf_enforce_hse ( int  lev,
amrex::MultiFab &  dens,
amrex::MultiFab &  pres,
amrex::MultiFab &  pi,
amrex::MultiFab &  th,
amrex::MultiFab &  qv,
std::unique_ptr< amrex::MultiFab > &  z_cc 
)

Enforces hydrostatic equilibrium when using terrain.

Parameters
[in]levInteger specifying the current level
[out]densMultiFab storing base state density
[out]presMultiFab storing base state pressure
[out]piMultiFab storing base state Exner function
[in]z_ccPointer to MultiFab storing cell centered z-coordinates
164 {
165  Real l_gravity = solverChoice.gravity;
166  bool l_use_terrain = (solverChoice.mesh_type != MeshType::ConstantDz);
167 
168  const auto geomdata = geom[lev].data();
169  const Real dz = geomdata.CellSize(2);
170 
171  for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
172  {
173  // Create a flat box with same horizontal extent but only one cell in vertical
174  const Box& tbz = mfi.nodaltilebox(2);
175  int klo = tbz.smallEnd(2);
176  int khi = tbz.bigEnd(2);
177 
178  // Note we only grow by 1 because that is how big z_cc is.
179  Box b2d = tbz; // Copy constructor
180  b2d.grow(0,1);
181  b2d.grow(1,1);
182  b2d.setRange(2,0);
183 
184  // Intersect this box with the domain
185  Box zdomain = convert(geom[lev].Domain(),tbz.ixType());
186  b2d &= zdomain;
187 
188  // We integrate to the first cell (and below) by using rho in this cell
189  // If gravity == 0 this is constant pressure
190  // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
191  // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
192  // Note ng_pres_hse = 1
193 
194  // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
195  // Note that gravity is positive
196 
197  Array4<Real> rho_arr = dens.array(mfi);
198  Array4<Real> pres_arr = pres.array(mfi);
199  Array4<Real> pi_arr = pi.array(mfi);
200  Array4<Real> th_arr = theta.array(mfi);
201  Array4<Real> zcc_arr;
202  if (l_use_terrain) {
203  zcc_arr = z_cc->array(mfi);
204  }
205 
206  const Real rdOcp = solverChoice.rdOcp;
207 
208  ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
209  {
210  // Set value at surface from Newton iteration for rho
211  if (klo == 0)
212  {
213  // Physical height of the terrain at cell center
214  Real hz;
215  if (l_use_terrain) {
216  hz = zcc_arr(i,j,klo);
217  } else {
218  hz = 0.5*dz;
219  }
220 
221  pres_arr(i,j,klo) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
222  pi_arr(i,j,klo) = getExnergivenP(pres_arr(i,j,klo), rdOcp);
223  th_arr(i,j,klo) = getRhoThetagivenP(pres_arr(i,j,klo)) / rho_arr(i,j,klo);
224 
225  //
226  // Set ghost cell with dz and rho at boundary
227  // (We will set the rest of the ghost cells in the boundary condition routine)
228  //
229  pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
230  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
231  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
232 
233  } else {
234 
235  // If level > 0 and klo > 0, we need to use the value of pres_arr(i,j,klo-1) which was
236  // filled from FillPatch-ing it.
237  Real dz_loc;
238  if (l_use_terrain) {
239  dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
240  } else {
241  dz_loc = dz;
242  }
243 
244  Real dens_interp = 0.5*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
245  pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
246 
247  pi_arr(i,j,klo ) = getExnergivenP(pres_arr(i,j,klo ), rdOcp);
248  th_arr(i,j,klo ) = getRhoThetagivenP(pres_arr(i,j,klo )) / rho_arr(i,j,klo );
249 
250  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
251  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
252  }
253 
254  Real dens_interp;
255  if (l_use_terrain) {
256  for (int k = klo+1; k <= khi; k++) {
257  Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
258  dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
259  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
260  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
261  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
262  }
263  } else {
264  for (int k = klo+1; k <= khi; k++) {
265  dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
266  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
267  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
268  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
269  }
270  }
271  });
272 
273  } // mfi
274 
275  dens.FillBoundary(geom[lev].periodicity());
276  pres.FillBoundary(geom[lev].periodicity());
277  pi.FillBoundary(geom[lev].periodicity());
278  theta.FillBoundary(geom[lev].periodicity());
279  qv.FillBoundary(geom[lev].periodicity());
280 }
constexpr amrex::Real p_0
Definition: ERF_Constants.H:18
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getRhoThetagivenP(const amrex::Real p, const amrex::Real qv=0.0)
Definition: ERF_EOS.H:175
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getExnergivenP(const amrex::Real P, const amrex::Real rdOcp)
Definition: ERF_EOS.H:144
@ pres
Definition: ERF_Kessler.H:25
real(c_double), parameter, private pi
Definition: ERF_module_mp_morr_two_moment.F90:100
amrex::Real rdOcp
Definition: ERF_DataStruct.H:701
amrex::Real gravity
Definition: ERF_DataStruct.H:699
Here is the call graph for this function:

◆ ERF_shared()

void ERF::ERF_shared ( )
108 {
109  if (ParallelDescriptor::IOProcessor()) {
110  const char* erf_hash = buildInfoGetGitHash(1);
111  const char* amrex_hash = buildInfoGetGitHash(2);
112  const char* buildgithash = buildInfoGetBuildGitHash();
113  const char* buildgitname = buildInfoGetBuildGitName();
114 
115  if (strlen(erf_hash) > 0) {
116  Print() << "\n"
117  << "ERF git hash: " << erf_hash << "\n";
118  }
119  if (strlen(amrex_hash) > 0) {
120  Print() << "AMReX git hash: " << amrex_hash << "\n";
121  }
122  if (strlen(buildgithash) > 0) {
123  Print() << buildgitname << " git hash: " << buildgithash << "\n";
124  }
125 
126  Print() << "\n";
127  }
128 
129  int nlevs_max = max_level + 1;
130 
131 #ifdef ERF_USE_WINDFARM
132  Nturb.resize(nlevs_max);
133  vars_windfarm.resize(nlevs_max);
134  SMark.resize(nlevs_max);
135 #endif
136 
137 #if defined(ERF_USE_RRTMGP)
138  qheating_rates.resize(nlevs_max);
139  sw_lw_fluxes.resize(nlevs_max);
140  solar_zenith.resize(nlevs_max);
141 #endif
142 
143  // NOTE: size lsm before readparams (chooses the model at all levels)
144  lsm.ReSize(nlevs_max);
145  lsm_data.resize(nlevs_max);
146  lsm_flux.resize(nlevs_max);
147 
148  // NOTE: size canopy model before readparams (if file exists, we construct)
149  m_forest_drag.resize(nlevs_max);
150  for (int lev = 0; lev <= max_level; ++lev) { m_forest_drag[lev] = nullptr;}
151 
152  ReadParameters();
153  initializeMicrophysics(nlevs_max);
154 
155 #ifdef ERF_USE_WINDFARM
156  initializeWindFarm(nlevs_max);
157 #endif
158 
159 #ifdef ERF_USE_RRTMGP
160  rad.resize(nlevs_max);
161  for (int lev = 0; lev <= max_level; ++lev) { rad[lev] = std::make_unique<Radiation>(lev,solverChoice); }
162 #endif
163 
164  const std::string& pv1 = "plot_vars_1"; setPlotVariables(pv1,plot_var_names_1);
165  const std::string& pv2 = "plot_vars_2"; setPlotVariables(pv2,plot_var_names_2);
166 
167  // This is only used when we have mesh_type == MeshType::StretchedDz
168  stretched_dz_h.resize(nlevs_max);
169  stretched_dz_d.resize(nlevs_max);
170 
171  // Initialize staggered vertical levels for grid stretching or terrain, and
172  // to simplify Rayleigh damping layer calculations.
173  zlevels_stag.resize(max_level+1);
177  geom,
178  refRatio(),
181  solverChoice.dz0);
182 
183  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
184  SolverChoice::mesh_type == MeshType::VariableDz) {
185  int nz = geom[0].Domain().length(2) + 1; // staggered
186  if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
187  Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
188  << " does not match highest requested z level " << zlevels_stag[0][nz-1]
189  << std::endl;
190  }
191  if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > 1.0e-4) {
192  Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
193  << " does not match lowest requested level " << zlevels_stag[0][0]
194  << std::endl;
195  }
196 
197  // Redefine the problem domain here?
198  }
199 
200  prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
201 
202  // Geometry on all levels has been defined already.
203 
204  // No valid BoxArray and DistributionMapping have been defined.
205  // But the arrays for them have been resized.
206 
207  istep.resize(nlevs_max, 0);
208  nsubsteps.resize(nlevs_max, 1);
209  for (int lev = 1; lev <= max_level; ++lev) {
210  nsubsteps[lev] = MaxRefRatio(lev-1);
211  }
212 
213  t_new.resize(nlevs_max, 0.0);
214  t_old.resize(nlevs_max, -1.e100);
215  dt.resize(nlevs_max, std::min(1.e100,dt_max_initial));
216  dt_mri_ratio.resize(nlevs_max, 1);
217 
218  vars_new.resize(nlevs_max);
219  vars_old.resize(nlevs_max);
220 
221  // We resize this regardless in order to pass it without error
222  pp_inc.resize(nlevs_max);
223 
224  rU_new.resize(nlevs_max);
225  rV_new.resize(nlevs_max);
226  rW_new.resize(nlevs_max);
227 
228  rU_old.resize(nlevs_max);
229  rV_old.resize(nlevs_max);
230  rW_old.resize(nlevs_max);
231 
232  // xmom_crse_rhs.resize(nlevs_max);
233  // ymom_crse_rhs.resize(nlevs_max);
234  zmom_crse_rhs.resize(nlevs_max);
235 
236  for (int lev = 0; lev < nlevs_max; ++lev) {
237  vars_new[lev].resize(Vars::NumTypes);
238  vars_old[lev].resize(Vars::NumTypes);
239  }
240 
241  // Time integrator
242  mri_integrator_mem.resize(nlevs_max);
243 
244  // Physical boundary conditions
245  physbcs_cons.resize(nlevs_max);
246  physbcs_u.resize(nlevs_max);
247  physbcs_v.resize(nlevs_max);
248  physbcs_w.resize(nlevs_max);
249  physbcs_base.resize(nlevs_max);
250 
251  // Planes to hold Dirichlet values at boundaries
252  xvel_bc_data.resize(nlevs_max);
253  yvel_bc_data.resize(nlevs_max);
254  zvel_bc_data.resize(nlevs_max);
255  th_bc_data.resize(nlevs_max);
256 
257  advflux_reg.resize(nlevs_max);
258 
259  // Stresses
260  Tau.resize(nlevs_max);
261  SFS_hfx1_lev.resize(nlevs_max); SFS_hfx2_lev.resize(nlevs_max); SFS_hfx3_lev.resize(nlevs_max);
262  SFS_diss_lev.resize(nlevs_max);
263  SFS_q1fx1_lev.resize(nlevs_max); SFS_q1fx2_lev.resize(nlevs_max); SFS_q1fx3_lev.resize(nlevs_max);
264  SFS_q2fx3_lev.resize(nlevs_max);
265  eddyDiffs_lev.resize(nlevs_max);
266  SmnSmn_lev.resize(nlevs_max);
267 
268  // Sea surface temps
269  sst_lev.resize(nlevs_max);
270  tsk_lev.resize(nlevs_max);
271  lmask_lev.resize(nlevs_max);
272 
273  // Metric terms
274  z_phys_nd.resize(nlevs_max);
275  z_phys_cc.resize(nlevs_max);
276  detJ_cc.resize(nlevs_max);
277  ax.resize(nlevs_max);
278  ay.resize(nlevs_max);
279  az.resize(nlevs_max);
280 
281  z_phys_nd_new.resize(nlevs_max);
282  detJ_cc_new.resize(nlevs_max);
283  ax_new.resize(nlevs_max);
284  ay_new.resize(nlevs_max);
285  az_new.resize(nlevs_max);
286 
287  z_phys_nd_src.resize(nlevs_max);
288  detJ_cc_src.resize(nlevs_max);
289  ax_src.resize(nlevs_max);
290  ay_src.resize(nlevs_max);
291  az_src.resize(nlevs_max);
292 
293  z_t_rk.resize(nlevs_max);
294 
295  terrain_blanking.resize(nlevs_max);
296 
297  // Wall distance
298  walldist.resize(nlevs_max);
299 
300  // BoxArrays to make MultiFabs needed to convert WRFBdy data
301  ba1d.resize(nlevs_max);
302  ba2d.resize(nlevs_max);
303 
304  // MultiFabs needed to convert WRFBdy data
305  mf_C1H.resize(nlevs_max);
306  mf_C2H.resize(nlevs_max);
307  mf_MUB.resize(nlevs_max);
308 
309  // Mapfactors
310  mapfac_m.resize(nlevs_max);
311  mapfac_u.resize(nlevs_max);
312  mapfac_v.resize(nlevs_max);
313 
314  // Thin immersed body
315  xflux_imask.resize(nlevs_max);
316  yflux_imask.resize(nlevs_max);
317  zflux_imask.resize(nlevs_max);
318  //overset_imask.resize(nlevs_max);
319  thin_xforce.resize(nlevs_max);
320  thin_yforce.resize(nlevs_max);
321  thin_zforce.resize(nlevs_max);
322 
323  // Base state
324  base_state.resize(nlevs_max);
325  base_state_new.resize(nlevs_max);
326 
327  // Wave coupling data
328  Hwave.resize(nlevs_max);
329  Lwave.resize(nlevs_max);
330  for (int lev = 0; lev < max_level; ++lev)
331  {
332  Hwave[lev] = nullptr;
333  Lwave[lev] = nullptr;
334  }
335  Hwave_onegrid.resize(nlevs_max);
336  Lwave_onegrid.resize(nlevs_max);
337  for (int lev = 0; lev < max_level; ++lev)
338  {
339  Hwave_onegrid[lev] = nullptr;
340  Lwave_onegrid[lev] = nullptr;
341  }
342 
343  // Theta prim for MOST
344  Theta_prim.resize(nlevs_max);
345 
346  // Qv prim for MOST
347  Qv_prim.resize(nlevs_max);
348 
349  // Qr prim for MOST
350  Qr_prim.resize(nlevs_max);
351 
352  // Time averaged velocity field
353  vel_t_avg.resize(nlevs_max);
354  t_avg_cnt.resize(nlevs_max);
355 
356 #ifdef ERF_USE_NETCDF
357  // Size lat long arrays if using netcdf
358  lat_m.resize(nlevs_max);
359  lon_m.resize(nlevs_max);
360  for (int lev = 0; lev < max_level; ++lev) {
361  lat_m[lev] = nullptr;
362  lon_m[lev] = nullptr;
363  }
364 #endif
365 
366  // Variable coriolis
367  sinPhi_m.resize(nlevs_max);
368  cosPhi_m.resize(nlevs_max);
369  for (int lev = 0; lev < max_level; ++lev) {
370  sinPhi_m[lev] = nullptr;
371  cosPhi_m[lev] = nullptr;
372  }
373 
374  // Initialize tagging criteria for mesh refinement
376 
377  for (int lev = 0; lev < max_level; ++lev)
378  {
379  Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
380  ref_ratio[lev][0] << " " << ref_ratio[lev][1] << " " << ref_ratio[lev][2] << std::endl;
381  }
382 
383  // We will create each of these in MakeNewLevelFromScratch
384  eb.resize(max_level+1);
385  for (int lev = 0; lev < max_level + 1; lev++){
386  eb[lev] = std::make_unique<eb_>();
387  }
388 
389  //
390  // Construct the EB data structures and store in a separate class
391  //
392  // This is needed before initializing level MultiFabs
393  if ( solverChoice.terrain_type == TerrainType::EB ||
394  solverChoice.terrain_type == TerrainType::ImmersedForcing)
395  {
396  Box terrain_bx(surroundingNodes(geom[max_level].Domain())); terrain_bx.grow(3);
397  FArrayBox terrain_fab(makeSlab(terrain_bx,2,0),1);
398  Real dummy_time = 0.0;
399  prob->init_terrain_surface(geom[max_level], terrain_fab, dummy_time);
400  TerrainIF ebterrain(terrain_fab, geom[max_level], stretched_dz_d[max_level]);
401  auto gshop = EB2::makeShop(ebterrain);
402  bool build_coarse_level_by_coarsening(false);
403  amrex::EB2::Build(gshop, geom[max_level], max_level, max_level, build_coarse_level_by_coarsening);
404  }
405 }
void init_zlevels(Vector< Vector< Real >> &zlevels_stag, Vector< Vector< Real >> &stretched_dz_h, Vector< Gpu::DeviceVector< Real >> &stretched_dz_d, Vector< Geometry > const &geom, Vector< IntVect > const &ref_ratio, const Real grid_stretching_ratio, const Real zsurf, const Real dz0)
Definition: ERF_InitZLevels.cpp:11
std::unique_ptr< ProblemBase > amrex_probinit(const amrex_real *problo, const amrex_real *probhi) AMREX_ATTRIBUTE_WEAK
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
Definition: ERF.H:873
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
Definition: ERF.H:906
void setPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:13
amrex::Vector< amrex::BoxArray > ba2d
Definition: ERF.H:1110
void ReadParameters()
Definition: ERF.cpp:1556
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_new
Definition: ERF.H:848
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
Definition: ERF.H:839
amrex::Vector< amrex::MultiFab > base_state_new
Definition: ERF.H:868
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
Definition: ERF.H:837
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
Definition: ERF.H:819
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
Definition: ERF.H:854
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
Definition: ERF.H:845
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
Definition: ERF.H:907
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
Definition: ERF.H:817
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
Definition: ERF.H:905
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
Definition: ERF.H:696
amrex::Vector< std::string > plot_var_names_1
Definition: ERF.H:981
amrex::Vector< amrex::Real > t_old
Definition: ERF.H:730
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
Definition: ERF.H:851
amrex::Vector< std::string > plot_var_names_2
Definition: ERF.H:982
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
Definition: ERF.H:874
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_C1H
Definition: ERF.H:1111
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
Definition: ERF.H:1189
amrex::Vector< amrex::BoxArray > ba1d
Definition: ERF.H:1109
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
Definition: ERF.H:693
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
Definition: ERF.H:840
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
Definition: ERF.H:842
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
Definition: ERF.H:900
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_new
Definition: ERF.H:849
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
Definition: ERF.H:790
void refinement_criteria_setup()
Definition: ERF_Tagging.cpp:165
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
Definition: ERF.H:684
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
Definition: ERF.H:841
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
Definition: ERF.H:828
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
Definition: ERF.H:789
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
Definition: ERF.H:864
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
Definition: ERF.H:843
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_C2H
Definition: ERF.H:1112
static amrex::Real dt_max_initial
Definition: ERF.H:941
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
Definition: ERF.H:872
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
Definition: ERF.H:684
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
Definition: ERF.H:901
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_new
Definition: ERF.H:847
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
Definition: ERF.H:695
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
Definition: ERF.H:846
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
Definition: ERF.H:694
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_MUB
Definition: ERF.H:1113
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
Definition: ERF.H:871
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
Definition: ERF.H:899
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
Definition: ERF.H:818
void initializeMicrophysics(const int &)
Definition: ERF.cpp:1359
void ReSize(const int &nlev)
Definition: ERF_LandSurface.H:24
Definition: ERF_EBIFTerrain.H:14
const char * buildInfoGetGitHash(int i)
amrex::Real dz0
Definition: ERF_DataStruct.H:706
amrex::Real grid_stretching_ratio
Definition: ERF_DataStruct.H:704
amrex::Real zsurf
Definition: ERF_DataStruct.H:705
Here is the call graph for this function:

◆ ErrorEst()

void ERF::ErrorEst ( int  lev,
amrex::TagBoxArray &  tags,
amrex::Real  time,
int  ngrow 
)
override

Function to tag cells for refinement – this overrides the pure virtual function in AmrCore

Parameters
[in]levclevel of refinement at which we tag cells (0 is coarsest level)
[out]tagsarray of tagged cells
[in]timecurrent time
16 {
17  const int clearval = TagBox::CLEAR;
18  const int tagval = TagBox::SET;
19 
20  //
21  // Make sure the ghost cells of the level we are tagging at are filled
22  // in case we take differences that require them
23  // NOTE: We are Fillpatching only the cell-centered variables here
24  //
25  MultiFab& S_new = vars_new[levc][Vars::cons];
26  MultiFab& U_new = vars_new[levc][Vars::xvel];
27  MultiFab& V_new = vars_new[levc][Vars::yvel];
28  MultiFab& W_new = vars_new[levc][Vars::zvel];
29  //
30  if (levc == 0) {
31  FillPatch(levc, time, {&S_new, &U_new, &V_new, &W_new});
32  } else {
33  FillPatch(levc, time, {&S_new, &U_new, &V_new, &W_new},
34  {&S_new, &rU_new[levc], &rV_new[levc], &rW_new[levc]},
35  base_state[levc], base_state[levc],
36  false, true);
37  }
38 
39  for (int j=0; j < ref_tags.size(); ++j)
40  {
41  //
42  // This mf must have ghost cells because we may take differences between adjacent values
43  //
44  std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 1);
45 
46  // This allows dynamic refinement based on the value of the density
47  if (ref_tags[j].Field() == "density")
48  {
49  MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,1);
50 
51  // This allows dynamic refinement based on the value of qv
52  } else if ( ref_tags[j].Field() == "qv" ) {
53  MultiFab::Copy( *mf, vars_new[levc][Vars::cons], RhoQ1_comp, 0, 1, 1);
54  MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
55 
56 
57  // This allows dynamic refinement based on the value of qc
58  } else if (ref_tags[j].Field() == "qc" ) {
59  MultiFab::Copy( *mf, vars_new[levc][Vars::cons], RhoQ2_comp, 0, 1, 1);
60  MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
61 
62  // This allows dynamic refinement based on the value of the z-component of vorticity
63  } else if (ref_tags[j].Field() == "vorticity" ) {
64  Vector<MultiFab> mf_cc_vel(1);
65  mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
66  average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
67 
68  // Impose bc's at domain boundaries at all levels
69  FillBdyCCVels(mf_cc_vel,levc);
70 
71  mf->setVal(0.);
72 
73  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
74  {
75  const Box& bx = mfi.tilebox();
76  auto& dfab = (*mf)[mfi];
77  auto& sfab = mf_cc_vel[0][mfi];
78  derived::erf_dervortz(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
79  }
80 
81  // This allows dynamic refinement based on the value of the scalar/theta
82  } else if ( (ref_tags[j].Field() == "scalar" ) ||
83  (ref_tags[j].Field() == "theta" ) )
84  {
85  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
86  {
87  const Box& bx = mfi.growntilebox();
88  auto& dfab = (*mf)[mfi];
89  auto& sfab = vars_new[levc][Vars::cons][mfi];
90  if (ref_tags[j].Field() == "scalar") {
91  derived::erf_derscalar(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
92  } else if (ref_tags[j].Field() == "theta") {
93  derived::erf_dertheta(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
94  }
95  } // mfi
96  // This allows dynamic refinement based on the value of the density
97  } else if ( (SolverChoice::terrain_type == TerrainType::ImmersedForcing) &&
98  (ref_tags[j].Field() == "terrain_blanking") )
99  {
100  MultiFab::Copy(*mf,*terrain_blanking[levc],0,0,1,1);
101  } else if (ref_tags[j].Field() == "velmag") {
102  mf->setVal(0.0);
103  ParmParse pp(pp_prefix);
104  Vector<std::string> refinement_indicators;
105  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
106  Real velmag_threshold = 1e10;
107  for (int i=0; i<refinement_indicators.size(); ++i)
108  {
109  if(refinement_indicators[i]=="hurricane_tracker"){
110  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
111  ParmParse ppr(ref_prefix);
112  ppr.get("value_greater",velmag_threshold);
113  break;
114  }
115  }
116  HurricaneTracker(levc, U_new, V_new, W_new, velmag_threshold, false, &tags);
117 #ifdef ERF_USE_PARTICLES
118  } else {
119  //
120  // This allows dynamic refinement based on the number of particles per cell
121  //
122  // Note that we must count all the particles in levels both at and above the current,
123  // since otherwise, e.g., if the particles are all at level 1, counting particles at
124  // level 0 will not trigger refinement when regridding so level 1 will disappear,
125  // then come back at the next regridding
126  //
127  const auto& particles_namelist( particleData.getNames() );
128  mf->setVal(0.0);
129  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
130  {
131  std::string tmp_string(particles_namelist[i]+"_count");
132  IntVect rr = IntVect::TheUnitVector();
133  if (ref_tags[j].Field() == tmp_string) {
134  for (int lev = levc; lev <= finest_level; lev++)
135  {
136  MultiFab temp_dat(grids[lev], dmap[lev], 1, 0); temp_dat.setVal(0);
137  particleData[particles_namelist[i]]->IncrementWithTotal(temp_dat, lev);
138 
139  MultiFab temp_dat_crse(grids[levc], dmap[levc], 1, 0); temp_dat_crse.setVal(0);
140 
141  if (lev == levc) {
142  MultiFab::Copy(*mf, temp_dat, 0, 0, 1, 0);
143  } else {
144  for (int d = 0; d < AMREX_SPACEDIM; d++) {
145  rr[d] *= ref_ratio[levc][d];
146  }
147  average_down(temp_dat, temp_dat_crse, 0, 1, rr);
148  MultiFab::Add(*mf, temp_dat_crse, 0, 0, 1, 0);
149  }
150  }
151  }
152  }
153 #endif
154  }
155 
156  ref_tags[j](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
157  } // loop over j
158 }
void FillBdyCCVels(amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)
Definition: ERF_FillBdyCCVels.cpp:11
void HurricaneTracker(int lev, const amrex::MultiFab &U_new, const amrex::MultiFab &V_new, const amrex::MultiFab &W_new, const amrex::Real velmag_threshold, const bool is_track_io, amrex::TagBoxArray *tags=nullptr)
Definition: ERF_Tagging.cpp:391
static amrex::Vector< amrex::AMRErrorTag > ref_tags
Definition: ERF.H:1194
void erf_derscalar(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:165
void erf_dervortz(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:256
void erf_dertheta(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:144
real(c_double), private rr
Definition: ERF_module_mp_morr_two_moment.F90:223
Here is the call graph for this function:

◆ estTimeStep()

Real ERF::estTimeStep ( int  level,
long &  dt_fast_ratio 
) const

Function that calls estTimeStep for each level

Parameters
[in]levellevel of refinement (coarsest level i 0)
[out]dt_fast_ratioratio of slow to fast time step
55 {
56  BL_PROFILE("ERF::estTimeStep()");
57 
58  Real estdt_comp = 1.e20;
59  Real estdt_lowM = 1.e20;
60 
61  // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation
62  const int nxc = geom[0].Domain().length(0);
63  const int nyc = geom[0].Domain().length(1);
64 
65  auto const dxinv = geom[level].InvCellSizeArray();
66  auto const dzinv = 1.0 / dz_min[level];
67 
68  MultiFab const& S_new = vars_new[level][Vars::cons];
69 
70  MultiFab ccvel(grids[level],dmap[level],3,0);
71 
72  average_face_to_cellcenter(ccvel,0,
73  Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
74  &vars_new[level][Vars::yvel],
75  &vars_new[level][Vars::zvel]});
76 
77  int l_implicit_substepping = (solverChoice.substepping_type[level] == SubsteppingType::Implicit);
78  int l_anelastic = solverChoice.anelastic[level];
79 
80  Real estdt_comp_inv;
81 
82  if (solverChoice.terrain_type == TerrainType::EB)
83  {
84  const eb_& eb_lev = get_eb(level);
85  const MultiFab& detJ = (eb_lev.get_const_factory())->getVolFrac();
86 
87  estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
88  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
89  Array4<Real const> const& s,
90  Array4<Real const> const& u,
91  Array4<Real const> const& vf) -> Real
92  {
93  Real new_comp_dt = -1.e100;
94  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
95  {
96  if (vf(i,j,k) > 0.)
97  {
98  const Real rho = s(i, j, k, Rho_comp);
99  const Real rhotheta = s(i, j, k, RhoTheta_comp);
100 
101  // NOTE: even when moisture is present,
102  // we only use the partial pressure of the dry air
103  // to compute the soundspeed
104  Real pressure = getPgivenRTh(rhotheta);
105  Real c = std::sqrt(Gamma * pressure / rho);
106 
107  // If we are doing implicit acoustic substepping, then the z-direction does not contribute
108  // to the computation of the time step
109  if (l_implicit_substepping) {
110  if ((nxc > 1) && (nyc==1)) {
111  // 2-D in x-z
112  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
113  } else if ((nyc > 1) && (nxc==1)) {
114  // 2-D in y-z
115  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
116  } else {
117  // 3-D or SCM
118  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
119  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
120  }
121 
122  // If we are not doing implicit acoustic substepping, then the z-direction contributes
123  // to the computation of the time step
124  } else {
125  if (nxc > 1 && nyc > 1) {
126  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
127  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
128  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
129  } else if (nxc > 1) {
130  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
131  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
132  } else if (nyc > 1) {
133  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
134  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
135  } else {
136  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
137  }
138 
139  }
140  }
141  });
142  return new_comp_dt;
143  });
144 
145  } else {
146  estdt_comp_inv = ReduceMax(S_new, ccvel, 0,
147  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
148  Array4<Real const> const& s,
149  Array4<Real const> const& u) -> Real
150  {
151  Real new_comp_dt = -1.e100;
152  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
153  {
154  {
155  const Real rho = s(i, j, k, Rho_comp);
156  const Real rhotheta = s(i, j, k, RhoTheta_comp);
157 
158  // NOTE: even when moisture is present,
159  // we only use the partial pressure of the dry air
160  // to compute the soundspeed
161  Real pressure = getPgivenRTh(rhotheta);
162  Real c = std::sqrt(Gamma * pressure / rho);
163 
164  // If we are doing implicit acoustic substepping, then the z-direction does not contribute
165  // to the computation of the time step
166  if (l_implicit_substepping) {
167  if ((nxc > 1) && (nyc==1)) {
168  // 2-D in x-z
169  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
170  } else if ((nyc > 1) && (nxc==1)) {
171  // 2-D in y-z
172  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
173  } else {
174  // 3-D or SCM
175  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
176  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
177  }
178 
179  // If we are not doing implicit acoustic substepping, then the z-direction contributes
180  // to the computation of the time step
181  } else {
182  if (nxc > 1 && nyc > 1) {
183  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
184  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
185  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
186  } else if (nxc > 1) {
187  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
188  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
189  } else if (nyc > 1) {
190  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
191  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
192  } else {
193  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
194  }
195 
196  }
197  }
198  });
199  return new_comp_dt;
200  });
201  } // not EB
202 
203  ParallelDescriptor::ReduceRealMax(estdt_comp_inv);
204  estdt_comp = cfl / estdt_comp_inv;
205 
206  Real estdt_lowM_inv = ReduceMax(ccvel, 0,
207  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
208  Array4<Real const> const& u) -> Real
209  {
210  Real new_lm_dt = -1.e100;
211  Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
212  {
213  new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
214  ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
215  ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
216  });
217  return new_lm_dt;
218  });
219 
220  ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
221  if (estdt_lowM_inv > 0.0_rt)
222  estdt_lowM = cfl / estdt_lowM_inv;
223 
224  if (verbose) {
225  if (fixed_dt[level] <= 0.0) {
226  Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " <<
227  1.0/dxinv[0] << " " << 1.0/dxinv[1] << " " << dz_min[level] << std::endl;
228  Print() << "Compressible dt at level " << level << ": " << estdt_comp << std::endl;
229  if (estdt_lowM_inv > 0.0_rt) {
230  Print() << "Anelastic dt at level " << level << ": " << estdt_lowM << std::endl;
231  } else {
232  Print() << "Anelastic dt at level " << level << ": undefined " << std::endl;
233  }
234  }
235 
236  if (fixed_dt[level] > 0.0) {
237  Print() << "Based on cfl of 1.0 " << std::endl;
238  Print() << "Compressible dt at level " << level << " would be: " << estdt_comp/cfl << std::endl;
239  if (estdt_lowM_inv > 0.0_rt) {
240  Print() << "Anelastic dt at level " << level << " would be: " << estdt_lowM/cfl << std::endl;
241  } else {
242  Print() << "Anelastic dt at level " << level << " would be undefined " << std::endl;
243  }
244  Print() << "Fixed dt at level " << level << " is: " << fixed_dt[level] << std::endl;
245  if (fixed_fast_dt[level] > 0.0) {
246  Print() << "Fixed fast dt at level " << level << " is: " << fixed_fast_dt[level] << std::endl;
247  }
248  }
249  }
250 
251  if (solverChoice.substepping_type[level] != SubsteppingType::None) {
252  if (fixed_dt[level] > 0. && fixed_fast_dt[level] > 0.) {
253  dt_fast_ratio = static_cast<long>( fixed_dt[level] / fixed_fast_dt[level] );
254  } else if (fixed_dt[level] > 0.) {
255  // Max CFL_c = 1.0 for substeps by default, but we enforce a min of 4 substeps
256  auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
257  dt_fast_ratio = static_cast<long>( std::max(fixed_dt[level]/dt_sub_max,4.) );
258  } else {
259  // auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
260  // dt_fast_ratio = static_cast<long>( std::max(estdt_comp/dt_sub_max,4.) );
261  dt_fast_ratio = static_cast<long>( std::max(cfl / sub_cfl, 4.) );
262  }
263 
264  // Force time step ratio to be an even value
266  if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
267  } else {
268  if ( dt_fast_ratio%6 != 0) {
269  Print() << "mri_dt_ratio = " << dt_fast_ratio
270  << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
271  dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/6.0) * 6);
272  }
273  }
274 
275  if (verbose) {
276  Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
277  }
278  } // if substepping, either explicit or implicit
279 
280  if (fixed_dt[level] > 0.0) {
281  return fixed_dt[level];
282  } else {
283  // Anelastic (substepping is not allowed)
284  if (l_anelastic) {
285 
286  // Make sure that timestep is less than the dt_max
287  estdt_lowM = amrex::min(estdt_lowM, dt_max);
288 
289  // On the first timestep enforce dt_max_initial
290  if(istep[level] == 0){
291  return amrex::min(dt_max_initial, estdt_lowM);
292  }
293  else{
294  return estdt_lowM;
295  }
296 
297 
298  // Compressible with or without substepping
299  } else {
300  return estdt_comp;
301  }
302  }
303 }
constexpr amrex::Real Gamma
Definition: ERF_Constants.H:19
amrex::Vector< amrex::Real > dz_min
Definition: ERF.H:1202
eb_ const & get_eb(int lev) const noexcept
Definition: ERF.H:1476
amrex::Vector< amrex::Real > fixed_dt
Definition: ERF.H:945
static amrex::Real dt_max
Definition: ERF.H:942
amrex::Vector< amrex::Real > fixed_fast_dt
Definition: ERF.H:946
static amrex::Real cfl
Definition: ERF.H:937
static amrex::Real sub_cfl
Definition: ERF.H:938
Definition: ERF_EB.H:13
const std::unique_ptr< amrex::EBFArrayBoxFactory > & get_const_factory() const noexcept
Definition: ERF_EB.H:40
@ rho
Definition: ERF_Kessler.H:22
int force_stage1_single_substep
Definition: ERF_DataStruct.H:668
Here is the call graph for this function:

◆ Evolve()

void ERF::Evolve ( )
412 {
413  BL_PROFILE_VAR("ERF::Evolve()", evolve);
414 
415  Real cur_time = t_new[0];
416 
417  // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
418  // for finer levels (with or without subcycling)
419  for (int step = istep[0]; step < max_step && cur_time < stop_time; ++step)
420  {
421  if (use_datetime) {
422  Print() << "\n" << getTimestamp(cur_time, datetime_format)
423  << " (" << cur_time-start_time << " s elapsed)"
424  << std::endl;
425  }
426  Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
427 
428  ComputeDt(step);
429 
430  // Make sure we have read enough of the boundary plane data to make it through this timestep
431  if (input_bndry_planes)
432  {
433  m_r2d->read_input_files(cur_time,dt[0],m_bc_extdir_vals);
434  }
435 
436  int lev = 0;
437  int iteration = 1;
438  timeStep(lev, cur_time, iteration);
439 
440  cur_time += dt[0];
441 
442  Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
443  << " DT = " << dt[0] << std::endl;
444 
445  post_timestep(step, cur_time, dt[0]);
446 
447  if (writeNow(cur_time, dt[0], step+1, m_plot_int_1, m_plot_per_1)) {
448  last_plot_file_step_1 = step+1;
450  }
451  if (writeNow(cur_time, dt[0], step+1, m_plot_int_2, m_plot_per_2)) {
452  last_plot_file_step_2 = step+1;
454  }
455  if (writeNow(cur_time, dt[0], step+1, m_subvol_int, m_subvol_per)) {
456  last_subvol = step+1;
457  WriteSubvolume();
458  }
459 
460  if (writeNow(cur_time, dt[0], step+1, m_check_int, m_check_per)) {
461  last_check_file_step = step+1;
463  }
464 
465 #ifdef AMREX_MEM_PROFILING
466  {
467  std::ostringstream ss;
468  ss << "[STEP " << step+1 << "]";
469  MemProfiler::report(ss.str());
470  }
471 #endif
472 
473  if (cur_time >= stop_time - 1.e-6*dt[0]) break;
474  }
475 
476  // Write plotfiles at final time
477  if ( (m_plot_int_1 > 0 || m_plot_per_1 > 0.) && istep[0] > last_plot_file_step_1 ) {
479  }
480  if ( (m_plot_int_2 > 0 || m_plot_per_2 > 0.) && istep[0] > last_plot_file_step_2) {
482  }
483  if ( (m_subvol_int > 0 || m_subvol_per > 0.) && istep[0] > last_subvol) {
484  WriteSubvolume();
485  }
486 
487  if ( (m_check_int > 0 || m_check_per > 0.) && istep[0] > last_check_file_step) {
489  }
490 
491  BL_PROFILE_VAR_STOP(evolve);
492 }
AMREX_FORCE_INLINE std::string getTimestamp(const amrex::Real epoch_real, const std::string &datetime_format)
Definition: ERF_EpochTime.H:35
int last_check_file_step
Definition: ERF.H:913
int max_step
Definition: ERF.H:926
int last_plot_file_step_2
Definition: ERF.H:910
static PlotFileType plotfile_type_1
Definition: ERF.H:1071
int m_subvol_int
Definition: ERF.H:964
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_extdir_vals
Definition: ERF.H:890
amrex::Real m_plot_per_1
Definition: ERF.H:965
void WriteSubvolume()
Definition: ERF_WriteSubvolume.cpp:9
int m_plot_int_1
Definition: ERF.H:962
void post_timestep(int nstep, amrex::Real time, amrex::Real dt_lev)
Definition: ERF.cpp:496
amrex::Real m_subvol_per
Definition: ERF.H:967
amrex::Real m_check_per
Definition: ERF.H:979
int m_check_int
Definition: ERF.H:978
int last_plot_file_step_1
Definition: ERF.H:909
static int input_bndry_planes
Definition: ERF.H:1130
int last_subvol
Definition: ERF.H:911
const std::string datetime_format
Definition: ERF.H:931
bool use_datetime
Definition: ERF.H:930
static PlotFileType plotfile_type_2
Definition: ERF.H:1072
void WritePlotFile(int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:231
amrex::Real start_time
Definition: ERF.H:927
void ComputeDt(int step=-1)
Definition: ERF_ComputeTimestep.cpp:11
void WriteCheckpointFile() const
Definition: ERF_Checkpoint.cpp:26
int m_plot_int_2
Definition: ERF.H:963
std::unique_ptr< ReadBndryPlanes > m_r2d
Definition: ERF.H:1187
bool writeNow(const amrex::Real cur_time, const amrex::Real dt, const int nstep, const int plot_int, const amrex::Real plot_per)
Definition: ERF.cpp:2153
void timeStep(int lev, amrex::Real time, int iteration)
Definition: ERF_TimeStep.cpp:17
amrex::Real m_plot_per_2
Definition: ERF.H:966

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fill_from_bndryregs()

void ERF::fill_from_bndryregs ( const amrex::Vector< amrex::MultiFab * > &  mfs,
amrex::Real  time 
)
14 {
15  //
16  // We now assume that if we read in on one face, we read in on all faces
17  //
18  AMREX_ALWAYS_ASSERT(m_r2d);
19 
20  int lev = 0;
21  const Box& domain = geom[lev].Domain();
22 
23  const auto& dom_lo = lbound(domain);
24  const auto& dom_hi = ubound(domain);
25 
26  Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time);
27 
28  const BCRec* bc_ptr = domain_bcs_type_d.data();
29 
30  // xlo: ori = 0
31  // ylo: ori = 1
32  // zlo: ori = 2
33  // xhi: ori = 3
34  // yhi: ori = 4
35  // zhi: ori = 5
36  const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
37  const auto& bdatylo = (*bndry_data[1])[lev].const_array();
38  const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
39  const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
40 
41  int bccomp;
42 
43  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
44  {
45  MultiFab& mf = *mfs[var_idx];
46  const int icomp = 0;
47  const int ncomp = mf.nComp();
48 
49  if (var_idx == Vars::xvel) {
50  bccomp = BCVars::xvel_bc;
51  } else if (var_idx == Vars::yvel) {
52  bccomp = BCVars::yvel_bc;
53  } else if (var_idx == Vars::zvel) {
54  bccomp = BCVars::zvel_bc;
55  } else if (var_idx == Vars::cons) {
56  bccomp = BCVars::cons_bc;
57  }
58 
59 #ifdef AMREX_USE_OMP
60 #pragma omp parallel if (Gpu::notInLaunchRegion())
61 #endif
62  for (MFIter mfi(mf); mfi.isValid(); ++mfi)
63  {
64  const Array4<Real>& dest_arr = mf.array(mfi);
65  Box bx = mfi.growntilebox();
66 
67  // x-faces
68  {
69  Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
70  if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
71 
72  Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
73  if (var_idx == Vars::xvel) bx_xhi.setSmall(0,dom_hi.x);
74 
75  ParallelFor(
76  bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
77  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
78  BCVars::RhoScalar_bc_comp : icomp+n;
79  if (bc_ptr[bc_comp].lo(0) == ERFBCType::ext_dir_ingested) {
80  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
81  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
82  dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
83  }
84  },
85  bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
86  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
87  BCVars::RhoScalar_bc_comp : icomp+n;
88  if (bc_ptr[bc_comp].hi(0) == ERFBCType::ext_dir_ingested) {
89  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
90  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
91  dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
92  }
93  }
94  );
95  } // x-faces
96 
97  // y-faces
98  {
99  Box bx_ylo(bx); bx_ylo.setBig (1,dom_lo.y-1);
100  if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
101 
102  Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
103  if (var_idx == Vars::yvel) bx_yhi.setSmall(1,dom_hi.y);
104 
105  ParallelFor(
106  bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
107  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
108  BCVars::RhoScalar_bc_comp : icomp+n;
109  if (bc_ptr[bc_comp].lo(1) == ERFBCType::ext_dir_ingested) {
110  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
111  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
112  dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
113  }
114  },
115  bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
116  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
117  BCVars::RhoScalar_bc_comp : icomp+n;
118  if (bc_ptr[bc_comp].hi(1) == ERFBCType::ext_dir_ingested) {
119  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
120  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
121  dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
122  }
123  }
124  );
125  } // y-faces
126  } // mf
127  } // var_idx
128 }
#define RhoScalar_comp
Definition: ERF_IndexDefines.H:40
#define NSCALARS
Definition: ERF_IndexDefines.H:16
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
Definition: ERF.H:884
@ RhoScalar_bc_comp
Definition: ERF_IndexDefines.H:80
@ ext_dir_ingested
Definition: ERF_IndexDefines.H:204

◆ fill_rhs()

void ERF::fill_rhs ( amrex::MultiFab &  rhs_mf,
const amrex::MultiFab &  state_mf,
amrex::Real  time,
const amrex::Geometry &  geom 
)
private

◆ FillBdyCCVels()

void ERF::FillBdyCCVels ( amrex::Vector< amrex::MultiFab > &  mf_cc_vel,
int  levc = 0 
)
12 {
13  // Impose bc's at domain boundaries
14  for (int ilev(0); ilev < mf_cc_vel.size(); ++ilev)
15  {
16  int lev = ilev + levc;
17  Box domain(Geom(lev).Domain());
18 
19  int ihi = domain.bigEnd(0);
20  int jhi = domain.bigEnd(1);
21  int khi = domain.bigEnd(2);
22 
23  // Impose periodicity first
24  mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
25 
26  int jper = (Geom(lev).isPeriodic(1));
27  int kper = (Geom(lev).isPeriodic(2));
28 
29  for (MFIter mfi(mf_cc_vel[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
30  {
31  const Box& bx = mfi.tilebox();
32  const Array4<Real>& vel_arr = mf_cc_vel[lev].array(mfi);
33 
34  if (!Geom(lev).isPeriodic(0)) {
35  // Low-x side
36  if (bx.smallEnd(0) <= domain.smallEnd(0)) {
37  Real multn = ( (phys_bc_type[0] == ERF_BC::slip_wall ) ||
39  (phys_bc_type[0] == ERF_BC::symmetry ) ) ? -1. : 1.;
40  Real multt = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -1. : 1.;
41  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
42  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
43  {
44  vel_arr(-1,j,k,0) = multn*vel_arr(0,j,k,0); // u
45  vel_arr(-1,j,k,1) = multt*vel_arr(0,j,k,1); // v
46  vel_arr(-1,j,k,2) = multt*vel_arr(0,j,k,2); // w
47  });
48  }
49 
50  // High-x side
51  if (bx.bigEnd(0) >= domain.bigEnd(0)) {
52  Real multn = ( (phys_bc_type[3] == ERF_BC::slip_wall ) ||
54  (phys_bc_type[3] == ERF_BC::symmetry ) ) ? -1. : 1.;
55  Real multt = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -1. : 1.;
56  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
57  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
58  {
59  vel_arr(ihi+1,j,k,0) = multn*vel_arr(ihi,j,k,0); // u
60  vel_arr(ihi+1,j,k,1) = multt*vel_arr(ihi,j,k,1); // v
61  vel_arr(ihi+1,j,k,2) = multt*vel_arr(ihi,j,k,2); // w
62  });
63  }
64  } // !periodic
65 
66  if (!Geom(lev).isPeriodic(1)) {
67  // Low-y side
68  if (bx.smallEnd(1) <= domain.smallEnd(1)) {
69  Real multn = ( (phys_bc_type[1] == ERF_BC::slip_wall ) ||
71  (phys_bc_type[1] == ERF_BC::symmetry ) ) ? -1. : 1.;
72  Real multt = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -1. : 1.;
73  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
74  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
75  {
76  vel_arr(i,-1,k,0) = multt*vel_arr(i,0,k,0); // u
77  vel_arr(i,-1,k,1) = multn*vel_arr(i,0,k,1); // u
78  vel_arr(i,-1,k,2) = multt*vel_arr(i,0,k,2); // w
79  });
80  }
81 
82  // High-y side
83  if (bx.bigEnd(1) >= domain.bigEnd(1)) {
84  Real multn = ( (phys_bc_type[4] == ERF_BC::slip_wall ) ||
86  (phys_bc_type[4] == ERF_BC::symmetry ) ) ? -1. : 1.;
87  Real multt = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -1. : 1.;
88  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
89  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
90  {
91  vel_arr(i,jhi+1,k,0) = multt*vel_arr(i,jhi,k,0); // u
92  vel_arr(i,jhi+1,k,1) = multn*vel_arr(i,jhi,k,1); // v
93  vel_arr(i,jhi+1,k,2) = multt*vel_arr(i,jhi,k,2); // w
94  });
95  }
96  } // !periodic
97 
98  if (!Geom(lev).isPeriodic(2)) {
99  // Low-z side
100  if (bx.smallEnd(2) <= domain.smallEnd(2)) {
101  Real multn = ( (phys_bc_type[2] == ERF_BC::slip_wall ) ||
103  (phys_bc_type[2] == ERF_BC::symmetry ) ) ? -1. : 1.;
104  Real multt = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -1. : 1.;
105  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
106  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
107  {
108  vel_arr(i,j,-1,0) = multt*vel_arr(i,j,0,0); // u
109  vel_arr(i,j,-1,1) = multt*vel_arr(i,j,0,1); // v
110  vel_arr(i,j,-1,2) = multn*vel_arr(i,j,0,2); // w
111  });
112  }
113 
114  // High-z side
115  if (bx.bigEnd(2) >= domain.bigEnd(2)) {
116  Real multn = ( (phys_bc_type[5] == ERF_BC::slip_wall ) ||
118  (phys_bc_type[5] == ERF_BC::symmetry ) ) ? -1. : 1.;
119  Real multt = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -1. : 1.;
120  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
121  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
122  {
123  vel_arr(i,j,khi+1,0) = multt*vel_arr(i,j,khi,0); // u
124  vel_arr(i,j,khi+1,1) = multt*vel_arr(i,j,khi,1); // v
125  vel_arr(i,j,khi+1,2) = multn*vel_arr(i,j,khi,2); // w
126  });
127  }
128  } // !periodic
129  } // MFIter
130 
131  // Impose periodicity again
132  mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
133  } // lev
134 }
@ no_slip_wall

◆ FillCoarsePatch()

void ERF::FillCoarsePatch ( int  lev,
amrex::Real  time 
)
private
22 {
23  BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
24  AMREX_ASSERT(lev > 0);
25 
26  //
27  //****************************************************************************************************************
28  // First fill velocities and density at the COARSE level so we can convert velocity to momenta at the COARSE level
29  //****************************************************************************************************************
30  //
31  bool cons_only = false;
32  if (lev == 1) {
33  FillPatch(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
34  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
35  cons_only);
36  } else {
37  FillPatch(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
38  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
39  {&vars_new[lev-1][Vars::cons],
40  &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
41  base_state[lev-1], base_state[lev-1],
42  false, cons_only);
43  }
44 
45  //
46  // ************************************************
47  // Convert velocity to momentum at the COARSE level
48  // ************************************************
49  //
50  VelocityToMomentum(vars_new[lev-1][Vars::xvel], IntVect{0},
51  vars_new[lev-1][Vars::yvel], IntVect{0},
52  vars_new[lev-1][Vars::zvel], IntVect{0},
53  vars_new[lev-1][Vars::cons],
54  rU_new[lev-1],
55  rV_new[lev-1],
56  rW_new[lev-1],
57  Geom(lev).Domain(),
59  //
60  // *****************************************************************
61  // Interpolate all cell-centered variables from coarse to fine level
62  // *****************************************************************
63  //
64  Interpolater* mapper_c = &cell_cons_interp;
65  Interpolater* mapper_f = &face_cons_linear_interp;
66 
67  //
68  //************************************************************************************************
69  // Interpolate cell-centered data from coarse to fine level
70  // with InterpFromCoarseLevel which ASSUMES that all ghost cells have already been filled
71  // ************************************************************************************************
72  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
73  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
74 
75  InterpFromCoarseLevel(vars_new[lev ][Vars::cons], ngvect_cons, IntVect(0,0,0),
76  vars_new[lev-1][Vars::cons], 0, 0, ncomp_cons,
77  geom[lev-1], geom[lev],
78  refRatio(lev-1), mapper_c, domain_bcs_type, BCVars::cons_bc);
79 
80  // ***************************************************************************
81  // Physical bc's for cell centered variables at domain boundary
82  // ***************************************************************************
84  0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc,true);
85 
86  //
87  //************************************************************************************************
88  // Interpolate x-momentum from coarse to fine level
89  // with InterpFromCoarseLevel which ASSUMES that all ghost cells have already been filled
90  // ************************************************************************************************
91  //
92  InterpFromCoarseLevel(rU_new[lev], IntVect{0}, IntVect{0}, rU_new[lev-1], 0, 0, 1,
93  geom[lev-1], geom[lev],
94  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::xvel_bc);
95 
96  //
97  //************************************************************************************************
98  // Interpolate y-momentum from coarse to fine level
99  // with InterpFromCoarseLevel which ASSUMES that all ghost cells have already been filled
100  // ************************************************************************************************
101  //
102  InterpFromCoarseLevel(rV_new[lev], IntVect{0}, IntVect{0}, rV_new[lev-1], 0, 0, 1,
103  geom[lev-1], geom[lev],
104  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::yvel_bc);
105 
106  //************************************************************************************************
107  // Interpolate z-momentum from coarse to fine level
108  // with InterpFromCoarseLevel which ASSUMES that all ghost cells have already been filled
109  // ************************************************************************************************
110  InterpFromCoarseLevel(rW_new[lev], IntVect{0}, IntVect{0}, rW_new[lev-1], 0, 0, 1,
111  geom[lev-1], geom[lev],
112  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::zvel_bc);
113  //
114  // *********************************************************
115  // After interpolation of momentum, convert back to velocity
116  // *********************************************************
117  //
118  for (int which_lev = lev-1; which_lev <= lev; which_lev++)
119  {
121  vars_new[which_lev][Vars::yvel],
122  vars_new[which_lev][Vars::zvel],
123  vars_new[which_lev][Vars::cons],
124  rU_new[which_lev],
125  rV_new[which_lev],
126  rW_new[which_lev],
127  Geom(lev).Domain(),
129  }
130 
131  // ***************************************************************************
132  // Physical bc's at domain boundary
133  // ***************************************************************************
134  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
135 
137  ngvect_vels,time,BCVars::xvel_bc,true);
139  ngvect_vels,time,BCVars::yvel_bc,true);
141  ngvect_vels,time,BCVars::zvel_bc,true);
142 
143  // ***************************************************************************
144  // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
145  // ***************************************************************************
146 }
void FillCoarsePatch(int lev, amrex::Real time)
Definition: ERF_FillCoarsePatch.cpp:21
Here is the call graph for this function:

◆ FillIntermediatePatch()

void ERF::FillIntermediatePatch ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
int  ng_cons,
int  ng_vel,
bool  cons_only,
int  icomp_cons,
int  ncomp_cons 
)
private
33 {
34  BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
35  Interpolater* mapper;
36 
37  PhysBCFunctNoOp null_bc;
38 
39  //
40  // ***************************************************************************
41  // The first thing we do is interpolate the momenta on the "valid" faces of
42  // the fine grids (where the interface is coarse/fine not fine/fine) -- this
43  // will not be over-written by interpolation below because the FillPatch
44  // operators see these as valid faces. But we must have these interpolated
45  // values in the fine data before we call FillPatchTwoLevels.
46  //
47  // Also -- note that we might be filling values by interpolation at physical boundaries
48  // here but that's ok because we will overwrite those values when we impose
49  // the physical bc's below
50  // ***************************************************************************
51  if (lev>0) {
52  if (cf_set_width > 0) {
53  // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
54  FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
55  }
56  if ( !cons_only && (cf_set_width >= 0) ) {
57  FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
58  FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
59  FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
60  }
61  }
62 
63  // amrex::Print() << "LEVEL " << lev << " CONS ONLY " << cons_only <<
64  // " ICOMP NCOMP " << icomp_cons << " " << ncomp_cons << " NGHOST " << ng_cons << std::endl;
65 
66  if (!cons_only) {
67  AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
68  AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
69  }
70 
71  // Enforce no penetration for thin immersed body
72  if (!cons_only) {
73  // Enforce no penetration for thin immersed body
74  if (xflux_imask[lev]) {
75  ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
76  }
77  if (yflux_imask[lev]) {
78  ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
79  }
80  if (zflux_imask[lev]) {
81  ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
82  }
83  }
84 
85  //
86  // We now start working on conserved quantities + VELOCITY
87  //
88  if (lev == 0)
89  {
90  // We don't do anything here because we will call the physbcs routines below,
91  // which calls FillBoundary and fills other domain boundary conditions
92  // Physical boundaries will be filled below
93 
94  if (!cons_only)
95  {
96  // ***************************************************************************
97  // We always come in to this call with updated momenta but we need to create updated velocity
98  // in order to impose the rest of the bc's
99  // ***************************************************************************
100  // This only fills VALID region of velocity
101  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
102  *mfs_vel[Vars::cons],
103  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
104  Geom(lev).Domain(), domain_bcs_type);
105  }
106  }
107  else
108  {
109  //
110  // We must fill a temporary then copy it back so we don't double add/subtract
111  //
112  MultiFab mf(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
113  mfs_vel[Vars::cons]->nComp() ,mfs_vel[Vars::cons]->nGrowVect());
114  //
115  // Set all components to 1.789e19, then copy just the density from *mfs_vel[Vars::cons]
116  //
117  mf.setVal(1.789e19);
118  MultiFab::Copy(mf,*mfs_vel[Vars::cons],Rho_comp,Rho_comp,1,mf.nGrowVect());
119 
120  Vector<MultiFab*> fmf = {mfs_vel[Vars::cons],mfs_vel[Vars::cons]};
121  Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
122  Vector<Real> ctime = {t_old[lev-1], t_new[lev-1]};
123  Vector<Real> ftime = {time,time};
124 
125  // Impose physical bc's on fine data (note time and 0 are not used)
126  bool do_fb = true; bool do_terrain_adjustment = false;
127  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
128  icomp_cons,ncomp_cons,IntVect{ng_cons},time,BCVars::cons_bc,
129  do_fb, do_terrain_adjustment);
130 
131  if ( (icomp_cons+ncomp_cons > 1) && (interpolation_type == StateInterpType::Perturbational) )
132  {
133  // Divide (rho theta) by rho to get theta
134  MultiFab::Divide(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
135 
136  // Subtract theta_0 from theta
137  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
138 
139  if (!amrex::almostEqual(time,ctime[1])) {
140  MultiFab::Divide(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
141  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
142  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
143  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
144  }
145  if (!amrex::almostEqual(time,ctime[0])) {
146  MultiFab::Divide(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
147  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
148  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
149  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
150  }
151  }
152 
153  // Subtract rho_0 from rho before we interpolate -- note we only subtract
154  // on valid region of mf since the ghost cells will be filled below
155  if (icomp_cons == 0 && (interpolation_type == StateInterpType::Perturbational))
156  {
157  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
158 
159  if (!amrex::almostEqual(time,ctime[1])) {
160  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
161  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
162  }
163  if (!amrex::almostEqual(time,ctime[0])) {
164  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
165  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
166  }
167  }
168 
169  // Call FillPatchTwoLevels which ASSUMES that all ghost cells have already been filled
170  mapper = &cell_cons_interp;
171  FillPatchTwoLevels(mf, IntVect{ng_cons}, IntVect(0,0,0),
172  time, cmf, ctime, fmf, ftime,
173  icomp_cons, icomp_cons, ncomp_cons, geom[lev-1], geom[lev],
174  refRatio(lev-1), mapper, domain_bcs_type,
175  icomp_cons);
176 
177  if (icomp_cons == 0 && (interpolation_type == StateInterpType::Perturbational))
178  {
179  // Restore the coarse values to what they were
180  if (!amrex::almostEqual(time,ctime[1])) {
181  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
182  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
183  }
184  if (!amrex::almostEqual(time,ctime[0])) {
185  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
186  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
187  }
188 
189  // Set values in the cells outside the domain boundary so that we can do the Add
190  // without worrying about uninitialized values outside the domain -- these
191  // will be filled in the physbcs call
192  mf.setDomainBndry(1.234e20,Rho_comp,1,geom[lev]);
193 
194  // Add rho_0 back to rho after we interpolate -- on all the valid + ghost region
195  MultiFab::Add(mf, base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{ng_cons});
196  }
197 
198  if ( (icomp_cons+ncomp_cons > 1) && (interpolation_type == StateInterpType::Perturbational) )
199  {
200  // Add theta_0 to theta
201  if (!amrex::almostEqual(time,ctime[1])) {
202  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
203  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
204  MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
205  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
206  }
207  if (!amrex::almostEqual(time,ctime[0])) {
208  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
209  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
210  MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
211  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
212  }
213 
214  // Multiply theta by rho to get (rho theta)
215  MultiFab::Multiply(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
216 
217  // Add theta_0 to theta
218  MultiFab::Add(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
219 
220  // Add theta_0 back to theta
221  MultiFab::Add(mf,base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{ng_cons});
222 
223  // Multiply (theta) by rho to get (rho theta)
224  MultiFab::Multiply(mf,mf,Rho_comp,RhoTheta_comp,1,IntVect{ng_cons});
225  }
226 
227  // Make sure to only copy back the components we worked on
228  MultiFab::Copy(*mfs_vel[Vars::cons],mf,icomp_cons,icomp_cons,ncomp_cons,IntVect{ng_cons});
229 
230  // *****************************************************************************************
231 
232  if (!cons_only)
233  {
234  // ***************************************************************************
235  // We always come in to this call with updated momenta but we need to create updated velocity
236  // in order to impose the rest of the bc's
237  // ***************************************************************************
238  // This only fills VALID region of velocity
239  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
240  *mfs_vel[Vars::cons],
241  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
242  Geom(lev).Domain(), domain_bcs_type);
243 
244  mapper = &face_cons_linear_interp;
245 
246  //
247  // NOTE: All interpolation here happens on velocities not momenta;
248  // note we only do the interpolation and FillBoundary here,
249  // physical bc's are imposed later
250  //
251  // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
252  // unlike the FillSet calls above which filled momenta on the coarse/fine bdy
253  //
254 
255  MultiFab& mfu = *mfs_vel[Vars::xvel];
256 
257  fmf = {&mfu,&mfu};
258  cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
259 
260  // Call FillPatchTwoLevels which ASSUMES that all ghost cells have already been filled
261  FillPatchTwoLevels(mfu, IntVect{ng_vel}, IntVect(0,0,0),
262  time, cmf, ctime, fmf, ftime,
263  0, 0, 1, geom[lev-1], geom[lev],
264  refRatio(lev-1), mapper, domain_bcs_type,
266 
267  // *****************************************************************************************
268 
269  MultiFab& mfv = *mfs_vel[Vars::yvel];
270 
271  fmf = {&mfv,&mfv};
272  cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
273 
274  // Call FillPatchTwoLevels which ASSUMES that all ghost cells have already been filled
275  FillPatchTwoLevels(mfv, IntVect{ng_vel}, IntVect(0,0,0),
276  time, cmf, ctime, fmf, ftime,
277  0, 0, 1, geom[lev-1], geom[lev],
278  refRatio(lev-1), mapper, domain_bcs_type,
280 
281  // *****************************************************************************************
282 
283  MultiFab& mfw = *mfs_vel[Vars::zvel];
284 
285  fmf = {&mfw,&mfw};
286  cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
287 
288  // Call FillPatchTwoLevels which ASSUMES that all ghost cells have already been filled
289  FillPatchTwoLevels(mfw, IntVect{ng_vel}, IntVect(0,0,0),
290  time, cmf, ctime, fmf, ftime,
291  0, 0, 1, geom[lev-1], geom[lev],
292  refRatio(lev-1), mapper, domain_bcs_type,
294  } // !cons_only
295  } // lev > 0
296 
297  // ***************************************************************************
298  // Physical bc's at domain boundary
299  // ***************************************************************************
300  IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
301  IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
302 
303  bool do_fb = true;
304 
305 #ifdef ERF_USE_NETCDF
306  // We call this here because it is an ERF routine
307  if (solverChoice.use_real_bcs && (lev==0)) {
308  fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
309  do_fb = false;
310  }
311 #endif
312 
313  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
314 
315  // We call this even if use_real_bcs is true because these will fill the vertical bcs
316  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
317  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
318  if (!cons_only) {
319  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
320  ngvect_vels,time,BCVars::xvel_bc, do_fb);
321  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
322  ngvect_vels,time,BCVars::yvel_bc, do_fb);
323  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
324  ngvect_vels,time,BCVars::zvel_bc, do_fb);
325  }
326  // ***************************************************************************
327 
328  // We always come in to this call with momenta so we need to leave with momenta!
329  // We need to make sure we convert back on all ghost cells/faces because this is
330  // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
331  if (!cons_only)
332  {
333  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
334  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
335  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
336 
337  VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
338  *mfs_vel[Vars::yvel], ngv,
339  *mfs_vel[Vars::zvel], ngw,
340  *mfs_vel[Vars::cons],
341  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
342  Geom(lev).Domain(),
344  }
345 
346  mfs_mom[IntVars::cons]->FillBoundary(geom[lev].periodicity());
347  mfs_mom[IntVars::xmom]->FillBoundary(geom[lev].periodicity());
348  mfs_mom[IntVars::ymom]->FillBoundary(geom[lev].periodicity());
349  mfs_mom[IntVars::zmom]->FillBoundary(geom[lev].periodicity());
350 }
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:387
void fill_from_bndryregs(const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
Definition: ERF_BoundaryConditionsBndryReg.cpp:13
void FillIntermediatePatch(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
Definition: ERF_FillIntermediatePatch.cpp:28
@ NumTypes
Definition: ERF_IndexDefines.H:154
static bool use_real_bcs
Definition: ERF_DataStruct.H:652
Here is the call graph for this function:

◆ FillPatch() [1/2]

void ERF::FillPatch ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
bool  cons_only = false 
)
private

◆ FillPatch() [2/2]

void ERF::FillPatch ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
const amrex::MultiFab &  old_base_state,
const amrex::MultiFab &  new_base_state,
bool  fillset = true,
bool  cons_only = false 
)
private

◆ get_eb()

eb_ const& ERF::get_eb ( int  lev) const
inlineprivatenoexcept
1476  {
1477  AMREX_ASSERT(lev >= 0 && lev < eb.size() && eb[lev] != nullptr);
1478  return *eb[lev];
1479  }

◆ get_projection_bc()

Array< LinOpBCType, AMREX_SPACEDIM > ERF::get_projection_bc ( amrex::Orientation::Side  side) const
noexcept
18 {
19  amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> r;
20  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
21  if (geom[0].isPeriodic(dir)) {
22  r[dir] = LinOpBCType::Periodic;
23  } else {
24  auto bc_type = domain_bc_type[Orientation(dir,side)];
25  if (bc_type == "Outflow") {
26  r[dir] = LinOpBCType::Dirichlet;
27  } else
28  {
29  r[dir] = LinOpBCType::Neumann;
30  }
31  }
32  }
33  return r;
34 }
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
Definition: ERF.H:887

◆ getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int  lev)
inlineprivate
1265  {
1266  return advflux_reg[lev];
1267  }

◆ getCPUTime()

static amrex::Real ERF::getCPUTime ( )
inlinestaticprivate
1357  {
1358  int numCores = amrex::ParallelDescriptor::NProcs();
1359 #ifdef _OPENMP
1360  numCores = numCores * omp_get_max_threads();
1361 #endif
1362 
1363  amrex::Real T =
1364  numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
1366 
1367  return T;
1368  }
static amrex::Real previousCPUTimeUsed
Definition: ERF.H:1353
static amrex::Real startCPUTime
Definition: ERF.H:1352
@ T
Definition: ERF_IndexDefines.H:110

◆ GotoNextLine()

void ERF::GotoNextLine ( std::istream &  is)
staticprivate

Utility to skip to next line in Header file input stream.

17 {
18  constexpr std::streamsize bl_ignore_max { 100000 };
19  is.ignore(bl_ignore_max, '\n');
20 }

◆ HurricaneTracker()

void ERF::HurricaneTracker ( int  lev,
const amrex::MultiFab &  U_new,
const amrex::MultiFab &  V_new,
const amrex::MultiFab &  W_new,
const amrex::Real  velmag_threshold,
const bool  is_track_io,
amrex::TagBoxArray *  tags = nullptr 
)
398 {
399  const auto dx = geom[levc].CellSizeArray();
400  const auto prob_lo = geom[levc].ProbLoArray();
401 
402  const int ncomp = AMREX_SPACEDIM; // Number of components (3 for 3D)
403 
404  Gpu::DeviceVector<Real> d_coords(3, 0.0); // Initialize to -1
405  Real* d_coords_ptr = d_coords.data(); // Get pointer to device vector
406  Gpu::DeviceVector<int> d_found(1,0);
407  int* d_found_ptr = d_found.data();
408 
409  MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
410  average_face_to_cellcenter(mf_cc_vel,0,{AMREX_D_DECL(&U_new,&V_new,&W_new)},0);
411 
412  // Loop through MultiFab using MFIter
413  for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi) {
414  const Box& box = mfi.validbox(); // Get the valid box for the current MFIter
415  const Array4<const Real>& vel_arr = mf_cc_vel.const_array(mfi); // Get the array for this MFIter
416 
417  // ParallelFor loop to check velocity magnitudes on the GPU
418  amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
419  // Access velocity components using ncomp
420  Real magnitude = 0.0; // Initialize magnitude
421 
422  for (int comp = 0; comp < ncomp; ++comp) {
423  Real vel = vel_arr(i, j, k, comp); // Access the component for each (i, j, k)
424  magnitude += vel * vel; // Sum the square of the components
425  }
426 
427  magnitude = std::sqrt(magnitude)*3.6; // Calculate magnitude
428  Real x = prob_lo[0] + (i + 0.5) * dx[0];
429  Real y = prob_lo[1] + (j + 0.5) * dx[1];
430  Real z = prob_lo[2] + (k + 0.5) * dx[2];
431 
432  // Check if magnitude exceeds threshold
433  if (z < 2.0e3 && magnitude > velmag_threshold) {
434  // Use atomic operations to set found flag and store coordinates
435  Gpu::Atomic::Add(&d_found_ptr[0], 1); // Mark as found
436 
437  // Store coordinates
438  Gpu::Atomic::Add(&d_coords_ptr[0],x); // Store x index
439  Gpu::Atomic::Add(&d_coords_ptr[1],y); // Store x index
440  Gpu::Atomic::Add(&d_coords_ptr[2],z); // Store x index
441  }
442  });
443  }
444 
445  // Synchronize to ensure all threads complete their execution
446  amrex::Gpu::streamSynchronize(); // Wait for all GPU threads to finish
447 
448  Vector<int> h_found(1,0);
449  Gpu::copy(Gpu::deviceToHost, d_found.begin(), d_found.end(), h_found.begin());
450  ParallelAllReduce::Sum(h_found.data(),
451  h_found.size(),
452  ParallelContext::CommunicatorAll());
453 
454  Real eye_x, eye_y;
455  // Broadcast coordinates if found
456  if (h_found[0] > 0) {
457  Vector<Real> h_coords(3,-1e10);
458  Gpu::copy(Gpu::deviceToHost, d_coords.begin(), d_coords.end(), h_coords.begin());
459 
460  ParallelAllReduce::Sum(h_coords.data(),
461  h_coords.size(),
462  ParallelContext::CommunicatorAll());
463 
464  eye_x = h_coords[0]/h_found[0];
465  eye_y = h_coords[1]/h_found[0];
466 
467  // Data structure to hold the hurricane track for I/O
468  if (amrex::ParallelDescriptor::IOProcessor() and is_track_io) {
469  hurricane_track_xy.push_back({eye_x, eye_y});
470  }
471 
472  if(is_track_io) {
473  return;
474  }
475 
476  Real rad_tag = 3e5*std::pow(2, max_level-1-levc);
477 
478  for (MFIter mfi(*tags); mfi.isValid(); ++mfi) {
479  TagBox& tag = (*tags)[mfi];
480  auto tag_arr = tag.array(); // Get device-accessible array
481 
482  const Box& tile_box = mfi.tilebox(); // The box for this tile
483 
484  ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
485  // Compute cell center coordinates
486  Real x = prob_lo[0] + (i + 0.5) * dx[0];
487  Real y = prob_lo[1] + (j + 0.5) * dx[1];
488 
489  Real dist = std::sqrt((x - eye_x)*(x - eye_x) + (y - eye_y)*(y - eye_y));
490 
491  if (dist < rad_tag) {
492  tag_arr(i,j,k) = TagBox::SET;
493  }
494  });
495  }
496  }
497 }
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_track_xy
Definition: ERF.H:161

◆ ImposeBCsOnPhi()

void ERF::ImposeBCsOnPhi ( int  lev,
amrex::MultiFab &  phi 
)

Impose bc's on the pressure that comes out of the solve

13 {
14  BL_PROFILE("ERF::ImposeBCsOnPhi()");
15 
16  auto const dom_lo = lbound(geom[lev].Domain());
17  auto const dom_hi = ubound(geom[lev].Domain());
18 
19  phi.FillBoundary(geom[lev].periodicity());
20 
21  // ****************************************************************************
22  // Impose bc's on pprime
23  // ****************************************************************************
24 #ifdef _OPENMP
25 #pragma omp parallel if (Gpu::notInLaunchRegion())
26 #endif
27  for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi)
28  {
29  Array4<Real> const& pp_arr = phi.array(mfi);
30  Box const& bx = mfi.tilebox();
31  auto const bx_lo = lbound(bx);
32  auto const bx_hi = ubound(bx);
33 
34  if (bx_lo.x == dom_lo.x) {
35  auto bc_type = domain_bc_type[Orientation(0,Orientation::low)];
36  if (bc_type == "Outflow" || bc_type == "Open") {
37  ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
38  {
39  pp_arr(i-1,j,k) = -pp_arr(i,j,k);
40  });
41  } else {
42  ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
43  {
44  pp_arr(i-1,j,k) = pp_arr(i,j,k);
45  });
46  }
47  }
48  if (bx_hi.x == dom_hi.x) {
49  auto bc_type = domain_bc_type[Orientation(0,Orientation::high)];
50  if (bc_type == "Outflow" || bc_type == "Open") {
51  ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
52  {
53  pp_arr(i+1,j,k) = -pp_arr(i,j,k);
54  });
55  } else {
56  ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
57  {
58  pp_arr(i+1,j,k) = pp_arr(i,j,k);
59  });
60  }
61  }
62 
63  if (bx_lo.y == dom_lo.y) {
64  auto bc_type = domain_bc_type[Orientation(1,Orientation::low)];
65  Box ybx(bx); ybx.grow(0,1); // Grow in x-dir because we have filled that above
66  if (bc_type == "Outflow" || bc_type == "Open") {
67  ParallelFor(makeSlab(ybx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
68  {
69  pp_arr(i,j-1,k) = -pp_arr(i,j,k);
70  });
71  } else {
72  ParallelFor(makeSlab(ybx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
73  {
74  pp_arr(i,j-1,k) = pp_arr(i,j,k);
75  });
76  }
77  }
78  if (bx_hi.y == dom_hi.y) {
79  auto bc_type = domain_bc_type[Orientation(1,Orientation::high)];
80  Box ybx(bx); ybx.grow(0,1); // Grow in x-dir because we have filled that above
81  if (bc_type == "Outflow" || bc_type == "Open") {
82  ParallelFor(makeSlab(ybx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
83  {
84  pp_arr(i,j+1,k) = -pp_arr(i,j,k);
85  });
86  } else {
87  ParallelFor(makeSlab(ybx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
88  {
89  pp_arr(i,j+1,k) = pp_arr(i,j,k);
90  });
91  }
92  }
93 
94  Box zbx(bx); zbx.grow(0,1); zbx.grow(1,1); // Grow in x-dir and y-dir because we have filled that above
95  if (bx_lo.z == dom_lo.z) {
96  ParallelFor(makeSlab(zbx,2,dom_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
97  {
98  pp_arr(i,j,k-1) = pp_arr(i,j,k);
99  });
100  }
101  if (bx_hi.z == dom_hi.z) {
102  ParallelFor(makeSlab(zbx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
103  {
104  pp_arr(i,j,k+1) = pp_arr(i,j,k);
105  });
106  }
107  } // mfi
108 
109  // Now overwrite with periodic fill outside domain and fine-fine fill inside
110  phi.FillBoundary(geom[lev].periodicity());
111 }

◆ init1DArrays()

void ERF::init1DArrays ( )
private

◆ init_bcs()

void ERF::init_bcs ( )
private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, InflowOutflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

21 {
22  bool rho_read = false;
23  bool read_prim_theta = true;
24  Vector<Real> cons_dir_init(NBCVAR_max,0.0);
25  cons_dir_init[BCVars::Rho_bc_comp] = 1.0;
26  cons_dir_init[BCVars::RhoTheta_bc_comp] = -1.0;
27  auto f = [this,&rho_read,&read_prim_theta] (std::string const& bcid, Orientation ori)
28  {
29  // These are simply defaults for Dirichlet faces -- they should be over-written below
31  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = -1.0; // It is important to set this negative
32  // because the sign is tested on below
33  for (int n = BCVars::RhoKE_bc_comp; n < BCVars::xvel_bc; n++) {
34  m_bc_extdir_vals[n][ori] = 0.0;
35  }
36 
37  m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.0; // default
40 
41  // These are simply defaults for Neumann gradients -- they should be over-written below
44 
53 
57 
58  std::string pp_text = pp_prefix + "." + bcid;
59  ParmParse pp(pp_text);
60 
61  std::string bc_type_in;
62  if (pp.query("type", bc_type_in) <= 0)
63  {
64  pp_text = bcid;
65  pp = ParmParse(pp_text);
66  pp.query("type", bc_type_in);
67  }
68 
69  std::string bc_type = amrex::toLower(bc_type_in);
70 
71  if (bc_type == "symmetry")
72  {
73  // Print() << bcid << " set to symmetry.\n";
75  domain_bc_type[ori] = "Symmetry";
76  }
77  else if (bc_type == "outflow")
78  {
79  // Print() << bcid << " set to outflow.\n";
81  domain_bc_type[ori] = "Outflow";
82  }
83  else if (bc_type == "open")
84  {
85  // Print() << bcid << " set to open.\n";
86  AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
88  domain_bc_type[ori] = "Open";
89  }
90  else if (bc_type == "ho_outflow")
91  {
93  domain_bc_type[ori] = "HO_Outflow";
94  }
95 
96  else if (bc_type == "inflow" || bc_type == "inflow_outflow")
97  {
98  if (bc_type == "inflow") {
99  // Print() << bcid << " set to inflow.\n";
101  domain_bc_type[ori] = "Inflow";
102  } else {
103  // Print() << bcid << " set to inflow_outflow.\n";
105  domain_bc_type[ori] = "InflowOutflow";
106  }
107 
108  std::vector<Real> v;
109  if (input_bndry_planes && m_r2d->ingested_velocity()) {
113  } else {
114  // Test for input data file if at xlo face
115  std::string dirichlet_file;
116  auto file_exists = pp.query("dirichlet_file", dirichlet_file);
117  if (file_exists) {
118  pp.query("read_prim_theta", read_prim_theta);
119  init_Dirichlet_bc_data(dirichlet_file);
120  } else {
121  pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
122  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
123  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
124  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
125  }
126  }
127 
128  Real rho_in = 0.;
129  if (input_bndry_planes && m_r2d->ingested_density()) {
131  } else {
132  if (!pp.query("density", rho_in)) {
133  amrex::Print() << "Using interior values to set conserved vars" << std::endl;
134  }
135  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
136  }
137 
138  bool th_read = (th_bc_data[0].data()!=nullptr);
139  Real theta_in = 0.;
140  if (input_bndry_planes && m_r2d->ingested_theta()) {
142  } else if (!th_read) {
143  if (rho_in > 0) {
144  pp.get("theta", theta_in);
145  }
146  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
147  }
148 
149  Real scalar_in = 0.;
150  if (input_bndry_planes && m_r2d->ingested_scalar()) {
152  } else {
153  if (pp.query("scalar", scalar_in))
154  m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
155  }
156 
157  if (solverChoice.moisture_type != MoistureType::None) {
158  Real qv_in = 0.;
159  if (input_bndry_planes && m_r2d->ingested_q1()) {
161  } else {
162  if (pp.query("qv", qv_in))
163  m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
164  }
165  Real qc_in = 0.;
166  if (input_bndry_planes && m_r2d->ingested_q2()) {
168  } else {
169  if (pp.query("qc", qc_in))
170  m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
171  }
172  }
173 
174  Real KE_in = 0.;
175  if (input_bndry_planes && m_r2d->ingested_KE()) {
177  } else {
178  if (pp.query("KE", KE_in))
179  m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
180  }
181  }
182  else if (bc_type == "noslipwall")
183  {
184  // Print() << bcid <<" set to no-slip wall.\n";
186  domain_bc_type[ori] = "NoSlipWall";
187 
188  std::vector<Real> v;
189 
190  // The values of m_bc_extdir_vals default to 0.
191  // But if we find "velocity" in the inputs file, use those values instead.
192  if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
193  {
194  v[ori.coordDir()] = 0.0;
195  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
196  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
197  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
198  }
199 
200  Real rho_in;
201  rho_read = pp.query("density", rho_in);
202  if (rho_read)
203  {
204  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
205  }
206 
207  Real theta_in;
208  if (pp.query("theta", theta_in))
209  {
211  }
212 
213  Real theta_grad_in;
214  if (pp.query("theta_grad", theta_grad_in))
215  {
216  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
217  }
218 
219  Real qv_in;
220  if (pp.query("qv", qv_in))
221  {
223  }
224  }
225  else if (bc_type == "slipwall")
226  {
227  // Print() << bcid <<" set to slip wall.\n";
228 
230  domain_bc_type[ori] = "SlipWall";
231 
232  Real rho_in;
233  rho_read = pp.query("density", rho_in);
234  if (rho_read)
235  {
236  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
237  }
238 
239  Real theta_in;
240  if (pp.query("theta", theta_in))
241  {
243  }
244 
245  Real rho_grad_in;
246  if (pp.query("density_grad", rho_grad_in))
247  {
248  m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = rho_grad_in;
249  }
250 
251  Real theta_grad_in;
252  if (pp.query("theta_grad", theta_grad_in))
253  {
254  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
255  }
256  }
257  else if (bc_type == "surface_layer")
258  {
260  domain_bc_type[ori] = "surface_layer";
261  }
262  else
263  {
265  }
266 
267  if (geom[0].isPeriodic(ori.coordDir())) {
268  domain_bc_type[ori] = "Periodic";
269  if (phys_bc_type[ori] == ERF_BC::undefined)
270  {
272  } else {
273  Abort("Wrong BC type for periodic boundary");
274  }
275  }
276 
277  if (phys_bc_type[ori] == ERF_BC::undefined)
278  {
279  Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
280  Abort("This BC type is unknown");
281  }
282  };
283 
284  f("xlo", Orientation(Direction::x,Orientation::low));
285  f("xhi", Orientation(Direction::x,Orientation::high));
286  f("ylo", Orientation(Direction::y,Orientation::low));
287  f("yhi", Orientation(Direction::y,Orientation::high));
288  f("zlo", Orientation(Direction::z,Orientation::low));
289  f("zhi", Orientation(Direction::z,Orientation::high));
290 
291  // *****************************************************************************
292  //
293  // Here we translate the physical boundary conditions -- one type per face --
294  // into logical boundary conditions for each velocity component
295  //
296  // *****************************************************************************
297  {
298  domain_bcs_type.resize(AMREX_SPACEDIM+NBCVAR_max);
299  domain_bcs_type_d.resize(AMREX_SPACEDIM+NBCVAR_max);
300 
301  for (OrientationIter oit; oit; ++oit) {
302  Orientation ori = oit();
303  int dir = ori.coordDir();
304  Orientation::Side side = ori.faceDir();
305  auto const bct = phys_bc_type[ori];
306  if ( bct == ERF_BC::symmetry )
307  {
308  if (side == Orientation::low) {
309  for (int i = 0; i < AMREX_SPACEDIM; i++) {
311  }
313  } else {
314  for (int i = 0; i < AMREX_SPACEDIM; i++) {
316  }
318  }
319  }
320  else if (bct == ERF_BC::outflow or bct == ERF_BC::ho_outflow )
321  {
322  if (side == Orientation::low) {
323  for (int i = 0; i < AMREX_SPACEDIM; i++) {
325  }
326  if (!solverChoice.anelastic[0]) {
328  }
329  } else {
330  for (int i = 0; i < AMREX_SPACEDIM; i++) {
332  }
333  if (!solverChoice.anelastic[0]) {
335  }
336  }
337  }
338  else if (bct == ERF_BC::open)
339  {
340  if (side == Orientation::low) {
341  for (int i = 0; i < AMREX_SPACEDIM; i++)
343  } else {
344  for (int i = 0; i < AMREX_SPACEDIM; i++)
346  }
347  }
348  else if (bct == ERF_BC::inflow)
349  {
350  if (side == Orientation::low) {
351  for (int i = 0; i < AMREX_SPACEDIM; i++) {
353  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
355  }
356  }
357  } else {
358  for (int i = 0; i < AMREX_SPACEDIM; i++) {
360  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
362  }
363  }
364  }
365  }
366  else if (bct == ERF_BC::inflow_outflow)
367  {
368  if (side == Orientation::low) {
369  for (int i = 0; i < AMREX_SPACEDIM; i++) {
371  }
372  } else {
373  for (int i = 0; i < AMREX_SPACEDIM; i++) {
375  }
376  }
377  }
378  else if (bct == ERF_BC::no_slip_wall)
379  {
380  if (side == Orientation::low) {
381  for (int i = 0; i < AMREX_SPACEDIM; i++) {
383  }
384  } else {
385  for (int i = 0; i < AMREX_SPACEDIM; i++) {
387  }
388  }
389  }
390  else if (bct == ERF_BC::slip_wall)
391  {
392  if (side == Orientation::low) {
393  for (int i = 0; i < AMREX_SPACEDIM; i++) {
395  }
396  // Only normal direction has ext_dir
398 
399  } else {
400  for (int i = 0; i < AMREX_SPACEDIM; i++) {
402  }
403  // Only normal direction has ext_dir
405  }
406  }
407  else if (bct == ERF_BC::periodic)
408  {
409  if (side == Orientation::low) {
410  for (int i = 0; i < AMREX_SPACEDIM; i++) {
412  }
413  } else {
414  for (int i = 0; i < AMREX_SPACEDIM; i++) {
416  }
417  }
418  }
419  else if ( bct == ERF_BC::surface_layer )
420  {
421  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
425  }
426  }
427  }
428 
429  // *****************************************************************************
430  //
431  // Here we translate the physical boundary conditions -- one type per face --
432  // into logical boundary conditions for each cell-centered variable
433  // (including the base state variables)
434  // NOTE: all "scalars" share the same type of boundary condition
435  //
436  // *****************************************************************************
437  {
438  for (OrientationIter oit; oit; ++oit) {
439  Orientation ori = oit();
440  int dir = ori.coordDir();
441  Orientation::Side side = ori.faceDir();
442  auto const bct = phys_bc_type[ori];
443  if ( bct == ERF_BC::symmetry )
444  {
445  if (side == Orientation::low) {
446  for (int i = 0; i < NBCVAR_max; i++) {
448  }
449  } else {
450  for (int i = 0; i < NBCVAR_max; i++) {
452  }
453  }
454  }
455  else if ( bct == ERF_BC::outflow )
456  {
457  if (side == Orientation::low) {
458  for (int i = 0; i < NBCVAR_max; i++) {
460  }
461  } else {
462  for (int i = 0; i < NBCVAR_max; i++) {
464  }
465  }
466  }
467  else if ( bct == ERF_BC::ho_outflow )
468  {
469  if (side == Orientation::low) {
470  for (int i = 0; i < NBCVAR_max; i++) {
472  }
473  } else {
474  for (int i = 0; i < NBCVAR_max; i++) {
476  }
477  }
478  }
479  else if ( bct == ERF_BC::open )
480  {
481  if (side == Orientation::low) {
482  for (int i = 0; i < NBCVAR_max; i++)
484  } else {
485  for (int i = 0; i < NBCVAR_max; i++)
487  }
488  }
489  else if ( bct == ERF_BC::no_slip_wall )
490  {
491  if (side == Orientation::low) {
492  for (int i = 0; i < NBCVAR_max; i++) {
494  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
495  if (rho_read) {
497  } else {
499  }
500  }
501  }
502  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
504  }
505  } else {
506  for (int i = 0; i < NBCVAR_max; i++) {
508  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
509  if (rho_read) {
511  } else {
513  }
514  }
515  }
516  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
518  }
519  }
520  }
521  else if (bct == ERF_BC::slip_wall)
522  {
523  if (side == Orientation::low) {
524  for (int i = 0; i < NBCVAR_max; i++) {
526  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
527  if (rho_read) {
529  } else {
531  }
532  }
533  }
534  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
536  }
537  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
539  }
540  } else {
541  for (int i = 0; i < NBCVAR_max; i++) {
543  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
544  if (rho_read) {
546  } else {
548  }
549  }
550  }
551  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
553  }
554  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
556  }
557  }
558  }
559  else if (bct == ERF_BC::inflow)
560  {
561  if (side == Orientation::low) {
562  for (int i = 0; i < NBCVAR_max; i++) {
564  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
565  (th_bc_data[0].data() != nullptr))
566  {
567  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
568  }
569  else if (input_bndry_planes && dir < 2 && (
570  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
571  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
572  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
573  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
574  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
575  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )) )
576  {
578  }
579  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
581  }
582  }
583  } else {
584  for (int i = 0; i < NBCVAR_max; i++) {
586  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
587  (th_bc_data[0].data() != nullptr))
588  {
589  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
590  }
591  else if (input_bndry_planes && dir < 2 && (
592  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
593  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
594  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
595  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
596  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
597  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )
598  ) )
599  {
601  }
602  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
604  }
605  }
606  }
607  }
608  else if (bct == ERF_BC::inflow_outflow )
609  {
610  if (side == Orientation::low) {
611  for (int i = 0; i < NBCVAR_max; i++) {
613  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
615  }
616  }
617  } else {
618  for (int i = 0; i < NBCVAR_max; i++) {
620  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
622  }
623  }
624  }
625  }
626  else if (bct == ERF_BC::periodic)
627  {
628  if (side == Orientation::low) {
629  for (int i = 0; i < NBCVAR_max; i++) {
631  }
632  } else {
633  for (int i = 0; i < NBCVAR_max; i++) {
635  }
636  }
637  }
638  else if ( bct == ERF_BC::surface_layer )
639  {
640  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
641  for (int i = 0; i < NBCVAR_max; i++) {
643  }
644  }
645  }
646  }
647 
648  // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
649  Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
650 }
#define NBCVAR_max
Definition: ERF_IndexDefines.H:29
@ ho_outflow
@ inflow_outflow
void init_Dirichlet_bc_data(const std::string input_file)
Definition: ERF_InitBCs.cpp:652
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_neumann_vals
Definition: ERF.H:893
@ RhoQ6_bc_comp
Definition: ERF_IndexDefines.H:86
@ RhoQ1_bc_comp
Definition: ERF_IndexDefines.H:81
@ RhoQ4_bc_comp
Definition: ERF_IndexDefines.H:84
@ RhoKE_bc_comp
Definition: ERF_IndexDefines.H:79
@ RhoQ3_bc_comp
Definition: ERF_IndexDefines.H:83
@ RhoTheta_bc_comp
Definition: ERF_IndexDefines.H:78
@ RhoQ2_bc_comp
Definition: ERF_IndexDefines.H:82
@ Rho_bc_comp
Definition: ERF_IndexDefines.H:77
@ RhoQ5_bc_comp
Definition: ERF_IndexDefines.H:85
@ neumann
Definition: ERF_IndexDefines.H:205
@ open
Definition: ERF_IndexDefines.H:207
@ reflect_odd
Definition: ERF_IndexDefines.H:197
@ hoextrap
Definition: ERF_IndexDefines.H:208
@ foextrap
Definition: ERF_IndexDefines.H:200
@ ext_dir
Definition: ERF_IndexDefines.H:201
@ ext_dir_prim
Definition: ERF_IndexDefines.H:203
@ ext_dir_upwind
Definition: ERF_IndexDefines.H:209
@ int_dir
Definition: ERF_IndexDefines.H:198
@ neumann_int
Definition: ERF_IndexDefines.H:206
@ reflect_even
Definition: ERF_IndexDefines.H:199
Here is the call graph for this function:

◆ init_custom()

void ERF::init_custom ( int  lev)
private

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters
levInteger specifying the current level
27 {
28  auto& lev_new = vars_new[lev];
29 
30  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
31  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
32 
33  MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
34  lev_new[Vars::cons].nComp() , lev_new[Vars::cons].nGrow());
35  MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
36  MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
37  MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
38 
39  // Default all perturbations to zero
40  cons_pert.setVal(0.);
41  xvel_pert.setVal(0.);
42  yvel_pert.setVal(0.);
43  zvel_pert.setVal(0.);
44 
45 #ifdef _OPENMP
46 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
47 #endif
48  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
49  {
50  const Box &bx = mfi.tilebox();
51  const Box &xbx = mfi.tilebox(IntVect(1,0,0));
52  const Box &ybx = mfi.tilebox(IntVect(0,1,0));
53  const Box &zbx = mfi.tilebox(IntVect(0,0,1));
54 
55  const auto &cons_pert_arr = cons_pert.array(mfi);
56  const auto &xvel_pert_arr = xvel_pert.array(mfi);
57  const auto &yvel_pert_arr = yvel_pert.array(mfi);
58  const auto &zvel_pert_arr = zvel_pert.array(mfi);
59 
60  Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
61  Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
62  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
63 
64  Array4<Real const> mf_m = mapfac_m[lev]->array(mfi);
65  Array4<Real const> mf_u = mapfac_m[lev]->array(mfi);
66  Array4<Real const> mf_v = mapfac_m[lev]->array(mfi);
67 
68  Array4<Real> r_hse_arr = r_hse.array(mfi);
69  Array4<Real> p_hse_arr = p_hse.array(mfi);
70 
71  prob->init_custom_pert(bx, xbx, ybx, zbx, cons_arr, cons_pert_arr,
72  xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
73  r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
74  geom[lev].data(), mf_m, mf_u, mf_v,
75  solverChoice);
76  } //mfi
77 
78  // Add problem-specific perturbation to background flow if not doing anelastic with fixed-in-time density
80  MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp, Rho_comp, 1, cons_pert.nGrow());
81  }
82  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp, 1, cons_pert.nGrow());
83  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,NSCALARS, cons_pert.nGrow());
84 
85  // RhoKE is relevant if using Deardorff with LES, k-equation for RANS, or MYNN with PBL
86  if (solverChoice.turbChoice[lev].use_tke) {
87  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp, RhoKE_comp, 1, cons_pert.nGrow());
88  }
89 
90  if (solverChoice.moisture_type != MoistureType::None) {
91  int qstate_size = micro->Get_Qstate_Size();
92  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoQ1_comp, RhoQ1_comp, 1, cons_pert.nGrow());
93  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoQ2_comp, RhoQ2_comp, 1, cons_pert.nGrow());
94  for (int q_offset(2); q_offset<qstate_size; ++q_offset) {
95  int q_idx = RhoQ1_comp+q_offset;
96  MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
97  }
98  }
99 
100  MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0, 0, 1, xvel_pert.nGrowVect());
101  MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0, 0, 1, yvel_pert.nGrowVect());
102  MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0, 0, 1, zvel_pert.nGrowVect());
103 }
bool fixed_density
Definition: ERF_DataStruct.H:673
Here is the call graph for this function:

◆ init_Dirichlet_bc_data()

void ERF::init_Dirichlet_bc_data ( const std::string  input_file)
private
653 {
654  const bool use_terrain = (solverChoice.terrain_type != TerrainType::None);
655 
656  // Read the dirichlet_input file
657  Print() << "dirichlet_input file location : " << input_file << std::endl;
658  std::ifstream input_reader(input_file);
659  if (!input_reader.is_open()) {
660  amrex::Abort("Error opening the dirichlet_input file.\n");
661  }
662 
663  Print() << "Successfully opened the dirichlet_input file. Now reading... " << std::endl;
664  std::string line;
665 
666  // Size of Ninp (number of z points in input file)
667  Vector<Real> z_inp_tmp, u_inp_tmp, v_inp_tmp, w_inp_tmp, th_inp_tmp;
668 
669  // Top and bot for domain
670  const int klo = geom[0].Domain().smallEnd()[2];
671  const int khi = geom[0].Domain().bigEnd()[2];
672  const Real zbot = (use_terrain) ? zlevels_stag[0][klo] : geom[0].ProbLo(2);
673  const Real ztop = (use_terrain) ? zlevels_stag[0][khi+1] : geom[0].ProbHi(2);
674 
675  // Flag if theta input
676  Real th_init = -300.0;
677  bool th_read{false};
678 
679  // Add surface
680  z_inp_tmp.push_back(zbot); // height above sea level [m]
681  u_inp_tmp.push_back(0.);
682  v_inp_tmp.push_back(0.);
683  w_inp_tmp.push_back(0.);
684  th_inp_tmp.push_back(th_init);
685 
686  // Read the vertical profile at each given height
687  Real z, u, v, w, th;
688  while(std::getline(input_reader, line)) {
689  std::istringstream iss_z(line);
690 
691  Vector<Real> rval_v;
692  Real rval;
693  while (iss_z >> rval) {
694  rval_v.push_back(rval);
695  }
696  z = rval_v[0];
697  u = rval_v[1];
698  v = rval_v[2];
699  w = rval_v[3];
700 
701  // Format without theta
702  if (rval_v.size() == 4) {
703  if (z == zbot) {
704  u_inp_tmp[0] = u;
705  v_inp_tmp[0] = v;
706  w_inp_tmp[0] = w;
707  } else {
708  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
709  z_inp_tmp.push_back(z);
710  u_inp_tmp.push_back(u);
711  v_inp_tmp.push_back(v);
712  w_inp_tmp.push_back(w);
713  if (z >= ztop) break;
714  }
715  } else if (rval_v.size() == 5) {
716  th_read = true;
717  th = rval_v[4];
718  if (z == zbot) {
719  u_inp_tmp[0] = u;
720  v_inp_tmp[0] = v;
721  w_inp_tmp[0] = w;
722  th_inp_tmp[0] = th;
723  } else {
724  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
725  z_inp_tmp.push_back(z);
726  u_inp_tmp.push_back(u);
727  v_inp_tmp.push_back(v);
728  w_inp_tmp.push_back(w);
729  th_inp_tmp.push_back(th);
730  if (z >= ztop) break;
731  }
732  } else {
733  Abort("Unknown inflow file format!");
734  }
735  }
736 
737  // Ensure we set a reasonable theta surface
738  if (th_read) {
739  if (th_inp_tmp[0] == th_init) {
740  Real slope = (th_inp_tmp[2] - th_inp_tmp[1]) / (z_inp_tmp[2] - z_inp_tmp[1]);
741  Real dz = z_inp_tmp[0] - z_inp_tmp[1];
742  th_inp_tmp[0] = slope * dz + th_inp_tmp[1];
743  }
744  }
745 
746  amrex::Print() << "Successfully read and interpolated the dirichlet_input file..." << std::endl;
747  input_reader.close();
748 
749  for (int lev = 0; lev <= max_level; lev++) {
750 
751  const int Nz = geom[lev].Domain().size()[2];
752  const Real dz = geom[lev].CellSize()[2];
753 
754  // Size of Nz (domain grid)
755  Vector<Real> zcc_inp(Nz );
756  Vector<Real> znd_inp(Nz+1);
757  Vector<Real> u_inp(Nz ); xvel_bc_data[lev].resize(Nz ,0.0);
758  Vector<Real> v_inp(Nz ); yvel_bc_data[lev].resize(Nz ,0.0);
759  Vector<Real> w_inp(Nz+1); zvel_bc_data[lev].resize(Nz+1,0.0);
760  Vector<Real> th_inp;
761  if (th_read) {
762  th_inp.resize(Nz);
763  th_bc_data[lev].resize(Nz, 0.0);
764  }
765 
766  // At this point, we have an input from zbot up to
767  // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights
768  const int Ninp = z_inp_tmp.size();
769  for (int k(0); k<Nz; ++k) {
770  zcc_inp[k] = (use_terrain) ? 0.5 * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1])
771  : zbot + (k + 0.5) * dz;
772  znd_inp[k] = (use_terrain) ? zlevels_stag[lev][k+1] : zbot + (k) * dz;
773  u_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), u_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
774  v_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), v_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
775  w_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), znd_inp[k], Ninp);
776  if (th_read) {
777  th_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), th_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
778  }
779  }
780  znd_inp[Nz] = ztop;
781  w_inp[Nz] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), ztop, Ninp);
782 
783  // Copy host data to the device
784  Gpu::copy(Gpu::hostToDevice, u_inp.begin(), u_inp.end(), xvel_bc_data[lev].begin());
785  Gpu::copy(Gpu::hostToDevice, v_inp.begin(), v_inp.end(), yvel_bc_data[lev].begin());
786  Gpu::copy(Gpu::hostToDevice, w_inp.begin(), w_inp.end(), zvel_bc_data[lev].begin());
787  if (th_read) {
788  Gpu::copy(Gpu::hostToDevice, th_inp.begin(), th_inp.end(), th_bc_data[lev].begin());
789  }
790 
791  // NOTE: These device vectors are passed to the PhysBC constructors when that
792  // class is instantiated in ERF_MakeNewArrays.cpp.
793  } // lev
794 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real interpolate_1d(const amrex::Real *alpha, const amrex::Real *beta, const amrex::Real alpha_interp, const int alpha_size)
Definition: ERF_Interpolation_1D.H:12
Here is the call graph for this function:

◆ init_from_hse()

void ERF::init_from_hse ( int  lev)

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

  • call prob->erf_init_dens_hse(...)
    • call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
    • save r_hse
  • call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse and th_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation
Parameters
levInteger specifying the current level
33 {
34  auto& lev_new = vars_new[lev];
35 
36  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
37  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
38 
39 #ifdef _OPENMP
40 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
41 #endif
42  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
43  {
44  const Box &gbx = mfi.growntilebox(1);
45  const Array4<Real >& cons_arr = lev_new[Vars::cons].array(mfi);
46  const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
47  const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
48 
49  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
50  {
51  cons_arr(i,j,k,Rho_comp) = r_hse_arr(i,j,k);
52  cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
53  });
54  } //mfi
55 }
Here is the call graph for this function:

◆ init_from_input_sounding()

void ERF::init_from_input_sounding ( int  lev)

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters
levInteger specifying the current level
52 {
53  // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
54  if (lev == 0) {
56  Error("input_sounding file name must be provided via input");
57  }
58 
60 
61  // this will interpolate the input profiles to the nominal height levels
62  // (ranging from 0 to the domain top)
63  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
65  }
66 
67  // this will calculate the hydrostatically balanced density and pressure
68  // profiles following WRF ideal.exe
70 
71  } else {
72  //
73  // We need to do this interp from coarse level in order to set the values of
74  // the base state inside the domain but outside of the fine region
75  //
76  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
77  //
78  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
79  // have been pre-filled - this includes ghost cells both inside and outside
80  // the domain
81  //
82  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
83  IntVect(0,0,0), // do not fill ghost cells outside the domain
84  base_state[lev-1], 0, 0, base_state[lev].nComp(),
85  geom[lev-1], geom[lev],
86  refRatio(lev-1), &cell_cons_interp,
88 
89  // We need to do this here because the interpolation above may leave corners unfilled
90  // when the corners need to be filled by, for example, reflection of the fine ghost
91  // cell outside the fine region but inide the domain.
92  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
93  }
94 
95  auto& lev_new = vars_new[lev];
96 
97  // update if init_sounding_ideal == true
98  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
99  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
100  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
101  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
102  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
103 
104  const Real l_gravity = solverChoice.gravity;
105  const Real l_rdOcp = solverChoice.rdOcp;
106  const bool l_moist = (solverChoice.moisture_type != MoistureType::None);
107 
108 #ifdef _OPENMP
109 #pragma omp parallel if (Gpu::notInLaunchRegion())
110 #endif
111  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
112  const Box &bx = mfi.tilebox();
113  const auto &cons_arr = lev_new[Vars::cons].array(mfi);
114  const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
115  const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
116  const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
117  Array4<Real> r_hse_arr = r_hse.array(mfi);
118  Array4<Real> p_hse_arr = p_hse.array(mfi);
119  Array4<Real> pi_hse_arr = pi_hse.array(mfi);
120  Array4<Real> th_hse_arr = th_hse.array(mfi);
121  Array4<Real> qv_hse_arr = qv_hse.array(mfi);
122 
123  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
124  Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
125 
127  {
128  // HSE will be initialized here, interpolated from values previously
129  // calculated by calc_rho_p()
131  bx, cons_arr,
132  r_hse_arr, p_hse_arr, pi_hse_arr, th_hse_arr, qv_hse_arr,
133  geom[lev].data(), z_cc_arr,
134  l_gravity, l_rdOcp, l_moist, input_sounding_data);
135  }
136  else
137  {
138  // HSE will be calculated later with call to initHSE
140  bx, cons_arr,
141  geom[lev].data(), z_cc_arr,
142  l_moist, input_sounding_data);
143  }
144 
146  bx, xvel_arr, yvel_arr, zvel_arr,
147  geom[lev].data(), z_nd_arr,
149 
150  } //mfi
151 }
void init_bx_scalars_from_input_sounding_hse(const Box &bx, Array4< Real > const &state, Array4< Real > const &r_hse_arr, Array4< Real > const &p_hse_arr, Array4< Real > const &pi_hse_arr, Array4< Real > const &th_hse_arr, Array4< Real > const &qv_hse_arr, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const Real &l_gravity, const Real &l_rdOcp, const bool &l_moist, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:225
void init_bx_scalars_from_input_sounding(const Box &bx, Array4< Real > const &state, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const bool &l_moist, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:163
void init_bx_velocities_from_input_sounding(const Box &bx, Array4< Real > const &x_vel, Array4< Real > const &y_vel, Array4< Real > const &z_vel, GeometryData const &geomdata, Array4< const Real > const &z_nd_arr, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:330
static bool init_sounding_ideal
Definition: ERF.H:1092
InputSoundingData input_sounding_data
Definition: ERF.H:687
@ rho0_bc_comp
Definition: ERF_IndexDefines.H:98
@ qv0_comp
Definition: ERF_IndexDefines.H:67
void resize_arrays()
Definition: ERF_InputSoundingData.H:60
int n_sounding_files
Definition: ERF_InputSoundingData.H:316
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag, int itime)
Definition: ERF_InputSoundingData.H:77
amrex::Vector< std::string > input_sounding_file
Definition: ERF_InputSoundingData.H:314
void calc_rho_p(int itime)
Definition: ERF_InputSoundingData.H:176
Here is the call graph for this function:

◆ init_geo_wind_profile()

void ERF::init_geo_wind_profile ( const std::string  input_file,
amrex::Vector< amrex::Real > &  u_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  u_geos_d,
amrex::Vector< amrex::Real > &  v_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  v_geos_d,
const amrex::Geometry &  lgeom,
const amrex::Vector< amrex::Real > &  zlev_stag 
)
private
17 {
18  const int klo = 0;
19  const int khi = lgeom.Domain().bigEnd()[AMREX_SPACEDIM-1];
20  const amrex::Real dz = lgeom.CellSize()[AMREX_SPACEDIM-1];
21 
22  const bool grid_stretch = (zlev_stag.size() > 0);
23  const Real zbot = (grid_stretch) ? zlev_stag[klo] : lgeom.ProbLo(AMREX_SPACEDIM-1);
24  const Real ztop = (grid_stretch) ? zlev_stag[khi+1] : lgeom.ProbHi(AMREX_SPACEDIM-1);
25 
26  amrex::Print() << "Reading geostrophic wind profile from " << input_file << std::endl;
27  std::ifstream profile_reader(input_file);
28  if(!profile_reader.is_open()) {
29  amrex::Error("Error opening the abl_geo_wind_table\n");
30  }
31 
32  // First, read the input data into temp vectors
33  std::string line;
34  Vector<Real> z_inp, Ug_inp, Vg_inp;
35  Real z, Ug, Vg;
36  amrex::Print() << "z Ug Vg" << std::endl;
37  while(std::getline(profile_reader, line)) {
38  std::istringstream iss(line);
39  iss >> z >> Ug >> Vg;
40  amrex::Print() << z << " " << Ug << " " << Vg << std::endl;
41  z_inp.push_back(z);
42  Ug_inp.push_back(Ug);
43  Vg_inp.push_back(Vg);
44  if (z >= ztop) break;
45  }
46 
47  const int Ninp = z_inp.size();
48  AMREX_ALWAYS_ASSERT(z_inp[0] <= zbot);
49  AMREX_ALWAYS_ASSERT(z_inp[Ninp-1] >= ztop);
50 
51  // Now, interpolate vectors to the cell centers
52  for (int k = 0; k <= khi; k++) {
53  z = (grid_stretch) ? 0.5 * (zlev_stag[k] + zlev_stag[k+1])
54  : zbot + (k + 0.5) * dz;
55  u_geos[k] = interpolate_1d(z_inp.dataPtr(), Ug_inp.dataPtr(), z, Ninp);
56  v_geos[k] = interpolate_1d(z_inp.dataPtr(), Vg_inp.dataPtr(), z, Ninp);
57  }
58 
59  // Copy from host version to device version
60  Gpu::copy(Gpu::hostToDevice, u_geos.begin(), u_geos.end(), u_geos_d.begin());
61  Gpu::copy(Gpu::hostToDevice, v_geos.begin(), v_geos.end(), v_geos_d.begin());
62 
63  profile_reader.close();
64 }
Here is the call graph for this function:

◆ init_only()

void ERF::init_only ( int  lev,
amrex::Real  time 
)
1433 {
1434  t_new[lev] = time;
1435  t_old[lev] = time - 1.e200;
1436 
1437  auto& lev_new = vars_new[lev];
1438  auto& lev_old = vars_old[lev];
1439 
1440  // Loop over grids at this level to initialize our grid data
1441  lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
1442  lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
1443  lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
1444  lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
1445 
1446  // Initialize background flow (optional)
1447  if (solverChoice.init_type == InitType::Input_Sounding) {
1448  // The base state is initialized by integrating vertically through the
1449  // input sounding, if the init_sounding_ideal flag is set; otherwise
1450  // it is set by initHSE()
1451 
1452  // The physbc's need the terrain but are needed for initHSE
1453  // We have already made the terrain in the call to init_zphys
1454  // in MakeNewLevelFromScratch
1455  make_physbcs(lev);
1456 
1457  // Now init the base state and the data itself
1459 
1460  if (init_sounding_ideal) {
1461  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
1462  "Gravity should be on to be consistent with sounding initialization.");
1463  } else {
1464  initHSE();
1465  }
1466 
1467 #ifdef ERF_USE_NETCDF
1468  }
1469  else if (solverChoice.init_type == InitType::WRFInput)
1470  {
1471  // The base state is initialized from WRF wrfinput data, output by
1472  // ideal.exe or real.exe
1473  init_from_wrfinput(lev, *mf_C1H[lev], *mf_C2H[lev], *mf_MUB[lev]);
1474  if (lev==0) {
1475  if ((start_time > 0) && (start_time != t_new[lev])) {
1476  Print() << "Ignoring specified start_time="
1477  << std::setprecision(timeprecision) << start_time
1478  << std::endl;
1479  }
1480  start_time = t_new[lev];
1481  }
1482  use_datetime = true;
1483 
1484  // The physbc's need the terrain but are needed for initHSE
1485  if (!solverChoice.use_real_bcs) {
1486  make_physbcs(lev);
1487  }
1488  }
1489  else if (solverChoice.init_type == InitType::NCFile)
1490  {
1491  // The state is initialized by reading from a Netcdf file
1492  init_from_ncfile(lev);
1493 
1494  // The physbc's need the terrain but are needed for initHSE
1495  make_physbcs(lev);
1496  }
1497  else if (solverChoice.init_type == InitType::Metgrid)
1498  {
1499  // The base state is initialized from data output by WPS metgrid;
1500  // we will rebalance after interpolation
1501  init_from_metgrid(lev);
1502 #endif
1503  } else if (solverChoice.init_type == InitType::Uniform) {
1504  // Initialize a uniform background field and base state based on the
1505  // problem-specified reference density and temperature
1506 
1507  // The physbc's need the terrain but are needed for initHSE
1508  make_physbcs(lev);
1509 
1510  init_uniform(lev);
1511  initHSE(lev);
1512  } else {
1513  // No background flow initialization specified, initialize the
1514  // background field to be equal to the base state, calculated from the
1515  // problem-specific erf_init_dens_hse
1516 
1517  // The bc's need the terrain but are needed for initHSE
1518  make_physbcs(lev);
1519 
1520  // We will initialize the state from the background state so must set that first
1521  initHSE(lev);
1522  init_from_hse(lev);
1523  }
1524 
1525  // Add problem-specific flow features
1526  //
1527  // Notes:
1528  // - This calls init_custom_pert that is defined for each problem
1529  // - This may modify the base state
1530  // - The fields set by init_custom_pert are **perturbations** to the
1531  // background flow set based on init_type
1532  if (solverChoice.init_type != InitType::NCFile) {
1533  init_custom(lev);
1534  }
1535 
1536  // Ensure that the face-based data are the same on both sides of a periodic domain.
1537  // The data associated with the lower grid ID is considered the correct value.
1538  lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
1539  lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
1540  lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
1541 
1542  if(solverChoice.spongeChoice.sponge_type == "input_sponge"){
1543  input_sponge(lev);
1544  }
1545 
1546  // Initialize turbulent perturbation
1547  if (solverChoice.pert_type == PerturbationType::Source ||
1548  solverChoice.pert_type == PerturbationType::Direct) {
1549  turbPert_update(lev, 0.);
1550  turbPert_amplitude(lev);
1551  }
1552 }
const int timeprecision
Definition: ERF.H:919
void init_from_input_sounding(int lev)
Definition: ERF_InitFromInputSounding.cpp:51
void init_custom(int lev)
Definition: ERF_InitCustom.cpp:26
void init_from_hse(int lev)
Definition: ERF_InitFromHSE.cpp:32
void initHSE()
Initialize HSE.
Definition: ERF_Init1D.cpp:142
void turbPert_update(const int lev, const amrex::Real dt)
Definition: ERF_InitTurbPert.cpp:12
void input_sponge(int lev)
Definition: ERF_InitSponge.cpp:17
void make_physbcs(int lev)
Definition: ERF_MakeNewArrays.cpp:629
void init_uniform(int lev)
Definition: ERF_InitUniform.cpp:17
void turbPert_amplitude(const int lev)
Definition: ERF_InitTurbPert.cpp:32
bool use_gravity
Definition: ERF_DataStruct.H:683
static InitType init_type
Definition: ERF_DataStruct.H:646

◆ init_stuff()

void ERF::init_stuff ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm,
amrex::Vector< amrex::MultiFab > &  lev_new,
amrex::Vector< amrex::MultiFab > &  lev_old,
amrex::MultiFab &  tmp_base_state,
std::unique_ptr< amrex::MultiFab > &  tmp_zphys_nd 
)
private
28 {
29  // ********************************************************************************************
30  // Base state holds r_0, pres_0, pi_0, th_0 (in that order)
31  //
32  // Here is where we set 3 ghost cells for the base state!
33  //
34  // ********************************************************************************************
35  tmp_base_state.define(ba,dm,BaseState::num_comps,3);
36  tmp_base_state.setVal(0.);
37 
38  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
39  base_state_new[lev].define(ba,dm,BaseState::num_comps,base_state[lev].nGrowVect());
40  base_state_new[lev].setVal(0.);
41  }
42 
43  // ********************************************************************************************
44  // Allocate terrain arrays
45  // ********************************************************************************************
46 
47  BoxArray ba_nd(ba);
48  ba_nd.surroundingNodes();
49 
50  // NOTE: this is where we actually allocate z_phys_nd -- but here it's called "tmp_zphys_nd"
51  // We need this to be one greater than the ghost cells to handle levels > 0
52 
53  int ngrow = ComputeGhostCells(solverChoice) + 2;
54  tmp_zphys_nd = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
55 
56  z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
57  init_default_zphys(lev, geom[lev], *tmp_zphys_nd, *z_phys_cc[lev]);
58 
59  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh)
60  {
61  detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
62  detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
63 
64  ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
65  ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
66  az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
67 
68  ax_new[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
69  ay_new[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
70  az_new[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
71 
72  z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
73 
74  z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
75  z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
76  }
77  else
78  {
79  z_phys_nd_new[lev] = nullptr;
80  detJ_cc_new[lev] = nullptr;
81 
82  z_phys_nd_src[lev] = nullptr;
83  detJ_cc_src[lev] = nullptr;
84 
85  z_t_rk[lev] = nullptr;
86  }
87 
88  if (SolverChoice::terrain_type == TerrainType::ImmersedForcing)
89  {
90  terrain_blanking[lev] = std::make_unique<MultiFab>(ba,dm,1,ngrow);
91  terrain_blanking[lev]->setVal(1.0);
92  }
93 
94  // We use these area arrays regardless of terrain, EB or none of the above
95  detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
96  ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
97  ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
98  az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
99 
100  detJ_cc[lev]->setVal(1.0);
101  ax[lev]->setVal(1.0);
102  ay[lev]->setVal(1.0);
103  az[lev]->setVal(1.0);
104 
105  // ********************************************************************************************
106  // Create wall distance array for RANS modeling
107  // ********************************************************************************************
108  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
109  walldist[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
110  walldist[lev]->setVal(1e23);
111  } else {
112  walldist[lev] = nullptr;
113  }
114 
115  // ********************************************************************************************
116  // These are the persistent containers for the old and new data
117  // ********************************************************************************************
118  int ncomp;
119  if (lev > 0) {
120  ncomp = vars_new[lev-1][Vars::cons].nComp();
121  } else {
122  int n_qstate = micro->Get_Qstate_Size();
123  ncomp = NDRY + NSCALARS + n_qstate;
124  }
125 
126  // ********************************************************************************************
127  // The number of ghost cells for density must be 1 greater than that for velocity
128  // so that we can go back in forth between velocity and momentum on all faces
129  // ********************************************************************************************
130  int ngrow_state = ComputeGhostCells(solverChoice) + 1;
131  int ngrow_vels = ComputeGhostCells(solverChoice);
132 
133  // ********************************************************************************************
134  // New solution data containers
135  // ********************************************************************************************
136  lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
137  lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
138 
139  lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
140  lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
141 
142  lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
143  lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
144 
145  // Note that we need the ghost cells in the z-direction if we are doing any
146  // kind of domain decomposition in the vertical (at level 0 or above)
147  lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
148  lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
149 
150  if (solverChoice.anelastic[lev] == 1) {
151  pp_inc[lev].define(ba, dm, 1, 1);
152  pp_inc[lev].setVal(0.0);
153  }
154 
155  // ********************************************************************************************
156  // These are just used for scratch in the time integrator but we might as well define them here
157  // ********************************************************************************************
158  rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
159  rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
160 
161  rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
162  rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
163 
164  rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
165  rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
166 
167  if (lev > 0) {
168  //xmom_crse_rhs[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, IntVect{0});
169  //ymom_crse_rhs[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, IntVect{0});
170  zmom_crse_rhs[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect{0});
171  }
172 
173  // We do this here just so they won't be undefined in the initial FillPatch
174  rU_old[lev].setVal(1.2e21);
175  rV_old[lev].setVal(3.4e22);
176  rW_old[lev].setVal(5.6e23);
177  rU_new[lev].setVal(1.2e21);
178  rV_new[lev].setVal(3.4e22);
179  rW_new[lev].setVal(5.6e23);
180 
181  // ********************************************************************************************
182  // These are just time averaged fields for diagnostics
183  // ********************************************************************************************
184 
185  // NOTE: We are not completing a fillpach call on the time averaged data;
186  // which would copy on intersection and interpolate from coarse.
187  // Therefore, we are restarting the averaging when the ba changes,
188  // this may give poor statistics for dynamic mesh refinement.
189  vel_t_avg[lev] = nullptr;
191  vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
192  vel_t_avg[lev]->setVal(0.0);
193  t_avg_cnt[lev] = 0.0;
194  }
195 
196  // ********************************************************************************************
197  // Initialize flux registers whenever we create/re-create a level
198  // ********************************************************************************************
199  if (solverChoice.coupling_type == CouplingType::TwoWay) {
200  if (lev == 0) {
201  advflux_reg[0] = nullptr;
202  } else {
203  int ncomp_reflux = vars_new[0][Vars::cons].nComp();
204  advflux_reg[lev] = new YAFluxRegister(ba , grids[lev-1],
205  dm , dmap[lev-1],
206  geom[lev], geom[lev-1],
207  ref_ratio[lev-1], lev, ncomp_reflux);
208  }
209  }
210 
211  // ********************************************************************************************
212  // Define Theta_prim storage if using surface_layer BC
213  // ********************************************************************************************
214  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
215  Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
216  if (solverChoice.moisture_type != MoistureType::None) {
217  Qv_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
218  Qr_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
219  } else {
220  Qv_prim[lev] = nullptr;
221  Qr_prim[lev] = nullptr;
222  }
223  } else {
224  Theta_prim[lev] = nullptr;
225  Qv_prim[lev] = nullptr;
226  Qr_prim[lev] = nullptr;
227  }
228 
229  // ********************************************************************************************
230  // Map factors
231  // ********************************************************************************************
232  BoxList bl2d_mf = ba.boxList();
233  for (auto& b : bl2d_mf) {
234  b.setRange(2,0);
235  }
236  BoxArray ba2d_mf(std::move(bl2d_mf));
237 
238  mapfac_m[lev] = std::make_unique<MultiFab>(ba2d_mf,dm,1,3);
239  mapfac_u[lev] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,3);
240  mapfac_v[lev] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,3);
242  mapfac_m[lev]->setVal(0.5);
243  mapfac_u[lev]->setVal(0.5);
244  mapfac_v[lev]->setVal(0.5);
245  }
246  else {
247  mapfac_m[lev]->setVal(1.);
248  mapfac_u[lev]->setVal(1.);
249  mapfac_v[lev]->setVal(1.);
250  }
251 
252  // ********************************************************************************************
253  // Build 1D BA and 2D BA
254  // ********************************************************************************************
255  BoxList bl1d = ba.boxList();
256  for (auto& b : bl1d) {
257  b.setRange(0,0);
258  b.setRange(1,0);
259  }
260  ba1d[lev] = BoxArray(std::move(bl1d));
261 
262  // Build 2D BA
263  BoxList bl2d = ba.boxList();
264  for (auto& b : bl2d) {
265  b.setRange(2,0);
266  }
267  ba2d[lev] = BoxArray(std::move(bl2d));
268 
269  IntVect ng = vars_new[lev][Vars::cons].nGrowVect();
270 
271  mf_C1H[lev] = std::make_unique<MultiFab>(ba1d[lev],dm,1,ng);
272  mf_C2H[lev] = std::make_unique<MultiFab>(ba1d[lev],dm,1,ng);
273  mf_MUB[lev] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ng);
274 
275  //*********************************************************
276  // Variables for Fitch model for windfarm parametrization
277  //*********************************************************
278 #if defined(ERF_USE_WINDFARM)
279  if (solverChoice.windfarm_type == WindFarmType::Fitch){
280  vars_windfarm[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
281  }
282  if (solverChoice.windfarm_type == WindFarmType::EWP){
283  vars_windfarm[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
284  }
285  if (solverChoice.windfarm_type == WindFarmType::SimpleAD) {
286  vars_windfarm[lev].define(ba, dm, 2, ngrow_state);// dudt, dvdt
287  }
288  if (solverChoice.windfarm_type == WindFarmType::GeneralAD) {
289  vars_windfarm[lev].define(ba, dm, 3, ngrow_state);// dudt, dvdt, dwdt
290  }
291  Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
292  SMark[lev].define(ba, dm, 2, 1); // Free stream velocity/source term
293  // sampling marker in a cell - 2 components
294 #endif
295 
296 
297 #ifdef ERF_USE_WW3_COUPLING
298  // create a new BoxArray and DistributionMapping for a MultiFab with 1 box
299  BoxArray ba_onegrid(geom[lev].Domain());
300  BoxList bl2d_onegrid = ba_onegrid.boxList();
301  for (auto& b : bl2d_onegrid) {
302  b.setRange(2,0);
303  }
304  BoxArray ba2d_onegrid(std::move(bl2d_onegrid));
305  Vector<int> pmap;
306  pmap.resize(1);
307  pmap[0]=0;
308  DistributionMapping dm_onegrid(ba2d_onegrid);
309  dm_onegrid.define(pmap);
310 
311  Hwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
312  Lwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
313 
314  BoxList bl2d_wave = ba.boxList();
315  for (auto& b : bl2d_wave) {
316  b.setRange(2,0);
317  }
318  BoxArray ba2d_wave(std::move(bl2d_wave));
319 
320  Hwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
321  Lwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
322 
323  std::cout<<ba_onegrid<<std::endl;
324  std::cout<<ba2d_onegrid<<std::endl;
325  std::cout<<dm_onegrid<<std::endl;
326 #endif
327 
328 
329 #if defined(ERF_USE_RRTMGP)
330  //*********************************************************
331  // Radiation heating source terms
332  //*********************************************************
333  qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, ngrow_state);
334  qheating_rates[lev]->setVal(0.);
335 
336  //*********************************************************
337  // Radiation fluxes for coupling to LSM
338  //*********************************************************
339 
340  // NOTE: Finer levels do not need to coincide with the bottom domain boundary
341  // at k=0. We make slabs here with the kmin for a given box. Therefore,
342  // care must be taken before applying these fluxes to an LSM model. For
343 
344  // Radiative fluxes for LSM
345  if (solverChoice.lsm_type != LandSurfaceType::None)
346  {
347  BoxList m_bl = ba.boxList();
348  for (auto& b : m_bl) {
349  int kmin = b.smallEnd(2);
350  b.setRange(2,kmin);
351  }
352  BoxArray m_ba(std::move(m_bl));
353 
354  sw_lw_fluxes[lev] = std::make_unique<MultiFab>(m_ba, dm, 5, ngrow_state); // SW direct (2), SW diffuse (2), LW
355  solar_zenith[lev] = std::make_unique<MultiFab>(m_ba, dm, 2, ngrow_state);
356 
357  sw_lw_fluxes[lev]->setVal(0.);
358  solar_zenith[lev]->setVal(0.);
359  }
360 #endif
361 
362  //*********************************************************
363  // Turbulent perturbation region initialization
364  //*********************************************************
365  if (solverChoice.pert_type == PerturbationType::Source ||
366  solverChoice.pert_type == PerturbationType::Direct)
367  {
368  amrex::Box bnd_bx = ba.minimalBox();
370  turbPert.init_tpi(lev, bnd_bx.smallEnd(), bnd_bx.bigEnd(), geom[lev].CellSizeArray(),
371  ba, dm, ngrow_state, pp_prefix, refRatio(), max_level);
372  }
373 
374  //
375  // Define the land mask here and set it to all land by default
376  // NOTE: the logic below will BREAK if we have any grids not touching the bottom boundary
377  //
378  {
379  lmask_lev[lev].resize(1);
380  auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0;
381  BoxList bl2d_mask = ba.boxList();
382  for (auto& b : bl2d_mask) {
383  b.setRange(2,0);
384  }
385  BoxArray ba2d_mask(std::move(bl2d_mask));
386  lmask_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
387  lmask_lev[lev][0]->setVal(1);
388  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
389  }
390 
391  // Read in tables needed for windfarm simulations
392  // fill in Nturb multifab - number of turbines in each mesh cell
393  // write out the vtk files for wind turbine location and/or
394  // actuator disks
395  #ifdef ERF_USE_WINDFARM
396  //init_windfarm(lev);
397  #endif
398 }
#define NDRY
Definition: ERF_IndexDefines.H:13
void init_default_zphys(int, const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc)
Definition: ERF_TerrainMetrics.cpp:15
static AMREX_FORCE_INLINE int ComputeGhostCells(const SolverChoice &sc)
Definition: ERF.H:1206
@ num_comps
Definition: ERF_IndexDefines.H:68
bool test_mapfactor
Definition: ERF_DataStruct.H:678
void init_tpi_type(const PerturbationType &pert_type)
Definition: ERF_TurbPertStruct.H:28
void init_tpi(const int lev, const amrex::IntVect &valid_box_lo, const amrex::IntVect &valid_box_hi, const amrex::GpuArray< amrex::Real, 3 > dx, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, const int ngrow_state, std::string pp_prefix, const amrex::Vector< amrex::IntVect > refRatio, const int max_level)
Definition: ERF_TurbPertStruct.H:43
Here is the call graph for this function:

◆ init_thin_body()

void ERF::init_thin_body ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
599 {
600  //********************************************************************************************
601  // Thin immersed body
602  // *******************************************************************************************
603 #if 0
604  if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
605  (solverChoice.advChoice.zero_yflux.size() > 0) ||
606  (solverChoice.advChoice.zero_zflux.size() > 0))
607  {
608  overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
609  overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
610  }
611 #endif
612 
613  if (solverChoice.advChoice.zero_xflux.size() > 0) {
614  amrex::Print() << "Setting up thin immersed body for "
615  << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
616  BoxArray ba_xf(ba);
617  ba_xf.surroundingNodes(0);
618  thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
619  thin_xforce[lev]->setVal(0.0);
620  xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
621  xflux_imask[lev]->setVal(1);
622  for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
623  {
624  Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
625  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
626  Box xbx = mfi.nodaltilebox(0);
627  for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
628  const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
629  if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
630  (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
631  (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
632  {
633  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
634  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
635  //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
636  amrex::AllPrint() << " mask xface at " << faceidx << std::endl;
637  }
638  }
639  }
640  } else {
641  thin_xforce[lev] = nullptr;
642  xflux_imask[lev] = nullptr;
643  }
644 
645  if (solverChoice.advChoice.zero_yflux.size() > 0) {
646  amrex::Print() << "Setting up thin immersed body for "
647  << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
648  BoxArray ba_yf(ba);
649  ba_yf.surroundingNodes(1);
650  thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
651  thin_yforce[lev]->setVal(0.0);
652  yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
653  yflux_imask[lev]->setVal(1);
654  for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
655  {
656  Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
657  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
658  Box ybx = mfi.nodaltilebox(1);
659  for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
660  const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
661  if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
662  (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
663  (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
664  {
665  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
666  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
667  //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
668  amrex::AllPrint() << " mask yface at " << faceidx << std::endl;
669  }
670  }
671  }
672  } else {
673  thin_yforce[lev] = nullptr;
674  yflux_imask[lev] = nullptr;
675  }
676 
677  if (solverChoice.advChoice.zero_zflux.size() > 0) {
678  amrex::Print() << "Setting up thin immersed body for "
679  << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
680  BoxArray ba_zf(ba);
681  ba_zf.surroundingNodes(2);
682  thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
683  thin_zforce[lev]->setVal(0.0);
684  zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
685  zflux_imask[lev]->setVal(1);
686  for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
687  {
688  Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
689  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
690  Box zbx = mfi.nodaltilebox(2);
691  for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
692  const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
693  if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
694  (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
695  (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
696  {
697  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
698  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
699  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
700  amrex::AllPrint() << " mask zface at " << faceidx << std::endl;
701  }
702  }
703  }
704  } else {
705  thin_zforce[lev] = nullptr;
706  zflux_imask[lev] = nullptr;
707  }
708 }
amrex::Vector< amrex::IntVect > zero_yflux
Definition: ERF_AdvStruct.H:358
amrex::Vector< amrex::IntVect > zero_xflux
Definition: ERF_AdvStruct.H:357
amrex::Vector< amrex::IntVect > zero_zflux
Definition: ERF_AdvStruct.H:359

◆ init_uniform()

void ERF::init_uniform ( int  lev)
private

Use problem-specific reference density and temperature to set the background state to a uniform value.

Parameters
levInteger specifying the current level
18 {
19  auto& lev_new = vars_new[lev];
20  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
21  const Box &gbx = mfi.growntilebox(1);
22  const auto &cons_arr = lev_new[Vars::cons].array(mfi);
23  prob->init_uniform(gbx, cons_arr);
24  }
25 }

◆ init_zphys()

void ERF::init_zphys ( int  lev,
amrex::Real  time 
)
492 {
493  if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
494  {
495  if (lev > 0) {
496  //
497  // First interpolate from coarser level if there is one
498  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
499  // have been pre-filled - this includes ghost cells both inside and outside
500  // the domain
501  //
502  InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
503  IntVect(0,0,0), // do not fill ghost cells outside the domain
504  *z_phys_nd[lev-1], 0, 0, 1,
505  geom[lev-1], geom[lev],
506  refRatio(lev-1), &node_bilinear_interp,
508  }
509 
510  int ngrow = ComputeGhostCells(solverChoice) + 2;
511  Box bx(surroundingNodes(Geom(lev).Domain())); bx.grow(ngrow);
512  FArrayBox terrain_fab(makeSlab(bx,2,0),1);
513 
514  //
515  // If we are using fitted mesh then we use the surface as defined above
516  // If we are not using fitted mesh but are using z_levels, we still need z_phys (for now)
517  // but we need to use a flat terrain for the mesh itself (the EB data has already been made
518  // from the correct terrain)
519  //
520  if (solverChoice.terrain_type != TerrainType::StaticFittedMesh &&
521  solverChoice.terrain_type != TerrainType::MovingFittedMesh) {
522  terrain_fab.template setVal<RunOn::Device>(0.0);
523  } else {
524  //
525  // Fill the values of the terrain height at k=0 only
526  //
527  prob->init_terrain_surface(geom[lev],terrain_fab,time);
528  }
529 
530  for (MFIter mfi(*z_phys_nd[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
531  {
532  Box isect = terrain_fab.box() & (*z_phys_nd[lev])[mfi].box();
533  if (!isect.isEmpty()) {
534  (*z_phys_nd[lev])[mfi].template copy<RunOn::Device>(terrain_fab,isect,0,isect,0,1);
535  }
536  }
537 
539 
540  z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
541 
542  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
543  terrain_blanking[lev]->setVal(1.0);
544  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
545  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
546  }
547 
548  if (lev == 0) {
549  Real zmax = z_phys_nd[0]->max(0,0,false);
550  Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
551  if (rel_diff < 1.e-8) {
552  amrex::Print() << "max of zphys_nd " << zmax << std::endl;
553  amrex::Print() << "max of zlevels " << zlevels_stag[0][zlevels_stag[0].size()-1] << std::endl;
554  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < 1.e-8, "Terrain is taller than domain top!");
555  }
556  } // lev == 0
557 
558  } // init_type
559 }
void make_terrain_fitted_coords(int lev, const Geometry &geom, MultiFab &z_phys_nd, Vector< Real > const &z_levels_h, GpuArray< ERF_BC, AMREX_SPACEDIM *2 > &phys_bc_type)
Definition: ERF_TerrainMetrics.cpp:46
amrex::EBFArrayBoxFactory const & EBFactory(int lev) const noexcept
Definition: ERF.H:1482
Here is the call graph for this function:

◆ InitData()

void ERF::InitData ( )
670 {
671  BL_PROFILE_VAR("ERF::InitData()", InitData);
672  InitData_pre();
673  InitData_post();
674  BL_PROFILE_VAR_STOP(InitData);
675 }
void InitData_pre()
Definition: ERF.cpp:678
void InitData_post()
Definition: ERF.cpp:736
void InitData()
Definition: ERF.cpp:669

Referenced by main().

Here is the caller graph for this function:

◆ InitData_post()

void ERF::InitData_post ( )
737 {
738  if (restart_chkfile.empty()) {
739  //
740  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
741  //
742  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
743  for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
744  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
745  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
746  }
747  }
748 
749  if (solverChoice.coupling_type == CouplingType::TwoWay) {
750  AverageDown();
751  }
752 
754  {
755  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
756  "Thin immersed body with refinement not currently supported.");
757  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
758  amrex::Print() << "NOTE: Thin immersed body with non-constant dz has not been tested." << std::endl;
759  }
760  }
761 
762 #ifdef ERF_USE_PARTICLES
763  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
764  for (int lev = 0; lev <= finest_level; lev++) {
765  dynamic_cast<LagrangianMicrophysics&>(*micro).initParticles(z_phys_nd[lev]);
766  }
767  }
768 #endif
769 
770  } else { // Restart from a checkpoint
771 
772  restart();
773 
774  // Create the physbc objects for {cons, u, v, w, base state}
775  // We fill the additional base state ghost cells just in case we have read the old format
776  for (int lev(0); lev <= finest_level; ++lev) {
777  make_physbcs(lev);
778  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
779  }
780 
782  for (int lev(0); lev <= finest_level; ++lev) {
783  m_forest_drag[lev]->define_drag_field(grids[lev], dmap[lev], geom[lev],
784  z_phys_cc[lev].get(), z_phys_nd[lev].get());
785  }
786  }
787 
788 #ifdef ERF_USE_NETCDF
789  //
790  // Create the needed bdy_data_xlo etc ... since we don't read it in from checkpoint any more
791  //
793 
794  bdy_time_interval = read_times_from_wrfbdy(nc_bdy_file,
795  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
796  start_bdy_time);
797  Real dT = bdy_time_interval;
798 
799  Real time_since_start_old = t_new[0] - start_bdy_time;
800  int n_time_old = static_cast<int>(time_since_start_old / dT);
801 
802  // I don't think this works if lev > 0 ...?
803  AMREX_ALWAYS_ASSERT(finest_level == 0);
804  int lev = 0;
805  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
806  for (int itime = n_time_old; itime < n_time_old+3; itime++)
807  {
808  read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
809  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
810  real_width);
811  convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
812  *mf_MUB[lev], *mf_C1H[lev], *mf_C2H[lev],
814  geom[lev], use_moist);
815  } // itime
816  } // use_real_bcs && lev == 0
817 #endif
818  } // end restart
819 
820 #ifdef ERF_USE_PARTICLES
821  /* If using a Lagrangian microphysics model, its particle container has now been
822  constructed and initialized (calls to micro->Init). So, add its pointer to
823  ERF::particleData and remove its name from list of unallocated particle containers. */
824  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
825  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
826  const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
827  particleData.pushBack(pc_name, pc_ptr);
828  particleData.getNamesUnalloc().remove(pc_name);
829  }
830 #endif
831 
832  if (input_bndry_planes) {
833  // Read the "time.dat" file to know what data is available
834  m_r2d->read_time_file();
835 
836  // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
837  Real dt_dummy = 0.0;
838  m_r2d->read_input_files(t_new[0],dt_dummy,m_bc_extdir_vals);
839  }
840 
842  {
843  h_rhotheta_src.resize(max_level+1, Vector<Real>(0));
844  d_rhotheta_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
845  for (int lev = 0; lev <= finest_level; lev++) {
846  const int domlen = geom[lev].Domain().length(2);
847  h_rhotheta_src[lev].resize(domlen, 0.0_rt);
848  d_rhotheta_src[lev].resize(domlen, 0.0_rt);
849  prob->update_rhotheta_sources(t_new[0],
850  h_rhotheta_src[lev], d_rhotheta_src[lev],
851  geom[lev], z_phys_cc[lev]);
852  }
853  }
854 
856  {
857  h_u_geos.resize(max_level+1, Vector<Real>(0));
858  d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
859  h_v_geos.resize(max_level+1, Vector<Real>(0));
860  d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
861  for (int lev = 0; lev <= finest_level; lev++) {
862  const int domlen = geom[lev].Domain().length(2);
863  h_u_geos[lev].resize(domlen, 0.0_rt);
864  d_u_geos[lev].resize(domlen, 0.0_rt);
865  h_v_geos[lev].resize(domlen, 0.0_rt);
866  d_v_geos[lev].resize(domlen, 0.0_rt);
868  prob->update_geostrophic_profile(t_new[0],
869  h_u_geos[lev], d_u_geos[lev],
870  h_v_geos[lev], d_v_geos[lev],
871  geom[lev], z_phys_cc[lev]);
872  } else {
873  if (SolverChoice::mesh_type == MeshType::VariableDz) {
874  amrex::Print() << "Note: 1-D geostrophic wind profile input is not defined for real terrain" << std::endl;
875  }
877  h_u_geos[lev], d_u_geos[lev],
878  h_v_geos[lev], d_v_geos[lev],
879  geom[lev],
880  zlevels_stag[0]);
881  }
882  }
883  }
884 
886  {
887  h_rhoqt_src.resize(max_level+1, Vector<Real>(0));
888  d_rhoqt_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
889  for (int lev = 0; lev <= finest_level; lev++) {
890  const int domlen = geom[lev].Domain().length(2);
891  h_rhoqt_src[lev].resize(domlen, 0.0_rt);
892  d_rhoqt_src[lev].resize(domlen, 0.0_rt);
893  prob->update_rhoqt_sources(t_new[0],
894  h_rhoqt_src[lev], d_rhoqt_src[lev],
895  geom[lev], z_phys_cc[lev]);
896  }
897  }
898 
900  {
901  h_w_subsid.resize(max_level+1, Vector<Real>(0));
902  d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
903  for (int lev = 0; lev <= finest_level; lev++) {
904  const int domlen = geom[lev].Domain().length(2) + 1; // lives on z-faces
905  h_w_subsid[lev].resize(domlen, 0.0_rt);
906  d_w_subsid[lev].resize(domlen, 0.0_rt);
907  prob->update_w_subsidence(t_new[0],
908  h_w_subsid[lev], d_w_subsid[lev],
909  geom[lev], z_phys_nd[lev]);
910  }
911  }
912 
915  {
916  initRayleigh();
917  if (solverChoice.init_type == InitType::Input_Sounding)
918  {
919  // Overwrite ubar, vbar, and thetabar with input profiles;
920  // wbar is assumed to be 0. Note: the tau coefficient set by
921  // prob->erf_init_rayleigh() is still used
922  bool restarting = (!restart_chkfile.empty());
923  setRayleighRefFromSounding(restarting);
924  }
925  }
926 
927  // Read in sponge data from input file
928  if(solverChoice.spongeChoice.sponge_type == "input_sponge")
929  {
930  initSponge();
931  bool restarting = (!restart_chkfile.empty());
932  setSpongeRefFromSounding(restarting);
933  }
934 
935  if (solverChoice.pert_type == PerturbationType::Source ||
936  solverChoice.pert_type == PerturbationType::Direct) {
937  if (is_it_time_for_action(istep[0], t_new[0], dt[0], pert_interval, -1.)) {
938  turbPert.debug(t_new[0]);
939  }
940  }
941 
942  // We only write the file at level 0 for now
944  {
945  // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
946  m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
947 
948  Real time = 0.;
949  if (time >= bndry_output_planes_start_time) {
950  bool is_moist = (micro->Get_Qstate_Size() > 0);
951  m_w2d->write_planes(0, time, vars_new, is_moist);
952  }
953  }
954 
955  //
956  // If we are starting from scratch, we have the option to project the initial velocity field
957  // regardless of how we initialized.
958  // pp_inc is used as scratch space here; we zero it out after the projection
959  //
960  if (restart_chkfile == "")
961  {
963  Real dummy_dt = 1.0;
964  if (verbose > 0) {
965  amrex::Print() << "Projecting initial velocity field" << std::endl;
966  }
967  for (int lev = 0; lev <= finest_level; ++lev)
968  {
969  project_velocities(lev, dummy_dt, vars_new[lev], pp_inc[lev]);
970  pp_inc[lev].setVal(0.);
971  }
972  }
973  }
974 
975  // Copy from new into old just in case
976  for (int lev = 0; lev <= finest_level; ++lev)
977  {
978  auto& lev_new = vars_new[lev];
979  auto& lev_old = vars_old[lev];
980 
981  // ***************************************************************************
982  // Physical bc's at domain boundary
983  // ***************************************************************************
984  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
985  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
986 
987  int ncomp_cons = lev_new[Vars::cons].nComp();
988  bool do_fb = true;
989 
990 #ifdef ERF_USE_NETCDF
991  // We call this here because it is an ERF routine
992  if (solverChoice.use_real_bcs && (lev==0)) {
993  int icomp_cons = 0;
994  bool cons_only = false;
995  Vector<MultiFab*> mfs_vec = {&lev_new[Vars::cons],&lev_new[Vars::xvel],
996  &lev_new[Vars::yvel],&lev_new[Vars::zvel]};
997  fill_from_realbdy(mfs_vec,t_new[lev],cons_only,icomp_cons,
998  ncomp_cons,ngvect_cons,ngvect_vels);
999  do_fb = false;
1000  }
1001 #endif
1002 
1003  (*physbcs_cons[lev])(lev_new[Vars::cons],lev_new[Vars::xvel],lev_new[Vars::yvel],0,ncomp_cons,
1004  ngvect_cons,t_new[lev],BCVars::cons_bc,do_fb);
1005  ( *physbcs_u[lev])(lev_new[Vars::xvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1006  ngvect_vels,t_new[lev],BCVars::xvel_bc,do_fb);
1007  ( *physbcs_v[lev])(lev_new[Vars::yvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1008  ngvect_vels,t_new[lev],BCVars::yvel_bc,do_fb);
1009  ( *physbcs_w[lev])(lev_new[Vars::zvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1010  ngvect_vels,t_new[lev],BCVars::zvel_bc,do_fb);
1011 
1012  MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,ncomp_cons,lev_new[Vars::cons].nGrowVect());
1013  MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0, 1,lev_new[Vars::xvel].nGrowVect());
1014  MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0, 1,lev_new[Vars::yvel].nGrowVect());
1015  MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0, 1,lev_new[Vars::zvel].nGrowVect());
1016  }
1017 
1018  // Compute the minimum dz in the domain at each level (to be used for setting the timestep)
1019  dz_min.resize(max_level+1);
1020  for (int lev = 0; lev <= finest_level; ++lev)
1021  {
1022  dz_min[lev] = geom[lev].CellSize(2);
1023  if ( SolverChoice::mesh_type != MeshType::ConstantDz ) {
1024  dz_min[lev] *= (*detJ_cc[lev]).min(0);
1025  }
1026  }
1027 
1028  ComputeDt();
1029 
1030  // Fill ghost cells/faces
1031  for (int lev = 0; lev <= finest_level; ++lev)
1032  {
1033  if (lev > 0 && cf_width >= 0) {
1035  }
1036 
1037  auto& lev_new = vars_new[lev];
1038 
1039  //
1040  // Fill boundary conditions -- not sure why we need this here
1041  //
1042  bool fillset = false;
1043  if (lev == 0) {
1044  FillPatch(lev, t_new[lev],
1045  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]});
1046  } else {
1047  FillPatch(lev, t_new[lev],
1048  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
1049  {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
1050  base_state[lev], base_state[lev],
1051  fillset);
1052  }
1053 
1054  //
1055  // We do this here to make sure level (lev-1) boundary conditions are filled
1056  // before we interpolate to level (lev) ghost cells
1057  //
1058  if (lev < finest_level) {
1059  auto& lev_old = vars_old[lev];
1060  MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
1061  MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
1062  MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
1063  MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
1064  }
1065 
1066  //
1067  // We fill the ghost cell values of the base state in case it wasn't done in the initialization
1068  //
1069  base_state[lev].FillBoundary(geom[lev].periodicity());
1070 
1071  // For moving terrain only
1072  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
1073  MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
1074  base_state_new[lev].FillBoundary(geom[lev].periodicity());
1075  }
1076 
1077  }
1078 
1079  // Allow idealized cases over water, used to set lmask
1080  ParmParse pp("erf");
1081  int is_land;
1082  for (int lev = 0; lev <= finest_level; ++lev)
1083  {
1084  if (pp.query("is_land", is_land, lev)) {
1085  if (is_land == 1) {
1086  amrex::Print() << "Level " << lev << " is land" << std::endl;
1087  } else if (is_land == 0) {
1088  amrex::Print() << "Level " << lev << " is water" << std::endl;
1089  } else {
1090  Error("is_land should be 0 or 1");
1091  }
1092  lmask_lev[lev][0]->setVal(is_land);
1093  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
1094  }
1095  }
1096 
1097 #ifdef ERF_USE_WW3_COUPLING
1098  int my_lev = 0;
1099  amrex::Print() << " About to call send_to_ww3 from ERF.cpp" << std::endl;
1100  send_to_ww3(my_lev);
1101  amrex::Print() << " About to call read_waves from ERF.cpp" << std::endl;
1102  read_waves(my_lev);
1103  // send_to_ww3(my_lev);
1104 #endif
1105 
1106  // Configure SurfaceLayer params if used
1107  // NOTE: we must set up the MOST routine after calling FillPatch
1108  // in order to have lateral ghost cells filled (MOST + terrain interp).
1109  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
1110  {
1111  bool rotate = solverChoice.use_rotate_surface_flux;
1112  if (rotate) {
1113  Print() << "Using surface layer model with stress rotations" << std::endl;
1114  }
1115 
1116  //
1117  // This constructor will make the ABLMost object but not allocate the arrays at each level.
1118  //
1119  m_SurfaceLayer = std::make_unique<SurfaceLayer>(geom, rotate, pp_prefix, Qv_prim,
1121 #ifdef ERF_USE_NETCDF
1122  ,start_bdy_time, bdy_time_interval
1123 #endif
1124  );
1125  // This call will allocate the arrays at each level. If we regrid later, either changing
1126  // the number of level sor just the grids at each existing level, we will call an update routine
1127  // to redefine the internal arrays in m_SurfaceLayer.
1128  int nlevs = geom.size();
1129  for (int lev = 0; lev < nlevs; lev++)
1130  {
1131  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
1132  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
1133  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
1134  mfv_old, Theta_prim[lev], Qv_prim[lev],
1135  Qr_prim[lev], z_phys_nd[lev],
1136  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
1137  lsm_data[lev], lsm_flux[lev], sst_lev[lev],
1138  tsk_lev[lev], lmask_lev[lev]);
1139  }
1140 
1141 
1142  if (restart_chkfile != "") {
1143  // Update surface fields if needed
1145  }
1146 
1147  // We now configure ABLMost params here so that we can print the averages at t=0
1148  // Note we don't fill ghost cells here because this is just for diagnostics
1149  for (int lev = 0; lev <= finest_level; ++lev)
1150  {
1151  Real time = t_new[lev];
1152  IntVect ng = Theta_prim[lev]->nGrowVect();
1153 
1154  MultiFab::Copy( *Theta_prim[lev], vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, ng);
1155  MultiFab::Divide(*Theta_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1156 
1157  if (solverChoice.moisture_type != MoistureType::None) {
1158  ng = Qv_prim[lev]->nGrowVect();
1159 
1160  MultiFab::Copy( *Qv_prim[lev], vars_new[lev][Vars::cons], RhoQ1_comp, 0, 1, ng);
1161  MultiFab::Divide(*Qv_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1162 
1163  int rhoqr_comp = solverChoice.RhoQr_comp;
1164  if (rhoqr_comp > -1) {
1165  MultiFab::Copy( *Qr_prim[lev], vars_new[lev][Vars::cons], rhoqr_comp, 0, 1, ng);
1166  MultiFab::Divide(*Qr_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1167  } else {
1168  Qr_prim[lev]->setVal(0.0);
1169  }
1170  }
1171  m_SurfaceLayer->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim, Qr_prim);
1172 
1173  if (restart_chkfile == "") {
1174  // Only do this if starting from scratch; if restarting, then
1175  // we don't want to call update_fluxes multiple times because
1176  // it will change u* and theta* from their previous values
1177  m_SurfaceLayer->update_pblh(lev, vars_new, z_phys_cc[lev].get(),
1181  m_SurfaceLayer->update_fluxes(lev, time);
1182  }
1183  }
1184  }
1185 
1186  // Update micro vars before first plot file
1187  if (solverChoice.moisture_type != MoistureType::None) {
1188  for (int lev = 0; lev <= finest_level; ++lev) micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons]);
1189  }
1190 
1191  // Fill time averaged velocities before first plot file
1192  if (solverChoice.time_avg_vel) {
1193  for (int lev = 0; lev <= finest_level; ++lev) {
1194  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
1195  vars_new[lev][Vars::xvel],
1196  vars_new[lev][Vars::yvel],
1197  vars_new[lev][Vars::zvel]);
1198  }
1199  }
1200 
1201  // check for additional plotting variables that are available after particle containers
1202  // are setup.
1203  const std::string& pv1 = "plot_vars_1"; appendPlotVariables(pv1,plot_var_names_1);
1204  const std::string& pv2 = "plot_vars_2"; appendPlotVariables(pv2,plot_var_names_2);
1205 
1206  if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > 0.) )
1207  {
1210  }
1211 
1212  if ( (restart_chkfile.empty()) ||
1213  (!restart_chkfile.empty() && plot_file_on_restart) )
1214  {
1215  if (m_plot_int_1 > 0 || m_plot_per_1 > 0.)
1216  {
1219  }
1220  if (m_plot_int_2 > 0 || m_plot_per_2 > 0.)
1221  {
1224  }
1225  if (m_subvol_int > 0 || m_subvol_per > 0.) {
1226  WriteSubvolume();
1227  last_subvol = istep[0];
1228  }
1229  }
1230 
1231  // Set these up here because we need to know which MPI rank "cell" is on...
1232  if (pp.contains("data_log"))
1233  {
1234  int num_datalogs = pp.countval("data_log");
1235  datalog.resize(num_datalogs);
1236  datalogname.resize(num_datalogs);
1237  pp.queryarr("data_log",datalogname,0,num_datalogs);
1238  for (int i = 0; i < num_datalogs; i++) {
1240  }
1241  }
1242 
1243  if (pp.contains("der_data_log"))
1244  {
1245  int num_der_datalogs = pp.countval("der_data_log");
1246  der_datalog.resize(num_der_datalogs);
1247  der_datalogname.resize(num_der_datalogs);
1248  pp.queryarr("der_data_log",der_datalogname,0,num_der_datalogs);
1249  for (int i = 0; i < num_der_datalogs; i++) {
1251  }
1252  }
1253 
1254  if (pp.contains("energy_data_log"))
1255  {
1256  int num_energy_datalogs = pp.countval("energy_data_log");
1257  tot_e_datalog.resize(num_energy_datalogs);
1258  tot_e_datalogname.resize(num_energy_datalogs);
1259  pp.queryarr("energy_data_log",tot_e_datalogname,0,num_energy_datalogs);
1260  for (int i = 0; i < num_energy_datalogs; i++) {
1262  }
1263  }
1264 
1265 
1266  if (restart_chkfile.empty() && profile_int > 0) {
1267  if (destag_profiles) {
1268  // all variables cell-centered
1270  } else {
1271  // some variables staggered
1273  }
1274  }
1275 
1276  if (pp.contains("sample_point_log") && pp.contains("sample_point"))
1277  {
1278  int lev = 0;
1279 
1280  int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
1281  if (num_samplepts > 0) {
1282  Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
1283  samplepoint.resize(num_samplepts);
1284 
1285  pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
1286  for (int i = 0; i < num_samplepts; i++) {
1287  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1288  samplepoint[i] = iv;
1289  }
1290  }
1291 
1292  int num_sampleptlogs = pp.countval("sample_point_log");
1293  AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
1294  if (num_sampleptlogs > 0) {
1295  sampleptlog.resize(num_sampleptlogs);
1296  sampleptlogname.resize(num_sampleptlogs);
1297  pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
1298 
1299  for (int i = 0; i < num_sampleptlogs; i++) {
1301  }
1302  }
1303 
1304  }
1305 
1306  if (pp.contains("sample_line_log") && pp.contains("sample_line"))
1307  {
1308  int lev = 0;
1309 
1310  int num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
1311  if (num_samplelines > 0) {
1312  Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
1313  sampleline.resize(num_samplelines);
1314 
1315  pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
1316  for (int i = 0; i < num_samplelines; i++) {
1317  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1318  sampleline[i] = iv;
1319  }
1320  }
1321 
1322  int num_samplelinelogs = pp.countval("sample_line_log");
1323  AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
1324  if (num_samplelinelogs > 0) {
1325  samplelinelog.resize(num_samplelinelogs);
1326  samplelinelogname.resize(num_samplelinelogs);
1327  pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
1328 
1329  for (int i = 0; i < num_samplelinelogs; i++) {
1331  }
1332  }
1333 
1334  }
1335 
1340  }
1341 
1342  // Create object to do line and plane sampling if needed
1343  bool do_line = false; bool do_plane = false;
1344  pp.query("do_line_sampling",do_line); pp.query("do_plane_sampling",do_plane);
1345  if (do_line || do_plane) { data_sampler = std::make_unique<SampleData>(do_line, do_plane); }
1346 
1347  if ( solverChoice.terrain_type == TerrainType::EB ||
1348  solverChoice.terrain_type == TerrainType::ImmersedForcing)
1349  {
1350  bool write_eb_surface = false;
1351  pp.query("write_eb_surface", write_eb_surface);
1352  if (write_eb_surface) WriteMyEBSurface();
1353  }
1354 
1355 }
void initRayleigh()
Initialize Rayleigh damping profiles.
Definition: ERF_InitRayleigh.cpp:14
amrex::Vector< std::string > samplelinelogname
Definition: ERF.H:1460
void setRayleighRefFromSounding(bool restarting)
Set Rayleigh mean profiles from input sounding.
Definition: ERF_InitRayleigh.cpp:55
amrex::Vector< amrex::IntVect > sampleline
Definition: ERF.H:1461
static amrex::Real sum_per
Definition: ERF.H:1068
void setRecordDataInfo(int i, const std::string &filename)
Definition: ERF.H:1370
void WriteMyEBSurface()
Definition: ERF_EBWriteSurface.cpp:5
void write_1D_profiles_stag(amrex::Real time)
Definition: ERF_Write1DProfiles_stag.cpp:25
void sum_energy_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:275
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
Definition: ERF.H:1459
static int sum_interval
Definition: ERF.H:1066
static int pert_interval
Definition: ERF.H:1067
void restart()
Definition: ERF.cpp:1394
void write_1D_profiles(amrex::Real time)
Definition: ERF_Write1DProfiles.cpp:17
int profile_int
Definition: ERF.H:973
bool destag_profiles
Definition: ERF.H:974
void appendPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:173
amrex::Vector< std::string > tot_e_datalogname
Definition: ERF.H:1453
static int output_bndry_planes
Definition: ERF.H:1124
static std::string nc_bdy_file
Definition: ERF.H:1083
void AverageDown()
Definition: ERF_AverageDown.cpp:16
std::unique_ptr< SampleData > data_sampler
Definition: ERF.H:1446
static amrex::Real bndry_output_planes_start_time
Definition: ERF.H:1127
std::string restart_chkfile
Definition: ERF.H:934
amrex::Vector< std::string > sampleptlogname
Definition: ERF.H:1456
void sum_derived_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:169
void sum_integrated_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:15
void setRecordDerDataInfo(int i, const std::string &filename)
Definition: ERF.H:1383
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
Definition: ERF.H:1455
std::unique_ptr< WriteBndryPlanes > m_w2d
Definition: ERF.H:1186
void init_geo_wind_profile(const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
Definition: ERF_InitGeowind.cpp:10
void initSponge()
Initialize sponge profiles.
Definition: ERF_InitSponge.cpp:35
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
Definition: ERF.H:1450
int real_width
Definition: ERF.H:1084
void setRecordEnergyDataInfo(int i, const std::string &filename)
Definition: ERF.H:1396
static bool is_it_time_for_action(int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
Definition: ERF_WriteScalarProfiles.cpp:685
void project_velocities(int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars, amrex::MultiFab &p)
Definition: ERF_PoissonSolve.cpp:10
int plot_file_on_restart
Definition: ERF.H:914
void Construct_ERFFillPatchers(int lev)
Definition: ERF.cpp:2076
void setRecordSampleLineInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1426
void setSpongeRefFromSounding(bool restarting)
Set sponge mean profiles from input sounding.
Definition: ERF_InitSponge.cpp:65
amrex::Vector< amrex::IntVect > samplepoint
Definition: ERF.H:1457
void setRecordSamplePointInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1409
void ReadCheckpointFileSurfaceLayer()
Definition: ERF_Checkpoint.cpp:877
static MoistureModelType modelType(const MoistureType a_moisture_type)
query if a specified moisture model is Eulerian or Lagrangian
Definition: ERF_Microphysics.H:69
bool have_zero_flux_faces
Definition: ERF_AdvStruct.H:360
bool have_geo_wind_profile
Definition: ERF_DataStruct.H:755
bool project_initial_velocity
Definition: ERF_DataStruct.H:708
std::string abl_geo_wind_table
Definition: ERF_DataStruct.H:754
bool use_rotate_surface_flux
Definition: ERF_DataStruct.H:727
bool do_forest_drag
Definition: ERF_DataStruct.H:781
void debug(amrex::Real)
Definition: ERF_TurbPertStruct.H:555
Here is the call graph for this function:

◆ InitData_pre()

void ERF::InitData_pre ( )
679 {
680  // Initialize the start time for our CPU-time tracker
681  startCPUTime = ParallelDescriptor::second();
682 
683  // Create the ReadBndryPlanes object so we can read boundary plane data
684  // m_r2d is used by init_bcs so we must instantiate this class before
685  if (input_bndry_planes) {
686  Print() << "Defining r2d for the first time " << std::endl;
687  m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
688  }
689 
693 
694  if (restart_chkfile.empty()) {
695  // start simulation from the beginning
696 
697  const Real time = start_time;
698  InitFromScratch(time);
699  } else {
700  // For initialization this is done in init_only; it is done here for restart
701  init_bcs();
702  }
703 
704  // Verify solver choices
705  for (int lev(0); lev <= max_level; ++lev) {
706  // BC compatibility
707  if ( ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
708  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ||
709  (solverChoice.turbChoice[lev].pbl_type == PBLType::YSU) ) &&
710  phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::surface_layer ) {
711  Abort("MYNN2.5/MYNNEDMF/YSU PBL Model requires MOST at lower boundary");
712  }
713  if ( (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) &&
714  (solverChoice.turbChoice[lev].Ce_wall > 0) &&
715  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::surface_layer) &&
716  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::slip_wall) &&
717  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::no_slip_wall) )
718  {
719  Warning("Deardorff LES assumes wall at zlo when applying Ce_wall");
720  }
721 
722  // mesoscale diffusion
723  if ((geom[lev].CellSize(0) > 2000.) || (geom[lev].CellSize(1) > 2000.))
724  {
725  if ( (solverChoice.turbChoice[lev].les_type == LESType::Smagorinsky) &&
726  (!solverChoice.turbChoice[lev].smag2d)) {
727  Warning("Should use 2-D Smagorinsky for mesoscale resolution");
728  } else if (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) {
729  Warning("Should not use Deardorff LES for mesoscale resolution");
730  }
731  }
732  }
733 }
void init_bcs()
Definition: ERF_InitBCs.cpp:20

◆ initHSE() [1/2]

void ERF::initHSE ( )
private

Initialize HSE.

143 {
144  AMREX_ALWAYS_ASSERT(!init_sounding_ideal);
145  for (int lev = 0; lev <= finest_level; lev++)
146  {
147  initHSE(lev);
148  }
149 }

◆ initHSE() [2/2]

void ERF::initHSE ( int  lev)
private

Initialize density and pressure base state in hydrostatic equilibrium.

21 {
22  // This integrates up through column to update p_hse, pi_hse, th_hse;
23  // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
24 
25  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
26  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
27  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
28  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
29  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
30 
31  bool all_boxes_touch_bottom = true;
32  Box domain(geom[lev].Domain());
33 
34  int icomp = 0; int ncomp = BaseState::num_comps;
35 
36  if (lev == 0) {
37  BoxArray ba(base_state[lev].boxArray());
38  for (int i = 0; i < ba.size(); i++) {
39  if (ba[i].smallEnd(2) != domain.smallEnd(2)) {
40  all_boxes_touch_bottom = false;
41  }
42  }
43  }
44  else
45  {
46  //
47  // We need to do this interp from coarse level in order to set the values of
48  // the base state inside the domain but outside of the fine region
49  //
50  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
51  //
52  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
53  // have been pre-filled - this includes ghost cells both inside and outside
54  // the domain
55  //
56  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
57  IntVect(0,0,0), // do not fill ghost cells outside the domain
58  base_state[lev-1], icomp, icomp, ncomp,
59  geom[lev-1], geom[lev],
60  refRatio(lev-1), &cell_cons_interp,
62 
63  // We need to do this here because the interpolation above may leave corners unfilled
64  // when the corners need to be filled by, for example, reflection of the fine ghost
65  // cell outside the fine region but inide the domain.
66  (*physbcs_base[lev])(base_state[lev],icomp,ncomp,base_state[lev].nGrowVect());
67  }
68 
69  if (all_boxes_touch_bottom || lev > 0) {
70 
71  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
73  prob->erf_init_dens_hse_moist(r_hse, z_phys_nd[lev], geom[lev]);
74  } else {
75  prob->erf_init_dens_hse(r_hse, z_phys_nd[lev], z_phys_cc[lev], geom[lev]);
76  }
77 
78  erf_enforce_hse(lev, r_hse, p_hse, pi_hse, th_hse, qv_hse, z_phys_cc[lev]);
79 
80  //
81  // Impose physical bc's on the base state
82  //
83  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
84 
85  } else {
86 
87  BoxArray ba_new(domain);
88 
89  ChopGrids2D(ba_new, domain, ParallelDescriptor::NProcs());
90 
91  DistributionMapping dm_new(ba_new);
92 
93  MultiFab new_base_state(ba_new, dm_new, BaseState::num_comps, base_state[lev].nGrowVect());
94  new_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
95  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
96 
97  MultiFab new_r_hse (new_base_state, make_alias, BaseState::r0_comp, 1);
98  MultiFab new_p_hse (new_base_state, make_alias, BaseState::p0_comp, 1);
99  MultiFab new_pi_hse(new_base_state, make_alias, BaseState::pi0_comp, 1);
100  MultiFab new_th_hse(new_base_state, make_alias, BaseState::th0_comp, 1);
101  MultiFab new_qv_hse(new_base_state, make_alias, BaseState::qv0_comp, 1);
102 
103  std::unique_ptr<MultiFab> new_z_phys_cc;
104  std::unique_ptr<MultiFab> new_z_phys_nd;
105  if (solverChoice.mesh_type != MeshType::ConstantDz) {
106  new_z_phys_cc = std::make_unique<MultiFab>(ba_new,dm_new,1,1);
107  new_z_phys_cc->ParallelCopy(*z_phys_cc[lev],0,0,1,1,1);
108 
109  BoxArray ba_new_nd(ba_new);
110  ba_new_nd.surroundingNodes();
111  new_z_phys_nd = std::make_unique<MultiFab>(ba_new_nd,dm_new,1,1);
112  new_z_phys_nd->ParallelCopy(*z_phys_nd[lev],0,0,1,1,1);
113  }
114 
115  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
117  prob->erf_init_dens_hse_moist(new_r_hse, new_z_phys_nd, geom[lev]);
118  } else {
119  prob->erf_init_dens_hse(new_r_hse, new_z_phys_nd, new_z_phys_cc, geom[lev]);
120  }
121 
122  erf_enforce_hse(lev, new_r_hse, new_p_hse, new_pi_hse, new_th_hse, new_qv_hse, new_z_phys_cc);
123 
124  //
125  // Impose physical bc's on the base state
126  //
127  (*physbcs_base[lev])(new_base_state,0,new_base_state.nComp(),new_base_state.nGrowVect());
128 
129  // Now copy back into the original arrays
130  base_state[lev].ParallelCopy(new_base_state,0,0,base_state[lev].nComp(),
131  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
132  }
133 
134  //
135  // Impose physical bc's on the base state -- the values outside the fine region
136  // but inside the domain have already been filled in the call above to InterpFromCoarseLevel
137  //
138  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
139 }
void ChopGrids2D(BoxArray &ba, const Box &domain, int target_size)
Definition: ERF_ChopGrids.cpp:21
void erf_enforce_hse(int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
Definition: ERF_Init1D.cpp:161
bool use_moist_background
Definition: ERF_DataStruct.H:763
Here is the call graph for this function:

◆ initialize_integrator()

void ERF::initialize_integrator ( int  lev,
amrex::MultiFab &  cons_mf,
amrex::MultiFab &  vel_mf 
)
private
608 {
609  const BoxArray& ba(cons_mf.boxArray());
610  const DistributionMapping& dm(cons_mf.DistributionMap());
611 
612  int ncomp_cons = cons_mf.nComp();
613 
614  // Initialize the integrator memory
615  Vector<MultiFab> int_state; // integration state data structure example
616  int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons)); // cons
617  int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
618  int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
619  int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
620 
621  mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
622  mri_integrator_mem[lev]->setNoSubstepping((solverChoice.substepping_type[lev] == SubsteppingType::None));
623  mri_integrator_mem[lev]->setAnelastic(solverChoice.anelastic[lev]);
624  mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
625  mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
626 }

◆ InitializeFromFile()

void ERF::InitializeFromFile ( )
private

◆ InitializeLevelFromData()

void ERF::InitializeLevelFromData ( int  lev,
const amrex::MultiFab &  initial_data 
)
private

◆ initializeMicrophysics()

void ERF::initializeMicrophysics ( const int &  a_nlevsmax)
private
Parameters
a_nlevsmaxnumber of AMR levels
1360 {
1361  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
1362 
1363  micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1364 
1365  } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
1366 #ifdef ERF_USE_PARTICLES
1367 
1368  micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1369  /* Lagrangian microphysics models will have a particle container; it needs to be added
1370  to ERF::particleData */
1371  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
1372  /* The particle container has not yet been constructed and initialized, so just add
1373  its name here for now (so that functions to set plotting variables can see it). */
1374  particleData.addName( pc_name );
1375 
1376 #else
1377  Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
1378 #endif
1379  }
1380 
1381  qmoist.resize(a_nlevsmax);
1382  return;
1383 }
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
Definition: ERF.H:773
Here is the call graph for this function:

◆ initRayleigh()

void ERF::initRayleigh ( )
private

Initialize Rayleigh damping profiles.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

15 {
16  const int khi = geom[0].Domain().bigEnd(2);
17  solverChoice.rayleigh_ztop = (solverChoice.terrain_type == TerrainType::None) ? geom[0].ProbHi(2) : zlevels_stag[0][khi+1];
18 
19  h_rayleigh_ptrs.resize(max_level+1);
20  d_rayleigh_ptrs.resize(max_level+1);
21 
22  for (int lev = 0; lev <= finest_level; lev++)
23  {
24  // These have 4 components: ubar, vbar, wbar, thetabar
25  h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
26  d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
27 
28  const int zlen_rayleigh = geom[lev].Domain().length(2);
29 
30  // Allocate space for these 1D vectors
31  for (int n = 0; n < Rayleigh::nvars; n++) {
32  h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
33  d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
34  }
35 
36  // Init the host vectors
37  prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_nd[lev], solverChoice.rayleigh_zdamp);
38 
39  // Copy from host vectors to device vectors
40  for (int n = 0; n < Rayleigh::nvars; n++) {
41  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
42  d_rayleigh_ptrs[lev][n].begin());
43  }
44  }
45 }
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
Definition: ERF.H:1161
amrex::Real rayleigh_zdamp
Definition: ERF_DataStruct.H:692
amrex::Real rayleigh_ztop
Definition: ERF_DataStruct.H:693

◆ initSponge()

void ERF::initSponge ( )
private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

36 {
37  h_sponge_ptrs.resize(max_level+1);
38  d_sponge_ptrs.resize(max_level+1);
39 
40  for (int lev = 0; lev <= finest_level; lev++)
41  {
42  // These have 2 components: ubar, vbar
45 
46  const int zlen_sponge = geom[lev].Domain().length(2);
47 
48  // Allocate space for these 1D vectors
49  for (int n = 0; n < Sponge::nvars_sponge; n++) {
50  h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
51  d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
52  }
53 
54  }
55 }
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
Definition: ERF.H:1164

◆ input_sponge()

void ERF::input_sponge ( int  lev)

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters
levInteger specifying the current level
18 {
19  // We only want to read the file once
20  if (lev == 0) {
22  Error("input_sounding file name must be provided via input");
23 
24  // this will interpolate the input profiles to the nominal height levels
25  // (ranging from 0 to the domain top)
27  }
28 }
InputSpongeData input_sponge_data
Definition: ERF.H:690
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag)
Definition: ERF_InputSpongeData.H:28
std::string input_sponge_file
Definition: ERF_InputSpongeData.H:111

◆ is_it_time_for_action()

bool ERF::is_it_time_for_action ( int  nstep,
amrex::Real  time,
amrex::Real  dt,
int  action_interval,
amrex::Real  action_per 
)
static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters
nstepTimestep number
timeCurrent time
dtlevTimestep for the current level
action_intervalInterval in number of timesteps for taking action
action_perInterval in simulation time for taking action
686 {
687  bool int_test = (action_interval > 0 && nstep % action_interval == 0);
688 
689  bool per_test = false;
690  if (action_per > 0.0) {
691  const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
692  const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
693 
694  if (num_per_old != num_per_new) {
695  per_test = true;
696  }
697  }
698 
699  return int_test || per_test;
700 }

◆ make_eb_box()

void ERF::make_eb_box ( )

◆ make_eb_regular()

void ERF::make_eb_regular ( )

◆ make_physbcs()

void ERF::make_physbcs ( int  lev)
private
630 {
631  if (SolverChoice::mesh_type == MeshType::VariableDz) {
632  AMREX_ALWAYS_ASSERT(z_phys_nd[lev] != nullptr);
633  }
634 
635  physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
637  z_phys_nd[lev], solverChoice.use_real_bcs, th_bc_data[lev].data());
638  physbcs_u[lev] = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
640  z_phys_nd[lev], solverChoice.use_real_bcs, xvel_bc_data[lev].data());
641  physbcs_v[lev] = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
643  z_phys_nd[lev], solverChoice.use_real_bcs, yvel_bc_data[lev].data());
644  physbcs_w[lev] = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
647  solverChoice.use_real_bcs, zvel_bc_data[lev].data());
648  physbcs_base[lev] = std::make_unique<ERFPhysBCFunct_base> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
649  (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
650 }

◆ MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage ( amrex::Vector< amrex::Real > &  h_havg,
amrex::MultiFab &  S,
int  n 
)
2031 {
2032  // Get the number of cells in z at level 0
2033  int dir_z = AMREX_SPACEDIM-1;
2034  auto domain = geom[0].Domain();
2035  int size_z = domain.length(dir_z);
2036  int start_z = domain.smallEnd()[dir_z];
2037  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
2038 
2039  // resize the level 0 horizontal average vectors
2040  h_havg.resize(size_z, 0.0_rt);
2041 
2042  // Get the cell centered data and construct sums
2043 #ifdef _OPENMP
2044 #pragma omp parallel if (Gpu::notInLaunchRegion())
2045 #endif
2046  for (MFIter mfi(S); mfi.isValid(); ++mfi) {
2047  const Box& box = mfi.validbox();
2048  const IntVect& se = box.smallEnd();
2049  const IntVect& be = box.bigEnd();
2050 
2051  auto fab_arr = S[mfi].array();
2052 
2053  FArrayBox fab_reduce(box, 1, The_Async_Arena());
2054  auto arr_reduce = fab_reduce.array();
2055 
2056  ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2057  arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
2058  });
2059 
2060  for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
2061  Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
2062  h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
2063  }
2064  }
2065 
2066  // combine sums from different MPI ranks
2067  ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), h_havg.size());
2068 
2069  // divide by the total number of cells we are averaging over
2070  for (int k = 0; k < size_z; ++k) {
2071  h_havg[k] /= area_z;
2072  }
2073 }

◆ MakeEBGeometry()

void ERF::MakeEBGeometry ( )

◆ MakeHorizontalAverages()

void ERF::MakeHorizontalAverages ( )
1925 {
1926  int lev = 0;
1927 
1928  // First, average down all levels (if doing two-way coupling)
1929  if (solverChoice.coupling_type == CouplingType::TwoWay) {
1930  AverageDown();
1931  }
1932 
1933  MultiFab mf(grids[lev], dmap[lev], 5, 0);
1934 
1935  int zdir = 2;
1936  auto domain = geom[0].Domain();
1937 
1938  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
1939  bool is_anelastic = (solverChoice.anelastic[lev] == 1);
1940 
1941  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
1942  const Box& bx = mfi.validbox();
1943  auto fab_arr = mf.array(mfi);
1944  auto const hse_arr = base_state[lev].const_array(mfi);
1945  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
1946  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
1947  Real dens = cons_arr(i, j, k, Rho_comp);
1948  fab_arr(i, j, k, 0) = dens;
1949  fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
1950  if (!use_moisture) {
1951  if (is_anelastic) {
1952  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
1953  } else {
1954  fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp));
1955  }
1956  }
1957  });
1958  }
1959 
1960  if (use_moisture)
1961  {
1962  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
1963  const Box& bx = mfi.validbox();
1964  auto fab_arr = mf.array(mfi);
1965  auto const hse_arr = base_state[lev].const_array(mfi);
1966  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
1967  int ncomp = vars_new[lev][Vars::cons].nComp();
1968 
1969  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
1970  Real dens = cons_arr(i, j, k, Rho_comp);
1971  if (is_anelastic) {
1972  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
1973  } else {
1974  Real qv = cons_arr(i, j, k, RhoQ1_comp) / dens;
1975  fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
1976  }
1977  fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : 0.0);
1978  fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : 0.0);
1979  });
1980  }
1981 
1982  Gpu::HostVector<Real> h_avg_qv = sumToLine(mf,3,1,domain,zdir);
1983  Gpu::HostVector<Real> h_avg_qc = sumToLine(mf,4,1,domain,zdir);
1984  }
1985 
1986  // Sum in the horizontal plane
1987  Gpu::HostVector<Real> h_avg_density = sumToLine(mf,0,1,domain,zdir);
1988  Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
1989  Gpu::HostVector<Real> h_avg_pressure = sumToLine(mf,2,1,domain,zdir);
1990 
1991  // Divide by the total number of cells we are averaging over
1992  int size_z = domain.length(zdir);
1993  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
1994  int klen = static_cast<int>(h_avg_density.size());
1995 
1996  for (int k = 0; k < klen; ++k) {
1997  h_havg_density[k] /= area_z;
1998  h_havg_temperature[k] /= area_z;
1999  h_havg_pressure[k] /= area_z;
2000  if (solverChoice.moisture_type != MoistureType::None)
2001  {
2002  h_havg_qc[k] /= area_z;
2003  h_havg_qv[k] /= area_z;
2004  }
2005  } // k
2006 
2007  // resize device vectors
2008  d_havg_density.resize(size_z, 0.0_rt);
2009  d_havg_temperature.resize(size_z, 0.0_rt);
2010  d_havg_pressure.resize(size_z, 0.0_rt);
2011 
2012  // copy host vectors to device vectors
2013  Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
2014  Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
2015  Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
2016 
2017  if (solverChoice.moisture_type != MoistureType::None)
2018  {
2019  d_havg_qv.resize(size_z, 0.0_rt);
2020  d_havg_qc.resize(size_z, 0.0_rt);
2021  Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
2022  Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
2023  }
2024 }
amrex::Gpu::DeviceVector< amrex::Real > d_havg_temperature
Definition: ERF.H:1179
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qv
Definition: ERF.H:1181
amrex::Vector< amrex::Real > h_havg_pressure
Definition: ERF.H:1174
amrex::Vector< amrex::Real > h_havg_qc
Definition: ERF.H:1176
amrex::Vector< amrex::Real > h_havg_density
Definition: ERF.H:1172
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qc
Definition: ERF.H:1182
amrex::Gpu::DeviceVector< amrex::Real > d_havg_density
Definition: ERF.H:1178
amrex::Vector< amrex::Real > h_havg_temperature
Definition: ERF.H:1173
amrex::Gpu::DeviceVector< amrex::Real > d_havg_pressure
Definition: ERF.H:1180
amrex::Vector< amrex::Real > h_havg_qv
Definition: ERF.H:1175
Here is the call graph for this function:

◆ MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
248 {
249  AMREX_ALWAYS_ASSERT(lev > 0);
250 
251  if (verbose) {
252  amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
253  }
254 
255  //********************************************************************************************
256  // This allocates all kinds of things, including but not limited to: solution arrays,
257  // terrain arrays, metric terms and base state.
258  // *******************************************************************************************
259  init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev], base_state[lev], z_phys_nd[lev]);
260 
261  t_new[lev] = time;
262  t_old[lev] = time - 1.e200;
263 
264  // ********************************************************************************************
265  // Build the data structures for metric quantities used with terrain-fitted coordinates
266  // ********************************************************************************************
267  init_zphys(lev, time);
269 
270  //
271  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
272  //
273  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
274  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
275  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
276  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
277  }
278  }
279 
280  // ********************************************************************************************
281  // Build the data structures for canopy model (depends upon z_phys)
282  // ********************************************************************************************
284  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
285  }
286 
287  //********************************************************************************************
288  // Microphysics
289  // *******************************************************************************************
290  int q_size = micro->Get_Qmoist_Size(lev);
291  qmoist[lev].resize(q_size);
292  micro->Define(lev, solverChoice);
293  if (solverChoice.moisture_type != MoistureType::None)
294  {
295  micro->Init(lev, vars_new[lev][Vars::cons],
296  grids[lev], Geom(lev), 0.0,
297  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
298  }
299  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
300  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
301  }
302 
303  // *****************************************************************************************************
304  // Initialize the boundary conditions (after initializing the terrain but before calling
305  // initHSE or FillCoarsePatch)
306  // *****************************************************************************************************
307  make_physbcs(lev);
308 
309  // ********************************************************************************************
310  // Update the base state at this level by interpolation from coarser level (inside initHSE)
311  // ********************************************************************************************
312  initHSE(lev);
313 
314  // ********************************************************************************************
315  // Build the data structures for calculating diffusive/turbulent terms
316  // ********************************************************************************************
317  update_diffusive_arrays(lev, ba, dm);
318 
319  // ********************************************************************************************
320  // Fill data at the new level by interpolation from the coarser level
321  // Note that internal to FillCoarsePatch we will convert velocity to momentum,
322  // then interpolate momentum, then convert momentum back to velocity
323  // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
324  // ********************************************************************************************
325  FillCoarsePatch(lev, time);
326 
327  // ********************************************************************************************
328  // Initialize the integrator class
329  // ********************************************************************************************
330  dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
332 
333  // ********************************************************************************************
334  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
335  // ********************************************************************************************
336  if (lev > 0 && cf_width >= 0) {
339  }
340 
341  // ********************************************************************************************
342  // Create the SurfaceLayer arrays at this (new) level
343  // ********************************************************************************************
344  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
345  int nlevs = finest_level+1;
346  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
347  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
348  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
349  mfv_old, Theta_prim[lev], Qv_prim[lev],
350  Qr_prim[lev], z_phys_nd[lev],
351  Hwave[lev].get(), Lwave[lev].get(), eddyDiffs_lev[lev].get(),
352  lsm_data[lev], lsm_flux[lev], sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
353  }
354 
355 #ifdef ERF_USE_PARTICLES
356  // particleData.Redistribute();
357 #endif
358 }
void update_diffusive_arrays(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewArrays.cpp:401
void initialize_integrator(int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
Definition: ERF_MakeNewArrays.cpp:607
void update_terrain_arrays(int lev)
Definition: ERF_MakeNewArrays.cpp:596
void init_zphys(int lev, amrex::Real time)
Definition: ERF_MakeNewArrays.cpp:491
void init_stuff(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:24
void Define_ERFFillPatchers(int lev)
Definition: ERF.cpp:2102

◆ MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
27 {
28  BoxArray ba;
29  DistributionMapping dm;
30  Box domain(Geom(0).Domain());
31  if (lev == 0 && restart_chkfile.empty() &&
32  (max_grid_size[0][0] >= domain.length(0)) &&
33  (max_grid_size[0][1] >= domain.length(1)) &&
34  ba_in.size() != ParallelDescriptor::NProcs())
35  {
36  // We only decompose in z if max_grid_size_z indicates we should
37  bool decompose_in_z = (max_grid_size[0][2] < domain.length(2));
38 
39  ba = ERFPostProcessBaseGrids(Geom(0).Domain(),decompose_in_z);
40  dm = DistributionMapping(ba);
41  } else {
42  ba = ba_in;
43  dm = dm_in;
44  }
45 
46  // ********************************************************************************************
47  // Define grids[lev] to be ba
48  // ********************************************************************************************
49  SetBoxArray(lev, ba);
50 
51  // ********************************************************************************************
52  // Define dmap[lev] to be dm
53  // ********************************************************************************************
54  SetDistributionMap(lev, dm);
55 
56  if (verbose) {
57  amrex::Print() <<" BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
58  }
59 
60  if (lev == 0) init_bcs();
61 
62  if ( solverChoice.terrain_type == TerrainType::EB ||
63  solverChoice.terrain_type == TerrainType::ImmersedForcing)
64  {
65  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
66  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
67  eb[lev]->make_factory(lev, geom[lev], grids[lev], dmap[lev], eb_level);
68  } else {
69  // m_factory[lev] = std::make_unique<FabFactory<FArrayBox>>();
70  }
71 
72  auto& lev_new = vars_new[lev];
73  auto& lev_old = vars_old[lev];
74 
75  //********************************************************************************************
76  // This allocates all kinds of things, including but not limited to: solution arrays,
77  // terrain arrays, metric terms and base state.
78  // *******************************************************************************************
79  init_stuff(lev, ba, dm, lev_new, lev_old, base_state[lev], z_phys_nd[lev]);
80 
81  //********************************************************************************************
82  // Land Surface Model
83  // *******************************************************************************************
84  int lsm_size = lsm.Get_Data_Size();
85  lsm_data[lev].resize(lsm_size);
86  lsm_flux[lev].resize(lsm_size);
87  lsm.Define(lev, solverChoice);
88  if (solverChoice.lsm_type != LandSurfaceType::None)
89  {
90  lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
91  }
92  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
93  lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
94  lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
95  }
96 
97  // ********************************************************************************************
98  // Build the data structures for calculating diffusive/turbulent terms
99  // ********************************************************************************************
100  update_diffusive_arrays(lev, ba, dm);
101 
102  // ********************************************************************************************
103  // Build the data structures for holding sea surface temps and skin temps
104  // ********************************************************************************************
105  sst_lev[lev].resize(1); sst_lev[lev][0] = nullptr;
106  tsk_lev[lev].resize(1); tsk_lev[lev][0] = nullptr;
107 
108  // ********************************************************************************************
109  // Thin immersed body
110  // *******************************************************************************************
111  init_thin_body(lev, ba, dm);
112 
113  // ********************************************************************************************
114  // Initialize the integrator class
115  // ********************************************************************************************
116  initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
117 
118  // ********************************************************************************************
119  // Initialize the data itself
120  // If (init_type == InitType::WRFInput) then we are initializing terrain and the initial data in
121  // the same call so we must call init_only before update_terrain_arrays
122  // If (init_type != InitType::WRFInput) then we want to initialize the terrain before the initial data
123  // since we may need to use the grid information before constructing
124  // initial idealized data
125  // ********************************************************************************************
126  if (restart_chkfile.empty()) {
127  if ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) )
128  {
129  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
130  init_only(lev, start_time);
131  init_zphys(lev, time);
133  make_physbcs(lev);
134  } else {
135  init_zphys(lev, time);
137  // Note that for init_type != InitType::WRFInput and != InitType::Metgrid,
138  // make_physbcs is called inside init_only
139  init_only(lev, start_time);
140  }
141  } else {
142  // if restarting and nudging from input sounding, load the input sounding files
143  if (lev == 0 && solverChoice.init_type == InitType::Input_Sounding && solverChoice.nudging_from_input_sounding)
144  {
146  Error("input_sounding file name must be provided via input");
147  }
148 
150 
151  // this will interpolate the input profiles to the nominal height levels
152  // (ranging from 0 to the domain top)
153  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
154  input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n);
155  }
156 
157  // this will calculate the hydrostatically balanced density and pressure
158  // profiles following WRF ideal.exe
160  }
161 
162  // We re-create terrain_blanking on restart rather than storing it in the checkpoint
163  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
164  int ngrow = ComputeGhostCells(solverChoice) + 2;
165  terrain_blanking[lev]->setVal(1.0);
166  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
167  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
168  }
169  }
170 
171  // Read in tables needed for windfarm simulations
172  // fill in Nturb multifab - number of turbines in each mesh cell
173  // write out the vtk files for wind turbine location and/or
174  // actuator disks
175  #ifdef ERF_USE_WINDFARM
176  init_windfarm(lev);
177  #endif
178 
179  // ********************************************************************************************
180  // Build the data structures for canopy model (depends upon z_phys)
181  // ********************************************************************************************
182  if (restart_chkfile.empty()) {
184  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
185  }
186  }
187 
188  //********************************************************************************************
189  // Create wall distance field for RANS model (depends upon z_phys)
190  // *******************************************************************************************
191  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
192  // Handle bottom boundary
193  poisson_wall_dist(lev);
194 
195  // Correct the wall distance for immersed bodies
201  geom[lev],
202  z_phys_cc[lev]);
203  }
204  }
205 
206  //********************************************************************************************
207  // Microphysics
208  // *******************************************************************************************
209  int q_size = micro->Get_Qmoist_Size(lev);
210  qmoist[lev].resize(q_size);
211  micro->Define(lev, solverChoice);
212  if (solverChoice.moisture_type != MoistureType::None)
213  {
214  micro->Init(lev, vars_new[lev][Vars::cons],
215  grids[lev], Geom(lev), 0.0,
216  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
217  }
218  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
219  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
220  }
221 
222  // ********************************************************************************************
223  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
224  // ********************************************************************************************
225  if (lev > 0 && cf_width >= 0) {
228  }
229 
230 #ifdef ERF_USE_PARTICLES
231  if (restart_chkfile.empty()) {
232  if (lev == 0) {
233  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd);
234  } else {
235  particleData.Redistribute();
236  }
237  }
238 #endif
239 }
BoxArray ERFPostProcessBaseGrids(const Box &domain, bool decompose_in_z)
Definition: ERF_ChopGrids.cpp:6
void thinbody_wall_dist(std::unique_ptr< MultiFab > &wdist, Vector< IntVect > &xfaces, Vector< IntVect > &yfaces, Vector< IntVect > &zfaces, const Geometry &geomdata, std::unique_ptr< MultiFab > &z_phys_cc)
Definition: ERF_ThinBodyWallDist.cpp:12
void init_only(int lev, amrex::Real time)
Definition: ERF.cpp:1432
void poisson_wall_dist(int lev)
Definition: ERF_PoissonWallDist.cpp:20
void init_thin_body(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewLevel.cpp:598
int Get_Data_Size()
Definition: ERF_LandSurface.H:92
amrex::MultiFab * Get_Flux_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:86
void Init(const int &lev, const amrex::MultiFab &cons_in, const amrex::Geometry &geom, const amrex::Real &dt_advance)
Definition: ERF_LandSurface.H:43
void Define(const int &lev, SolverChoice &sc)
Definition: ERF_LandSurface.H:36
amrex::MultiFab * Get_Data_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:83
bool nudging_from_input_sounding
Definition: ERF_DataStruct.H:724
Here is the call graph for this function:

◆ MakeVTKFilename()

std::string ERF::MakeVTKFilename ( int  nstep)
574  {
575  // Ensure output directory exists
576  const std::string dir = "Output_HurricaneTracker";
577  if (!fs::exists(dir)) {
578  fs::create_directory(dir);
579  }
580 
581  // Construct filename with zero-padded step
582  std::ostringstream oss;
583  if(nstep==0){
584  oss << dir << "/hurricane_track_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
585  } else {
586  oss << dir << "/hurricane_track_" << std::setw(7) << std::setfill('0') << nstep+1 << ".vtk";
587  }
588 
589  return oss.str();
590 }

◆ nghost_eb_basic()

static int ERF::nghost_eb_basic ( )
inlinestaticprivate
1487  { return 5; }

◆ nghost_eb_full()

static int ERF::nghost_eb_full ( )
inlinestaticprivate
1494  { return 4; }

◆ nghost_eb_volume()

static int ERF::nghost_eb_volume ( )
inlinestaticprivate
1491  { return 5; }

◆ NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )
inlineprivatenoexcept
1286  {
1287  return datalog.size();
1288  }

◆ NumDerDataLogs()

AMREX_FORCE_INLINE int ERF::NumDerDataLogs ( )
inlineprivatenoexcept
1293  {
1294  return der_datalog.size();
1295  }

◆ NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )
inlineprivatenoexcept
1322  {
1323  return samplelinelog.size();
1324  }

◆ NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )
inlineprivatenoexcept
1348  {
1349  return sampleline.size();
1350  }

◆ NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )
inlineprivatenoexcept
1308  {
1309  return sampleptlog.size();
1310  }

◆ NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )
inlineprivatenoexcept
1335  {
1336  return samplepoint.size();
1337  }

◆ operator=() [1/2]

ERF& ERF::operator= ( const ERF other)
delete

◆ operator=() [2/2]

ERF& ERF::operator= ( ERF &&  other)
deletenoexcept

◆ ParameterSanityChecks()

void ERF::ParameterSanityChecks ( )
private
1858 {
1859  AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt[0] > 0.);
1860 
1861  // We don't allow use_real_bcs to be true if init_type is not either InitType::WRFInput or InitType::Metgrid
1862  AMREX_ALWAYS_ASSERT( !solverChoice.use_real_bcs ||
1863  ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) );
1864 
1865  AMREX_ALWAYS_ASSERT(real_width >= 0);
1866  AMREX_ALWAYS_ASSERT(real_set_width >= 0);
1867  AMREX_ALWAYS_ASSERT(real_width >= real_set_width);
1868 
1869  if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
1870  Abort("You must set cf_width >= cf_set_width >= 0");
1871  }
1872  if (max_level > 0 && cf_set_width > 0) {
1873  for (int lev = 1; lev <= max_level; lev++) {
1874  if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
1875  cf_set_width%ref_ratio[lev-1][1] != 0 ||
1876  cf_set_width%ref_ratio[lev-1][2] != 0 ) {
1877  Abort("You must set cf_width to be a multiple of ref_ratio");
1878  }
1879  }
1880  }
1881 
1882  // If fixed_mri_dt_ratio is set, it must be even
1883  if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
1884  {
1885  Abort("If you specify fixed_mri_dt_ratio, it must be even");
1886  }
1887 
1888  for (int lev = 0; lev <= max_level; lev++)
1889  {
1890  // We ignore fixed_fast_dt if not substepping
1891  if (solverChoice.substepping_type[lev] == SubsteppingType::None) {
1892  fixed_fast_dt[lev] = -1.0;
1893  }
1894 
1895  // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
1896  if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio <= 0)
1897  {
1898  Real eps = 1.e-12;
1899  int ratio = static_cast<int>( ( (1.0+eps) * fixed_dt[lev] ) / fixed_fast_dt[lev] );
1900  if (fixed_dt[lev] / fixed_fast_dt[lev] != ratio)
1901  {
1902  Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
1903  }
1904  }
1905 
1906  // If all three are specified, they must be consistent
1907  if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio > 0)
1908  {
1909  if (fixed_dt[lev] / fixed_fast_dt[lev] != fixed_mri_dt_ratio)
1910  {
1911  Abort("Dt is over-specfied");
1912  }
1913  }
1914  } // lev
1915 
1916  if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
1917  Abort("For two-way coupling you must set cf_width = 0");
1918  }
1919 }
int real_set_width
Definition: ERF.H:1085

◆ PlotFileName()

std::string ERF::PlotFileName ( int  lev) const
private

◆ PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string >  plot_var_names)
staticprivate
220 {
221  Vector<std::string> names;
222 
223  names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
224 
225  return names;
226 
227 }

◆ poisson_wall_dist()

void ERF::poisson_wall_dist ( int  lev)

Calculate wall distances using the Poisson equation

The zlo boundary is assumed to correspond to the land surface. If there are no boundary walls, then the other use case is to calculate wall distances for immersed boundaries (embedded or thin body).

See Tucker, P. G. (2003). Differential equation-based wall distance computation for DES and RANS. Journal of Computational Physics, 190(1), 229–248. https://doi.org/10.1016/S0021-9991(03)00272-9

21 {
22  BL_PROFILE("ERF::poisson_wall_dist()");
23 
24  bool havewall{false};
25  Orientation zlo(Direction::z, Orientation::low);
26  if ( ( phys_bc_type[zlo] == ERF_BC::surface_layer ) ||
27  ( phys_bc_type[zlo] == ERF_BC::no_slip_wall ) )/*||
28  ((phys_bc_type[zlo] == ERF_BC::slip_wall) && (dom_hi.z > dom_lo.z)) )*/
29  {
30  havewall = true;
31  }
32 
33  auto const& geomdata = geom[lev];
34 
35  if (havewall) {
36  if (solverChoice.mesh_type == MeshType::ConstantDz) {
37 // Comment this out to test the wall dist calc in the trivial case:
38 //#if 0
39  Print() << "Directly calculating direct wall distance for constant dz" << std::endl;
40  const Real* prob_lo = geomdata.ProbLo();
41  const Real* dx = geomdata.CellSize();
42  for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
43  const Box& bx = mfi.validbox();
44  auto dist_arr = walldist[lev]->array(mfi);
45  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
46  dist_arr(i, j, k) = prob_lo[2] + (k + 0.5) * dx[2];
47  });
48  }
49  return;
50 //#endif
51  } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
52  // TODO: Handle this trivial case
53  Error("Wall dist calc not implemented with grid stretching yet");
54  } else {
55  // TODO
56  Error("Wall dist calc not implemented over terrain yet");
57  }
58  }
59 
60  Print() << "Calculating Poisson wall distance" << std::endl;
61 
62  // Make sure the solver only sees the levels over which we are solving
63  BoxArray nba = walldist[lev]->boxArray();
64  nba.surroundingNodes();
65  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
66  Vector<BoxArray> ba_tmp; ba_tmp.push_back(nba);
67  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(walldist[lev]->DistributionMap());
68 
69  Vector<MultiFab> rhs;
70  Vector<MultiFab> phi;
71 
72  if (solverChoice.terrain_type == TerrainType::EB) {
73  amrex::Error("Wall dist calc not implemented for EB");
74  } else {
75  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
76  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
77  }
78 
79  rhs[0].setVal(-1.0);
80 
81  // Define an overset mask to set dirichlet nodes on walls
82  iMultiFab mask(ba_tmp[0], dm_tmp[0], 1, 0);
83  Vector<const iMultiFab*> overset_mask = {&mask};
84 
85  auto const dom_lo = lbound(geom[lev].Domain());
86  auto const dom_hi = ubound(geom[lev].Domain());
87 
88  // ****************************************************************************
89  // Initialize phi
90  // (It is essential that we do this in order to fill the corners; this is
91  // used if we include blanking.)
92  // ****************************************************************************
93  phi[0].setVal(0.0);
94 
95  // ****************************************************************************
96  // Interior boundaries are marked with phi=0
97  // ****************************************************************************
98  // Overset mask is 0/1: 1 means the node is an unknown. 0 means it's known.
99  mask.setVal(1);
101  Warning("Poisson distance is inaccurate for bodies in open domains that are small compared to the domain size, skipping");
102  return;
103 #if 0
104  Gpu::DeviceVector<IntVect> xfacelist, yfacelist, zfacelist;
105 
106  xfacelist.resize(solverChoice.advChoice.zero_xflux.size());
107  yfacelist.resize(solverChoice.advChoice.zero_yflux.size());
108  zfacelist.resize(solverChoice.advChoice.zero_zflux.size());
109 
110  if (xfacelist.size() > 0) {
111  Gpu::copy(amrex::Gpu::hostToDevice,
114  xfacelist.begin());
115  Print() << " masking interior xfaces" << std::endl;
116  }
117  if (yfacelist.size() > 0) {
118  Gpu::copy(amrex::Gpu::hostToDevice,
121  yfacelist.begin());
122  Print() << " masking interior yfaces" << std::endl;
123  }
124  if (zfacelist.size() > 0) {
125  Gpu::copy(amrex::Gpu::hostToDevice,
128  zfacelist.begin());
129  Print() << " masking interior zfaces" << std::endl;
130  }
131 
132  for (MFIter mfi(phi[0]); mfi.isValid(); ++mfi) {
133  const Box& bx = mfi.validbox();
134 
135  auto phi_arr = phi[0].array(mfi);
136  auto mask_arr = mask.array(mfi);
137 
138  if (xfacelist.size() > 0) {
139  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
140  for (int iface=0; iface < xfacelist.size(); ++iface) {
141  if ((i == xfacelist[iface][0]) &&
142  (j == xfacelist[iface][1]) &&
143  (k == xfacelist[iface][2]))
144  {
145  mask_arr(i, j , k ) = 0;
146  mask_arr(i, j , k+1) = 0;
147  mask_arr(i, j+1, k ) = 0;
148  mask_arr(i, j+1, k+1) = 0;
149  }
150  }
151  });
152  }
153 
154  if (yfacelist.size() > 0) {
155  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
156  for (int iface=0; iface < yfacelist.size(); ++iface) {
157  if ((i == yfacelist[iface][0]) &&
158  (j == yfacelist[iface][1]) &&
159  (k == yfacelist[iface][2]))
160  {
161  mask_arr(i , j, k ) = 0;
162  mask_arr(i , j, k+1) = 0;
163  mask_arr(i+1, j, k ) = 0;
164  mask_arr(i+1, j, k+1) = 0;
165  }
166  }
167  });
168  }
169 
170  if (zfacelist.size() > 0) {
171  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
172  for (int iface=0; iface < zfacelist.size(); ++iface) {
173  if ((i == xfacelist[iface][0]) &&
174  (j == xfacelist[iface][1]) &&
175  (k == xfacelist[iface][2]))
176  {
177  mask_arr(i , j , k) = 0;
178  mask_arr(i , j+1, k) = 0;
179  mask_arr(i+1, j , k) = 0;
180  mask_arr(i+1, j+1, k) = 0;
181  }
182  }
183  });
184  }
185  }
186 #endif
187  }
188 
189  // ****************************************************************************
190  // Setup BCs, with solid domain boundaries being dirichlet
191  // ****************************************************************************
192  amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bc3d_lo, bc3d_hi;
193  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
194  if (geom[0].isPeriodic(dir)) {
195  bc3d_lo[dir] = LinOpBCType::Periodic;
196  bc3d_hi[dir] = LinOpBCType::Periodic;
197  } else {
198  bc3d_lo[dir] = LinOpBCType::Neumann;
199  bc3d_hi[dir] = LinOpBCType::Neumann;
200  }
201  }
202  if (havewall) {
203  Print() << " Poisson zlo BC is dirichlet" << std::endl;
204  bc3d_lo[2] = LinOpBCType::Dirichlet;
205  }
206  Print() << " bc lo : " << bc3d_lo << std::endl;
207  Print() << " bc hi : " << bc3d_hi << std::endl;
208 
209  if (!solverChoice.advChoice.have_zero_flux_faces && !havewall) {
210  Error("No solid boundaries in the computational domain");
211  }
212 
213  LPInfo info;
214 /* Nodal solver cannot have hidden dimensions */
215 #if 0
216  // Allow a hidden direction if the domain is one cell wide
217  if (dom_lo.x == dom_hi.x) {
218  info.setHiddenDirection(0);
219  Print() << " domain is 2D in yz" << std::endl;
220  } else if (dom_lo.y == dom_hi.y) {
221  info.setHiddenDirection(1);
222  Print() << " domain is 2D in xz" << std::endl;
223  } else if (dom_lo.z == dom_hi.z) {
224  info.setHiddenDirection(2);
225  Print() << " domain is 2D in xy" << std::endl;
226  }
227 #endif
228 
229  // ****************************************************************************
230  // Solve nodal masked Poisson problem with MLMG
231  // TODO: different solver for terrain?
232  // ****************************************************************************
233  const Real reltol = solverChoice.poisson_reltol;
234  const Real abstol = solverChoice.poisson_abstol;
235 
236  Real sigma = 1.0;
237  Vector<EBFArrayBoxFactory const*> factory_vec = { &EBFactory(lev) };
238  MLNodeLaplacian mlpoisson(geom_tmp, ba_tmp, dm_tmp, info, factory_vec, sigma);
239 
240  mlpoisson.setDomainBC(bc3d_lo, bc3d_hi);
241 
242  if (lev > 0) {
243  mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
244  }
245 
246  mlpoisson.setLevelBC(0, nullptr);
247 
248  mlpoisson.setOversetMask(0, mask);
249 
250  // Solve
251  MLMG mlmg(mlpoisson);
252  int max_iter = 100;
253  mlmg.setMaxIter(max_iter);
254 
255  mlmg.setVerbose(mg_verbose);
256  mlmg.setBottomVerbose(0);
257 
258  mlmg.solve(GetVecOfPtrs(phi),
259  GetVecOfConstPtrs(rhs),
260  reltol, abstol);
261 
262  // Now overwrite with periodic fill outside domain and fine-fine fill inside
263  phi[0].FillBoundary(geom[lev].periodicity());
264 
265  // ****************************************************************************
266  // Compute grad(phi) to get distances
267  // - Note that phi is nodal and walldist is cell-centered
268  // - TODO: include terrain metrics for dphi/dz
269  // ****************************************************************************
270  for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
271  const Box& bx = mfi.validbox();
272 
273  const auto invCellSize = geomdata.InvCellSizeArray();
274 
275  auto const& phi_arr = phi[0].const_array(mfi);
276  auto dist_arr = walldist[lev]->array(mfi);
277 
278  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
279  Real dpdx{0}, dpdy{0}, dpdz{0};
280 
281  // dphi/dx
282  if (dom_lo.x != dom_hi.x) {
283  dpdx = 0.25 * invCellSize[0] * (
284  (phi_arr(i+1, j , k ) - phi_arr(i, j , k ))
285  + (phi_arr(i+1, j , k+1) - phi_arr(i, j , k+1))
286  + (phi_arr(i+1, j+1, k ) - phi_arr(i, j+1, k ))
287  + (phi_arr(i+1, j+1, k+1) - phi_arr(i, j+1, k+1)) );
288  }
289 
290  // dphi/dy
291  if (dom_lo.y != dom_hi.y) {
292  dpdy = 0.25 * invCellSize[1] * (
293  (phi_arr(i , j+1, k ) - phi_arr(i , j, k ))
294  + (phi_arr(i , j+1, k+1) - phi_arr(i , j, k+1))
295  + (phi_arr(i+1, j+1, k ) - phi_arr(i+1, j, k ))
296  + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j, k+1)) );
297  }
298 
299  // dphi/dz
300  if (dom_lo.z != dom_hi.z) {
301  dpdz = 0.25 * invCellSize[2] * (
302  (phi_arr(i , j , k+1) - phi_arr(i , j , k))
303  + (phi_arr(i , j+1, k+1) - phi_arr(i , j+1, k))
304  + (phi_arr(i+1, j , k+1) - phi_arr(i+1, j , k))
305  + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j+1, k)) );
306  }
307 
308  Real dp_dot_dp = dpdx*dpdx + dpdy*dpdy + dpdz*dpdz;
309  Real phi_avg = 0.125 * (
310  phi_arr(i , j , k ) + phi_arr(i , j , k+1) + phi_arr(i , j+1, k ) + phi_arr(i , j+1, k+1)
311  + phi_arr(i+1, j , k ) + phi_arr(i+1, j , k+1) + phi_arr(i+1, j+1, k ) + phi_arr(i+1, j+1, k+1) );
312  dist_arr(i, j, k) = -std::sqrt(dp_dot_dp) + std::sqrt(dp_dot_dp + 2*phi_avg);
313 
314  // DEBUG: output phi instead
315  //dist_arr(i, j, k) = phi_arr(i, j, k);
316  });
317  }
318 }
static int mg_verbose
Definition: ERF.H:1062
amrex::Real poisson_reltol
Definition: ERF_DataStruct.H:676
amrex::Real poisson_abstol
Definition: ERF_DataStruct.H:675

◆ post_timestep()

void ERF::post_timestep ( int  nstep,
amrex::Real  time,
amrex::Real  dt_lev 
)
497 {
498  BL_PROFILE("ERF::post_timestep()");
499 
500 #ifdef ERF_USE_PARTICLES
501  particleData.Redistribute();
502 #endif
503 
504  if (solverChoice.coupling_type == CouplingType::TwoWay)
505  {
506  int ncomp = vars_new[0][Vars::cons].nComp();
507  for (int lev = finest_level-1; lev >= 0; lev--)
508  {
509  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
510  // m is the map scale factor at cell centers
511  // Here we pre-divide (rho S) by m^2 before refluxing
512  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
513  const Box& bx = mfi.tilebox();
514  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
515  const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
516  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
517  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
518  {
519  cons_arr(i,j,k,n) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
520  });
521  } else {
522  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
523  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
524  {
525  cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
526  });
527  }
528  } // mfi
529 
530  // This call refluxes all "slow" cell-centered variables
531  // (i.e. not density or (rho theta) or velocities) from the lev/lev+1 interface onto lev
532  getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 2, 2, ncomp-2);
533 
534  // Here we multiply (rho S) by m^2 after refluxing
535  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
536  const Box& bx = mfi.tilebox();
537  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
538  const Array4<const Real> mapfac_arr = mapfac_m[lev]->const_array(mfi);
539  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
540  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
541  {
542  cons_arr(i,j,k,n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
543  });
544  } else {
545  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
546  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
547  {
548  cons_arr(i,j,k,n) *= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0)) / detJ_arr(i,j,k);
549  });
550  }
551  } // mfi
552 
553  // We need to do this before anything else because refluxing changes the
554  // values of coarse cells underneath fine grids with the assumption they'll
555  // be over-written by averaging down
556  int src_comp;
557  if (solverChoice.anelastic[lev]) {
558  src_comp = 1;
559  } else {
560  src_comp = 0;
561  }
562  int num_comp = ncomp - src_comp;
563  AverageDownTo(lev,src_comp,num_comp);
564  }
565  }
566 
567  if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
570  sum_energy_quantities(time);
571  }
572 
573  if (solverChoice.pert_type == PerturbationType::Source ||
574  solverChoice.pert_type == PerturbationType::Direct) {
575  if (is_it_time_for_action(nstep, time, dt_lev0, pert_interval, -1.)) {
576  turbPert.debug(time);
577  }
578  }
579 
580  if (profile_int > 0 && (nstep+1) % profile_int == 0) {
581  if (destag_profiles) {
582  // all variables cell-centered
583  write_1D_profiles(time);
584  } else {
585  // some variables staggered
587  }
588  }
589 
590  if (output_1d_column) {
591 #ifdef ERF_USE_NETCDF
592  if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
593  {
594  int lev_column = 0;
595  for (int lev = finest_level; lev >= 0; lev--)
596  {
597  Real dx_lev = geom[lev].CellSize(0);
598  Real dy_lev = geom[lev].CellSize(1);
599  int i_lev = static_cast<int>(std::floor(column_loc_x / dx_lev));
600  int j_lev = static_cast<int>(std::floor(column_loc_y / dy_lev));
601  if (grids[lev].contains(IntVect(i_lev,j_lev,0))) lev_column = lev;
602  }
603  writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
604  }
605 #else
606  Abort("To output 1D column files ERF must be compiled with NetCDF");
607 #endif
608  }
609 
611  {
614  {
615  bool is_moist = (micro->Get_Qstate_Size() > 0);
616  m_w2d->write_planes(istep[0], time, vars_new, is_moist);
617  }
618  }
619 
620  // Write plane/line sampler data
621  if (is_it_time_for_action(nstep+1, time, dt_lev0, sampler_interval, sampler_per) && (data_sampler) ) {
622  data_sampler->get_sample_data(geom, vars_new);
623  data_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
624  }
625 
626  // Moving terrain
627  if ( solverChoice.terrain_type == TerrainType::MovingFittedMesh )
628  {
629  for (int lev = finest_level; lev >= 0; lev--)
630  {
631  // Copy z_phs_nd and detJ_cc at end of timestep
632  MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
633  MultiFab::Copy( *detJ_cc[lev], *detJ_cc_new[lev], 0, 0, 1, detJ_cc[lev]->nGrowVect());
634  MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
635 
636  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
637  }
638  }
639 
640  bool is_hurricane_tracker_io=false;
641  ParmParse pp("erf");
642  pp.query("is_hurricane_tracker_io", is_hurricane_tracker_io);
643 
644  if(is_hurricane_tracker_io) {
645  if(nstep == 0 or (nstep+1)%m_plot_int_1 == 0){
646  std::string filename = MakeVTKFilename(nstep);
647  Real velmag_threshold = 1e10;
648  pp.query("hurr_track_io_velmag_greater_than", velmag_threshold);
649  if(velmag_threshold==1e10) {
650  Abort("As hurricane tracking IO is active using erf.is_hurricane_tracker_io = true"
651  " there needs to be an input erf.hurr_track_io_velmag_greater_than which specifies the"
652  " magnitude of velocity above which cells will be tagged for refinement.");
653  }
654  int levc=finest_level;
655  MultiFab& U_new = vars_new[levc][Vars::xvel];
656  MultiFab& V_new = vars_new[levc][Vars::yvel];
657  MultiFab& W_new = vars_new[levc][Vars::zvel];
658 
659  HurricaneTracker(levc, U_new, V_new, W_new, velmag_threshold, true);
660  if (ParallelDescriptor::IOProcessor()) {
662  }
663  }
664  }
665 } // post_timestep
void make_zcc(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc)
Definition: ERF_TerrainMetrics.cpp:623
static amrex::Real column_loc_y
Definition: ERF.H:1120
static std::string column_file_name
Definition: ERF.H:1121
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg(int lev)
Definition: ERF.H:1264
static amrex::Real bndry_output_planes_per
Definition: ERF.H:1126
std::string MakeVTKFilename(int nstep)
Definition: ERF_Write1DProfiles.cpp:574
static amrex::Real column_per
Definition: ERF.H:1118
amrex::Real sampler_per
Definition: ERF.H:1445
static amrex::Real column_loc_x
Definition: ERF.H:1119
static int bndry_output_planes_interval
Definition: ERF.H:1125
int sampler_interval
Definition: ERF.H:1444
static int output_1d_column
Definition: ERF.H:1116
void WriteVTKPolyline(const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
Definition: ERF_Write1DProfiles.cpp:593
static int column_interval
Definition: ERF.H:1117
Here is the call graph for this function:

◆ post_update()

void ERF::post_update ( amrex::MultiFab &  state_mf,
amrex::Real  time,
const amrex::Geometry &  geom 
)
private

◆ print_banner()

void ERF::print_banner ( MPI_Comm  comm,
std::ostream &  out 
)
static
65  : " << msg << std::endl;
66 }
67 
68 void ERF::print_banner (MPI_Comm comm, std::ostream& out)
69 {
70 #ifdef AMREX_USE_MPI
71  int irank = 0;
72  int num_ranks = 1;
73  MPI_Comm_size(comm, &num_ranks);
74  MPI_Comm_rank(comm, &irank);
75 
76  // Only root process does the printing
77  if (irank != 0) return;
78 #else
79  amrex::ignore_unused(comm);
80 #endif
81 
82  auto etime = std::chrono::system_clock::now();
83  auto etimet = std::chrono::system_clock::to_time_t(etime);
84 #ifndef _WIN32
85  char time_buf[64];
86  ctime_r(&etimet, time_buf);
87  const std::string tstamp(time_buf);
88 #else
89  char* time_buf = new char[64];
90  ctime_s(time_buf, 64, &etimet);
91  const std::string tstamp(time_buf);
92 #endif
93 
94  const char* githash1 = amrex::buildInfoGetGitHash(1);
95  const char* githash2 = amrex::buildInfoGetGitHash(2);
96 
97  // clang-format off
98  out << dbl_line
99  << " ERF (https://github.com/erf-model/ERF)"
100  << std::endl << std::endl
101  << " ERF Git SHA :: " << githash1 << std::endl
102  << " AMReX Git SHA :: " << githash2 << std::endl
103  << " AMReX version :: " << amrex::Version() << std::endl << std::endl
104  << " Exec. time :: " << tstamp
105  << " Build time :: " << amrex::buildInfoGetBuildDate() << std::endl
106  << " C++ compiler :: " << amrex::buildInfoGetComp()
107  << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
108  << " MPI :: "
109 #ifdef AMREX_USE_MPI
110  << "ON (Num. ranks = " << num_ranks << ")" << std::endl
111 #else
112  << "OFF " << std::endl
113 #endif
114  << " GPU :: "
115 #ifdef AMREX_USE_GPU
116  << "ON "
117 #if defined(AMREX_USE_CUDA)
118  << "(Backend: CUDA)"
119 #elif defined(AMREX_USE_HIP)
120  << "(Backend: HIP)"
121 #elif defined(AMREX_USE_SYCL)
122  << "(Backend: SYCL)"
123 #endif
124  << std::endl
125 #else
126  << "OFF" << std::endl
127 #endif
128  << " OpenMP :: "
129 #ifdef AMREX_USE_OMP
130  << "ON (Num. threads = " << omp_get_max_threads() << ")" << std::endl
131 #else
132  << "OFF" << std::endl
133 #endif
134  << std::endl;
135 
ERF()
Definition: ERF.cpp:92
const char * buildInfoGetBuildDate()
const char * buildInfoGetComp()
const char * buildInfoGetCompVersion()

Referenced by main().

Here is the caller graph for this function:

◆ print_error()

void ERF::print_error ( MPI_Comm  comm,
const std::string &  msg 
)
static
43  :
44  input_file : Input file with simulation settings
45 
46 Optional:
47  param=value : Overrides for parameters during runtime
48 )doc" << std::endl;
49 }
50 
51 void ERF::print_error (MPI_Comm comm, const std::string& msg)
52 {
53 #ifdef AMREX_USE_MPI
54  int irank = 0;
55  int num_ranks = 1;
56  MPI_Comm_size(comm, &num_ranks);
57  MPI_Comm_rank(comm, &irank);
58 

Referenced by main().

Here is the caller graph for this function:

◆ print_summary()

static void ERF::print_summary ( std::ostream &  )
static

◆ print_tpls()

void ERF::print_tpls ( std::ostream &  out)
static
140  ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
141  << dash_line << std::endl;
142  // clang-format on
143 }
144 
145 void ERF::print_tpls (std::ostream& out)
146 {
147  amrex::Vector<std::string> tpls;
148 
149 #ifdef ERF_USE_NETCDF
150  tpls.push_back(std::string("NetCDF ") + NC_VERSION);
151 #endif
152 #ifdef AMREX_USE_SUNDIALS
153  tpls.push_back(std::string("SUNDIALS ") + SUNDIALS_VERSION);
154 #endif
155 
156  if (!tpls.empty()) {
157  out << " Enabled third-party libraries: ";
158  for (const auto& val : tpls) {
static void print_tpls(std::ostream &)
Definition: ERF_ConsoleIO.cpp:137

◆ print_usage()

void ERF::print_usage ( MPI_Comm  comm,
std::ostream &  out 
)
static
27 {
28 #ifdef AMREX_USE_MPI
29  int irank = 0;
30  int num_ranks = 1;
31  MPI_Comm_size(comm, &num_ranks);
32  MPI_Comm_rank(comm, &irank);
33 
34  // Only root process does the printing
35  if (irank != 0) return;
36 #else
37  amrex::ignore_unused(comm);
38 #endif
39 
40  out << R"doc(Usage:
41  ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

◆ project_velocities()

void ERF::project_velocities ( int  lev,
amrex::Real  dt,
amrex::Vector< amrex::MultiFab > &  vars,
amrex::MultiFab &  p 
)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level 0.

11 {
12  BL_PROFILE("ERF::project_velocities()");
13 
14  auto const dom_lo = lbound(geom[lev].Domain());
15  auto const dom_hi = ubound(geom[lev].Domain());
16 
17  // Make sure the solver only sees the levels over which we are solving
18  Vector<BoxArray> ba_tmp; ba_tmp.push_back(mom_mf[Vars::cons].boxArray());
19  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap());
20  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
21 
22  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
23 
24  Vector<MultiFab> rhs;
25  Vector<MultiFab> phi;
26 
27  if (solverChoice.terrain_type == TerrainType::EB)
28  {
29  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
30  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1, MFInfo(), EBFactory(lev));
31  } else {
32  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
33  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
34  }
35 
36  auto dxInv = geom[lev].InvCellSizeArray();
37 
38  // If !fixed_density, we must convert (rho u) which came in
39  // to (rho0 u) which is what we will project
41  ConvertForProjection(mom_mf[Vars::cons], r_hse,
42  mom_mf[IntVars::xmom],
43  mom_mf[IntVars::ymom],
44  mom_mf[IntVars::zmom],
45  Geom(lev).Domain(),
47  }
48 
49  //
50  // ****************************************************************************
51  // Now convert the rho0w MultiFab to hold Omega rather than rhow
52  // ****************************************************************************
53  //
54  if (solverChoice.mesh_type == MeshType::VariableDz)
55  {
56  for ( MFIter mfi(rhs[0],TilingIfNotGPU()); mfi.isValid(); ++mfi)
57  {
58  const Array4<Real const>& rho0u_arr = mom_mf[IntVars::xmom].const_array(mfi);
59  const Array4<Real const>& rho0v_arr = mom_mf[IntVars::ymom].const_array(mfi);
60  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
61 
62  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
63  const Array4<Real const>& mf_u = mapfac_u[lev]->const_array(mfi);
64  const Array4<Real const>& mf_v = mapfac_v[lev]->const_array(mfi);
65 
66  //
67  // Define Omega from (rho0 W) but store it in the same array
68  //
69  Box tbz = mfi.nodaltilebox(2);
70  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
71  if (k > dom_lo.z && k <= dom_hi.z) {
72  Real rho0w = rho0w_arr(i,j,k);
73  rho0w_arr(i,j,k) = OmegaFromW(i,j,k,rho0w,
74  rho0u_arr,rho0v_arr,
75  mf_u,mf_v,z_nd,dxInv);
76  } else {
77  rho0w_arr(i,j,k) = Real(0.0);
78  }
79  });
80  } // mfi
81  }
82 
83  // ****************************************************************************
84  // Compute divergence which will form RHS
85  // Note that we replace "rho0w" with the contravariant momentum, Omega
86  // ****************************************************************************
87  Array<MultiFab const*, AMREX_SPACEDIM> rho0_u_const;
88  rho0_u_const[0] = &mom_mf[IntVars::xmom];
89  rho0_u_const[1] = &mom_mf[IntVars::ymom];
90  rho0_u_const[2] = &mom_mf[IntVars::zmom];
91 
92  compute_divergence(lev, rhs[0], rho0_u_const, geom_tmp[0]);
93 
94  Real rhsnorm = rhs[0].norm0();
95 
96  if (mg_verbose > 0) {
97  Print() << "Max/L2 norm of divergence before solve at level " << lev << " : " << rhsnorm << " " << rhs[0].norm2() << std::endl;
98  }
99 
100  // ****************************************************************************
101  //
102  // No need to build the solver if RHS == 0
103  //
104  if (rhsnorm <= solverChoice.poisson_abstol) return;
105  // ****************************************************************************
106 
107  // ****************************************************************************
108  // Initialize phi to 0
109  // (It is essential that we do this in order to fill the corners; these are never
110  // used but the Saxpy requires the values to be initialized.)
111  // ****************************************************************************
112  phi[0].setVal(0.0);
113 
114  Real start_step = static_cast<Real>(ParallelDescriptor::second());
115 
116  // ****************************************************************************
117  // Allocate fluxes
118  // ****************************************************************************
119  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
120  fluxes.resize(1);
121  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
122  if (solverChoice.terrain_type == TerrainType::EB) {
123  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
124  } else {
125  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
126  }
127  }
128 
129  // ****************************************************************************
130  // Choose the solver and solve
131  // ****************************************************************************
132 
133  // ****************************************************************************
134  // EB
135  // ****************************************************************************
136  if (solverChoice.terrain_type == TerrainType::EB) {
137  solve_with_EB_mlmg(lev, rhs, phi, fluxes);
138  } else {
139 
140 #ifdef ERF_USE_FFT
141  Box my_region(ba_tmp[0].minimalBox());
142  bool boxes_make_rectangle = (my_region.numPts() == ba_tmp[0].numPts());
143 #endif
144 
145  // ****************************************************************************
146  // No terrain or grid stretching
147  // ****************************************************************************
148  if (solverChoice.mesh_type == MeshType::ConstantDz) {
149 #ifdef ERF_USE_FFT
150  if (use_fft) {
151  if (boxes_make_rectangle) {
152  solve_with_fft(lev, rhs[0], phi[0], fluxes[0]);
153  } else {
154  amrex::Warning("FFT won't work unless the union of boxes is rectangular: defaulting to MLMG");
155  solve_with_mlmg(lev, rhs, phi, fluxes);
156  }
157  } else {
158  solve_with_mlmg(lev, rhs, phi, fluxes);
159  }
160 #else
161  if (use_fft) {
162  amrex::Warning("You set use_fft=true but didn't build with USE_FFT = TRUE; defaulting to MLMG");
163  }
164  solve_with_mlmg(lev, rhs, phi, fluxes);
165 #endif
166  } // No terrain or grid stretching
167 
168  // ****************************************************************************
169  // Grid stretching (flat terrain)
170  // ****************************************************************************
171  else if (solverChoice.mesh_type == MeshType::StretchedDz) {
172 #ifndef ERF_USE_FFT
173  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT solver");
174 #else
175  if (!boxes_make_rectangle) {
176  amrex::Abort("FFT won't work unless the union of boxes is rectangular");
177  } else {
178  if (!use_fft) {
179  amrex::Warning("Using FFT even though you didn't set use_fft to true; it's the best choice");
180  }
181  solve_with_fft(lev, rhs[0], phi[0], fluxes[0]);
182  }
183 #endif
184  } // grid stretching
185 
186  // ****************************************************************************
187  // General terrain
188  // ****************************************************************************
189  else if (solverChoice.mesh_type == MeshType::VariableDz) {
190 #ifdef ERF_USE_FFT
191  if (use_fft)
192  {
193  amrex::Warning("FFT solver does not work for general terrain: switching to FFT-preconditioned GMRES");
194  }
195  if (!boxes_make_rectangle) {
196  amrex::Abort("FFT preconditioner for GMRES won't work unless the union of boxes is rectangular");
197  } else {
198  solve_with_gmres(lev, rhs, phi, fluxes);
199  }
200 #else
201  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT preconditioner for GMRES");
202 #endif
203  } // general terrain
204 
205  } // not EB
206 
207  // ****************************************************************************
208  // Print time in solve
209  // ****************************************************************************
210  Real end_step = static_cast<Real>(ParallelDescriptor::second());
211  if (mg_verbose > 0) {
212  amrex::Print() << "Time in solve " << end_step - start_step << std::endl;
213  }
214 
215  // ****************************************************************************
216  // Subtract dt grad(phi) from the momenta (rho0u, rho0v, Omega)
217  // ****************************************************************************
218  MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0);
219  MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0);
220  MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0);
221 
222  //
223  // This call is only to verify the divergence after the solve
224  // It is important we do this before computing the rho0w_arr from Omega back to rho0w
225  //
226  // ****************************************************************************
227  // THIS IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE
228  // ****************************************************************************
229  //
230  if (mg_verbose > 0)
231  {
232  compute_divergence(lev, rhs[0], rho0_u_const, geom_tmp[0]);
233 
234  Print() << "Max/L2 norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << " " << rhs[0].norm2() << std::endl;
235 
236 #if 0
237  // FOR DEBUGGING ONLY
238  for ( MFIter mfi(rhs[0],TilingIfNotGPU()); mfi.isValid(); ++mfi)
239  {
240  const Array4<Real const>& rhs_arr = rhs[0].const_array(mfi);
241  Box bx = mfi.validbox();
242  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
243  if (std::abs(rhs_arr(i,j,k)) > 1.e-10) {
244  amrex::AllPrint() << "RHS AFTER SOLVE AT " <<
245  IntVect(i,j,k) << " " << rhs_arr(i,j,k) << std::endl;
246  }
247  });
248  } // mfi
249 #endif
250 
251  } // mg_verbose
252 
253  //
254  // ****************************************************************************
255  // Now convert the rho0w MultiFab back to holding (rho0w) rather than Omega
256  // ****************************************************************************
257  //
258  if (solverChoice.mesh_type == MeshType::VariableDz)
259  {
260  for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi)
261  {
262  Box tbz = mfi.nodaltilebox(2);
263  const Array4<Real >& rho0u_arr = mom_mf[IntVars::xmom].array(mfi);
264  const Array4<Real >& rho0v_arr = mom_mf[IntVars::ymom].array(mfi);
265  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
266  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
267  const Array4<Real const>& mf_u = mapfac_u[lev]->const_array(mfi);
268  const Array4<Real const>& mf_v = mapfac_v[lev]->const_array(mfi);
269  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
270  Real omega = rho0w_arr(i,j,k);
271  rho0w_arr(i,j,k) = WFromOmega(i,j,k,omega,
272  rho0u_arr,rho0v_arr,
273  mf_u,mf_v,z_nd,dxInv);
274  });
275  } // mfi
276  }
277 
278  // If !fixed_density, we must convert (rho0 u) back
279  // to (rho0 u) which is what we will pass back out
281  ConvertForProjection(r_hse, mom_mf[Vars::cons],
282  mom_mf[IntVars::xmom],
283  mom_mf[IntVars::ymom],
284  mom_mf[IntVars::zmom],
285  Geom(lev).Domain(),
287  }
288 
289  // ****************************************************************************
290  // Update pressure variable with phi -- note that phi is dt * change in pressure
291  // ****************************************************************************
292  MultiFab::Saxpy(pmf, 1.0/l_dt, phi[0],0,0,1,1);
293 }
void ConvertForProjection(const MultiFab &den_div, const MultiFab &den_mlt, MultiFab &xmom, MultiFab &ymom, MultiFab &zmom, const Box &domain, const Vector< BCRec > &domain_bcs_type_h)
Definition: ERF_ConvertForProjection.cpp:25
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real WFromOmega(int &i, int &j, int &k, amrex::Real omega, const amrex::Array4< const amrex::Real > &u_arr, const amrex::Array4< const amrex::Real > &v_arr, const amrex::Array4< const amrex::Real > &mf_u, const amrex::Array4< const amrex::Real > &mf_v, const amrex::Array4< const amrex::Real > &z_nd, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &dxInv)
Definition: ERF_TerrainMetrics.H:465
static bool use_fft
Definition: ERF.H:1063
void compute_divergence(int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
Definition: ERF_ComputeDivergence.cpp:10
void solve_with_mlmg(int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
Definition: ERF_SolveWithMLMG.cpp:40
void solve_with_gmres(int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
Definition: ERF_SolveWithGMRES.cpp:12
void solve_with_EB_mlmg(int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
Definition: ERF_SolveWithEBMLMG.cpp:19
@ omega
Definition: ERF_Morrison.H:53
Here is the call graph for this function:

◆ project_velocities_tb()

void ERF::project_velocities_tb ( int  lev,
amrex::Real  dt,
amrex::Vector< amrex::MultiFab > &  vars,
amrex::MultiFab &  p 
)

Project the single-level velocity field to enforce incompressibility with a thin body

21 {
22  BL_PROFILE("ERF::project_velocities_tb()");
23  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
24 
25  // Make sure the solver only sees the levels over which we are solving
26  Vector<BoxArray> ba_tmp; ba_tmp.push_back(vmf[Vars::cons].boxArray());
27  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(vmf[Vars::cons].DistributionMap());
28  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
29 
30  // Use the default settings
31  LPInfo info;
32  std::unique_ptr<MLPoisson> p_mlpoisson;
33 #if 0
34  if (overset_imask[0]) {
35  // Add overset mask around thin body
36  p_mlpoisson = std::make_unique<MLPoisson>(geom, grids, dmap, GetVecOfConstPtrs(overset_imask), info);
37  }
38  else
39 #endif
40  {
41  // Use the default settings
42  p_mlpoisson = std::make_unique<MLPoisson>(geom_tmp, ba_tmp, dm_tmp, info);
43  }
44 
45  auto bclo = get_projection_bc(Orientation::low);
46  auto bchi = get_projection_bc(Orientation::high);
47  bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true;
48  p_mlpoisson->setDomainBC(bclo, bchi);
49 
50  if (lev > 0) {
51  p_mlpoisson->setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
52  }
53 
54  p_mlpoisson->setLevelBC(0, nullptr);
55 
56  Vector<MultiFab> rhs;
57  Vector<MultiFab> phi;
58  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
59  Vector<Array<MultiFab,AMREX_SPACEDIM> > deltaf; // f^* - f^{n-1}
60  Vector<Array<MultiFab,AMREX_SPACEDIM> > u_plus_dtdf; // u + dt*deltaf
61 
62  // Used to pass array of const MFs to ComputeDivergence
63  Array<MultiFab const*, AMREX_SPACEDIM> u;
64 
65  rhs.resize(1);
66  phi.resize(1);
67  fluxes.resize(1);
68  deltaf.resize(1);
69  u_plus_dtdf.resize(1);
70 
71  rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
72  phi[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
73  rhs[0].setVal(0.0);
74  phi[0].setVal(0.0);
75 
76  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
77  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
78  u_plus_dtdf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
79 
80  deltaf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
81  deltaf[0][idim].setVal(0.0); // start with f^* == f^{n-1}
82  }
83 
84 #if 0
85  // DEBUG
86  u[0] = &(vmf[Vars::xvel]);
87  u[1] = &(vmf[Vars::yvel]);
88  u[2] = &(vmf[Vars::zvel]);
89  computeDivergence(rhs[0], u, geom[0]);
90  Print() << "Max norm of divergence before solve at level 0 : " << rhs[0].norm0() << std::endl;
91 #endif
92 
93  for (int itp = 0; itp < solverChoice.ncorr; ++itp)
94  {
95  // Calculate u + dt*deltaf
96  for (int idim = 0; idim < 3; ++idim) {
97  MultiFab::Copy(u_plus_dtdf[0][idim], deltaf[0][idim], 0, 0, 1, 0);
98  u_plus_dtdf[0][0].mult(-l_dt,0,1,0);
99  }
100  MultiFab::Add(u_plus_dtdf[0][0], vmf[Vars::xvel], 0, 0, 1, 0);
101  MultiFab::Add(u_plus_dtdf[0][1], vmf[Vars::yvel], 0, 0, 1, 0);
102  MultiFab::Add(u_plus_dtdf[0][2], vmf[Vars::zvel], 0, 0, 1, 0);
103 
104  u[0] = &(u_plus_dtdf[0][0]);
105  u[1] = &(u_plus_dtdf[0][1]);
106  u[2] = &(u_plus_dtdf[0][2]);
107  computeDivergence(rhs[0], u, geom_tmp[0]);
108 
109 #if 0
110  // DEBUG
111  if (itp==0) {
112  for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
113  {
114  const Box& bx = mfi.tilebox();
115  const Array4<Real const>& divU = rhs[0].const_array(mfi);
116  const Array4<Real const>& uarr = vmf[Vars::xvel].const_array(mfi);
117  const Array4<Real const>& varr = vmf[Vars::yvel].const_array(mfi);
118  const Array4<Real const>& warr = vmf[Vars::zvel].const_array(mfi);
119  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
120  {
121  if ((i>=120) && (i<=139) && (j==0) && ((k>=127)&&(k<=128))) {
122  amrex::AllPrint() << "before project div"<<IntVect(i,j,k)<<" = "<< divU(i,j,k)
123  << " u: " << uarr(i,j,k) << " " << uarr(i+1,j,k)
124  << " v: " << varr(i,j,k) << " " << varr(i,j+1,k)
125  << " w: " << warr(i,j,k) << " " << warr(i,j,k+1)
126  << std::endl;
127  }
128  });
129  }
130  }
131 #endif
132 
133  // If all Neumann BCs, adjust RHS to make sure we can converge
134  if (need_adjust_rhs) {
135  Real offset = volWgtSumMF(lev, rhs[0], 0, *mapfac_m[lev], false);
136  // amrex::Print() << "Poisson solvability offset = " << offset << std::endl;
137  rhs[0].plus(-offset, 0, 1);
138  }
139 
140  // Initialize phi to 0
141  phi[0].setVal(0.0);
142 
143  MLMG mlmg(*p_mlpoisson);
144  int max_iter = 100;
145  mlmg.setMaxIter(max_iter);
146 
147  mlmg.setVerbose(mg_verbose);
148  //mlmg.setBottomVerbose(mg_verbose);
149 
150  // solve for dt*p
151  mlmg.solve(GetVecOfPtrs(phi),
152  GetVecOfConstPtrs(rhs),
155 
156  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
157 
158  // Calculate new intermediate body force with updated gradp
159  if (thin_xforce[lev]) {
160  MultiFab::Copy( deltaf[0][0], fluxes[0][0], 0, 0, 1, 0);
161  ApplyInvertedMask(deltaf[0][0], *xflux_imask[0]);
162  }
163  if (thin_yforce[lev]) {
164  MultiFab::Copy( deltaf[0][1], fluxes[0][1], 0, 0, 1, 0);
165  ApplyInvertedMask(deltaf[0][1], *yflux_imask[0]);
166  }
167  if (thin_zforce[lev]) {
168  MultiFab::Copy( deltaf[0][2], fluxes[0][2], 0, 0, 1, 0);
169  ApplyInvertedMask(deltaf[0][2], *zflux_imask[0]);
170  }
171 
172  // DEBUG
173  // for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
174  // {
175  // const Box& bx = mfi.tilebox();
176  // const Array4<Real const>& dfz_arr = deltaf[0][2].const_array(mfi);
177  // ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
178  // {
179  // if ((i>=120) && (i<=139) && (j==0) && (k==128)) {
180  // amrex::AllPrint()
181  // << " piter" << itp
182  // << " dfz"<<IntVect(i,j,k)<<" = "<< dfz_arr(i,j,k)
183  // << std::endl;
184  // }
185  // });
186  // }
187 
188  // Update pressure variable with phi -- note that phi is change in pressure, not the full pressure
189  MultiFab::Saxpy(pmf, 1.0, phi[0],0,0,1,0);
190 
191  // Subtract grad(phi) from the velocity components
192  Real beta = 1.0;
193  MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
194  MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
195  MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
196  if (thin_xforce[lev]) {
197  ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
198  }
199  if (thin_yforce[lev]) {
200  ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
201  }
202  if (thin_zforce[lev]) {
203  ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
204  }
205  } // itp: pressure-force iterations
206 
207  // Subtract grad(phi) from the velocity components
208 // Real beta = 1.0;
209 // for (int ilev = lev_min; ilev <= lev_max; ++ilev) {
210 // MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
211 // MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
212 // MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
213 // if (thin_xforce[lev]) {
214 // ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
215 // }
216 // if (thin_yforce[lev]) {
217 // ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
218 // }
219 // if (thin_zforce[lev]) {
220 // ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
221 // }
222 // }
223 
224 #if 0
225  // Confirm that the velocity is now divergence free
226  u[0] = &(vmf[Vars::xvel]);
227  u[1] = &(vmf[Vars::yvel]);
228  u[2] = &(vmf[Vars::zvel]);
229  computeDivergence(rhs[0], u, geom_tmp[0]);
230  Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
231 
232 #endif
233 }
AMREX_FORCE_INLINE IntVect offset(const int face_dir, const int normal)
Definition: ERF_ReadBndryPlanes.cpp:28
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyInvertedMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:409
amrex::Real volWgtSumMF(int lev, const amrex::MultiFab &mf, int comp, const amrex::MultiFab &mapfac, bool finemask)
Definition: ERF_WriteScalarProfiles.cpp:595
amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > get_projection_bc(amrex::Orientation::Side side) const noexcept
Definition: ERF_SolveWithMLMG.cpp:17
bool projection_has_dirichlet(amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > bcs) const
Definition: ERF_PoissonSolve_tb.cpp:8
int ncorr
Definition: ERF_DataStruct.H:674
Here is the call graph for this function:

◆ projection_has_dirichlet()

bool ERF::projection_has_dirichlet ( amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM >  bcs) const
9 {
10  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
11  if (bcs[dir] == LinOpBCType::Dirichlet) return true;
12  }
13  return false;
14 }

◆ ReadCheckpointFile()

void ERF::ReadCheckpointFile ( )

ERF function for reading data from a checkpoint file during restart.

399 {
400  Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
401 
402  // Header
403  std::string File(restart_chkfile + "/Header");
404 
405  VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
406 
407  Vector<char> fileCharPtr;
408  ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
409  std::string fileCharPtrString(fileCharPtr.dataPtr());
410  std::istringstream is(fileCharPtrString, std::istringstream::in);
411 
412  std::string line, word;
413 
414  int chk_ncomp_cons, chk_ncomp;
415 
416  // read in title line
417  std::getline(is, line);
418 
419  // read in finest_level
420  is >> finest_level;
421  GotoNextLine(is);
422 
423  // read the number of components
424  // for each variable we store
425 
426  // conservative, cell-centered vars
427  is >> chk_ncomp_cons;
428  GotoNextLine(is);
429 
430  // x-velocity on faces
431  is >> chk_ncomp;
432  GotoNextLine(is);
433  AMREX_ASSERT(chk_ncomp == 1);
434 
435  // y-velocity on faces
436  is >> chk_ncomp;
437  GotoNextLine(is);
438  AMREX_ASSERT(chk_ncomp == 1);
439 
440  // z-velocity on faces
441  is >> chk_ncomp;
442  GotoNextLine(is);
443  AMREX_ASSERT(chk_ncomp == 1);
444 
445  // read in array of istep
446  std::getline(is, line);
447  {
448  std::istringstream lis(line);
449  int i = 0;
450  while (lis >> word) {
451  istep[i++] = std::stoi(word);
452  }
453  }
454 
455  // read in array of dt
456  std::getline(is, line);
457  {
458  std::istringstream lis(line);
459  int i = 0;
460  while (lis >> word) {
461  dt[i++] = std::stod(word);
462  }
463  }
464 
465  // read in array of t_new
466  std::getline(is, line);
467  {
468  std::istringstream lis(line);
469  int i = 0;
470  while (lis >> word) {
471  t_new[i++] = std::stod(word);
472  }
473  }
474 
475  for (int lev = 0; lev <= finest_level; ++lev) {
476  // read in level 'lev' BoxArray from Header
477  BoxArray ba;
478  ba.readFrom(is);
479  GotoNextLine(is);
480 
481  // create a distribution mapping
482  DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
483 
484  MakeNewLevelFromScratch (lev, t_new[lev], ba, dm);
485  }
486 
487  // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
488  // NOTE: Data is written over ncomp, so check that we match the header file
489  int ncomp_cons = vars_new[0][Vars::cons].nComp();
490 
491  // NOTE: QKE was removed so this is for backward compatibility
492  AMREX_ASSERT((chk_ncomp_cons==ncomp_cons) || ((chk_ncomp_cons-1)==ncomp_cons));
493  //
494  // See if we have a written separate file that tells how many components and how many ghost cells
495  // we have of the base state
496  //
497  // If we can't find the file, then set the number of components to the original number = 3
498  //
499  int ncomp_base_to_read = 3;
500  IntVect ng_base = IntVect{1};
501  {
502  std::string BaseStateFile(restart_chkfile + "/num_base_state_comps");
503 
504  if (amrex::FileExists(BaseStateFile))
505  {
506  Vector<char> BaseStatefileCharPtr;
507  ParallelDescriptor::ReadAndBcastFile(BaseStateFile, BaseStatefileCharPtr);
508  std::string BaseStatefileCharPtrString(BaseStatefileCharPtr.dataPtr());
509 
510  // We set this to the default value of 3 but allow it be larger if th0 and qv0 were written
511  std::istringstream isb(BaseStatefileCharPtrString, std::istringstream::in);
512  isb >> ncomp_base_to_read;
513  isb >> ng_base;
514  }
515  }
516 
517  // read in the MultiFab data
518  for (int lev = 0; lev <= finest_level; ++lev)
519  {
520  // NOTE: For backward compatibility (chk file has QKE)
521  if ((chk_ncomp_cons-1)==ncomp_cons) {
522  MultiFab cons(grids[lev],dmap[lev],chk_ncomp_cons,0);
523  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
524 
525  // Copy up to RhoKE_comp
526  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,(RhoKE_comp+1),0);
527 
528  // Only if we have a PBL model do we need to copy QKE is src to KE in dst
529  if ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
530  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ) {
531  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+1),RhoKE_comp,1,0);
532  vars_new[lev][Vars::cons].mult(0.5,RhoKE_comp,1,0);
533  }
534 
535  // Copy other components
536  int ncomp_remainder = ncomp_cons - (RhoKE_comp + 1);
537  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+2),(RhoKE_comp+1),ncomp_remainder,0);
538 
539  vars_new[lev][Vars::cons].setBndry(1.0e34);
540  } else {
541  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
542  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
543  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
544  vars_new[lev][Vars::cons].setBndry(1.0e34);
545  }
546 
547  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
548  VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
549  MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
550  vars_new[lev][Vars::xvel].setBndry(1.0e34);
551 
552  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
553  VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
554  MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
555  vars_new[lev][Vars::yvel].setBndry(1.0e34);
556 
557  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
558  VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
559  MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
560  vars_new[lev][Vars::zvel].setBndry(1.0e34);
561 
562  // Note that we read the ghost cells of the base state (unlike above)
563 
564  // The original base state only had 3 components and 1 ghost cell -- we read this
565  // here to be consistent with the old style
566  MultiFab base(grids[lev],dmap[lev],ncomp_base_to_read,ng_base);
567  VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
568 
569  MultiFab::Copy(base_state[lev],base,0,0,ncomp_base_to_read,ng_base);
570 
571  // Create theta0 from p0, rh0
572  if (ncomp_base_to_read < 4) {
573  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
574  {
575  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
576  const Box& bx = mfi.tilebox();
577  Array4<Real> const& fab = base_state[lev].array(mfi);
578  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
579  {
581  / fab(i,j,k,BaseState::r0_comp);
582  });
583  }
584  }
585  // Default theta0 to 0
586  if (ncomp_base_to_read < 5) {
587  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
588  {
589  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
590  const Box& bx = mfi.tilebox();
591  Array4<Real> const& fab = base_state[lev].array(mfi);
592  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
593  {
594  fab(i,j,k,BaseState::qv0_comp) = 0.0;
595  });
596  }
597  }
598  base_state[lev].FillBoundary(geom[lev].periodicity());
599 
600  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
601  // Note that we also read the ghost cells of z_phys_nd
602  IntVect ng = z_phys_nd[lev]->nGrowVect();
603  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
604  VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
605  MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
607  }
608 
609  // Read in the moisture model restart variables
610  std::vector<int> qmoist_indices;
611  std::vector<std::string> qmoist_names;
612  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
613  int qmoist_nvar = qmoist_indices.size();
614  for (int var = 0; var < qmoist_nvar; var++) {
615  const int ncomp = 1;
616  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
617  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
618  VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", qmoist_names[var]));
619  MultiFab::Copy(*(qmoist[lev][qmoist_indices[var]]),moist_vars,0,0,ncomp,ng_moist);
620  }
621 
622 #if defined(ERF_USE_WINDFARM)
623  if(solverChoice.windfarm_type == WindFarmType::Fitch or
624  solverChoice.windfarm_type == WindFarmType::EWP or
625  solverChoice.windfarm_type == WindFarmType::SimpleAD){
626  IntVect ng = Nturb[lev].nGrowVect();
627  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
628  VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
629  MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
630  }
631 #endif
632 
633  if (solverChoice.lsm_type != LandSurfaceType::None) {
634  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
635  BoxArray ba = lsm_data[lev][mvar]->boxArray();
636  DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
637  IntVect ng = lsm_data[lev][mvar]->nGrowVect();
638  int nvar = lsm_data[lev][mvar]->nComp();
639  MultiFab lsm_vars(ba,dm,nvar,ng);
640  VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmVars"));
641  MultiFab::Copy(*(lsm_data[lev][mvar]),lsm_vars,0,0,nvar,ng);
642  }
643  }
644 
645 
646  IntVect ng = mapfac_m[lev]->nGrowVect();
647  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
648  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
649  MultiFab::Copy(*mapfac_m[lev],mf_m,0,0,1,ng);
650 
651  ng = mapfac_u[lev]->nGrowVect();
652  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
653  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
654  MultiFab::Copy(*mapfac_u[lev],mf_u,0,0,1,ng);
655 
656  ng = mapfac_v[lev]->nGrowVect();
657  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
658  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
659  MultiFab::Copy(*mapfac_v[lev],mf_v,0,0,1,ng);
660 
661 
662  // NOTE: We read MOST data in ReadCheckpointFileMOST (see below)!
663 
664  // See if we wrote out SST data
665  std::string FirstSSTFileName(restart_chkfile + "/Level_0/SST_0_H");
666  if (amrex::FileExists(FirstSSTFileName))
667  {
668  amrex::Print() << "Reading SST data" << std::endl;
669  int ntimes = 1;
670  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
671  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
672  sst_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
673  for (int nt(0); nt<ntimes; ++nt) {
674  VisMF::Read(sst_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
675  "SST_" + std::to_string(nt)));
676  MultiFab::Copy(*sst_lev[lev][nt],sst_at_t,0,0,1,ng);
677  }
678  }
679 
680  // See if we wrote out TSK data
681  std::string FirstTSKFileName(restart_chkfile + "/Level_0/TSK_0_H");
682  if (amrex::FileExists(FirstTSKFileName))
683  {
684  amrex::Print() << "Reading TSK data" << std::endl;
685  int ntimes = 1;
686  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
687  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
688  tsk_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
689  for (int nt(0); nt<ntimes; ++nt) {
690  VisMF::Read(tsk_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
691  "TSK_" + std::to_string(nt)));
692  MultiFab::Copy(*tsk_lev[lev][nt],tsk_at_t,0,0,1,ng);
693  }
694  }
695 
696  {
697  amrex::Print() << "Reading LMASK data" << std::endl;
698  int ntimes = 1;
699  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
700  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
701  for (int nt(0); nt<ntimes; ++nt) {
702  VisMF::Read(lmask_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
703  "LMASK_" + std::to_string(nt)));
704  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
705  const Box& bx = mfi.growntilebox();
706  Array4<int> const& dst_arr = lmask_lev[lev][nt]->array(mfi);
707  Array4<Real> const& src_arr = lmask_at_t.array(mfi);
708  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
709  {
710  dst_arr(i,j,k) = int(src_arr(i,j,k));
711  });
712  }
713  }
714  }
715 
716 #ifdef ERF_USE_NETCDF
717  IntVect ngv = ng; ngv[2] = 0;
718 
719  // Read lat/lon if it exists
721  amrex::Print() << "Reading Lat/Lon variables" << std::endl;
722  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
723  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
724  VisMF::Read(lat, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LAT"));
725  VisMF::Read(lon, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LON"));
726  lat_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
727  lon_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
728  MultiFab::Copy(*lat_m[lev],lat,0,0,1,ngv);
729  MultiFab::Copy(*lon_m[lev],lon,0,0,1,ngv);
730  }
731 
732  // Read sinPhi and cosPhi if it exists
734  amrex::Print() << "Reading Coriolis factors" << std::endl;
735  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
736  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
737  VisMF::Read(sphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SinPhi"));
738  VisMF::Read(cphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "CosPhi"));
739  sinPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
740  cosPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
741  MultiFab::Copy(*sinPhi_m[lev],sphi,0,0,1,ngv);
742  MultiFab::Copy(*cosPhi_m[lev],cphi,0,0,1,ngv);
743  }
744 
745  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
746  MultiFab tmp1d(ba1d[lev],dmap[lev],1,ngv);
747  MultiFab tmp2d(ba2d[lev],dmap[lev],1,ngv);
748 
749  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C1H"));
750  MultiFab::Copy(*mf_C1H[lev],tmp1d,0,0,1,ngv);
751 
752  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C2H"));
753  MultiFab::Copy(*mf_C2H[lev],tmp1d,0,0,1,ngv);
754 
755  VisMF::Read(tmp2d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MUB"));
756  MultiFab::Copy(*mf_MUB[lev],tmp2d,0,0,1,ngv);
757  }
758 #endif
759 
760  } // for lev
761 
762 #ifdef ERF_USE_PARTICLES
763  restartTracers((ParGDBBase*)GetParGDB(),restart_chkfile);
764  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
765  dynamic_cast<LagrangianMicrophysics&>(*micro).restartParticles((ParGDBBase*)GetParGDB(),restart_chkfile);
766  }
767 #endif
768 
769 #if 0
770 #ifdef ERF_USE_NETCDF
771  // Read bdy_data files
772  if ( ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
774  {
775  int ioproc = ParallelDescriptor::IOProcessorNumber(); // I/O rank
776  int num_time;
777  int num_var;
778  Vector<Box> bx_v;
779  if (ParallelDescriptor::IOProcessor()) {
780  // Open header file and read from it
781  std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
782  bdy_h_file >> num_time;
783  bdy_h_file >> num_var;
784  bdy_h_file >> start_bdy_time;
785  bdy_h_file >> bdy_time_interval;
786  bdy_h_file >> real_width;
787  bx_v.resize(4*num_var);
788  for (int ivar(0); ivar<num_var; ++ivar) {
789  bdy_h_file >> bx_v[4*ivar ];
790  bdy_h_file >> bx_v[4*ivar+1];
791  bdy_h_file >> bx_v[4*ivar+2];
792  bdy_h_file >> bx_v[4*ivar+3];
793  }
794 
795  // IO size the FABs
796  bdy_data_xlo.resize(num_time);
797  bdy_data_xhi.resize(num_time);
798  bdy_data_ylo.resize(num_time);
799  bdy_data_yhi.resize(num_time);
800  for (int itime(0); itime<num_time; ++itime) {
801  bdy_data_xlo[itime].resize(num_var);
802  bdy_data_xhi[itime].resize(num_var);
803  bdy_data_ylo[itime].resize(num_var);
804  bdy_data_yhi[itime].resize(num_var);
805  for (int ivar(0); ivar<num_var; ++ivar) {
806  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
807  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
808  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
809  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
810  }
811  }
812 
813  // Open data file and read from it
814  std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
815  for (int itime(0); itime<num_time; ++itime) {
816  for (int ivar(0); ivar<num_var; ++ivar) {
817  bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
818  bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
819  bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
820  bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
821  }
822  }
823  } // IO
824 
825  // Broadcast the data
826  ParallelDescriptor::Barrier();
827  ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
828  ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
829  ParallelDescriptor::Bcast(&real_width,1,ioproc);
830  ParallelDescriptor::Bcast(&num_time,1,ioproc);
831  ParallelDescriptor::Bcast(&num_var,1,ioproc);
832 
833  // Everyone size their boxes
834  bx_v.resize(4*num_var);
835 
836  ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
837 
838  // Everyone but IO size their FABs
839  if (!ParallelDescriptor::IOProcessor()) {
840  bdy_data_xlo.resize(num_time);
841  bdy_data_xhi.resize(num_time);
842  bdy_data_ylo.resize(num_time);
843  bdy_data_yhi.resize(num_time);
844  for (int itime(0); itime<num_time; ++itime) {
845  bdy_data_xlo[itime].resize(num_var);
846  bdy_data_xhi[itime].resize(num_var);
847  bdy_data_ylo[itime].resize(num_var);
848  bdy_data_yhi[itime].resize(num_var);
849  for (int ivar(0); ivar<num_var; ++ivar) {
850  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
851  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
852  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
853  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
854  }
855  }
856  }
857 
858  for (int itime(0); itime<num_time; ++itime) {
859  for (int ivar(0); ivar<num_var; ++ivar) {
860  ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
861  ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
862  ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
863  ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
864  }
865  }
866  } // init_type == WRFInput or Metgrid
867 #endif
868 #endif
869 }
static void GotoNextLine(std::istream &is)
Definition: ERF_Checkpoint.cpp:16
void MakeNewLevelFromScratch(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:25
bool variable_coriolis
Definition: ERF_DataStruct.H:758
bool has_lat_lon
Definition: ERF_DataStruct.H:757
Here is the call graph for this function:

◆ ReadCheckpointFileSurfaceLayer()

void ERF::ReadCheckpointFileSurfaceLayer ( )

ERF function for reading additional data for MOST from a checkpoint file during restart.

This is called after the ABLMost object is instantiated.

878 {
879  for (int lev = 0; lev <= finest_level; ++lev)
880  {
881  amrex::Print() << "Reading MOST variables" << std::endl;
882 
883  IntVect ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
884  MultiFab m_var(ba2d[lev],dmap[lev],1,ng);
885  MultiFab* dst = nullptr;
886 
887  // U*
888  dst = m_SurfaceLayer->get_u_star(lev);
889  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Ustar"));
890  MultiFab::Copy(*dst,m_var,0,0,1,ng);
891 
892  // W*
893  dst = m_SurfaceLayer->get_w_star(lev);
894  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Wstar"));
895  MultiFab::Copy(*dst,m_var,0,0,1,ng);
896 
897  // T*
898  dst = m_SurfaceLayer->get_t_star(lev);
899  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Tstar"));
900  MultiFab::Copy(*dst,m_var,0,0,1,ng);
901 
902  // Q*
903  dst = m_SurfaceLayer->get_q_star(lev);
904  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qstar"));
905  MultiFab::Copy(*dst,m_var,0,0,1,ng);
906 
907  // Olen
908  dst = m_SurfaceLayer->get_olen(lev);
909  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Olen"));
910  MultiFab::Copy(*dst,m_var,0,0,1,ng);
911 
912  // PBLH
913  dst = m_SurfaceLayer->get_pblh(lev);
914  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PBLH"));
915  MultiFab::Copy(*dst,m_var,0,0,1,ng);
916 
917  // Z0
918  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z0"));
919  for (amrex::MFIter mfi(m_var); mfi.isValid(); ++mfi) {
920  const Box& bx = mfi.growntilebox();
921  FArrayBox* most_z0 = (m_SurfaceLayer->get_z0(lev));
922  most_z0->copy<RunOn::Device>(m_var[mfi], bx);
923  }
924  }
925 }

◆ ReadParameters()

void ERF::ReadParameters ( )
private
1557 {
1558  {
1559  ParmParse pp; // Traditionally, max_step and stop_time do not have prefix.
1560  pp.query("max_step", max_step);
1561 
1562  std::string start_datetime, stop_datetime;
1563  if (pp.query("start_datetime", start_datetime)) {
1564  start_time = getEpochTime(start_datetime, datetime_format);
1565  if (pp.query("stop_datetime", stop_datetime)) {
1566  stop_time = getEpochTime(stop_datetime, datetime_format);
1567  }
1568  use_datetime = true;
1569  } else {
1570  pp.query("stop_time", stop_time);
1571  pp.query("start_time", start_time); // This is optional, it defaults to 0
1572  }
1573  }
1574 
1575  ParmParse pp(pp_prefix);
1576  ParmParse pp_amr("amr");
1577  {
1578  pp.query("regrid_level_0_on_restart", regrid_level_0_on_restart);
1579  pp.query("regrid_int", regrid_int);
1580  pp.query("check_file", check_file);
1581 
1582  // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
1583  // for those or "erf.restart" / "erf.m_check_int" with the former taking
1584  // precedence if both are specified
1585  pp.query("check_int", m_check_int);
1586  pp.query("check_per", m_check_per);
1587  pp_amr.query("check_int", m_check_int);
1588  pp_amr.query("check_per", m_check_per);
1589 
1590  pp.query("restart", restart_chkfile);
1591  pp_amr.query("restart", restart_chkfile);
1592 
1593  // Verbosity
1594  pp.query("v", verbose);
1595  pp.query("mg_v", mg_verbose);
1596  pp.query("use_fft", use_fft);
1597 #ifndef ERF_USE_FFT
1598  if (use_fft) {
1599  amrex::Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
1600  }
1601 #endif
1602 
1603  // Frequency of diagnostic output
1604  pp.query("sum_interval", sum_interval);
1605  pp.query("sum_period" , sum_per);
1606 
1607  pp.query("pert_interval", pert_interval);
1608 
1609  // Time step controls
1610  pp.query("cfl", cfl);
1611  pp.query("substepping_cfl", sub_cfl);
1612  pp.query("init_shrink", init_shrink);
1613  pp.query("change_max", change_max);
1614  pp.query("dt_max_initial", dt_max_initial);
1615  pp.query("dt_max", dt_max);
1616 
1617  fixed_dt.resize(max_level+1,-1.);
1618  fixed_fast_dt.resize(max_level+1,-1.);
1619 
1620  pp.query("fixed_dt", fixed_dt[0]);
1621  pp.query("fixed_fast_dt", fixed_fast_dt[0]);
1622 
1623  for (int lev = 1; lev <= max_level; lev++)
1624  {
1625  fixed_dt[lev] = fixed_dt[lev-1] / static_cast<Real>(MaxRefRatio(lev-1));
1626  fixed_fast_dt[lev] = fixed_fast_dt[lev-1] / static_cast<Real>(MaxRefRatio(lev-1));
1627  }
1628 
1629  pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
1630 
1631  // We use this to keep track of how many boxes we read in from WRF initialization
1632  num_files_at_level.resize(max_level+1,0);
1633 
1634  // We use this to keep track of how many boxes are specified thru the refinement indicators
1635  num_boxes_at_level.resize(max_level+1,0);
1636  boxes_at_level.resize(max_level+1);
1637 
1638  // We always have exactly one file at level 0
1639  num_boxes_at_level[0] = 1;
1640  boxes_at_level[0].resize(1);
1641  boxes_at_level[0][0] = geom[0].Domain();
1642 
1643 #ifdef ERF_USE_NETCDF
1644  nc_init_file.resize(max_level+1);
1645 
1646  // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
1647  // but we always have exactly one file at level 0
1648  for (int lev = 0; lev <= max_level; lev++) {
1649  const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
1650  if (pp.contains(nc_file_names.c_str())) {
1651  int num_files = pp.countval(nc_file_names.c_str());
1652  num_files_at_level[lev] = num_files;
1653  nc_init_file[lev].resize(num_files);
1654  pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
1655  for (int j = 0; j < num_files; j++) {
1656  Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
1657  } // j
1658  } // if pp.contains
1659  } // lev
1660 
1661  // NetCDF wrfbdy lateral boundary file
1662  if (pp.query("nc_bdy_file", nc_bdy_file)) {
1663  Print() << "Reading NC bdy file name " << nc_bdy_file << std::endl;
1664  }
1665 
1666  // NetCDF wrflow lateral boundary file
1667  if (pp.query("nc_low_file", nc_low_file)) {
1668  Print() << "Reading NC low file name " << nc_low_file << std::endl;
1669  }
1670 
1671 #endif
1672 
1673  // Flag to trigger initialization from input_sounding like WRF's ideal.exe
1674  pp.query("init_sounding_ideal", init_sounding_ideal);
1675 
1676  // Options for vertical interpolation of met_em*.nc data.
1677  pp.query("metgrid_debug_quiescent", metgrid_debug_quiescent);
1678  pp.query("metgrid_debug_isothermal", metgrid_debug_isothermal);
1679  pp.query("metgrid_debug_dry", metgrid_debug_dry);
1680  pp.query("metgrid_debug_psfc", metgrid_debug_psfc);
1681  pp.query("metgrid_debug_msf", metgrid_debug_msf);
1682  pp.query("metgrid_interp_theta", metgrid_interp_theta);
1683  pp.query("metgrid_basic_linear", metgrid_basic_linear);
1684  pp.query("metgrid_use_below_sfc", metgrid_use_below_sfc);
1685  pp.query("metgrid_use_sfc", metgrid_use_sfc);
1686  pp.query("metgrid_retain_sfc", metgrid_retain_sfc);
1687  pp.query("metgrid_proximity", metgrid_proximity);
1688  pp.query("metgrid_order", metgrid_order);
1689  pp.query("metgrid_force_sfc_k", metgrid_force_sfc_k);
1690 
1691  // Set default to FullState for now ... later we will try Perturbation
1692  interpolation_type = StateInterpType::FullState;
1693  pp.query_enum_case_insensitive("interpolation_type" ,interpolation_type);
1694 
1695  PlotFileType plotfile_type_temp = PlotFileType::None;
1696  pp.query_enum_case_insensitive("plotfile_type" ,plotfile_type_temp);
1697  pp.query_enum_case_insensitive("plotfile_type_1",plotfile_type_1);
1698  pp.query_enum_case_insensitive("plotfile_type_2",plotfile_type_2);
1699  //
1700  // This option is for backward consistency -- if only plotfile_type is set,
1701  // then it will be used for both 1 and 2 if and only if they are not set
1702  //
1703  // Default is native amrex if no type is specified
1704  //
1705  if (plotfile_type_temp == PlotFileType::None) {
1706  if (plotfile_type_1 == PlotFileType::None) {
1707  plotfile_type_1 = PlotFileType::Amrex;
1708  }
1709  if (plotfile_type_2 == PlotFileType::None) {
1710  plotfile_type_2 = PlotFileType::Amrex;
1711  }
1712  } else {
1713  if (plotfile_type_1 == PlotFileType::None) {
1714  plotfile_type_1 = plotfile_type_temp;
1715  } else {
1716  amrex::Abort("You must set either plotfile_type or plotfile_type_1, not both");
1717  }
1718  if (plotfile_type_2 == PlotFileType::None) {
1719  plotfile_type_2 = plotfile_type_temp;
1720  } else {
1721  amrex::Abort("You must set either plotfile_type or plotfile_type_2, not both");
1722  }
1723  }
1724 #ifndef ERF_USE_NETCDF
1725  if (plotfile_type_1 == PlotFileType::Netcdf ||
1726  plotfile_type_2 == PlotFileType::Netcdf) {
1727  amrex::Abort("Plotfile type = Netcdf is not allowed without USE_NETCDF = TRUE");
1728  }
1729 #endif
1730 
1731  pp.query("plot_file_1", plot_file_1);
1732  pp.query("plot_file_2", plot_file_2);
1733  pp.query("plot_int_1" , m_plot_int_1);
1734  pp.query("plot_int_2" , m_plot_int_2);
1735  pp.query("plot_per_1", m_plot_per_1);
1736  pp.query("plot_per_2", m_plot_per_2);
1737 
1738  pp.query("subvol_file", subvol_file);
1739  pp.query("subvol_int" , m_subvol_int);
1740  pp.query("subvol_per" , m_subvol_per);
1741 
1742  pp.query("expand_plotvars_to_unif_rr",m_expand_plotvars_to_unif_rr);
1743 
1744  pp.query("plot_face_vels",m_plot_face_vels);
1745 
1746  if ( (m_plot_int_1 > 0 && m_plot_per_1 > 0) ||
1747  (m_plot_int_2 > 0 && m_plot_per_2 > 0.) ) {
1748  Abort("Must choose only one of plot_int or plot_per");
1749  }
1750 
1751  pp.query("profile_int", profile_int);
1752  pp.query("destag_profiles", destag_profiles);
1753 
1754  pp.query("plot_lsm", plot_lsm);
1755 #ifdef ERF_USE_RRTMGP
1756  pp.query("plot_rad", plot_rad);
1757 #endif
1758 
1759  pp.query("output_1d_column", output_1d_column);
1760  pp.query("column_per", column_per);
1761  pp.query("column_interval", column_interval);
1762  pp.query("column_loc_x", column_loc_x);
1763  pp.query("column_loc_y", column_loc_y);
1764  pp.query("column_file_name", column_file_name);
1765 
1766  // Sampler output frequency
1767  pp.query("sampler_per", sampler_per);
1768  pp.query("sampler_interval", sampler_interval);
1769 
1770  // Specify information about outputting planes of data
1771  pp.query("output_bndry_planes", output_bndry_planes);
1772  pp.query("bndry_output_planes_interval", bndry_output_planes_interval);
1773  pp.query("bndry_output_planes_per", bndry_output_planes_per);
1774  pp.query("bndry_output_start_time", bndry_output_planes_start_time);
1775 
1776  // Specify whether ingest boundary planes of data
1777  pp.query("input_bndry_planes", input_bndry_planes);
1778 
1779  // Query the set and total widths for wrfbdy interior ghost cells
1780  pp.query("real_width", real_width);
1781  pp.query("real_set_width", real_set_width);
1782 
1783  // Query the set and total widths for crse-fine interior ghost cells
1784  pp.query("cf_width", cf_width);
1785  pp.query("cf_set_width", cf_set_width);
1786 
1787  // AmrMesh iterate on grids?
1788  bool iterate(true);
1789  pp_amr.query("iterate_grids",iterate);
1790  if (!iterate) SetIterateToFalse();
1791  }
1792 
1793 #ifdef ERF_USE_PARTICLES
1794  readTracersParams();
1795 #endif
1796 
1797  solverChoice.init_params(max_level,pp_prefix);
1798 
1799 #ifndef ERF_USE_NETCDF
1800  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(( (solverChoice.init_type != InitType::WRFInput) &&
1801  (solverChoice.init_type != InitType::Metgrid ) &&
1802  (solverChoice.init_type != InitType::NCFile ) ),
1803  "init_type cannot be 'WRFInput', 'MetGrid' or 'NCFile' if we don't build with netcdf!");
1804 #endif
1805 
1806  // Query the canopy model file name
1807  std::string forestfile;
1808  solverChoice.do_forest_drag = pp.query("forest_file", forestfile);
1810  for (int lev = 0; lev <= max_level; ++lev) {
1811  m_forest_drag[lev] = std::make_unique<ForestDrag>(forestfile);
1812  }
1813  }
1814 
1815  // If init from WRFInput or Metgrid make sure a valid file name is present
1816  if ((solverChoice.init_type == InitType::WRFInput) ||
1817  (solverChoice.init_type == InitType::Metgrid) ||
1818  (solverChoice.init_type == InitType::NCFile) ) {
1819  for (int lev = 0; lev <= max_level; lev++) {
1820  int num_files = nc_init_file[lev].size();
1821  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(num_files>0, "A file name must be present for init type WRFInput, Metgrid or NCFile.");
1822  for (int j = 0; j < num_files; j++) {
1823  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!nc_init_file[lev][j].empty(), "Valid file name must be present for init type WRFInput, Metgrid or NCFile.");
1824  } //j
1825  } // lev
1826  } // InitType
1827 
1828  // What type of land surface model to use
1829  // NOTE: Must be checked after init_params
1830  if (solverChoice.lsm_type == LandSurfaceType::SLM) {
1831  lsm.SetModel<SLM>();
1832  Print() << "SLM land surface model!\n";
1833  } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
1834  lsm.SetModel<MM5>();
1835  Print() << "MM5 land surface model!\n";
1836 #ifdef ERF_USE_NOAH
1837  } else if (solverChoice.lsm_type == LandSurfaceType::NOAH) {
1838  lsm.SetModel<NOAH>();
1839  Print() << "NOAH land surface model!\n";
1840 #endif
1841  } else if (solverChoice.lsm_type == LandSurfaceType::None) {
1842  lsm.SetModel<NullSurf>();
1843  Print() << "Null land surface model!\n";
1844  } else {
1845  Abort("Dont know this LandSurfaceType!") ;
1846  }
1847 
1848  if (verbose > 0) {
1849  solverChoice.display(max_level,pp_prefix);
1850  }
1851 
1853 }
AMREX_FORCE_INLINE std::time_t getEpochTime(const std::string &dateTime, const std::string &dateTimeFormat)
Definition: ERF_EpochTime.H:14
bool metgrid_basic_linear
Definition: ERF.H:1101
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
Definition: ERF.H:723
bool metgrid_debug_msf
Definition: ERF.H:1099
std::string plot_file_2
Definition: ERF.H:959
bool m_plot_face_vels
Definition: ERF.H:968
int regrid_int
Definition: ERF.H:951
bool metgrid_retain_sfc
Definition: ERF.H:1104
bool metgrid_use_sfc
Definition: ERF.H:1103
amrex::Vector< int > num_files_at_level
Definition: ERF.H:722
bool metgrid_debug_quiescent
Definition: ERF.H:1095
bool metgrid_interp_theta
Definition: ERF.H:1100
static amrex::Vector< amrex::Vector< std::string > > nc_init_file
Definition: ERF.H:1080
bool regrid_level_0_on_restart
Definition: ERF.H:955
int metgrid_force_sfc_k
Definition: ERF.H:1107
bool metgrid_use_below_sfc
Definition: ERF.H:1102
std::string subvol_file
Definition: ERF.H:960
amrex::Real metgrid_proximity
Definition: ERF.H:1105
std::string plot_file_1
Definition: ERF.H:958
bool metgrid_debug_dry
Definition: ERF.H:1097
bool metgrid_debug_isothermal
Definition: ERF.H:1096
bool metgrid_debug_psfc
Definition: ERF.H:1098
static std::string nc_low_file
Definition: ERF.H:1088
void ParameterSanityChecks()
Definition: ERF.cpp:1857
bool m_expand_plotvars_to_unif_rr
Definition: ERF.H:961
amrex::Vector< int > num_boxes_at_level
Definition: ERF.H:721
std::string check_file
Definition: ERF.H:977
int metgrid_order
Definition: ERF.H:1106
bool plot_lsm
Definition: ERF.H:970
void SetModel()
Definition: ERF_LandSurface.H:28
Definition: ERF_MM5.H:26
Definition: ERF_NOAH.H:30
Definition: ERF_NullSurf.H:8
Definition: ERF_SLM.H:26
void display(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:495
void init_params(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:88
Here is the call graph for this function:

◆ refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )
private

Function to define the refinement criteria based on user input

166 {
167  if (max_level > 0)
168  {
169  ParmParse pp(pp_prefix);
170  Vector<std::string> refinement_indicators;
171  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
172 
173  for (int i=0; i<refinement_indicators.size(); ++i)
174  {
175  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
176 
177  ParmParse ppr(ref_prefix);
178  RealBox realbox;
179  int lev_for_box;
180 
181  int num_real_lo = ppr.countval("in_box_lo");
182  int num_indx_lo = ppr.countval("in_box_lo_indices");
183  int num_real_hi = ppr.countval("in_box_hi");
184  int num_indx_hi = ppr.countval("in_box_hi_indices");
185 
186  AMREX_ALWAYS_ASSERT(num_real_lo == num_real_hi);
187  AMREX_ALWAYS_ASSERT(num_indx_lo == num_indx_hi);
188 
189  if ( !((num_real_lo >= AMREX_SPACEDIM-1 && num_indx_lo == 0) ||
190  (num_indx_lo >= AMREX_SPACEDIM-1 && num_real_lo == 0) ||
191  (num_indx_lo == 0 && num_real_lo == 0)) )
192  {
193  amrex::Abort("Must only specify box for refinement using real OR index space");
194  }
195 
196  if (num_real_lo > 0) {
197  std::vector<Real> rbox_lo(3), rbox_hi(3);
198  ppr.get("max_level",lev_for_box);
199  if (lev_for_box <= max_level)
200  {
201  if (n_error_buf[0] != IntVect::TheZeroVector()) {
202  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
203  }
204 
205  const Real* plo = geom[lev_for_box].ProbLo();
206  const Real* phi = geom[lev_for_box].ProbHi();
207 
208  ppr.getarr("in_box_lo",rbox_lo,0,num_real_lo);
209  ppr.getarr("in_box_hi",rbox_hi,0,num_real_hi);
210 
211  if (rbox_lo[0] < plo[0]) rbox_lo[0] = plo[0];
212  if (rbox_lo[1] < plo[1]) rbox_lo[1] = plo[1];
213  if (rbox_hi[0] > phi[0]) rbox_hi[0] = phi[0];
214  if (rbox_hi[1] > phi[1]) rbox_hi[1] = phi[1];
215  if (num_real_lo < AMREX_SPACEDIM) {
216  rbox_lo[2] = plo[2];
217  rbox_hi[2] = phi[2];
218  }
219 
220  realbox = RealBox(&(rbox_lo[0]),&(rbox_hi[0]));
221 
222  Print() << "Realbox read in and intersected laterally with domain is " << realbox << std::endl;
223 
224  num_boxes_at_level[lev_for_box] += 1;
225 
226  int ilo, jlo, klo;
227  int ihi, jhi, khi;
228  const auto* dx = geom[lev_for_box].CellSize();
229  ilo = static_cast<int>((rbox_lo[0] - plo[0])/dx[0]);
230  jlo = static_cast<int>((rbox_lo[1] - plo[1])/dx[1]);
231  ihi = static_cast<int>((rbox_hi[0] - plo[0])/dx[0]-1);
232  jhi = static_cast<int>((rbox_hi[1] - plo[1])/dx[1]-1);
233  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
234  // Search for k indices corresponding to nominal grid
235  // AGL heights
236  const Box& domain = geom[lev_for_box].Domain();
237  klo = domain.smallEnd(2) - 1;
238  khi = domain.smallEnd(2) - 1;
239 
240  if (rbox_lo[2] <= zlevels_stag[lev_for_box][domain.smallEnd(2)])
241  {
242  klo = domain.smallEnd(2);
243  }
244  else
245  {
246  for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
247  if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
248  klo = k-1;
249  break;
250  }
251  }
252  }
253  AMREX_ASSERT(klo >= domain.smallEnd(2));
254 
255  if (rbox_hi[2] >= zlevels_stag[lev_for_box][domain.bigEnd(2)+1])
256  {
257  khi = domain.bigEnd(2);
258  }
259  else
260  {
261  for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
262  if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
263  khi = k-1;
264  break;
265  }
266  }
267  }
268  AMREX_ASSERT((khi <= domain.bigEnd(2)) && (khi > klo));
269 
270  // Need to update realbox because tagging is based on
271  // the initial _un_deformed grid
272  realbox = RealBox(plo[0]+ ilo *dx[0], plo[1]+ jlo *dx[1], plo[2]+ klo *dx[2],
273  plo[0]+(ihi+1)*dx[0], plo[1]+(jhi+1)*dx[1], plo[2]+(khi+1)*dx[2]);
274  } else {
275  klo = static_cast<int>((rbox_lo[2] - plo[2])/dx[2]);
276  khi = static_cast<int>((rbox_hi[2] - plo[2])/dx[2]-1);
277  }
278 
279  Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
280  if ( (ilo%ref_ratio[lev_for_box-1][0] != 0) || ((ihi+1)%ref_ratio[lev_for_box-1][0] != 0) ||
281  (jlo%ref_ratio[lev_for_box-1][1] != 0) || ((jhi+1)%ref_ratio[lev_for_box-1][1] != 0) ||
282  (klo%ref_ratio[lev_for_box-1][2] != 0) || ((khi+1)%ref_ratio[lev_for_box-1][2] != 0) )
283  {
284  amrex::Print() << "Box : " << bx << std::endl;
285  amrex::Print() << "RealBox : " << realbox << std::endl;
286  amrex::Print() << "ilo, ihi+1, jlo, jhi+1, klo, khi+1 by ref_ratio : "
287  << ilo%ref_ratio[lev_for_box-1][0] << " " << (ihi+1)%ref_ratio[lev_for_box-1][0] << " "
288  << jlo%ref_ratio[lev_for_box-1][1] << " " << (jhi+1)%ref_ratio[lev_for_box-1][1] << " "
289  << klo%ref_ratio[lev_for_box-1][2] << " " << (khi+1)%ref_ratio[lev_for_box-1][2] << std::endl;
290  amrex::Error("Fine box is not legit with this ref_ratio");
291  }
292  boxes_at_level[lev_for_box].push_back(bx);
293  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
294  } // lev
295  if (solverChoice.init_type == InitType::WRFInput) {
296  if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
297  amrex::Error("Number of boxes doesn't match number of input files");
298 
299  }
300  }
301 
302  } else if (num_indx_lo > 0) {
303 
304  std::vector<int> box_lo(3), box_hi(3);
305  ppr.get("max_level",lev_for_box);
306  if (lev_for_box <= max_level)
307  {
308  if (n_error_buf[0] != IntVect::TheZeroVector()) {
309  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
310  }
311 
312  ppr.getarr("in_box_lo_indices",box_lo,0,AMREX_SPACEDIM);
313  ppr.getarr("in_box_hi_indices",box_hi,0,AMREX_SPACEDIM);
314 
315  Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
316  amrex::Print() << "BOX " << bx << std::endl;
317 
318  const auto* dx = geom[lev_for_box].CellSize();
319  const Real* plo = geom[lev_for_box].ProbLo();
320  realbox = RealBox(plo[0]+ box_lo[0] *dx[0], plo[1]+ box_lo[1] *dx[1], plo[2]+ box_lo[2] *dx[2],
321  plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
322 
323  Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
324  num_boxes_at_level[lev_for_box] += 1;
325 
326  if ( (box_lo[0]%ref_ratio[lev_for_box-1][0] != 0) || ((box_hi[0]+1)%ref_ratio[lev_for_box-1][0] != 0) ||
327  (box_lo[1]%ref_ratio[lev_for_box-1][1] != 0) || ((box_hi[1]+1)%ref_ratio[lev_for_box-1][1] != 0) ||
328  (box_lo[2]%ref_ratio[lev_for_box-1][2] != 0) || ((box_hi[2]+1)%ref_ratio[lev_for_box-1][2] != 0) )
329  amrex::Error("Fine box is not legit with this ref_ratio");
330  boxes_at_level[lev_for_box].push_back(bx);
331  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
332  } // lev
333  if (solverChoice.init_type == InitType::WRFInput) {
334  if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
335  amrex::Error("Number of boxes doesn't match number of input files");
336 
337  }
338  }
339  }
340 
341  AMRErrorTagInfo info;
342 
343  if (realbox.ok()) {
344  info.SetRealBox(realbox);
345  }
346  if (ppr.countval("start_time") > 0) {
347  Real ref_min_time; ppr.get("start_time",ref_min_time);
348  info.SetMinTime(ref_min_time);
349  }
350  if (ppr.countval("end_time") > 0) {
351  Real ref_max_time; ppr.get("end_time",ref_max_time);
352  info.SetMaxTime(ref_max_time);
353  }
354  if (ppr.countval("max_level") > 0) {
355  int ref_max_level; ppr.get("max_level",ref_max_level);
356  info.SetMaxLevel(ref_max_level);
357  }
358 
359  if (ppr.countval("value_greater")) {
360  int num_val = ppr.countval("value_greater");
361  Vector<Real> value(num_val);
362  ppr.getarr("value_greater",value,0,num_val);
363  std::string field; ppr.get("field_name",field);
364  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
365  }
366  else if (ppr.countval("value_less")) {
367  int num_val = ppr.countval("value_less");
368  Vector<Real> value(num_val);
369  ppr.getarr("value_less",value,0,num_val);
370  std::string field; ppr.get("field_name",field);
371  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
372  }
373  else if (ppr.countval("adjacent_difference_greater")) {
374  int num_val = ppr.countval("adjacent_difference_greater");
375  Vector<Real> value(num_val);
376  ppr.getarr("adjacent_difference_greater",value,0,num_val);
377  std::string field; ppr.get("field_name",field);
378  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
379  }
380  else if (realbox.ok())
381  {
382  ref_tags.push_back(AMRErrorTag(info));
383  } else {
384  Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
385  }
386  } // loop over criteria
387  } // if max_level > 0
388 }
Here is the call graph for this function:

◆ remake_zphys()

void ERF::remake_zphys ( int  lev,
amrex::Real  time,
std::unique_ptr< amrex::MultiFab > &  temp_zphys_nd 
)
563 {
564  if (lev > 0)
565  {
566  //
567  // First interpolate from coarser level
568  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
569  // have been pre-filled - this includes ghost cells both inside and outside
570  // the domain
571  //
572  InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
573  IntVect(0,0,0), // do not fill ghost cells outside the domain
574  *z_phys_nd[lev-1], 0, 0, 1,
575  geom[lev-1], geom[lev],
576  refRatio(lev-1), &node_bilinear_interp,
578 
579  // This recomputes the fine values using the bottom terrain at the fine resolution,
580  // and also fills values of z_phys_nd outside the domain
581  make_terrain_fitted_coords(lev,geom[lev],*temp_zphys_nd,zlevels_stag[lev],phys_bc_type);
582 
583  std::swap(temp_zphys_nd, z_phys_nd[lev]);
584 
585  } // lev > 0
586 
587  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
588  //
589  // This assumes we have already remade the EBGeometry
590  //
591  terrain_blanking[lev]->setVal(1.0);
592  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, z_phys_nd[lev]->nGrowVect());
593  }
594 }
Here is the call graph for this function:

◆ RemakeLevel()

void ERF::RemakeLevel ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
366 {
367  if (verbose) {
368  amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
369  }
370 
371  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::MovingFittedMesh);
372 
373  BoxArray ba_old(vars_new[lev][Vars::cons].boxArray());
374  DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
375 
376  if (verbose) {
377  amrex::Print() <<" OLD BA AT LEVEL " << lev << " " << ba_old << std::endl;
378  }
379 
380  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
381  IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
382 
383  int ngrow_vels = ComputeGhostCells(solverChoice);
384 
385  Vector<MultiFab> temp_lev_new(Vars::NumTypes);
386  Vector<MultiFab> temp_lev_old(Vars::NumTypes);
387  MultiFab temp_base_state;
388 
389  std::unique_ptr<MultiFab> temp_zphys_nd;
390 
391  //********************************************************************************************
392  // This allocates all kinds of things, including but not limited to: solution arrays,
393  // terrain arrays and metrics, and base state.
394  // *******************************************************************************************
395  init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old, temp_base_state, temp_zphys_nd);
396 
397  // ********************************************************************************************
398  // Build the data structures for terrain-related quantities
399  // ********************************************************************************************
400  remake_zphys(lev, time, temp_zphys_nd);
402 
403  // ********************************************************************************************
404  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
405  // ********************************************************************************************
406  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
407  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
408  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
409  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
410  }
411  }
412 
413  // ********************************************************************************************
414  // Build the data structures for canopy model (depends upon z_phys)
415  // ********************************************************************************************
417  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
418  }
419 
420  // *****************************************************************************************************
421  // Create the physbcs objects (after initializing the terrain but before calling FillCoarsePatch
422  // *****************************************************************************************************
423  make_physbcs(lev);
424 
425  // ********************************************************************************************
426  // Update the base state at this level by interpolation from coarser level AND copy
427  // from previous (pre-regrid) base_state array
428  // ********************************************************************************************
429  if (lev > 0) {
430  Interpolater* mapper = &cell_cons_interp;
431 
432  Vector<MultiFab*> fmf = {&base_state[lev ], &base_state[lev ]};
433  Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
434  Vector<Real> ftime = {time, time};
435  Vector<Real> ctime = {time, time};
436 
437  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
438  FillPatchTwoLevels(temp_base_state, temp_base_state.nGrowVect(), IntVect(0,0,0),
439  time, cmf, ctime, fmf, ftime,
440  0, 0, temp_base_state.nComp(), geom[lev-1], geom[lev],
441  refRatio(lev-1), mapper, domain_bcs_type,
443 
444  // Impose bc's outside the domain
445  (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
446 
447  // *************************************************************************************************
448  // This will fill the temporary MultiFabs with data from vars_new
449  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
450  // NOTE: we must create the new base state before calling FillPatch because we will
451  // interpolate perturbational quantities
452  // *************************************************************************************************
453  FillPatch(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
454  &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
455  {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
456  base_state[lev], temp_base_state, false);
457  } else {
458  temp_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
459  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
460  temp_lev_new[Vars::cons].ParallelCopy(vars_new[lev][Vars::cons],0,0,ncomp_cons,ngrow_state,ngrow_state);
461  temp_lev_new[Vars::xvel].ParallelCopy(vars_new[lev][Vars::xvel],0,0, 1,ngrow_vels,ngrow_vels);
462  temp_lev_new[Vars::yvel].ParallelCopy(vars_new[lev][Vars::yvel],0,0, 1,ngrow_vels,ngrow_vels);
463 
464  temp_lev_new[Vars::zvel].setVal(0.);
465  temp_lev_new[Vars::zvel].ParallelCopy(vars_new[lev][Vars::zvel],0,0, 1,
466  IntVect(ngrow_vels,ngrow_vels,0),IntVect(ngrow_vels,ngrow_vels,0));
467  }
468 
469  // Now swap the pointers since we needed both old and new in the FillPatch
470  std::swap(temp_base_state, base_state[lev]);
471 
472  // ********************************************************************************************
473  // Copy from new into old just in case
474  // ********************************************************************************************
475  MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
476  MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0, 1,ngrow_vels);
477  MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0, 1,ngrow_vels);
478  MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0, 1,IntVect(ngrow_vels,ngrow_vels,0));
479 
480  // ********************************************************************************************
481  // Now swap the pointers
482  // ********************************************************************************************
483  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
484  std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
485  std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
486  }
487 
488  t_new[lev] = time;
489  t_old[lev] = time - 1.e200;
490 
491  // ********************************************************************************************
492  // Build the data structures for calculating diffusive/turbulent terms
493  // ********************************************************************************************
494  update_diffusive_arrays(lev, ba, dm);
495 
496  //********************************************************************************************
497  // Microphysics
498  // *******************************************************************************************
499  int q_size = micro->Get_Qmoist_Size(lev);
500  qmoist[lev].resize(q_size);
501  micro->Define(lev, solverChoice);
502  if (solverChoice.moisture_type != MoistureType::None)
503  {
504  micro->Init(lev, vars_new[lev][Vars::cons],
505  grids[lev], Geom(lev), 0.0,
506  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
507  }
508  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
509  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
510  }
511 
512  // ********************************************************************************************
513  // Initialize the integrator class
514  // ********************************************************************************************
516 
517  // We need to re-define the FillPatcher if the grids have changed
518  if (lev > 0 && cf_width >= 0) {
519  bool ba_changed = (ba != ba_old);
520  bool dm_changed = (dm != dm_old);
521  if (ba_changed || dm_changed) {
523  }
524  }
525 
526  // ********************************************************************************************
527  // Update the SurfaceLayer arrays at this level
528  // ********************************************************************************************
529  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
530  int nlevs = finest_level+1;
531  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
532  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
533  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
534  mfv_old, Theta_prim[lev], Qv_prim[lev],
535  Qr_prim[lev], z_phys_nd[lev],
536  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
537  lsm_data[lev], lsm_flux[lev], sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
538  }
539 
540  // These calls are done in AmrCore::regrid if this is a regrid at lev > 0
541  // For a level 0 regrid we must explicitly do them here
542  if (lev == 0) {
543  // Define grids[lev] to be ba
544  SetBoxArray(lev, ba);
545 
546  // Define dmap[lev] to be dm
547  SetDistributionMap(lev, dm);
548  }
549 
550 #ifdef ERF_USE_PARTICLES
551  particleData.Redistribute();
552 #endif
553 }
void remake_zphys(int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:562

◆ restart()

void ERF::restart ( )
1395 {
1397 
1398  // We set this here so that we don't over-write the checkpoint file we just started from
1400 
1402  //
1403  // Coarsening before we split the grids ensures that each resulting
1404  // grid will have an even number of cells in each direction.
1405  //
1406  BoxArray new_ba(amrex::coarsen(Geom(0).Domain(),2));
1407  //
1408  // Now split up into list of grids within max_grid_size[0] limit.
1409  //
1410  new_ba.maxSize(max_grid_size[0]/2);
1411  //
1412  // Now refine these boxes back to level 0.
1413  //
1414  new_ba.refine(2);
1415 
1416  if (refine_grid_layout) {
1417  ChopGrids(0, new_ba, ParallelDescriptor::NProcs());
1418  }
1419 
1420  if (new_ba != grids[0]) {
1421  DistributionMapping new_dm(new_ba);
1422  RemakeLevel(0,t_new[0],new_ba,new_dm);
1423  }
1424  }
1425 }
void RemakeLevel(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:365
void ReadCheckpointFile()
Definition: ERF_Checkpoint.cpp:398

◆ sample_lines()

void ERF::sample_lines ( int  lev,
amrex::Real  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters
levCurrent level
timeCurrent time
cellIntVect containing the x,y-dimension indices to sample along z
mfMultiFab from which we sample the data
508 {
509  int ifile = 0;
510 
511  const int ncomp = mf.nComp(); // cell-centered state vars
512 
513  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
514  average_face_to_cellcenter(mf_vels, 0,
515  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
516 
517  //
518  // Sample the data at a line (in direction "dir") in space
519  // In this case we sample in the vertical direction so dir = 2
520  // The "k" value of "cell" is ignored
521  //
522  int dir = 2;
523  MultiFab my_line = get_line_data(mf, dir, cell);
524  MultiFab my_line_vels = get_line_data(mf_vels, dir, cell);
525  MultiFab my_line_tau11 = get_line_data(*Tau[lev][TauType::tau11], dir, cell);
526  MultiFab my_line_tau12 = get_line_data(*Tau[lev][TauType::tau12], dir, cell);
527  MultiFab my_line_tau13 = get_line_data(*Tau[lev][TauType::tau13], dir, cell);
528  MultiFab my_line_tau22 = get_line_data(*Tau[lev][TauType::tau22], dir, cell);
529  MultiFab my_line_tau23 = get_line_data(*Tau[lev][TauType::tau23], dir, cell);
530  MultiFab my_line_tau33 = get_line_data(*Tau[lev][TauType::tau33], dir, cell);
531 
532  for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
533  {
534  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
535 
536  std::ostream& sample_log = SampleLineLog(ifile);
537  if (sample_log.good()) {
538  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
539  const auto& my_line_arr = my_line[0].const_array();
540  const auto& my_line_vels_arr = my_line_vels[0].const_array();
541  const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
542  const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
543  const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
544  const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
545  const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
546  const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
547  const Box& my_box = my_line[0].box();
548  const int klo = my_box.smallEnd(2);
549  const int khi = my_box.bigEnd(2);
550  int i = cell[0];
551  int j = cell[1];
552  for (int n = 0; n < ncomp; n++) {
553  for (int k = klo; k <= khi; k++) {
554  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
555  }
556  }
557  for (int n = 0; n < AMREX_SPACEDIM; n++) {
558  for (int k = klo; k <= khi; k++) {
559  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
560  }
561  }
562  for (int k = klo; k <= khi; k++) {
563  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
564  }
565  for (int k = klo; k <= khi; k++) {
566  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
567  }
568  for (int k = klo; k <= khi; k++) {
569  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
570  }
571  for (int k = klo; k <= khi; k++) {
572  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
573  }
574  for (int k = klo; k <= khi; k++) {
575  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
576  }
577  for (int k = klo; k <= khi; k++) {
578  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
579  }
580  sample_log << std::endl;
581  } // if good
582  } // mfi
583 }
const int datwidth
Definition: ERF.H:917
AMREX_FORCE_INLINE std::ostream & SampleLineLog(int i)
Definition: ERF.H:1314
const int datprecision
Definition: ERF.H:918

◆ sample_points()

void ERF::sample_points ( int  lev,
amrex::Real  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling MultiFab data at a specified cell index.

Parameters
levLevel for the associated MultiFab data
timeCurrent time
cellIntVect containing the indexes for the cell where we want to sample
mfMultiFab from which we wish to sample data
472 {
473  int ifile = 0;
474 
475  //
476  // Sample the data at a single point in space
477  //
478  int ncomp = mf.nComp();
479  Vector<Real> my_point = get_cell_data(mf, cell);
480 
481  if (!my_point.empty()) {
482 
483  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
484 
485  std::ostream& sample_log = SamplePointLog(ifile);
486  if (sample_log.good()) {
487  sample_log << std::setw(datwidth) << time;
488  for (int i = 0; i < ncomp; ++i)
489  {
490  sample_log << std::setw(datwidth) << my_point[i];
491  }
492  sample_log << std::endl;
493  } // if good
494  } // only write from processor that holds the cell
495 }
AMREX_FORCE_INLINE std::ostream & SamplePointLog(int i)
Definition: ERF.H:1300

◆ SampleLine()

amrex::IntVect& ERF::SampleLine ( int  i)
inlineprivate
1341  {
1342  return sampleline[i];
1343  }

◆ SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int  i)
inlineprivate
1315  {
1316  return *samplelinelog[i];
1317  }

◆ SampleLineLogName()

std::string ERF::SampleLineLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith samplelinelog file.

1471 { return samplelinelogname[i]; }

◆ SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int  i)
inlineprivate
1328  {
1329  return samplepoint[i];
1330  }

◆ SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int  i)
inlineprivate
1301  {
1302  return *sampleptlog[i];
1303  }

◆ SamplePointLogName()

std::string ERF::SamplePointLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith sampleptlog file.

1468 { return sampleptlogname[i]; }

◆ setPlotVariables()

void ERF::setPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
14 {
15  ParmParse pp(pp_prefix);
16 
17  if (pp.contains(pp_plot_var_names.c_str()))
18  {
19  std::string nm;
20 
21  int nPltVars = pp.countval(pp_plot_var_names.c_str());
22 
23  for (int i = 0; i < nPltVars; i++)
24  {
25  pp.get(pp_plot_var_names.c_str(), nm, i);
26 
27  // Add the named variable to our list of plot variables
28  // if it is not already in the list
29  if (!containerHasElement(plot_var_names, nm)) {
30  plot_var_names.push_back(nm);
31  }
32  }
33  } else {
34  //
35  // The default is to add none of the variables to the list
36  //
37  plot_var_names.clear();
38  }
39 
40  // Get state variables in the same order as we define them,
41  // since they may be in any order in the input list
42  Vector<std::string> tmp_plot_names;
43 
44  for (int i = 0; i < cons_names.size(); ++i) {
45  if ( containerHasElement(plot_var_names, cons_names[i]) ) {
46  if (solverChoice.moisture_type == MoistureType::None) {
47  if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
48  cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
49  {
50  tmp_plot_names.push_back(cons_names[i]);
51  }
52  } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
53  if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
54  {
55  tmp_plot_names.push_back(cons_names[i]);
56  }
57  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
58  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
59  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
60  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
61  cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
62  {
63  tmp_plot_names.push_back(cons_names[i]);
64  }
65  } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
66  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow rhoQ1, rhoQ2, rhoQ4
67  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
68  {
69  tmp_plot_names.push_back(cons_names[i]);
70  }
71  } else
72  {
73  // For moisture_type SAM and Morrison we have all six variables
74  tmp_plot_names.push_back(cons_names[i]);
75  }
76  }
77  }
78 
79  // check for velocity since it's not in cons_names
80  // if we are asked for any velocity component, we will need them all
81  if (containerHasElement(plot_var_names, "x_velocity") ||
82  containerHasElement(plot_var_names, "y_velocity") ||
83  containerHasElement(plot_var_names, "z_velocity")) {
84  tmp_plot_names.push_back("x_velocity");
85  tmp_plot_names.push_back("y_velocity");
86  tmp_plot_names.push_back("z_velocity");
87  }
88 
89  //
90  // If the model we are running doesn't have the variable listed in the inputs file,
91  // just ignore it rather than aborting
92  //
93  for (int i = 0; i < derived_names.size(); ++i) {
94  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
95  bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing) ||
96  (derived_names[i] != "terrain_IB_mask") );
97  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
98  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
99  (derived_names[i] != "detJ") );
100  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
101  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
102  (derived_names[i] != "z_phys") );
103 #ifndef ERF_USE_WINDFARM
104  ok_to_add &= (derived_names[i] != "SMark0" && derived_names[i] != "SMark1");
105 #endif
106  if (ok_to_add)
107  {
108  if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
109  if (derived_names[i] != "qv" && derived_names[i] != "qc" && derived_names[i] != "qrain" &&
110  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
111  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
112  {
113  tmp_plot_names.push_back(derived_names[i]);
114  }
115  } else if ( (solverChoice.moisture_type == MoistureType::Kessler ) ||
116  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
117  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow qv, qc, qrain
118  if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
119  derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
120  {
121  tmp_plot_names.push_back(derived_names[i]);
122  }
123  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
124  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
125  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
126  if (derived_names[i] != "qrain" &&
127  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
128  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
129  {
130  tmp_plot_names.push_back(derived_names[i]);
131  }
132  } else
133  {
134  // For moisture_type SAM and Morrison we have all moist quantities
135  tmp_plot_names.push_back(derived_names[i]);
136  }
137  } // use_terrain?
138  } // hasElement
139  }
140 
141 #ifdef ERF_USE_WINDFARM
142  for (int i = 0; i < derived_names.size(); ++i) {
143  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
144  if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP) {
145  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0") {
146  tmp_plot_names.push_back(derived_names[i]);
147  }
148  }
149  if( solverChoice.windfarm_type == WindFarmType::SimpleAD or
150  solverChoice.windfarm_type == WindFarmType::GeneralAD ) {
151  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0" or derived_names[i] == "SMark1") {
152  tmp_plot_names.push_back(derived_names[i]);
153  }
154  }
155  }
156  }
157 #endif
158 
159 #ifdef ERF_USE_PARTICLES
160  const auto& particles_namelist( particleData.getNamesUnalloc() );
161  for (auto it = particles_namelist.cbegin(); it != particles_namelist.cend(); ++it) {
162  std::string tmp( (*it)+"_count" );
163  if (containerHasElement(plot_var_names, tmp) ) {
164  tmp_plot_names.push_back(tmp);
165  }
166  }
167 #endif
168 
169  plot_var_names = tmp_plot_names;
170 }
const amrex::Vector< std::string > derived_names
Definition: ERF.H:988
const amrex::Vector< std::string > cons_names
Definition: ERF.H:983
Here is the call graph for this function:

◆ setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool  restarting)
private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
56 {
57  // If we are restarting then we haven't read the input_sounding file yet
58  // so we need to read it here
59  // TODO: should we store this information in the checkpoint file instead?
60  if (restarting) {
62  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
64  }
65  }
66 
67  const Real* z_inp_sound = input_sounding_data.z_inp_sound[0].dataPtr();
68  const Real* U_inp_sound = input_sounding_data.U_inp_sound[0].dataPtr();
69  const Real* V_inp_sound = input_sounding_data.V_inp_sound[0].dataPtr();
70  const Real* theta_inp_sound = input_sounding_data.theta_inp_sound[0].dataPtr();
71  const int inp_sound_size = input_sounding_data.size(0);
72 
73  int refine_fac{1};
74  for (int lev = 0; lev <= finest_level; lev++)
75  {
76  const int klo = geom[lev].Domain().smallEnd(2);
77  const int khi = geom[lev].Domain().bigEnd(2);
78  const int Nz = khi - klo + 1;
79 
80  Vector<Real> zcc(Nz);
81  Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
82  zlevels_stag[0].begin()+khi/refine_fac+2);
83  expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
84 #if 0
85  amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
86  for (int k = 0; k < zlevels_sub.size(); k++) { amrex::AllPrint() << zlevels_sub[k] << " "; }
87  amrex::AllPrint() << " --> ";
88  for (int k = 0; k < Nz; k++) { amrex::AllPrint() << zcc[k] << " "; }
89  amrex::AllPrint() << std::endl;
90 #endif
91 
92  for (int k = 0; k < Nz; k++)
93  {
94  h_rayleigh_ptrs[lev][Rayleigh::ubar][k] = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
95  h_rayleigh_ptrs[lev][Rayleigh::vbar][k] = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
96  h_rayleigh_ptrs[lev][Rayleigh::wbar][k] = Real(0.0);
97  h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
98  }
99 
100  // Copy from host version to device version
101  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
102  d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
103  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
104  d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
105  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
106  d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
107  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
108  d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
109 
110  refine_fac *= ref_ratio[lev][2];
111  }
112 }
AMREX_FORCE_INLINE void expand_and_interpolate_1d(amrex::Vector< amrex::Real > &znew, const amrex::Vector< amrex::Real > &zorig, int refine_fac, bool destag=false)
Definition: ERF_Interpolation_1D.H:85
amrex::Vector< amrex::Vector< amrex::Real > > theta_inp_sound
Definition: ERF_InputSoundingData.H:323
amrex::Vector< amrex::Vector< amrex::Real > > z_inp_sound
Definition: ERF_InputSoundingData.H:323
amrex::Vector< amrex::Vector< amrex::Real > > U_inp_sound
Definition: ERF_InputSoundingData.H:323
amrex::Vector< amrex::Vector< amrex::Real > > V_inp_sound
Definition: ERF_InputSoundingData.H:323
int size(int itime) const
Definition: ERF_InputSoundingData.H:300
Here is the call graph for this function:

◆ setRecordDataInfo()

void ERF::setRecordDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1371  {
1372  if (amrex::ParallelDescriptor::IOProcessor())
1373  {
1374  datalog[i] = std::make_unique<std::fstream>();
1375  datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1376  if (!datalog[i]->good()) {
1377  amrex::FileOpenFailed(filename);
1378  }
1379  }
1380  amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
1381  }

◆ setRecordDerDataInfo()

void ERF::setRecordDerDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1384  {
1385  if (amrex::ParallelDescriptor::IOProcessor())
1386  {
1387  der_datalog[i] = std::make_unique<std::fstream>();
1388  der_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1389  if (!der_datalog[i]->good()) {
1390  amrex::FileOpenFailed(filename);
1391  }
1392  }
1393  amrex::ParallelDescriptor::Barrier("ERF::setRecordDerDataInfo");
1394  }

◆ setRecordEnergyDataInfo()

void ERF::setRecordEnergyDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1397  {
1398  if (amrex::ParallelDescriptor::IOProcessor())
1399  {
1400  tot_e_datalog[i] = std::make_unique<std::fstream>();
1401  tot_e_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1402  if (!tot_e_datalog[i]->good()) {
1403  amrex::FileOpenFailed(filename);
1404  }
1405  }
1406  amrex::ParallelDescriptor::Barrier("ERF::setRecordEnergyDataInfo");
1407  }

◆ setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1427  {
1428  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1429  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1430  {
1431  const amrex::Box& bx = mfi.validbox();
1432  if (bx.contains(cell)) {
1433  samplelinelog[i] = std::make_unique<std::fstream>();
1434  samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1435  if (!samplelinelog[i]->good()) {
1436  amrex::FileOpenFailed(filename);
1437  }
1438  }
1439  }
1440  amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
1441  }

◆ setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1410  {
1411  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1412  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1413  {
1414  const amrex::Box& bx = mfi.validbox();
1415  if (bx.contains(cell)) {
1416  sampleptlog[i] = std::make_unique<std::fstream>();
1417  sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1418  if (!sampleptlog[i]->good()) {
1419  amrex::FileOpenFailed(filename);
1420  }
1421  }
1422  }
1423  amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
1424  }

◆ setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool  restarting)
private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
66 {
67  // If we are restarting then we haven't read the input_sponge file yet
68  // so we need to read it here
69  // TODO: should we store this information in the checkpoint file instead?
70  if (restarting) {
72  }
73 
74  const Real* z_inp_sponge = input_sponge_data.z_inp_sponge.dataPtr();
75  const Real* U_inp_sponge = input_sponge_data.U_inp_sponge.dataPtr();
76  const Real* V_inp_sponge = input_sponge_data.V_inp_sponge.dataPtr();
77  const int inp_sponge_size = input_sponge_data.size();
78 
79  for (int lev = 0; lev <= finest_level; lev++)
80  {
81  const int khi = geom[lev].Domain().bigEnd()[2];
82  Vector<Real> zcc(khi+1);
83 
84  if (z_phys_cc[lev]) {
85  // use_terrain=1
86  // calculate the damping strength based on the max height at each k
88  } else {
89  const auto *const prob_lo = geom[lev].ProbLo();
90  const auto *const dx = geom[lev].CellSize();
91  for (int k = 0; k <= khi; k++)
92  {
93  zcc[k] = prob_lo[2] + (k+0.5) * dx[2];
94  }
95  }
96 
97  for (int k = 0; k <= khi; k++)
98  {
99  h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
100  h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
101  }
102 
103  // Copy from host version to device version
104  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
105  d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
106  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
107  d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
108  }
109 }
AMREX_FORCE_INLINE void reduce_to_max_per_height(amrex::Vector< amrex::Real > &v, std::unique_ptr< amrex::MultiFab > &mf)
Definition: ERF_ParFunctions.H:8
amrex::Vector< amrex::Real > V_inp_sponge
Definition: ERF_InputSpongeData.H:114
amrex::Vector< amrex::Real > z_inp_sponge
Definition: ERF_InputSpongeData.H:114
amrex::Vector< amrex::Real > U_inp_sponge
Definition: ERF_InputSpongeData.H:114
int size() const
Definition: ERF_InputSpongeData.H:102
Here is the call graph for this function:

◆ solve_with_EB_mlmg()

void ERF::solve_with_EB_mlmg ( int  lev,
amrex::Vector< amrex::MultiFab > &  rhs,
amrex::Vector< amrex::MultiFab > &  p,
amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &  fluxes 
)

Solve the Poisson equation using EB_enabled MLMG Note that the level may or may not be level 0.

20 {
21  BL_PROFILE("ERF::solve_with_EB_mlmg()");
22 
23  auto const dom_lo = lbound(geom[lev].Domain());
24  auto const dom_hi = ubound(geom[lev].Domain());
25 
26  LPInfo info;
27  // Allow a hidden direction if the domain is one cell wide in any lateral direction
28  if (dom_lo.x == dom_hi.x) {
29  info.setHiddenDirection(0);
30  } else if (dom_lo.y == dom_hi.y) {
31  info.setHiddenDirection(1);
32  }
33 
34  // Make sure the solver only sees the levels over which we are solving
35  Vector<BoxArray> ba_tmp; ba_tmp.push_back(rhs[0].boxArray());
36  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(rhs[0].DistributionMap());
37  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
38 
39  auto bclo = get_projection_bc(Orientation::low);
40  auto bchi = get_projection_bc(Orientation::high);
41 
42  // amrex::Print() << "BCLO " << bclo[0] << " " << bclo[1] << " " << bclo[2] << std::endl;
43  // amrex::Print() << "BCHI " << bchi[0] << " " << bchi[1] << " " << bchi[2] << std::endl;
44 
45  Real reltol = solverChoice.poisson_reltol;
46  Real abstol = solverChoice.poisson_abstol;
47 
48  // ****************************************************************************
49  // Multigrid solve
50  // ****************************************************************************
51 
52  MLEBABecLap mleb (geom_tmp, ba_tmp, dm_tmp, info, {&EBFactory(lev)});
53 
54  mleb.setMaxOrder(2);
55  mleb.setDomainBC(bclo, bchi);
56  mleb.setLevelBC(0, nullptr);
57 
58  //
59  // This sets A = 0, B = 1 so that
60  // the operator A alpha - b del dot beta grad to b
61  // becomes - del dot beta grad
62  //
63  mleb.setScalars(0.0, 1.0);
64 
65  Array<MultiFab,AMREX_SPACEDIM> bcoef;
66  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
67  bcoef[idim].define(convert(ba_tmp[0],IntVect::TheDimensionVector(idim)),
68  dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
69  bcoef[idim].setVal(-1.0);
70  }
71  mleb.setBCoeffs(0, amrex::GetArrOfConstPtrs(bcoef));
72 
73  MLMG mlmg(mleb);
74 
75  int max_iter = 100;
76  mlmg.setMaxIter(max_iter);
77  mlmg.setVerbose(mg_verbose);
78  mlmg.setBottomVerbose(0);
79 
80  mlmg.solve(GetVecOfPtrs(phi), GetVecOfConstPtrs(rhs), reltol, abstol);
81 
82  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
83 
84  ImposeBCsOnPhi(lev,phi[0]);
85 
86  //
87  // This arises because we solve MINUS del dot beta grad phi = div (rho u)
88  //
89  fluxes[0][0].mult(-1.);
90  fluxes[0][1].mult(-1.);
91  fluxes[0][2].mult(-1.);
92 }
void ImposeBCsOnPhi(int lev, amrex::MultiFab &phi)
Definition: ERF_ImposeBCsOnPhi.cpp:12

◆ solve_with_gmres()

void ERF::solve_with_gmres ( int  lev,
amrex::Vector< amrex::MultiFab > &  rhs,
amrex::Vector< amrex::MultiFab > &  p,
amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &  fluxes 
)

Solve the Poisson equation using FFT-preconditioned GMRES

14 {
15 #ifdef ERF_USE_FFT
16  BL_PROFILE("ERF::solve_with_gmres()");
17 
18  Real reltol = solverChoice.poisson_reltol;
19  Real abstol = solverChoice.poisson_abstol;
20 
21  amrex::GMRES<MultiFab, TerrainPoisson> gmsolver;
22 
23  TerrainPoisson tp(geom[lev], rhs[0].boxArray(), rhs[0].DistributionMap(), stretched_dz_d[lev],
24  z_phys_nd[lev].get(), domain_bc_type);
25 
26  gmsolver.define(tp);
27 
28  gmsolver.setVerbose(mg_verbose);
29 
30  tp.usePrecond(true);
31 
32  gmsolver.solve(phi[0], rhs[0], reltol, abstol);
33 
34  tp.getFluxes(phi[0], fluxes[0]);
35 #else
36  amrex::ignore_unused(lev, rhs, phi, fluxes);
37 #endif
38 
39  // ****************************************************************************
40  // Impose bc's on pprime
41  // ****************************************************************************
42  ImposeBCsOnPhi(lev, phi[0]);
43 }

◆ solve_with_mlmg()

void ERF::solve_with_mlmg ( int  lev,
amrex::Vector< amrex::MultiFab > &  rhs,
amrex::Vector< amrex::MultiFab > &  p,
amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &  fluxes 
)

Solve the Poisson equation using MLMG Note that the level may or may not be level 0.

41 {
42  BL_PROFILE("ERF::solve_with_mlmg()");
43 
44  auto const dom_lo = lbound(geom[lev].Domain());
45  auto const dom_hi = ubound(geom[lev].Domain());
46 
47  LPInfo info;
48  // Allow a hidden direction if the domain is one cell wide in any lateral direction
49  if (dom_lo.x == dom_hi.x) {
50  info.setHiddenDirection(0);
51  } else if (dom_lo.y == dom_hi.y) {
52  info.setHiddenDirection(1);
53  }
54 
55  // Make sure the solver only sees the levels over which we are solving
56  Vector<BoxArray> ba_tmp; ba_tmp.push_back(rhs[0].boxArray());
57  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(rhs[0].DistributionMap());
58  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
59 
60  auto bclo = get_projection_bc(Orientation::low);
61  auto bchi = get_projection_bc(Orientation::high);
62 
63  // amrex::Print() << "BCLO " << bclo[0] << " " << bclo[1] << " " << bclo[2] << std::endl;
64  // amrex::Print() << "BCHI " << bchi[0] << " " << bchi[1] << " " << bchi[2] << std::endl;
65 
66  Real reltol = solverChoice.poisson_reltol;
67  Real abstol = solverChoice.poisson_abstol;
68 
69  // ****************************************************************************
70  // Multigrid solve
71  // ****************************************************************************
72 
73  MLPoisson mlpoisson(geom_tmp, ba_tmp, dm_tmp, info);
74  mlpoisson.setDomainBC(bclo, bchi);
75  if (lev > 0) {
76  mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
77  }
78  mlpoisson.setLevelBC(0, nullptr);
79 
80  MLMG mlmg(mlpoisson);
81  int max_iter = 100;
82  mlmg.setMaxIter(max_iter);
83 
84  mlmg.setVerbose(mg_verbose);
85  mlmg.setBottomVerbose(0);
86 
87  mlmg.solve(GetVecOfPtrs(phi),
88  GetVecOfConstPtrs(rhs),
89  reltol, abstol);
90  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
91 
92  // ****************************************************************************
93  // Impose bc's on pprime
94  // ****************************************************************************
95  ImposeBCsOnPhi(lev, phi[0]);
96 }

◆ sum_derived_quantities()

void ERF::sum_derived_quantities ( amrex::Real  time)
170 {
171  if (verbose <= 0 || NumDerDataLogs() <= 0) return;
172 
173  int lev = 0;
174 
175  AMREX_ALWAYS_ASSERT(lev == 0);
176 
177  // ************************************************************************
178  // WARNING: we are not filling ghost cells other than periodic outside the domain
179  // ************************************************************************
180 
181  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
182  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
183 
184  // Average all three components of velocity (on faces) to the cell center
185  average_face_to_cellcenter(mf_cc_vel,0,
186  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
187  &vars_new[lev][Vars::yvel],
188  &vars_new[lev][Vars::zvel]});
189  mf_cc_vel.FillBoundary(geom[lev].periodicity());
190 
191  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
192  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
193  }
194 
195  MultiFab r_wted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
196  MultiFab unwted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
197  MultiFab enstrophysq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
198 
199 #ifdef _OPENMP
200 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
201 #endif
202  for (MFIter mfi(unwted_magvelsq, TilingIfNotGPU()); mfi.isValid(); ++mfi)
203  {
204  const Box& bx = mfi.tilebox();
205  auto& src_fab = mf_cc_vel[mfi];
206 
207  auto& dest1_fab = unwted_magvelsq[mfi];
208  derived::erf_dermagvelsq(bx, dest1_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
209 
210  auto& dest2_fab = enstrophysq[mfi];
211  derived::erf_derenstrophysq(bx, dest2_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
212  }
213 
214  // Copy the MF holding 1/2(u^2 + v^2 + w^2) into the MF that will hold 1/2 rho (u^2 + v^2 + w^2)d
215  MultiFab::Copy(r_wted_magvelsq, unwted_magvelsq, 0, 0, 1, 0);
216 
217  // Multiply the MF holding 1/2(u^2 + v^2 + w^2) by rho to get 1/2 rho (u^2 + v^2 + w^2)
218  MultiFab::Multiply(r_wted_magvelsq, vars_new[lev][Vars::cons], 0, 0, 1, 0);
219 
220  Real unwted_avg = volWgtSumMF(lev, unwted_magvelsq, 0, *mapfac_m[lev],false);
221  Real r_wted_avg = volWgtSumMF(lev, r_wted_magvelsq, 0, *mapfac_m[lev],false);
222  Real enstrsq_avg = volWgtSumMF(lev, enstrophysq, 0, *mapfac_m[lev],false);
223 
224  // Get volume including terrain (consistent with volWgtSumMF routine)
225  Real vol = geom[lev].ProbDomain().volume();
226  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
227  MultiFab volume(grids[lev], dmap[lev], 1, 0);
228  auto const& dx = geom[lev].CellSizeArray();
229  Real cell_vol = dx[0]*dx[1]*dx[2];
230  volume.setVal(cell_vol);
231  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
232  vol = volume.sum();
233  }
234 
235  unwted_avg /= vol;
236  r_wted_avg /= vol;
237  enstrsq_avg /= vol;
238 
239  const int nfoo = 3;
240  Real foo[nfoo] = {unwted_avg,r_wted_avg,enstrsq_avg};
241 #ifdef AMREX_LAZY
242  Lazy::QueueReduction([=]() mutable {
243 #endif
244  ParallelDescriptor::ReduceRealSum(
245  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
246 
247  if (ParallelDescriptor::IOProcessor()) {
248  int i = 0;
249  unwted_avg = foo[i++];
250  r_wted_avg = foo[i++];
251  enstrsq_avg = foo[i++];
252 
253  std::ostream& data_log_der = DerDataLog(0);
254 
255  if (time == 0.0) {
256  data_log_der << std::setw(datwidth) << " time";
257  data_log_der << std::setw(datwidth) << " ke_den";
258  data_log_der << std::setw(datwidth) << " velsq";
259  data_log_der << std::setw(datwidth) << " enstrophy";
260  data_log_der << std::endl;
261  }
262  data_log_der << std::setw(datwidth) << std::setprecision(timeprecision) << time;
263  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << unwted_avg;
264  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << r_wted_avg;
265  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << enstrsq_avg;
266  data_log_der << std::endl;
267 
268  } // if IOProcessor
269 #ifdef AMREX_LAZY
270  }
271 #endif
272 }
AMREX_FORCE_INLINE std::ostream & DerDataLog(int i)
Definition: ERF.H:1278
AMREX_FORCE_INLINE int NumDerDataLogs() noexcept
Definition: ERF.H:1292
void erf_dermagvelsq(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:346
void erf_derenstrophysq(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:284
Here is the call graph for this function:

◆ sum_energy_quantities()

void ERF::sum_energy_quantities ( amrex::Real  time)
276 {
277  if ( (verbose <= 0) || (tot_e_datalog.size() < 1) ) { return; }
278 
279  int lev = 0;
280 
281  AMREX_ALWAYS_ASSERT(lev == 0);
282 
283  // ************************************************************************
284  // WARNING: we are not filling ghost cells other than periodic outside the domain
285  // ************************************************************************
286 
287  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
288  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
289 
290  // Average all three components of velocity (on faces) to the cell center
291  average_face_to_cellcenter(mf_cc_vel,0,
292  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
293  &vars_new[lev][Vars::yvel],
294  &vars_new[lev][Vars::zvel]});
295  mf_cc_vel.FillBoundary(geom[lev].periodicity());
296 
297  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
298  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
299  }
300 
301  MultiFab tot_mass (grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
302  MultiFab tot_energy(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
303 
304  auto const& dx = geom[lev].CellSizeArray();
305  bool is_moist = (solverChoice.moisture_type != MoistureType::None);
306 
307 #ifdef _OPENMP
308 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
309 #endif
310  for (MFIter mfi(tot_mass, TilingIfNotGPU()); mfi.isValid(); ++mfi)
311  {
312  const Box& bx = mfi.tilebox();
313 
314  const Array4<Real>& cc_vel_arr = mf_cc_vel.array(mfi);
315  const Array4<Real>& tot_mass_arr = tot_mass.array(mfi);
316  const Array4<Real>& tot_energy_arr = tot_energy.array(mfi);
317  const Array4<const Real>& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
318  const Array4<const Real>& z_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) :
319  Array4<const Real>{};
320  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
321  {
322  Real Qv = (is_moist) ? cons_arr(i,j,k,RhoQ1_comp) : 0.0;
323  Real Qc = (is_moist) ? cons_arr(i,j,k,RhoQ2_comp) : 0.0;
324  Real Qt = Qv + Qc;
325  Real Rhod = cons_arr(i,j,k,Rho_comp);
326  Real Rhot = Rhod * (1.0 + Qt);
327  Real Temp = getTgivenRandRTh(Rhod, cons_arr(i,j,k,RhoTheta_comp), Qv);
328  Real TKE = 0.5 * ( cc_vel_arr(i,j,k,0)*cc_vel_arr(i,j,k,0)
329  + cc_vel_arr(i,j,k,1)*cc_vel_arr(i,j,k,1)
330  + cc_vel_arr(i,j,k,2)*cc_vel_arr(i,j,k,2) );
331  Real zval = (z_arr) ? z_arr(i,j,k) : Real(k)*dx[2];
332 
333  Real Cv = Cp_d - R_d;
334  Real Cvv = Cp_v - R_v;
335  Real Cpv = Cp_v;
336 
337  tot_mass_arr(i,j,k) = Rhot;
338  tot_energy_arr(i,j,k) = Rhod * ( (Cv + Cvv*Qv + Cpv*Qc)*Temp - L_v*Qc
339  + (1.0 + Qt)*TKE + (1.0 + Qt)*CONST_GRAV*zval );
340 
341  });
342 
343  }
344 
345  Real tot_mass_avg = volWgtSumMF(lev, tot_mass , 0, *mapfac_m[lev],false);
346  Real tot_energy_avg = volWgtSumMF(lev, tot_energy, 0, *mapfac_m[lev],false);
347 
348  // Get volume including terrain (consistent with volWgtSumMF routine)
349  Real vol = geom[lev].ProbDomain().volume();
350  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
351  MultiFab volume(grids[lev], dmap[lev], 1, 0);
352  Real cell_vol = dx[0]*dx[1]*dx[2];
353  volume.setVal(cell_vol);
354  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
355  vol = volume.sum();
356  }
357 
358  // Divide by the volume
359  tot_mass_avg /= vol;
360  tot_energy_avg /= vol;
361 
362  const int nfoo = 2;
363  Real foo[nfoo] = {tot_mass_avg,tot_energy_avg};
364 #ifdef AMREX_LAZY
365  Lazy::QueueReduction([=]() mutable {
366 #endif
367  ParallelDescriptor::ReduceRealSum(
368  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
369 
370  if (ParallelDescriptor::IOProcessor()) {
371  int i = 0;
372  tot_mass_avg = foo[i++];
373  tot_energy_avg = foo[i++];
374 
375  std::ostream& data_log_energy = *tot_e_datalog[0];
376 
377  if (time == 0.0) {
378  data_log_energy << std::setw(datwidth) << " time";
379  data_log_energy << std::setw(datwidth) << " tot_mass";
380  data_log_energy << std::setw(datwidth) << " tot_energy";
381  data_log_energy << std::endl;
382  }
383  data_log_energy << std::setw(datwidth) << std::setprecision(timeprecision) << time;
384  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_mass_avg;
385  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_energy_avg;
386  data_log_energy << std::endl;
387 
388  } // if IOProcessor
389 #ifdef AMREX_LAZY
390  }
391 #endif
392 }
constexpr amrex::Real R_v
Definition: ERF_Constants.H:11
constexpr amrex::Real Cp_d
Definition: ERF_Constants.H:12
constexpr amrex::Real CONST_GRAV
Definition: ERF_Constants.H:21
constexpr amrex::Real Cp_v
Definition: ERF_Constants.H:13
constexpr amrex::Real R_d
Definition: ERF_Constants.H:10
constexpr amrex::Real L_v
Definition: ERF_Constants.H:16
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getTgivenRandRTh(const amrex::Real rho, const amrex::Real rhotheta, const amrex::Real qv=0.0)
Definition: ERF_EOS.H:46
Here is the call graph for this function:

◆ sum_integrated_quantities()

void ERF::sum_integrated_quantities ( amrex::Real  time)

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters
timeCurrent time
16 {
17  BL_PROFILE("ERF::sum_integrated_quantities()");
18 
19  if (verbose <= 0)
20  return;
21 
22  // Single level sum
23  Real mass_sl;
24 
25  // Multilevel sums
26  Real mass_ml = 0.0;
27  Real rhth_ml = 0.0;
28  Real scal_ml = 0.0;
29 
30 #if 1
31  mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,*mapfac_m[0],false);
32  for (int lev = 0; lev <= finest_level; lev++) {
33  mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,*mapfac_m[lev],true);
34  }
35 #else
36  for (int lev = 0; lev <= finest_level; lev++) {
37  MultiFab pert_dens(vars_new[lev][Vars::cons].boxArray(),
38  vars_new[lev][Vars::cons].DistributionMap(),
39  1,0);
40  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
41  for ( MFIter mfi(pert_dens,TilingIfNotGPU()); mfi.isValid(); ++mfi)
42  {
43  const Box& bx = mfi.tilebox();
44  const Array4<Real >& pert_dens_arr = pert_dens.array(mfi);
45  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
46  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
47  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
48  pert_dens_arr(i, j, k, 0) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
49  });
50  }
51  if (lev == 0) {
52  mass_sl = volWgtSumMF(0,pert_dens,0,*mapfac_m[0],false);
53  }
54  mass_ml += volWgtSumMF(lev,pert_dens,0,*mapfac_m[lev],true);
55  } // lev
56 #endif
57 
58  Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,*mapfac_m[0],false);
59  Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,*mapfac_m[0],false);
60 
61  for (int lev = 0; lev <= finest_level; lev++) {
62  rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,*mapfac_m[lev],true);
63  scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,*mapfac_m[lev],true);
64  }
65 
66  Gpu::HostVector<Real> h_avg_ustar; h_avg_ustar.resize(1);
67  Gpu::HostVector<Real> h_avg_tstar; h_avg_tstar.resize(1);
68  Gpu::HostVector<Real> h_avg_olen; h_avg_olen.resize(1);
69  if ((m_SurfaceLayer != nullptr) && (NumDataLogs() > 0)) {
70  Box domain = geom[0].Domain();
71  int zdir = 2;
72  h_avg_ustar = sumToLine(*m_SurfaceLayer->get_u_star(0),0,1,domain,zdir);
73  h_avg_tstar = sumToLine(*m_SurfaceLayer->get_t_star(0),0,1,domain,zdir);
74  h_avg_olen = sumToLine(*m_SurfaceLayer->get_olen(0) ,0,1,domain,zdir);
75 
76  // Divide by the total number of cells we are averaging over
77  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
78  h_avg_ustar[0] /= area_z;
79  h_avg_tstar[0] /= area_z;
80  h_avg_olen[0] /= area_z;
81 
82  } else {
83  h_avg_ustar[0] = 0.;
84  h_avg_tstar[0] = 0.;
85  h_avg_olen[0] = 0.;
86  }
87 
88  const int nfoo = 6;
89  Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mass_ml,rhth_ml,scal_ml};
90 #ifdef AMREX_LAZY
91  Lazy::QueueReduction([=]() mutable {
92 #endif
93  ParallelDescriptor::ReduceRealSum(
94  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
95 
96  if (ParallelDescriptor::IOProcessor()) {
97  int i = 0;
98  mass_sl = foo[i++];
99  rhth_sl = foo[i++];
100  scal_sl = foo[i++];
101  mass_ml = foo[i++];
102  rhth_ml = foo[i++];
103  scal_ml = foo[i++];
104 
105  Print() << '\n';
106  Print() << "TIME= " << std::setw(datwidth) << std::setprecision(timeprecision) << std::left << time << '\n';
107  if (finest_level == 0) {
108 #if 1
109  Print() << " MASS = " << mass_sl << '\n';
110 #else
111  Print() << " PERT MASS = " << mass_sl << '\n';
112 #endif
113  Print() << " RHO THETA = " << rhth_sl << '\n';
114  Print() << " RHO SCALAR = " << scal_sl << '\n';
115  } else {
116 #if 1
117  Print() << " MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
118 #else
119  Print() << " PERT MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
120 #endif
121  Print() << " RHO THETA SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
122  Print() << " RHO SCALAR SL/ML = " << scal_sl << " " << scal_ml << '\n';
123  }
124 
125  // The first data log only holds scalars
126  if (NumDataLogs() > 0)
127  {
128  int n_d = 0;
129  std::ostream& data_log1 = DataLog(n_d);
130  if (data_log1.good()) {
131  if (time == 0.0) {
132  data_log1 << std::setw(datwidth) << " time";
133  data_log1 << std::setw(datwidth) << " u_star";
134  data_log1 << std::setw(datwidth) << " t_star";
135  data_log1 << std::setw(datwidth) << " olen";
136  data_log1 << std::endl;
137  } // time = 0
138 
139  // Write the quantities at this time
140  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time;
141  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_ustar[0];
142  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_tstar[0];
143  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_olen[0];
144  data_log1 << std::endl;
145  } // if good
146  } // loop over i
147  } // if IOProcessor
148 #ifdef AMREX_LAZY
149  });
150 #endif
151 
152  // This is just an alias for convenience
153  int lev = 0;
154  if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
155  for (int i = 0; i < NumSamplePoints(); ++i)
156  {
157  sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
158  }
159  }
160  if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
161  for (int i = 0; i < NumSampleLines(); ++i)
162  {
163  sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
164  }
165  }
166 }
AMREX_FORCE_INLINE int NumSampleLineLogs() noexcept
Definition: ERF.H:1321
AMREX_FORCE_INLINE int NumSamplePointLogs() noexcept
Definition: ERF.H:1307
amrex::IntVect & SampleLine(int i)
Definition: ERF.H:1340
AMREX_FORCE_INLINE int NumSamplePoints() noexcept
Definition: ERF.H:1334
AMREX_FORCE_INLINE int NumSampleLines() noexcept
Definition: ERF.H:1347
amrex::IntVect & SamplePoint(int i)
Definition: ERF.H:1327
void sample_points(int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:471
AMREX_FORCE_INLINE std::ostream & DataLog(int i)
Definition: ERF.H:1271
AMREX_FORCE_INLINE int NumDataLogs() noexcept
Definition: ERF.H:1285
void sample_lines(int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:507

◆ timeStep()

void ERF::timeStep ( int  lev,
amrex::Real  time,
int  iteration 
)
private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters
[in]levlevel of refinement (coarsest level is 0)
[in]timestart time for time advance
[in]iterationtime step counter
18 {
19  //
20  // We need to FillPatch the coarse level before assessing whether to regrid
21  // We have not done the swap yet so we fill the "new" which will become the "old"
22  //
23  MultiFab& S_new = vars_new[lev][Vars::cons];
24  MultiFab& U_new = vars_new[lev][Vars::xvel];
25  MultiFab& V_new = vars_new[lev][Vars::yvel];
26  MultiFab& W_new = vars_new[lev][Vars::zvel];
27 
28 #ifdef ERF_USE_NETCDF
29  //
30  // Since we now only read in a subset of the time slices in wrfbdy we need to check
31  // whether it's time to read in more
32  //
33  if (solverChoice.use_real_bcs && (lev==0)) {
34  Real dT = bdy_time_interval;
35 
36  Real time_since_start_old = time - start_bdy_time;
37  int n_time_old = static_cast<int>( time_since_start_old / dT);
38 
39  Real time_since_start_new = time + dt[lev] - start_bdy_time;
40  int n_time_new = static_cast<int>( time_since_start_new / dT);
41 
42  int ntimes = bdy_data_xlo.size();
43  for (int itime = 0; itime < ntimes; itime++)
44  {
45  //if (bdy_data_xlo[itime].size() > 0) {
46  // amrex::Print() << "HAVE DATA AT TIME " << itime << std::endl;
47  //} else {
48  // amrex::Print() << " NO DATA AT TIME " << itime << std::endl;
49  //}
50 
51  bool clear_itime = (itime < n_time_old);
52 
53  if (clear_itime && bdy_data_xlo[itime].size() > 0) {
54  bdy_data_xlo[itime].clear();
55  //amrex::Print() << "CLEAR DATA AT TIME " << itime << std::endl;
56  }
57 
58  bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
59  //if (need_itime) amrex::Print() << "NEED DATA AT TIME " << itime << std::endl;
60 
61  if (bdy_data_xlo[itime].size() == 0 && need_itime) {
62  read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
63  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
64  real_width);
65 
66  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
67  convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
68  *mf_MUB[lev], *mf_C1H[lev], *mf_C2H[lev],
70  geom[lev], use_moist);
71  }
72  } // itime
73  } // use_real_bcs && lev == 0
74 #endif
75 
76  //
77  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
78  //
79  if (lev == 0) {
80  FillPatch(lev, time, {&S_new, &U_new, &V_new, &W_new});
81  } else if (lev < finest_level) {
82  FillPatch(lev, time, {&S_new, &U_new, &V_new, &W_new},
83  {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
84  base_state[lev], base_state[lev]);
85  }
86 
87  if (regrid_int > 0) // We may need to regrid
88  {
89  // help keep track of whether a level was already regridded
90  // from a coarser level call to regrid
91  static Vector<int> last_regrid_step(max_level+1, 0);
92 
93  // regrid changes level "lev+1" so we don't regrid on max_level
94  // also make sure we don't regrid fine levels again if
95  // it was taken care of during a coarser regrid
96  if (lev < max_level)
97  {
98  if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
99  {
100  // regrid could add newly refine levels (if finest_level < max_level)
101  // so we save the previous finest level index
102  int old_finest = finest_level;
103 
104  regrid(lev, time);
105 
106 #ifdef ERF_USE_PARTICLES
107  if (finest_level != old_finest) {
108  particleData.Redistribute();
109  }
110 #endif
111 
112  // mark that we have regridded this level already
113  for (int k = lev; k <= finest_level; ++k) {
114  last_regrid_step[k] = istep[k];
115  }
116 
117  // if there are newly created levels, set the time step
118  for (int k = old_finest+1; k <= finest_level; ++k) {
119  dt[k] = dt[k-1] / MaxRefRatio(k-1);
120  }
121  } // if
122  } // lev
123  }
124 
125  // Update what we call "old" and "new" time
126  t_old[lev] = t_new[lev];
127  t_new[lev] += dt[lev];
128 
129  if (Verbose()) {
130  amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
131  amrex::Print() << std::setprecision(timeprecision)
132  << "ADVANCE from time = " << t_old[lev] << " to " << t_new[lev]
133  << " with dt = " << dt[lev] << std::endl;
134  }
135 
136 #ifdef ERF_USE_WW3_COUPLING
137  amrex::Print() << " About to call send_to_ww3 from ERF_Timestep" << std::endl;
138  send_to_ww3(lev);
139  amrex::Print() << " About to call read_waves from ERF_Timestep" << std::endl;
140  read_waves(lev);
141  //send_to_ww3(lev);
142  //read_waves(lev);
143  //send_to_ww3(lev);
144 #endif
145 
146  // Advance a single level for a single time step
147  Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
148 
149  ++istep[lev];
150 
151  if (Verbose()) {
152  amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
153  amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
154  }
155 
156  if (lev < finest_level)
157  {
158  // recursive call for next-finer level
159  for (int i = 1; i <= nsubsteps[lev+1]; ++i)
160  {
161  Real strt_time_for_fine = time + (i-1)*dt[lev+1];
162  timeStep(lev+1, strt_time_for_fine, i);
163  }
164  }
165 
166  if (verbose && lev == 0 && solverChoice.moisture_type != MoistureType::None) {
167  amrex::Print() << "Cloud fraction " << time << " " << cloud_fraction(time) << std::endl;
168  }
169 }
amrex::Real cloud_fraction(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:395
void Advance(int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
Definition: ERF_Advance.cpp:20

◆ turbPert_amplitude()

void ERF::turbPert_amplitude ( const int  lev)
private
33 {
34  // Accessing data
35  auto& lev_new = vars_new[lev];
36 
37  // Creating local data
38  int ncons = lev_new[Vars::cons].nComp();
39  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
40 
41  // Defining BoxArray type
42  auto m_ixtype = cons_data.boxArray().ixType();
43 
44 #ifdef _OPENMP
45 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
46 #endif
47  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
48  const Box &bx = mfi.validbox();
49  const auto &cons_pert_arr = cons_data.array(mfi); // Address of perturbation array
50  const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
51 
52  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cons_pert_arr, pert_cell);
53  } // mfi
54 }
Here is the call graph for this function:

◆ turbPert_update()

void ERF::turbPert_update ( const int  lev,
const amrex::Real  dt 
)
private
13 {
14  // Accessing data
15  auto& lev_new = vars_new[lev];
16 
17  // Create aliases to state data to pass to calc_tpi_update
18  int ncons = lev_new[Vars::cons].nComp();
19  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
20  MultiFab xvel_data(lev_new[Vars::xvel], make_alias, 0, 1);
21  MultiFab yvel_data(lev_new[Vars::yvel], make_alias, 0, 1);
22 
23  // Computing perturbation update time
24  turbPert.calc_tpi_update(lev, local_dt, xvel_data, yvel_data, cons_data);
25 
26  Print() << "Successfully initialized turbulent perturbation update time and amplitude with type: "<< turbPert.pt_type <<"\n";
27 }
int pt_type
Definition: ERF_TurbPertStruct.H:573

◆ update_diffusive_arrays()

void ERF::update_diffusive_arrays ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
private
402 {
403  // ********************************************************************************************
404  // Diffusive terms
405  // ********************************************************************************************
406  bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None);
407  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
408  bool l_use_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
409  l_use_kturb );
410  bool l_need_SmnSmn = solverChoice.turbChoice[lev].use_keqn;
411  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
412  bool l_rotate = ( solverChoice.use_rotate_surface_flux );
413 
414  BoxArray ba12 = convert(ba, IntVect(1,1,0));
415  BoxArray ba13 = convert(ba, IntVect(1,0,1));
416  BoxArray ba23 = convert(ba, IntVect(0,1,1));
417 
418  Tau[lev].resize(9);
419 
420  if (l_use_diff) {
421  //
422  // NOTE: We require ghost cells in the vertical when allowing grids that don't
423  // cover the entire vertical extent of the domain at this level
424  //
425  for (int i = 0; i < 3; i++) {
426  Tau[lev][i] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) );
427  }
428  Tau[lev][TauType::tau12] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
429  Tau[lev][TauType::tau13] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
430  Tau[lev][TauType::tau23] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
431  if (l_use_terrain) {
432  Tau[lev][TauType::tau21] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
433  Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
434  Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
435  } else {
436  Tau[lev][TauType::tau21] = nullptr;
437  Tau[lev][TauType::tau31] = nullptr;
438  Tau[lev][TauType::tau32] = nullptr;
439  }
440  SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
441  SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
442  SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
443  SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) );
444  SFS_hfx1_lev[lev]->setVal(0.);
445  SFS_hfx2_lev[lev]->setVal(0.);
446  SFS_hfx3_lev[lev]->setVal(0.);
447  SFS_diss_lev[lev]->setVal(0.);
448  if (l_use_moist) {
449  SFS_q1fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
450  SFS_q2fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
451  SFS_q1fx3_lev[lev]->setVal(0.0);
452  SFS_q2fx3_lev[lev]->setVal(0.0);
453  if (l_rotate) {
454  SFS_q1fx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
455  SFS_q1fx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
456  SFS_q1fx1_lev[lev]->setVal(0.0);
457  SFS_q1fx2_lev[lev]->setVal(0.0);
458  } else {
459  SFS_q1fx1_lev[lev] = nullptr;
460  SFS_q1fx2_lev[lev] = nullptr;
461  }
462  } else {
463  SFS_q1fx1_lev[lev] = nullptr;
464  SFS_q1fx2_lev[lev] = nullptr;
465  SFS_q1fx3_lev[lev] = nullptr;
466  SFS_q2fx3_lev[lev] = nullptr;
467  }
468  } else {
469  for (int i = 0; i < 9; i++) {
470  Tau[lev][i] = nullptr;
471  }
472  SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
473  SFS_diss_lev[lev] = nullptr;
474  }
475 
476  if (l_use_kturb) {
477  eddyDiffs_lev[lev] = std::make_unique<MultiFab>(ba, dm, EddyDiff::NumDiffs, 2);
478  eddyDiffs_lev[lev]->setVal(0.0);
479  if(l_need_SmnSmn) {
480  SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
481  } else {
482  SmnSmn_lev[lev] = nullptr;
483  }
484  } else {
485  eddyDiffs_lev[lev] = nullptr;
486  SmnSmn_lev[lev] = nullptr;
487  }
488 }
@ NumDiffs
Definition: ERF_IndexDefines.H:173

◆ update_terrain_arrays()

void ERF::update_terrain_arrays ( int  lev)
597 {
598  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
599  SolverChoice::mesh_type == MeshType::VariableDz) {
600  make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
601  make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
602  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
603  }
604 }
void make_areas(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &ax, MultiFab &ay, MultiFab &az)
Definition: ERF_TerrainMetrics.cpp:558
void make_J(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &detJ_cc)
Definition: ERF_TerrainMetrics.cpp:520
Here is the call graph for this function:

◆ volWgtSumMF()

Real ERF::volWgtSumMF ( int  lev,
const amrex::MultiFab &  mf,
int  comp,
const amrex::MultiFab &  mapfac,
bool  finemask 
)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters
levCurrent level
mfMultiFab on which we do the volume weighted sum
compIndex of the component we want to sum
localBoolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)
finemaskIf a finer level is available, determines whether we mask fine data
598 {
599  BL_PROFILE("ERF::volWgtSumMF()");
600 
601  Real sum = 0.0;
602  MultiFab tmp(grids[lev], dmap[lev], 1, 0);
603  MultiFab::Copy(tmp, mf, comp, 0, 1, 0);
604 
605  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
606  // m is the map scale factor at cell centers
607  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
608  const Box& bx = mfi.tilebox();
609  const Array4< Real> tmp_arr = tmp.array(mfi);
610  const Array4<const Real> mapfac_arr = mapfac.const_array(mfi);
611  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
612  {
613  tmp_arr(i,j,k) /= (mapfac_arr(i,j,0)*mapfac_arr(i,j,0));
614  });
615  } // mfi
616 
617  if (lev < finest_level && finemask) {
618  const MultiFab& mask = build_fine_mask(lev+1);
619  MultiFab::Multiply(tmp, mask, 0, 0, 1, 0);
620  }
621 
622  MultiFab volume(grids[lev], dmap[lev], 1, 0);
623  auto const& dx = geom[lev].CellSizeArray();
624  Real cell_vol = dx[0]*dx[1]*dx[2];
625  volume.setVal(cell_vol);
626  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
627  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
628  }
629 
630  //
631  // Note that when we send in local = true, NO ParallelAllReduce::Sum
632  // is called inside the Dot product -- we will do that before we print
633  //
634  bool local = true;
635  sum = MultiFab::Dot(tmp, 0, volume, 0, 1, 0, local);
636 
637  return sum;
638 }
amrex::MultiFab & build_fine_mask(int lev)
Definition: ERF_WriteScalarProfiles.cpp:648

◆ write_1D_profiles()

void ERF::write_1D_profiles ( amrex::Real  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters
timeCurrent time
18 {
19  BL_PROFILE("ERF::write_1D_profiles()");
20 
21  if (NumDataLogs() > 1)
22  {
23  // Define the 1d arrays we will need
24  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
25  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
26  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
27  Gpu::HostVector<Real> h_avg_wthv;
28  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
29  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
30  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
31  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
32  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
33  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
34 
35  if (NumDataLogs() > 1) {
37  h_avg_u, h_avg_v, h_avg_w,
38  h_avg_rho, h_avg_th, h_avg_ksgs,
39  h_avg_Kmv, h_avg_Khv,
40  h_avg_qv, h_avg_qc, h_avg_qr,
41  h_avg_wqv, h_avg_wqc, h_avg_wqr,
42  h_avg_qi, h_avg_qs, h_avg_qg,
43  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
44  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
45  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
46  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
47  h_avg_wthv);
48  }
49 
50  if (NumDataLogs() > 3 && time > 0.) {
51  derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
52  h_avg_tau22, h_avg_tau23, h_avg_tau33,
53  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
54  h_avg_sgsdiss);
55  }
56 
57  int hu_size = h_avg_u.size();
58 
59  auto const& dx = geom[0].CellSizeArray();
60  if (ParallelDescriptor::IOProcessor()) {
61  if (NumDataLogs() > 1) {
62  std::ostream& data_log1 = DataLog(1);
63  if (data_log1.good()) {
64  // Write the quantities at this time
65  for (int k = 0; k < hu_size; k++) {
66  Real z;
67  if (zlevels_stag[0].size() > 1) {
68  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
69  } else {
70  z = (k + 0.5)* dx[2];
71  }
72  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
73  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
74  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
75  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
76  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
77  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
78  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
79  << std::endl;
80  } // loop over z
81  } // if good
82  } // NumDataLogs
83 
84  if (NumDataLogs() > 2) {
85  std::ostream& data_log2 = DataLog(2);
86  if (data_log2.good()) {
87  // Write the perturbational quantities at this time
88  for (int k = 0; k < hu_size; k++) {
89  Real z;
90  if (zlevels_stag[0].size() > 1) {
91  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
92  } else {
93  z = (k + 0.5)* dx[2];
94  }
95  Real thv = h_avg_th[k] * (1 + 0.61*h_avg_qv[k] - h_avg_qc[k] - h_avg_qr[k]);
96  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
97  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
98  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " "
99  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " "
100  << h_avg_uw[k] - h_avg_u[k]*h_avg_w[k] << " "
101  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " "
102  << h_avg_vw[k] - h_avg_v[k]*h_avg_w[k] << " "
103  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " "
104  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " "
105  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " "
106  << h_avg_wth[k] - h_avg_w[k]*h_avg_th[k] << " "
107  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
108  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
109  // - <u_i u_i> * <u_j>
110  // - 2*<u_i> * <u_i u_j>
111  // + 2*<u_i>*<u_i> * <u_j>
112  << h_avg_uiuiu[k]
113  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
114  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
115  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
116  << " " // (u'_i u'_i)u'
117  << h_avg_uiuiv[k]
118  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
119  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
120  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
121  << " " // (u'_i u'_i)v'
122  << h_avg_uiuiw[k]
123  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
124  - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
125  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
126  << " " // (u'_i u'_i)w'
127  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " "
128  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " "
129  << h_avg_pw[k] - h_avg_p[k]*h_avg_w[k] << " "
130  << h_avg_wqv[k] - h_avg_qv[k]*h_avg_w[k] << " "
131  << h_avg_wqc[k] - h_avg_qc[k]*h_avg_w[k] << " "
132  << h_avg_wqr[k] - h_avg_qr[k]*h_avg_w[k] << " "
133  << h_avg_wthv[k] - h_avg_w[k]*thv
134  << std::endl;
135  } // loop over z
136  } // if good
137  } // NumDataLogs
138 
139  if (NumDataLogs() > 3 && time > 0.) {
140  std::ostream& data_log3 = DataLog(3);
141  if (data_log3.good()) {
142  // Write the average stresses
143  for (int k = 0; k < hu_size; k++) {
144  Real z;
145  if (zlevels_stag[0].size() > 1) {
146  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
147  } else {
148  z = (k + 0.5)* dx[2];
149  }
150  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
151  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
152  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
153  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
154  << h_avg_sgshfx[k] << " "
155  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
156  << h_avg_sgsdiss[k]
157  << std::endl;
158  } // loop over z
159  } // if good
160  } // if (NumDataLogs() > 3)
161  } // if IOProcessor
162  } // if (NumDataLogs() > 1)
163 }
void derive_diag_profiles(amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles.cpp:190
void derive_stress_profiles(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles.cpp:475

◆ write_1D_profiles_stag()

void ERF::write_1D_profiles_stag ( amrex::Real  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Quantities are output at their native grid locations. Therefore, w and associated flux quantities <(•)'w'>, tau13, and tau23 (where '•' includes u, v, p, theta, ...) will be output at staggered heights (i.e., coincident with z faces) rather than cell-center heights to avoid performing additional averaging. Unstaggered (i.e., cell-centered) quantities are output alongside staggered quantities at the lower cell faces in the log file; these quantities will have a zero value at the big end, corresponding to k=Nz+1.

The structure of file should follow ERF_Write1DProfiles.cpp

Parameters
timeCurrent time
26 {
27  BL_PROFILE("ERF::write_1D_profiles()");
28 
29  if (NumDataLogs() > 1)
30  {
31  // Define the 1d arrays we will need
32  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
33  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
34  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
35  Gpu::HostVector<Real> h_avg_wthv;
36  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
37  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
38  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
39  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
40  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
41  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
42 
43  if (NumDataLogs() > 1) {
45  h_avg_u, h_avg_v, h_avg_w,
46  h_avg_rho, h_avg_th, h_avg_ksgs,
47  h_avg_Kmv, h_avg_Khv,
48  h_avg_qv, h_avg_qc, h_avg_qr,
49  h_avg_wqv, h_avg_wqc, h_avg_wqr,
50  h_avg_qi, h_avg_qs, h_avg_qg,
51  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
52  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
53  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
54  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
55  h_avg_wthv);
56  }
57 
58  if (NumDataLogs() > 3 && time > 0.) {
59  derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
60  h_avg_tau22, h_avg_tau23, h_avg_tau33,
61  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
62  h_avg_sgsdiss);
63  }
64 
65  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
66 
67  auto const& dx = geom[0].CellSizeArray();
68  if (ParallelDescriptor::IOProcessor()) {
69  if (NumDataLogs() > 1) {
70  std::ostream& data_log1 = DataLog(1);
71  if (data_log1.good()) {
72  // Write the quantities at this time
73  for (int k = 0; k < unstag_size; k++) {
74  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
75  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
76  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
77  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
78  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
79  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
80  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
81  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
82  << std::endl;
83  } // loop over z
84  // Write top face values
85  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
86  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
87  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
88  << 0 << " " << 0 << " " << h_avg_w[unstag_size] << " "
89  << 0 << " " << 0 << " " << 0 << " " // rho, theta, ksgs
90  << 0 << " " << 0 << " " // Kmv, Khv
91  << 0 << " " << 0 << " " << 0 << " " // qv, qc, qr
92  << 0 << " " << 0 << " " << 0 // qi, qs, qg
93  << std::endl;
94  } // if good
95  } // NumDataLogs
96 
97  if (NumDataLogs() > 2) {
98  std::ostream& data_log2 = DataLog(2);
99  if (data_log2.good()) {
100  // Write the perturbational quantities at this time
101  // For surface values (k=0), assume w = uw = vw = ww = 0
102  Real w_cc = h_avg_w[1] / 2; // w at first cell center
103  Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
104  Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
105  Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
106  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
107  << std::setw(datwidth) << std::setprecision(datprecision) << 0 << " "
108  << h_avg_uu[0] - h_avg_u[0]*h_avg_u[0] << " " // u'u'
109  << h_avg_uv[0] - h_avg_u[0]*h_avg_v[0] << " " // u'v'
110  << 0 << " " // u'w'
111  << h_avg_vv[0] - h_avg_v[0]*h_avg_v[0] << " " // v'v'
112  << 0 << " " // v'w'
113  << 0 << " " // w'w'
114  << h_avg_uth[0] - h_avg_u[0]*h_avg_th[0] << " " // u'th'
115  << h_avg_vth[0] - h_avg_v[0]*h_avg_th[0] << " " // v'th'
116  << 0 << " " // w'th'
117  << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
118  << h_avg_uiuiu[0]
119  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
120  - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
121  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
122  << " " // (u'_i u'_i)u'
123  << h_avg_uiuiv[0]
124  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
125  - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
126  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
127  << " " // (u'_i u'_i)v'
128  << 0 << " " // (u'_i u'_i)w'
129  << h_avg_pu[0] - h_avg_p[0]*h_avg_u[0] << " " // p'u'
130  << h_avg_pv[0] - h_avg_p[0]*h_avg_v[0] << " " // p'v'
131  << 0 << " " // p'w'
132  << 0 << " " // qv'w'
133  << 0 << " " // qc'w'
134  << 0 << " " // qr'w'
135  << 0 // thv'w'
136  << std::endl;
137 
138  // For internal values, interpolate scalar quantities to faces
139  for (int k = 1; k < unstag_size; k++) {
140  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
141  Real uface = 0.5*(h_avg_u[k] + h_avg_u[k-1]);
142  Real vface = 0.5*(h_avg_v[k] + h_avg_v[k-1]);
143  Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]);
144  Real pface = 0.5*(h_avg_p[k] + h_avg_p[k-1]);
145  Real qvface = 0.5*(h_avg_qv[k] + h_avg_qv[k-1]);
146  Real qcface = 0.5*(h_avg_qc[k] + h_avg_qc[k-1]);
147  Real qrface = 0.5*(h_avg_qr[k] + h_avg_qr[k-1]);
148  Real uuface = 0.5*(h_avg_uu[k] + h_avg_uu[k-1]);
149  Real vvface = 0.5*(h_avg_vv[k] + h_avg_vv[k-1]);
150  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
151  w_cc = 0.5*(h_avg_w[k-1] + h_avg_w[k]);
152  uw_cc = 0.5*(h_avg_uw[k-1] + h_avg_uw[k]);
153  vw_cc = 0.5*(h_avg_vw[k-1] + h_avg_vw[k]);
154  ww_cc = 0.5*(h_avg_ww[k-1] + h_avg_ww[k]);
155  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
156  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
157  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " " // u'u'
158  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " " // u'v'
159  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
160  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " " // v'v'
161  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
162  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
163  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " " // u'th'
164  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " " // v'th'
165  << h_avg_wth[k] - h_avg_w[k]*thface << " " // w'th'
166  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
167  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
168  // - <u_i u_i> * <u_j>
169  // - 2*<u_i> * <u_i u_j>
170  // + 2*<u_i>*<u_i> * <u_j>
171  << h_avg_uiuiu[k]
172  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
173  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
174  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
175  << " " // cell-centered (u'_i u'_i)u'
176  << h_avg_uiuiv[k]
177  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
178  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
179  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
180  << " " // cell-centered (u'_i u'_i)v'
181  << h_avg_uiuiw[k]
182  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
183  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
184  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
185  << " " // face-centered (u'_i u'_i)w'
186  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " " // cell-centered p'u'
187  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " " // cell-centered p'v'
188  << h_avg_pw[k] - pface*h_avg_w[k] << " " // face-centered p'w'
189  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
190  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
191  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
192  << h_avg_wthv[k] - thvface*h_avg_w[k]
193  << std::endl;
194  } // loop over z
195 
196  // Write top face values, extrapolating scalar quantities
197  const int k = unstag_size;
198  Real uface = 1.5*h_avg_u[k-1] - 0.5*h_avg_u[k-2];
199  Real vface = 1.5*h_avg_v[k-1] - 0.5*h_avg_v[k-2];
200  Real thface = 1.5*h_avg_th[k-1] - 0.5*h_avg_th[k-2];
201  Real pface = 1.5*h_avg_p[k-1] - 0.5*h_avg_p[k-2];
202  Real qvface = 1.5*h_avg_qv[k-1] - 0.5*h_avg_qv[k-2];
203  Real qcface = 1.5*h_avg_qc[k-1] - 0.5*h_avg_qc[k-2];
204  Real qrface = 1.5*h_avg_qr[k-1] - 0.5*h_avg_qr[k-2];
205  Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2];
206  Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2];
207  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
208  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
209  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
210  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
211  << 0 << " " // u'u'
212  << 0 << " " // u'v'
213  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
214  << 0 << " " // v'v'
215  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
216  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
217  << 0 << " " // u'th'
218  << 0 << " " // v'th'
219  << h_avg_wth[k] - thface*h_avg_w[k] << " " // w'th'
220  << 0 << " " // th'th'
221  << 0 << " " // (u'_i u'_i)u'
222  << 0 << " " // (u'_i u'_i)v'
223  << h_avg_uiuiw[k]
224  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
225  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
226  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
227  << " " // (u'_i u'_i)w'
228  << 0 << " " // pu'
229  << 0 << " " // pv'
230  << h_avg_pw[k] - pface*h_avg_w[k] << " " // pw'
231  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
232  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
233  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
234  << h_avg_wthv[k] - thvface*h_avg_w[k]
235  << std::endl;
236  } // if good
237  } // NumDataLogs
238 
239  if (NumDataLogs() > 3 && time > 0.) {
240  std::ostream& data_log3 = DataLog(3);
241  if (data_log3.good()) {
242  // Write the average stresses
243  for (int k = 0; k < unstag_size; k++) {
244  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
245  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
246  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
247  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
248  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
249  << h_avg_sgshfx[k] << " "
250  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
251  << h_avg_sgsdiss[k]
252  << std::endl;
253  } // loop over z
254  // Write top face values
255  Real NANval = 0.0;
256  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
257  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
258  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
259  << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
260  << NANval << " " << h_avg_tau23[unstag_size] << " " << NANval << " "
261  << h_avg_sgshfx[unstag_size] << " "
262  << h_avg_sgsq1fx[unstag_size] << " " << h_avg_sgsq2fx[unstag_size] << " "
263  << NANval
264  << std::endl;
265  } // if good
266  } // if (NumDataLogs() > 3)
267  } // if IOProcessor
268  } // if (NumDataLogs() > 1)
269 }
void derive_diag_profiles_stag(amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles_stag.cpp:296
void derive_stress_profiles_stag(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles_stag.cpp:605

◆ writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream &  os)
static
138 {
139  std::string PrettyLine = std::string(78, '=') + "\n";
140  std::string OtherLine = std::string(78, '-') + "\n";
141  std::string SkipSpace = std::string(8, ' ');
142 
143  // build information
144  os << PrettyLine;
145  os << " ERF Build Information\n";
146  os << PrettyLine;
147 
148  os << "build date: " << buildInfoGetBuildDate() << "\n";
149  os << "build machine: " << buildInfoGetBuildMachine() << "\n";
150  os << "build dir: " << buildInfoGetBuildDir() << "\n";
151  os << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
152 
153  os << "\n";
154 
155  os << "COMP: " << buildInfoGetComp() << "\n";
156  os << "COMP version: " << buildInfoGetCompVersion() << "\n";
157 
158  os << "C++ compiler: " << buildInfoGetCXXName() << "\n";
159  os << "C++ flags: " << buildInfoGetCXXFlags() << "\n";
160 
161  os << "\n";
162 
163  os << "Link flags: " << buildInfoGetLinkFlags() << "\n";
164  os << "Libraries: " << buildInfoGetLibraries() << "\n";
165 
166  os << "\n";
167 
168  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
169  os << buildInfoGetModuleName(n) << ": "
170  << buildInfoGetModuleVal(n) << "\n";
171  }
172 
173  os << "\n";
174  const char* githash1 = buildInfoGetGitHash(1);
175  const char* githash2 = buildInfoGetGitHash(2);
176  if (strlen(githash1) > 0) {
177  os << "ERF git hash: " << githash1 << "\n";
178  }
179  if (strlen(githash2) > 0) {
180  os << "AMReX git hash: " << githash2 << "\n";
181  }
182 
183  const char* buildgithash = buildInfoGetBuildGitHash();
184  const char* buildgitname = buildInfoGetBuildGitName();
185  if (strlen(buildgithash) > 0) {
186  os << buildgitname << " git hash: " << buildgithash << "\n";
187  }
188 
189  os << "\n";
190  os << " ERF Compile time variables: \n";
191 
192  os << "\n";
193  os << " ERF Defines: \n";
194 #ifdef _OPENMP
195  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
196  << std::endl;
197 #else
198  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
199  << std::endl;
200 #endif
201 
202 #ifdef MPI_VERSION
203  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
204  << MPI_VERSION << std::endl;
205 #else
206  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
207  << "UNDEFINED" << std::endl;
208 #endif
209 
210 #ifdef MPI_SUBVERSION
211  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
212  << MPI_SUBVERSION << std::endl;
213 #else
214  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
215  << "UNDEFINED" << std::endl;
216 #endif
217 
218  os << "\n\n";
219 }

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ WriteCheckpointFile()

void ERF::WriteCheckpointFile ( ) const

ERF function for writing a checkpoint file.

27 {
28  // chk00010 write a checkpoint file with this root directory
29  // chk00010/Header this contains information you need to save (e.g., finest_level, t_new, etc.) and also
30  // the BoxArrays at each level
31  // chk00010/Level_0/
32  // chk00010/Level_1/
33  // etc. these subdirectories will hold the MultiFab data at each level of refinement
34 
35  // checkpoint file name, e.g., chk00010
36  const std::string& checkpointname = Concatenate(check_file,istep[0],5);
37 
38  Print() << "Writing native checkpoint " << checkpointname << "\n";
39 
40  const int nlevels = finest_level+1;
41 
42  // ---- prebuild a hierarchy of directories
43  // ---- dirName is built first. if dirName exists, it is renamed. then build
44  // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
45  // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
46  // ---- after all directories are built
47  // ---- ParallelDescriptor::IOProcessor() creates the directories
48  PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
49 
50  int ncomp_cons = vars_new[0][Vars::cons].nComp();
51 
52  // write Header file
53  if (ParallelDescriptor::IOProcessor()) {
54 
55  std::string HeaderFileName(checkpointname + "/Header");
56  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
57  std::ofstream HeaderFile;
58  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
59  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
60  std::ofstream::trunc |
61  std::ofstream::binary);
62  if(! HeaderFile.good()) {
63  FileOpenFailed(HeaderFileName);
64  }
65 
66  HeaderFile.precision(17);
67 
68  // write out title line
69  HeaderFile << "Checkpoint file for ERF\n";
70 
71  // write out finest_level
72  HeaderFile << finest_level << "\n";
73 
74  // write the number of components
75  // for each variable we store
76 
77  // conservative, cell-centered vars
78  HeaderFile << ncomp_cons << "\n";
79 
80  // x-velocity on faces
81  HeaderFile << 1 << "\n";
82 
83  // y-velocity on faces
84  HeaderFile << 1 << "\n";
85 
86  // z-velocity on faces
87  HeaderFile << 1 << "\n";
88 
89  // write out array of istep
90  for (int i = 0; i < istep.size(); ++i) {
91  HeaderFile << istep[i] << " ";
92  }
93  HeaderFile << "\n";
94 
95  // write out array of dt
96  for (int i = 0; i < dt.size(); ++i) {
97  HeaderFile << dt[i] << " ";
98  }
99  HeaderFile << "\n";
100 
101  // write out array of t_new
102  for (int i = 0; i < t_new.size(); ++i) {
103  HeaderFile << t_new[i] << " ";
104  }
105  HeaderFile << "\n";
106 
107  // write the BoxArray at each level
108  for (int lev = 0; lev <= finest_level; ++lev) {
109  boxArray(lev).writeOn(HeaderFile);
110  HeaderFile << '\n';
111  }
112 
113  // Write separate file that tells how many components we have of the base state
114  std::string BaseStateFileName(checkpointname + "/num_base_state_comps");
115  std::ofstream BaseStateFile;
116  BaseStateFile.open(BaseStateFileName.c_str(), std::ofstream::out |
117  std::ofstream::trunc |
118  std::ofstream::binary);
119  if(! BaseStateFile.good()) {
120  FileOpenFailed(BaseStateFileName);
121  } else {
122  // write out number of components in base state
123  BaseStateFile << BaseState::num_comps << "\n";
124  BaseStateFile << base_state[0].nGrowVect() << "\n";
125  }
126  }
127 
128  // write the MultiFab data to, e.g., chk00010/Level_0/
129  // Here we make copies of the MultiFab with no ghost cells
130  for (int lev = 0; lev <= finest_level; ++lev)
131  {
132  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
133  MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
134  VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
135 
136  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
137  MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
138  VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
139 
140  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
141  MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
142  VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
143 
144  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
145  MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
146  VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
147 
148  // Note that we write the ghost cells of the base state (unlike above)
149  IntVect ng_base = base_state[lev].nGrowVect();
150  int ncomp_base = base_state[lev].nComp();
151  MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
152  MultiFab::Copy(base,base_state[lev],0,0,ncomp_base,ng_base);
153  VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
154 
155  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
156  // Note that we also write the ghost cells of z_phys_nd
157  IntVect ng = z_phys_nd[lev]->nGrowVect();
158  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
159  MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
160  VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
161  }
162 
163  // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
164  // Write the moisture model restart variables
165  std::vector<int> qmoist_indices;
166  std::vector<std::string> qmoist_names;
167  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
168  int qmoist_nvar = qmoist_indices.size();
169  for (int var = 0; var < qmoist_nvar; var++) {
170  const int ncomp = 1;
171  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
172  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
173  MultiFab::Copy(moist_vars,*(qmoist[lev][qmoist_indices[var]]),0,0,ncomp,ng_moist);
174  VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", qmoist_names[var]));
175  }
176 
177 #if defined(ERF_USE_WINDFARM)
178  if(solverChoice.windfarm_type == WindFarmType::Fitch or
179  solverChoice.windfarm_type == WindFarmType::EWP or
180  solverChoice.windfarm_type == WindFarmType::SimpleAD){
181  IntVect ng_turb = Nturb[lev].nGrowVect();
182  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng_turb);
183  MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng_turb);
184  VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
185  }
186 #endif
187 
188  if (solverChoice.lsm_type != LandSurfaceType::None) {
189  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
190  BoxArray ba = lsm_data[lev][mvar]->boxArray();
191  DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
192  IntVect ng = lsm_data[lev][mvar]->nGrowVect();
193  int nvar = lsm_data[lev][mvar]->nComp();
194  MultiFab lsm_vars(ba,dm,nvar,ng);
195  MultiFab::Copy(lsm_vars,*(lsm_data[lev][mvar]),0,0,nvar,ng);
196  VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmVars"));
197  }
198  }
199 
200  IntVect ng = mapfac_m[lev]->nGrowVect();
201  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
202  MultiFab::Copy(mf_m,*mapfac_m[lev],0,0,1,ng);
203  VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_m"));
204 
205  ng = mapfac_u[lev]->nGrowVect();
206  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
207  MultiFab::Copy(mf_u,*mapfac_u[lev],0,0,1,ng);
208  VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_u"));
209 
210  ng = mapfac_v[lev]->nGrowVect();
211  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
212  MultiFab::Copy(mf_v,*mapfac_v[lev],0,0,1,ng);
213  VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_v"));
214 
215  if (m_SurfaceLayer) {
216  amrex::Print() << "Writing SurfaceLayer variables" << std::endl;
217  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
218  MultiFab m_var(ba2d[lev],dmap[lev],1,ng);
219  MultiFab* src = nullptr;
220 
221  // U*
222  src = m_SurfaceLayer->get_u_star(lev);
223  MultiFab::Copy(m_var,*src,0,0,1,ng);
224  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Ustar"));
225 
226  // W*
227  src = m_SurfaceLayer->get_w_star(lev);
228  MultiFab::Copy(m_var,*src,0,0,1,ng);
229  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Wstar"));
230 
231  // T*
232  src = m_SurfaceLayer->get_t_star(lev);
233  MultiFab::Copy(m_var,*src,0,0,1,ng);
234  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Tstar"));
235 
236  // Q*
237  src = m_SurfaceLayer->get_q_star(lev);
238  MultiFab::Copy(m_var,*src,0,0,1,ng);
239  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qstar"));
240 
241  // Olen
242  src = m_SurfaceLayer->get_olen(lev);
243  MultiFab::Copy(m_var,*src,0,0,1,ng);
244  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Olen"));
245 
246  // PBLH
247  src = m_SurfaceLayer->get_pblh(lev);
248  MultiFab::Copy(m_var,*src,0,0,1,ng);
249  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PBLH"));
250 
251  // Z0
252  for (MFIter mfi(m_var); mfi.isValid(); ++mfi) {
253  const Box& bx = mfi.growntilebox();
254  Array4<const Real> const& fab_arr = m_SurfaceLayer->get_z0(lev)->const_array();
255  Array4< Real> const& mv_arr = m_var.array(mfi);
256  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
257  mv_arr(i,j,k) = fab_arr(i,j,k);
258  });
259  }
260  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z0"));
261  }
262 
263  if (sst_lev[lev][0]) {
264  int ntimes = 1;
265  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
266  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
267  for (int nt(0); nt<ntimes; ++nt) {
268  MultiFab::Copy(sst_at_t,*sst_lev[lev][nt],0,0,1,ng);
269  VisMF::Write(sst_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
270  "SST_" + std::to_string(nt)));
271  }
272  }
273 
274  if (tsk_lev[lev][0]) {
275  int ntimes = 1;
276  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
277  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
278  for (int nt(0); nt<ntimes; ++nt) {
279  MultiFab::Copy(tsk_at_t,*tsk_lev[lev][nt],0,0,1,ng);
280  VisMF::Write(tsk_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
281  "TSK_" + std::to_string(nt)));
282  }
283  }
284 
285  {
286  int ntimes = 1;
287  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
288  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
289  for (int nt(0); nt<ntimes; ++nt) {
290  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
291  const Box& bx = mfi.growntilebox();
292  Array4<int> const& src_arr = lmask_lev[lev][nt]->array(mfi);
293  Array4<Real> const& dst_arr = lmask_at_t.array(mfi);
294  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
295  {
296  dst_arr(i,j,k) = Real(src_arr(i,j,k));
297  });
298  }
299  VisMF::Write(lmask_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
300  "LMASK_" + std::to_string(nt)));
301  }
302  }
303 
304 #ifdef ERF_USE_NETCDF
305  IntVect ngv = ng; ngv[2] = 0;
306 
307  // Write lat/lon if it exists
308  if (lat_m[lev] && lon_m[lev] && solverChoice.has_lat_lon) {
309  amrex::Print() << "Writing Lat/Lon variables" << std::endl;
310  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
311  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
312  MultiFab::Copy(lat,*lat_m[lev],0,0,1,ngv);
313  MultiFab::Copy(lon,*lon_m[lev],0,0,1,ngv);
314  VisMF::Write(lat, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LAT"));
315  VisMF::Write(lon, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LON"));
316  }
317 
318  // Write sinPhi and cosPhi if it exists
319  if (cosPhi_m[lev] && sinPhi_m[lev] && solverChoice.variable_coriolis) {
320  amrex::Print() << "Writing Coriolis factors" << std::endl;
321  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
322  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
323  MultiFab::Copy(sphi,*sinPhi_m[lev],0,0,1,ngv);
324  MultiFab::Copy(cphi,*cosPhi_m[lev],0,0,1,ngv);
325  VisMF::Write(sphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SinPhi"));
326  VisMF::Write(cphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "CosPhi"));
327  }
328 
329  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
330  MultiFab tmp1d(ba1d[lev],dmap[lev],1,ngv);
331  MultiFab tmp2d(ba2d[lev],dmap[lev],1,ngv);
332 
333  MultiFab::Copy(tmp1d,*mf_C1H[lev],0,0,1,ngv);
334  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C1H"));
335 
336  MultiFab::Copy(tmp1d,*mf_C2H[lev],0,0,1,ngv);
337  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C2H"));
338 
339  MultiFab::Copy(tmp2d,*mf_MUB[lev],0,0,1,ngv);
340  VisMF::Write(tmp2d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MUB"));
341  }
342 #endif
343 
344  } // for lev
345 
346 #ifdef ERF_USE_PARTICLES
347  particleData.Checkpoint(checkpointname);
348 #endif
349 
350 #if 0
351 #ifdef ERF_USE_NETCDF
352  // Write bdy_data files
353  if ( ParallelDescriptor::IOProcessor() &&
354  ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
356  {
357  // Vector dimensions
358  int num_time = bdy_data_xlo.size();
359  int num_var = bdy_data_xlo[0].size();
360 
361  // Open header file and write to it
362  std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
363  bdy_h_file << std::setprecision(1) << std::fixed;
364  bdy_h_file << num_time << "\n";
365  bdy_h_file << num_var << "\n";
366  bdy_h_file << start_bdy_time << "\n";
367  bdy_h_file << bdy_time_interval << "\n";
368  bdy_h_file << real_width << "\n";
369  for (int ivar(0); ivar<num_var; ++ivar) {
370  bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
371  bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
372  bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
373  bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
374  }
375 
376  // Open data file and write to it
377  std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
378  for (int itime(0); itime<num_time; ++itime) {
379  if (bdy_data_xlo[itime].size() > 0) {
380  for (int ivar(0); ivar<num_var; ++ivar) {
381  bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
382  bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
383  bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
384  bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
385  }
386  }
387  }
388  }
389 #endif
390 #endif
391 
392 }

◆ WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain ( std::ostream &  HeaderFile,
int  nlevels,
const amrex::Vector< amrex::BoxArray > &  bArray,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
amrex::Real  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName,
const std::string &  levelPrefix,
const std::string &  mfPrefix 
) const
1820 {
1821  AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
1822  AMREX_ALWAYS_ASSERT(nlevels <= my_ref_ratio.size()+1);
1823  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
1824 
1825  HeaderFile.precision(17);
1826 
1827  // ---- this is the generic plot file type name
1828  HeaderFile << versionName << '\n';
1829 
1830  HeaderFile << varnames.size() << '\n';
1831 
1832  for (int ivar = 0; ivar < varnames.size(); ++ivar) {
1833  HeaderFile << varnames[ivar] << "\n";
1834  }
1835  HeaderFile << AMREX_SPACEDIM << '\n';
1836  HeaderFile << my_time << '\n';
1837  HeaderFile << finest_level << '\n';
1838  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1839  HeaderFile << my_geom[0].ProbLo(i) << ' ';
1840  }
1841  HeaderFile << '\n';
1842  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1843  HeaderFile << my_geom[0].ProbHi(i) << ' ';
1844  }
1845  HeaderFile << '\n';
1846  for (int i = 0; i < finest_level; ++i) {
1847  HeaderFile << my_ref_ratio[i][0] << ' ';
1848  }
1849  HeaderFile << '\n';
1850  for (int i = 0; i <= finest_level; ++i) {
1851  HeaderFile << my_geom[i].Domain() << ' ';
1852  }
1853  HeaderFile << '\n';
1854  for (int i = 0; i <= finest_level; ++i) {
1855  HeaderFile << level_steps[i] << ' ';
1856  }
1857  HeaderFile << '\n';
1858  for (int i = 0; i <= finest_level; ++i) {
1859  for (int k = 0; k < AMREX_SPACEDIM; ++k) {
1860  HeaderFile << my_geom[i].CellSize()[k] << ' ';
1861  }
1862  HeaderFile << '\n';
1863  }
1864  HeaderFile << (int) my_geom[0].Coord() << '\n';
1865  HeaderFile << "0\n";
1866 
1867  for (int level = 0; level <= finest_level; ++level) {
1868  HeaderFile << level << ' ' << bArray[level].size() << ' ' << my_time << '\n';
1869  HeaderFile << level_steps[level] << '\n';
1870 
1871  const IntVect& domain_lo = my_geom[level].Domain().smallEnd();
1872  for (int i = 0; i < bArray[level].size(); ++i)
1873  {
1874  // Need to shift because the RealBox ctor we call takes the
1875  // physical location of index (0,0,0). This does not affect
1876  // the usual cases where the domain index starts with 0.
1877  const Box& b = shift(bArray[level][i], -domain_lo);
1878  RealBox loc = RealBox(b, my_geom[level].CellSize(), my_geom[level].ProbLo());
1879  for (int n = 0; n < AMREX_SPACEDIM; ++n) {
1880  HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
1881  }
1882  }
1883 
1884  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
1885  }
1886  HeaderFile << "1" << "\n";
1887  HeaderFile << "3" << "\n";
1888  HeaderFile << "amrexvec_nu_x" << "\n";
1889  HeaderFile << "amrexvec_nu_y" << "\n";
1890  HeaderFile << "amrexvec_nu_z" << "\n";
1891  std::string mf_nodal_prefix = "Nu_nd";
1892  for (int level = 0; level <= finest_level; ++level) {
1893  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
1894  }
1895 }
Coord
Definition: ERF_DataStruct.H:68

◆ writeJobInfo()

void ERF::writeJobInfo ( const std::string &  dir) const
10 {
11  // job_info file with details about the run
12  std::ofstream jobInfoFile;
13  std::string FullPathJobInfoFile = dir;
14  FullPathJobInfoFile += "/job_info";
15  jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out);
16 
17  std::string PrettyLine = "==================================================="
18  "============================\n";
19  std::string OtherLine = "----------------------------------------------------"
20  "----------------------------\n";
21  std::string SkipSpace = " ";
22 
23  // job information
24  jobInfoFile << PrettyLine;
25  jobInfoFile << " ERF Job Information\n";
26  jobInfoFile << PrettyLine;
27 
28  jobInfoFile << "inputs file: " << inputs_name << "\n\n";
29 
30  jobInfoFile << "number of MPI processes: "
31  << ParallelDescriptor::NProcs() << "\n";
32 #ifdef _OPENMP
33  jobInfoFile << "number of threads: " << omp_get_max_threads() << "\n";
34 #endif
35 
36  jobInfoFile << "\n";
37  jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
38  << getCPUTime() / 3600.0;
39 
40  jobInfoFile << "\n\n";
41 
42  // plotfile information
43  jobInfoFile << PrettyLine;
44  jobInfoFile << " Plotfile Information\n";
45  jobInfoFile << PrettyLine;
46 
47  time_t now = time(nullptr);
48 
49  // Convert now to tm struct for local timezone
50  tm* localtm = localtime(&now);
51  jobInfoFile << "output data / time: " << asctime(localtm);
52 
53  std::string currentDir = FileSystem::CurrentPath();
54  jobInfoFile << "output dir: " << currentDir << "\n";
55 
56  jobInfoFile << "\n\n";
57 
58  // build information
59  jobInfoFile << PrettyLine;
60  jobInfoFile << " Build Information\n";
61  jobInfoFile << PrettyLine;
62 
63  jobInfoFile << "build date: " << buildInfoGetBuildDate() << "\n";
64  jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
65  jobInfoFile << "build dir: " << buildInfoGetBuildDir() << "\n";
66  jobInfoFile << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
67 
68  jobInfoFile << "\n";
69 
70  jobInfoFile << "COMP: " << buildInfoGetComp() << "\n";
71  jobInfoFile << "COMP version: " << buildInfoGetCompVersion() << "\n";
72 
73  jobInfoFile << "\n";
74 
75  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
76  jobInfoFile << buildInfoGetModuleName(n) << ": "
77  << buildInfoGetModuleVal(n) << "\n";
78  }
79 
80  jobInfoFile << "\n";
81 
82  const char* githash1 = buildInfoGetGitHash(1);
83  const char* githash2 = buildInfoGetGitHash(2);
84  if (strlen(githash1) > 0) {
85  jobInfoFile << "ERF git hash: " << githash1 << "\n";
86  }
87  if (strlen(githash2) > 0) {
88  jobInfoFile << "AMReX git hash: " << githash2 << "\n";
89  }
90 
91  const char* buildgithash = buildInfoGetBuildGitHash();
92  const char* buildgitname = buildInfoGetBuildGitName();
93  if (strlen(buildgithash) > 0) {
94  jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
95  }
96 
97  jobInfoFile << "\n\n";
98 
99  // grid information
100  jobInfoFile << PrettyLine;
101  jobInfoFile << " Grid Information\n";
102  jobInfoFile << PrettyLine;
103 
104  int f_lev = finest_level;
105 
106  for (int i = 0; i <= f_lev; i++) {
107  jobInfoFile << " level: " << i << "\n";
108  jobInfoFile << " number of boxes = " << grids[i].size() << "\n";
109  jobInfoFile << " maximum zones = ";
110  for (int n = 0; n < AMREX_SPACEDIM; n++) {
111  jobInfoFile << geom[i].Domain().length(n) << " ";
112  }
113  jobInfoFile << "\n\n";
114  }
115 
116  jobInfoFile << " Boundary conditions\n";
117 
118  jobInfoFile << " -x: " << domain_bc_type[0] << "\n";
119  jobInfoFile << " +x: " << domain_bc_type[3] << "\n";
120  jobInfoFile << " -y: " << domain_bc_type[1] << "\n";
121  jobInfoFile << " +y: " << domain_bc_type[4] << "\n";
122  jobInfoFile << " -z: " << domain_bc_type[2] << "\n";
123  jobInfoFile << " +z: " << domain_bc_type[5] << "\n";
124 
125  jobInfoFile << "\n\n";
126 
127  // runtime parameters
128  jobInfoFile << PrettyLine;
129  jobInfoFile << " Inputs File Parameters\n";
130  jobInfoFile << PrettyLine;
131 
132  ParmParse::dumpTable(jobInfoFile, true);
133  jobInfoFile.close();
134 }
std::string inputs_name
Definition: main.cpp:14
static amrex::Real getCPUTime()
Definition: ERF.H:1356
Here is the call graph for this function:

◆ WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain ( const std::string &  plotfilename,
int  nlevels,
const amrex::Vector< const amrex::MultiFab * > &  mf,
const amrex::Vector< const amrex::MultiFab * > &  mf_nd,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
amrex::Real  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName = "HyperCLaw-V1.1",
const std::string &  levelPrefix = "Level_",
const std::string &  mfPrefix = "Cell",
const amrex::Vector< std::string > &  extra_dirs = amrex::Vector<std::string>() 
) const
1734 {
1735  BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
1736 
1737  AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
1738  AMREX_ALWAYS_ASSERT(nlevels <= rr.size()+1);
1739  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
1740  AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
1741 
1742  bool callBarrier(false);
1743  PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
1744  if (!extra_dirs.empty()) {
1745  for (const auto& d : extra_dirs) {
1746  const std::string ed = plotfilename+"/"+d;
1747  PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
1748  }
1749  }
1750  ParallelDescriptor::Barrier();
1751 
1752  if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
1753  Vector<BoxArray> boxArrays(nlevels);
1754  for(int level(0); level < boxArrays.size(); ++level) {
1755  boxArrays[level] = mf[level]->boxArray();
1756  }
1757 
1758  auto f = [=]() {
1759  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
1760  std::string HeaderFileName(plotfilename + "/Header");
1761  std::ofstream HeaderFile;
1762  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
1763  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
1764  std::ofstream::trunc |
1765  std::ofstream::binary);
1766  if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
1767  WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
1768  my_geom, time, level_steps, rr, versionName,
1769  levelPrefix, mfPrefix);
1770  };
1771 
1772  if (AsyncOut::UseAsyncOut()) {
1773  AsyncOut::Submit(std::move(f));
1774  } else {
1775  f();
1776  }
1777  }
1778 
1779  std::string mf_nodal_prefix = "Nu_nd";
1780  for (int level = 0; level <= finest_level; ++level)
1781  {
1782  if (AsyncOut::UseAsyncOut()) {
1783  VisMF::AsyncWrite(*mf[level],
1784  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
1785  true);
1786  VisMF::AsyncWrite(*mf_nd[level],
1787  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
1788  true);
1789  } else {
1790  const MultiFab* data;
1791  std::unique_ptr<MultiFab> mf_tmp;
1792  if (mf[level]->nGrowVect() != 0) {
1793  mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
1794  mf[level]->DistributionMap(),
1795  mf[level]->nComp(), 0, MFInfo(),
1796  mf[level]->Factory());
1797  MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
1798  data = mf_tmp.get();
1799  } else {
1800  data = mf[level];
1801  }
1802  VisMF::Write(*data , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
1803  VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
1804  }
1805  }
1806 }
void WriteGenericPlotfileHeaderWithTerrain(std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
Definition: ERF_Plotfile.cpp:1809

◆ WriteMyEBSurface()

void ERF::WriteMyEBSurface ( )
6 {
7  using namespace amrex;
8 
9  amrex::Print() << "Writing the geometry to a vtp file.\n" << std::endl;
10 
11  // Only write at the finest level!
12  int lev = finest_level;
13 
14  BoxArray & ba = grids[lev];
15  DistributionMapping & dm = dmap[lev];
16 
17  const EBFArrayBoxFactory* ebfact = &EBFactory(lev);
18 
19  WriteEBSurface(ba,dm,Geom(lev),ebfact);
20 }
Definition: ERF_ConsoleIO.cpp:12
Here is the call graph for this function:

◆ writeNow()

bool ERF::writeNow ( const amrex::Real  cur_time,
const amrex::Real  dt,
const int  nstep,
const int  plot_int,
const amrex::Real  plot_per 
)
2154 {
2155  bool write_now = false;
2156 
2157  if ( plot_int > 0 && (nstep % plot_int == 0) ) {
2158  write_now = true;
2159 
2160  } else if (plot_per > 0.0) {
2161 
2162  // Check to see if we've crossed a plot_per interval by comparing
2163  // the number of intervals that have elapsed for both the current
2164  // time and the time at the beginning of this timestep.
2165 
2166  const Real eps = std::numeric_limits<Real>::epsilon() * Real(10.0) * std::abs(cur_time);
2167 
2168  int num_per_old = static_cast<int>(std::floor((cur_time-eps-dt_lev) / plot_per));
2169  int num_per_new = static_cast<int>(std::floor((cur_time-eps ) / plot_per));
2170 
2171  // Before using these, however, we must test for the case where we're
2172  // within machine epsilon of the next interval. In that case, increment
2173  // the counter, because we have indeed reached the next plot_per interval
2174  // at this point.
2175 
2176  const Real next_plot_time = (num_per_old + 1) * plot_per;
2177 
2178  if ((num_per_new == num_per_old) && std::abs(cur_time - next_plot_time) <= eps)
2179  {
2180  num_per_new += 1;
2181  }
2182 
2183  // Similarly, we have to account for the case where the old time is within
2184  // machine epsilon of the beginning of this interval, so that we don't double
2185  // count that time threshold -- we already plotted at that time on the last timestep.
2186 
2187  if ((num_per_new != num_per_old) && std::abs((cur_time - dt_lev) - next_plot_time) <= eps)
2188  num_per_old += 1;
2189 
2190  if (num_per_old != num_per_new)
2191  write_now = true;
2192  }
2193  return write_now;
2194 }
real(c_double), parameter epsilon
Definition: ERF_module_model_constants.F90:12

◆ WritePlotFile()

void ERF::WritePlotFile ( int  which,
PlotFileType  plotfile_type,
amrex::Vector< std::string >  plot_var_names 
)
232 {
233  const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
234  const int ncomp_mf = varnames.size();
235 
236  int ncomp_cons = vars_new[0][Vars::cons].nComp();
237 
238  if (ncomp_mf == 0) return;
239 
240  // We Fillpatch here because some of the derived quantities require derivatives
241  // which require ghost cells to be filled. We do not need to call FillPatcher
242  // because we don't need to set interior fine points.
243  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
244 
245  // Level 0 FilLPatch
247  &vars_new[0][Vars::yvel], &vars_new[0][Vars::zvel]});
248 
249  for (int lev = 1; lev <= finest_level; ++lev) {
250  bool fillset = false;
251  FillPatch(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
252  &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
253  {&vars_new[lev][Vars::cons],
254  &rU_new[lev], &rV_new[lev], &rW_new[lev]},
255  base_state[lev], base_state[lev], fillset);
256  }
257 
258  // Get qmoist pointers if using moisture
259  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
260  for (int lev = 0; lev <= finest_level; ++lev) {
261  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
262  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
263  }
264  }
265 
266  // Vector of MultiFabs for cell-centered data
267  Vector<MultiFab> mf(finest_level+1);
268  for (int lev = 0; lev <= finest_level; ++lev) {
269  mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
270  }
271 
272  // Vector of MultiFabs for nodal data
273  Vector<MultiFab> mf_nd(finest_level+1);
274  if ( SolverChoice::mesh_type != MeshType::ConstantDz) {
275  for (int lev = 0; lev <= finest_level; ++lev) {
276  BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
277  mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
278  mf_nd[lev].setVal(0.);
279  }
280  }
281 
282  // Vector of MultiFabs for face-centered velocity
283  Vector<MultiFab> mf_u(finest_level+1);
284  Vector<MultiFab> mf_v(finest_level+1);
285  Vector<MultiFab> mf_w(finest_level+1);
286  if (m_plot_face_vels) {
287  for (int lev = 0; lev <= finest_level; ++lev) {
288  BoxArray grid_stag_u(grids[lev]); grid_stag_u.surroundingNodes(0);
289  BoxArray grid_stag_v(grids[lev]); grid_stag_v.surroundingNodes(1);
290  BoxArray grid_stag_w(grids[lev]); grid_stag_w.surroundingNodes(2);
291  mf_u[lev].define(grid_stag_u, dmap[lev], 1, 0);
292  mf_v[lev].define(grid_stag_v, dmap[lev], 1, 0);
293  mf_w[lev].define(grid_stag_w, dmap[lev], 1, 0);
294  MultiFab::Copy(mf_u[lev],vars_new[lev][Vars::xvel],0,0,1,0);
295  MultiFab::Copy(mf_v[lev],vars_new[lev][Vars::yvel],0,0,1,0);
296  MultiFab::Copy(mf_w[lev],vars_new[lev][Vars::zvel],0,0,1,0);
297  }
298  }
299 
300  // Array of MultiFabs for cell-centered velocity
301  Vector<MultiFab> mf_cc_vel(finest_level+1);
302 
303  if (containerHasElement(plot_var_names, "x_velocity" ) ||
304  containerHasElement(plot_var_names, "y_velocity" ) ||
305  containerHasElement(plot_var_names, "z_velocity" ) ||
306  containerHasElement(plot_var_names, "magvel" ) ||
307  containerHasElement(plot_var_names, "vorticity_x") ||
308  containerHasElement(plot_var_names, "vorticity_y") ||
309  containerHasElement(plot_var_names, "vorticity_z") ) {
310 
311  for (int lev = 0; lev <= finest_level; ++lev) {
312  mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
313  mf_cc_vel[lev].setVal(-1.e20);
314  average_face_to_cellcenter(mf_cc_vel[lev],0,
315  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
316  &vars_new[lev][Vars::yvel],
317  &vars_new[lev][Vars::zvel]});
318  } // lev
319  } // if (vel or vort)
320 
321  // We need ghost cells if computing vorticity
322  if ( containerHasElement(plot_var_names, "vorticity_x")||
323  containerHasElement(plot_var_names, "vorticity_y") ||
324  containerHasElement(plot_var_names, "vorticity_z") )
325  {
326  amrex::Interpolater* mapper = &cell_cons_interp;
327  for (int lev = 1; lev <= finest_level; ++lev)
328  {
329  Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
330  Vector<Real> ftime = {t_new[lev], t_new[lev]};
331  Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
332  Vector<Real> ctime = {t_new[lev], t_new[lev]};
333 
334  FillBdyCCVels(mf_cc_vel,lev-1);
335 
336  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
337  FillPatchTwoLevels(mf_cc_vel[lev], mf_cc_vel[lev].nGrowVect(), IntVect(0,0,0),
338  t_new[lev], cmf, ctime, fmf, ftime,
339  0, 0, mf_cc_vel[lev].nComp(), geom[lev-1], geom[lev],
340  refRatio(lev-1), mapper, domain_bcs_type,
342  } // lev
343  FillBdyCCVels(mf_cc_vel);
344  } // if (vort)
345 
346 
347  for (int lev = 0; lev <= finest_level; ++lev)
348  {
349  int mf_comp = 0;
350 
351  // First, copy any of the conserved state variables into the output plotfile
352  for (int i = 0; i < cons_names.size(); ++i) {
353  if (containerHasElement(plot_var_names, cons_names[i])) {
354  MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,0);
355  mf_comp++;
356  }
357  }
358 
359  // Next, check for velocities
360  if (containerHasElement(plot_var_names, "x_velocity")) {
361  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 0, mf_comp, 1, 0);
362  mf_comp += 1;
363  }
364  if (containerHasElement(plot_var_names, "y_velocity")) {
365  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 1, mf_comp, 1, 0);
366  mf_comp += 1;
367  }
368  if (containerHasElement(plot_var_names, "z_velocity")) {
369  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 2, mf_comp, 1, 0);
370  mf_comp += 1;
371  }
372 
373  // Finally, check for any derived quantities and compute them, inserting
374  // them into our output multifab
375  auto calculate_derived = [&](const std::string& der_name,
376  MultiFab& src_mf,
377  decltype(derived::erf_dernull)& der_function)
378  {
379  if (containerHasElement(plot_var_names, der_name)) {
380  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
381 #ifdef _OPENMP
382 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
383 #endif
384  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
385  {
386  const Box& bx = mfi.tilebox();
387  auto& dfab = dmf[mfi];
388  auto& sfab = src_mf[mfi];
389  der_function(bx, dfab, 0, 1, sfab, Geom(lev), t_new[0], nullptr, lev);
390  }
391 
392  mf_comp++;
393  }
394  };
395 
396  bool ismoist = (solverChoice.moisture_type != MoistureType::None);
397 
398  // Note: All derived variables must be computed in order of "derived_names" defined in ERF.H
399  calculate_derived("soundspeed", vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
400  if (ismoist) {
401  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dermoisttemp);
402  } else {
403  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dertemp);
404  }
405  calculate_derived("theta", vars_new[lev][Vars::cons], derived::erf_dertheta);
406  calculate_derived("KE", vars_new[lev][Vars::cons], derived::erf_derKE);
407  calculate_derived("scalar", vars_new[lev][Vars::cons], derived::erf_derscalar);
408  calculate_derived("vorticity_x", mf_cc_vel[lev] , derived::erf_dervortx);
409  calculate_derived("vorticity_y", mf_cc_vel[lev] , derived::erf_dervorty);
410  calculate_derived("vorticity_z", mf_cc_vel[lev] , derived::erf_dervortz);
411  calculate_derived("magvel" , mf_cc_vel[lev] , derived::erf_dermagvel);
412 
413  if (containerHasElement(plot_var_names, "divU"))
414  {
415  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
416  Array<MultiFab const*, AMREX_SPACEDIM> u;
417  u[0] = &(vars_new[lev][Vars::xvel]);
418  u[1] = &(vars_new[lev][Vars::yvel]);
419  u[2] = &(vars_new[lev][Vars::zvel]);
420  compute_divergence (lev, dmf, u, geom[lev]);
421  mf_comp += 1;
422  }
423 
424  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
425  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
426  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
427  if (containerHasElement(plot_var_names, "pres_hse"))
428  {
429  MultiFab::Copy(mf[lev],p_hse,0,mf_comp,1,0);
430  mf_comp += 1;
431  }
432  if (containerHasElement(plot_var_names, "dens_hse"))
433  {
434  MultiFab::Copy(mf[lev],r_hse,0,mf_comp,1,0);
435  mf_comp += 1;
436  }
437  if (containerHasElement(plot_var_names, "theta_hse"))
438  {
439  MultiFab::Copy(mf[lev],th_hse,0,mf_comp,1,0);
440  mf_comp += 1;
441  }
442 
443  if (containerHasElement(plot_var_names, "pressure"))
444  {
445  if (solverChoice.anelastic[lev] == 1) {
446  MultiFab::Copy(mf[lev], p_hse, 0, mf_comp, 1, 0);
447  } else
448 #ifdef _OPENMP
449 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
450 #endif
451  {
452  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
453  {
454  const Box& bx = mfi.tilebox();
455  const Array4<Real >& derdat = mf[lev].array(mfi);
456  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
457  const int ncomp = vars_new[lev][Vars::cons].nComp();
458 
459  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
460  {
461  Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
462  const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
463  derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p);
464  });
465  }
466  } // not anelastic
467  mf_comp += 1;
468  } // pressure
469 
470  if (containerHasElement(plot_var_names, "pert_pres"))
471  {
472  if (solverChoice.anelastic[lev] == 1) {
473  MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0);
474  } else
475 #ifdef _OPENMP
476 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
477 #endif
478  {
479  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
480  {
481  const Box& bx = mfi.tilebox();
482  const Array4<Real>& derdat = mf[lev].array(mfi);
483  const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
484  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
485  const int ncomp = vars_new[lev][Vars::cons].nComp();
486 
487  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
488  {
489  Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
490  const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
491  derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p) - p0_arr(i,j,k);
492  });
493  }
494  } // not anelastic
495  mf_comp += 1;
496  }
497 
498  if (containerHasElement(plot_var_names, "pert_dens"))
499  {
500 #ifdef _OPENMP
501 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
502 #endif
503  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
504  {
505  const Box& bx = mfi.tilebox();
506  const Array4<Real>& derdat = mf[lev].array(mfi);
507  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
508  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
509  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
510  derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
511  });
512  }
513  mf_comp ++;
514  }
515 
516  if (containerHasElement(plot_var_names, "eq_pot_temp"))
517  {
518 #ifdef _OPENMP
519 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
520 #endif
521  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
522  {
523  const Box& bx = mfi.tilebox();
524  const Array4<Real>& derdat = mf[lev].array(mfi);
525  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
526  const int ncomp = vars_new[lev][Vars::cons].nComp();
527  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
528  Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
529  Real qc = (use_moisture && (ncomp > RhoQ2_comp)) ? S_arr(i,j,k,RhoQ2_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
530  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
531  Real pressure = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp), qv);
532  Real fac = Cp_d + Cp_l*(qv + qc);
533  Real pv = erf_esatw(T)*100.0;
534 
535  derdat(i, j, k, mf_comp) = T*std::pow((pressure - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
536  });
537  }
538  mf_comp ++;
539  }
540 
541  if (containerHasElement(plot_var_names, "terrain_IB_mask"))
542  {
543  MultiFab* terrain_blank = terrain_blanking[lev].get();
544  MultiFab::Copy(mf[lev],*terrain_blank,0,mf_comp,1,0);
545  mf_comp ++;
546  }
547 
548 #ifdef ERF_USE_WINDFARM
549  if ( containerHasElement(plot_var_names, "num_turb") and
550  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
551  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
552  {
553  MultiFab::Copy(mf[lev],Nturb[lev],0,mf_comp,1,0);
554  mf_comp ++;
555  }
556 
557  if ( containerHasElement(plot_var_names, "SMark0") and
558  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
559  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
560  {
561  MultiFab::Copy(mf[lev],SMark[lev],0,mf_comp,1,0);
562  mf_comp ++;
563  }
564 
565  if (containerHasElement(plot_var_names, "SMark1") and
566  (solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD))
567  {
568  MultiFab::Copy(mf[lev],SMark[lev],1,mf_comp,1,0);
569  mf_comp ++;
570  }
571 
572 #endif
573 
574  int klo = geom[lev].Domain().smallEnd(2);
575  int khi = geom[lev].Domain().bigEnd(2);
576 
577  if (containerHasElement(plot_var_names, "dpdx"))
578  {
579  auto dxInv = geom[lev].InvCellSizeArray();
580  MultiFab pres(vars_new[lev][Vars::cons].boxArray(), vars_new[lev][Vars::cons].DistributionMap(), 1, 1);
581 #ifdef _OPENMP
582 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
583 #endif
584  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
585  {
586  // First define pressure on grown box
587  const Box& gbx = mfi.growntilebox(1);
588  const Array4<Real > & p_arr = pres.array(mfi);
589  const Array4<Real const> & hse_arr = base_state[lev].const_array(mfi);
590  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
591  if (solverChoice.anelastic[lev] == 1) {
592  ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
593  p_arr(i,j,k) = hse_arr(i,j,k,1);
594  });
595  } else {
596  ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
597  p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp));
598  });
599  }
600  }
601  pres.FillBoundary(geom[lev].periodicity());
602 
603 #ifdef _OPENMP
604 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
605 #endif
606  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
607  {
608  // Now compute pressure gradient on valid box
609  const Box& bx = mfi.tilebox();
610  const Array4<Real>& derdat = mf[lev].array(mfi);
611  const Array4<Real> & p_arr = pres.array(mfi);
612 
613  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
614  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
615  const Array4<Real const>& z_cc = z_phys_cc[lev]->const_array(mfi);
616  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
617  {
618  // Pgrad at lower I face
619  Real gpx_lo = dxInv[0] * (p_arr(i,j,k) - p_arr(i-1,j,k));
620 
621  Real met_h_xi_hi = Compute_h_xi_AtCellCenter (i , j, k, dxInv, z_nd);
622  Real met_h_xi_lo = Compute_h_xi_AtCellCenter (i-1, j, k, dxInv, z_nd);
623 
624  Real dz_phys_hi, dz_phys_lo;
625  Real gpz_lo, gpz_hi;
626  if (k==klo) {
627  dz_phys_hi = z_cc(i ,j,k+1) - z_cc(i ,j,k );
628  dz_phys_lo = z_cc(i-1,j,k+1) - z_cc(i-1,j,k );
629  gpz_hi = (p_arr(i ,j,k+1) - p_arr(i ,j,k )) / dz_phys_hi;
630  gpz_lo = (p_arr(i-1,j,k+1) - p_arr(i-1,j,k )) / dz_phys_lo;
631  } else if (k==khi) {
632  dz_phys_hi = z_cc(i ,j,k ) - z_cc(i ,j,k-1);
633  dz_phys_lo = z_cc(i-1,j,k ) - z_cc(i-1,j,k-1);
634  gpz_hi = (p_arr(i ,j,k ) - p_arr(i ,j,k-1)) / dz_phys_hi;
635  gpz_lo = (p_arr(i-1,j,k ) - p_arr(i-1,j,k-1)) / dz_phys_lo;
636  } else {
637  dz_phys_hi = z_cc(i ,j,k+1) - z_cc(i ,j,k-1);
638  dz_phys_lo = z_cc(i-1,j,k+1) - z_cc(i-1,j,k-1);
639  gpz_hi = (p_arr(i ,j,k+1) - p_arr(i ,j,k-1)) / dz_phys_hi;
640  gpz_lo = (p_arr(i-1,j,k+1) - p_arr(i-1,j,k-1)) / dz_phys_lo;
641  }
642  Real gpx_metric = 0.5 * ( gpz_hi * met_h_xi_hi + gpz_lo * met_h_xi_lo );
643  gpx_lo -= gpx_metric;
644 
645  // Pgrad at higher I face
646  Real gpx_hi = dxInv[0] * (p_arr(i+1,j,k) - p_arr(i,j,k));
647 
648  met_h_xi_hi = Compute_h_xi_AtCellCenter (i+1, j, k, dxInv, z_nd);
649  met_h_xi_lo = Compute_h_xi_AtCellCenter (i , j, k, dxInv, z_nd);
650 
651  if (k==klo) {
652  dz_phys_hi = z_cc(i+1,j,k+1) - z_cc(i+1,j,k );
653  dz_phys_lo = z_cc(i ,j,k+1) - z_cc(i ,j,k );
654  gpz_hi = (p_arr(i+1,j,k+1) - p_arr(i+1,j,k )) / dz_phys_hi;
655  gpz_lo = (p_arr(i ,j,k+1) - p_arr(i ,j,k )) / dz_phys_lo;
656  } else if (k==khi) {
657  dz_phys_hi = z_cc(i+1,j,k ) - z_cc(i+1,j,k-1);
658  dz_phys_lo = z_cc(i ,j,k ) - z_cc(i ,j,k-1);
659  gpz_hi = (p_arr(i+1,j,k ) - p_arr(i+1,j,k-1)) / dz_phys_hi;
660  gpz_lo = (p_arr(i ,j,k ) - p_arr(i ,j,k-1)) / dz_phys_lo;
661  } else {
662  dz_phys_hi = z_cc(i+1,j,k+1) - z_cc(i+1,j,k-1);
663  dz_phys_lo = z_cc(i ,j,k+1) - z_cc(i ,j,k-1);
664  gpz_hi = (p_arr(i+1,j,k+1) - p_arr(i+1,j,k-1)) / dz_phys_hi;
665  gpz_lo = (p_arr(i ,j,k+1) - p_arr(i ,j,k-1)) / dz_phys_lo;
666  }
667  gpx_metric = 0.5 * ( gpz_hi * met_h_xi_hi + gpz_lo * met_h_xi_lo );
668  gpx_hi -= gpx_metric;
669 
670  // Average P grad to CC
671  derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_hi + gpx_lo);
672  });
673  } else {
674  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
675  derdat(i ,j ,k, mf_comp) = 0.5 * (p_arr(i+1,j,k) - p_arr(i-1,j,k)) * dxInv[0];
676  });
677  }
678  } // mfi
679  mf_comp ++;
680  } // dpdx
681 
682  if (containerHasElement(plot_var_names, "dpdy"))
683  {
684  auto dxInv = geom[lev].InvCellSizeArray();
685 
686  MultiFab pres(vars_new[lev][Vars::cons].boxArray(), vars_new[lev][Vars::cons].DistributionMap(), 1, 1);
687 #ifdef _OPENMP
688 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
689 #endif
690  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
691  {
692  // First define pressure on grown box
693  const Box& gbx = mfi.growntilebox(1);
694  const Array4<Real > & p_arr = pres.array(mfi);
695  const Array4<Real const> & hse_arr = base_state[lev].const_array(mfi);
696  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
697  if (solverChoice.anelastic[lev] == 1) {
698  ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
699  p_arr(i,j,k) = hse_arr(i,j,k,1);
700  });
701  } else {
702  ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
703  p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp));
704  });
705  }
706  }
707  pres.FillBoundary(geom[lev].periodicity());
708 
709 #ifdef _OPENMP
710 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
711 #endif
712  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
713  {
714  // Now compute pressure gradient on valid box
715  const Box& bx = mfi.tilebox();
716  const Array4<Real>& derdat = mf[lev].array(mfi);
717  const Array4<Real> & p_arr = pres.array(mfi);
718 
719  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
720  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
721  const Array4<Real const>& z_cc = z_phys_cc[lev]->const_array(mfi);
722  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
723  {
724  // Pgrad at lower J face
725  Real gpy_lo = dxInv[1] * (p_arr(i,j,k) - p_arr(i,j-1,k));
726 
727  Real met_h_eta_hi = Compute_h_eta_AtCellCenter (i, j , k, dxInv, z_nd);
728  Real met_h_eta_lo = Compute_h_eta_AtCellCenter (i, j-1, k, dxInv, z_nd);
729 
730  Real dz_phys_hi, dz_phys_lo;
731  Real gpz_lo, gpz_hi;
732  if (k==klo) {
733  dz_phys_hi = z_cc(i,j ,k+1) - z_cc(i,j ,k );
734  dz_phys_lo = z_cc(i,j-1,k+1) - z_cc(i,j-1,k );
735  gpz_hi = (p_arr(i,j ,k+1) - p_arr(i,j ,k )) / dz_phys_hi;
736  gpz_lo = (p_arr(i,j-1,k+1) - p_arr(i,j-1,k )) / dz_phys_lo;
737  } else if (k==khi) {
738  dz_phys_hi = z_cc(i,j ,k ) - z_cc(i,j ,k-1);
739  dz_phys_lo = z_cc(i,j-1,k ) - z_cc(i,j-1,k-1);
740  gpz_hi = (p_arr(i,j ,k ) - p_arr(i,j ,k-1)) / dz_phys_hi;
741  gpz_lo = (p_arr(i,j-1,k ) - p_arr(i,j-1,k-1)) / dz_phys_lo;
742  } else {
743  dz_phys_hi = z_cc(i,j ,k+1) - z_cc(i,j ,k-1);
744  dz_phys_lo = z_cc(i,j-1,k+1) - z_cc(i,j-1,k-1);
745  gpz_hi = (p_arr(i,j ,k+1) - p_arr(i,j ,k-1)) / dz_phys_hi;
746  gpz_lo = (p_arr(i,j-1,k+1) - p_arr(i,j-1,k-1)) / dz_phys_lo;
747  }
748  Real gpy_metric = 0.5 * ( gpz_hi * met_h_eta_hi + gpz_lo * met_h_eta_lo );
749  gpy_lo -= gpy_metric;
750 
751  // Pgrad at higher J face
752  Real gpy_hi = dxInv[1] * (p_arr(i,j+1,k) - p_arr(i,j,k));
753 
754  met_h_eta_hi = Compute_h_eta_AtCellCenter (i, j+1, k, dxInv, z_nd);
755  met_h_eta_lo = Compute_h_eta_AtCellCenter (i, j , k, dxInv, z_nd);
756 
757  if (k==klo) {
758  dz_phys_hi = z_cc(i,j+1,k+1) - z_cc(i,j+1,k );
759  dz_phys_lo = z_cc(i,j ,k+1) - z_cc(i,j ,k );
760  gpz_hi = (p_arr(i,j+1,k+1) - p_arr(i,j+1,k )) / dz_phys_hi;
761  gpz_lo = (p_arr(i,j ,k+1) - p_arr(i,j ,k )) / dz_phys_lo;
762  } else if (k==khi) {
763  dz_phys_hi = z_cc(i,j+1,k ) - z_cc(i,j+1,k-1);
764  dz_phys_lo = z_cc(i,j ,k ) - z_cc(i,j ,k-1);
765  gpz_hi = (p_arr(i,j+1,k ) - p_arr(i,j+1,k-1)) / dz_phys_hi;
766  gpz_lo = (p_arr(i,j ,k ) - p_arr(i,j ,k-1)) / dz_phys_lo;
767  } else {
768  dz_phys_hi = z_cc(i,j+1,k+1) - z_cc(i,j+1,k-1);
769  dz_phys_lo = z_cc(i,j ,k+1) - z_cc(i,j ,k-1);
770  gpz_hi = (p_arr(i,j+1,k+1) - p_arr(i,j+1,k-1)) / dz_phys_hi;
771  gpz_lo = (p_arr(i,j ,k+1) - p_arr(i,j ,k-1)) / dz_phys_lo;
772  }
773  gpy_metric = 0.5 * ( gpz_hi * met_h_eta_hi + gpz_lo * met_h_eta_lo );
774  gpy_hi -= gpy_metric;
775 
776  derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_lo + gpy_hi);
777  });
778  } else {
779  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
780  derdat(i ,j ,k, mf_comp) = 0.5 * (p_arr(i,j+1,k) - p_arr(i,j-1,k)) * dxInv[1];
781  });
782  }
783  } // mf
784  mf_comp ++;
785  } // dpdy
786 
787  if (containerHasElement(plot_var_names, "pres_hse_x"))
788  {
789  auto dxInv = geom[lev].InvCellSizeArray();
790  if (z_phys_nd[lev]) {
791 #ifdef _OPENMP
792 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
793 #endif
794  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
795  {
796  const Box& bx = mfi.tilebox();
797  const Array4<Real >& derdat = mf[lev].array(mfi);
798  const Array4<Real const>& p_arr = p_hse.const_array(mfi);
799 
800  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
801 
802  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
803  Real met_h_xi_lo = Compute_h_xi_AtIface (i, j, k, dxInv, z_nd);
804  Real met_h_zeta_lo = Compute_h_zeta_AtIface(i, j, k, dxInv, z_nd);
805  Real gp_xi_lo = dxInv[0] * (p_arr(i,j,k) - p_arr(i-1,j,k));
806  Real gp_zeta_on_iface_lo;
807  if (k == klo) {
808  gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
809  p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
810  - p_arr(i-1,j,k ) - p_arr(i,j,k ) );
811  } else if (k == khi) {
812  gp_zeta_on_iface_lo = 0.5 * dxInv[2] * (
813  p_arr(i-1,j,k ) + p_arr(i,j,k )
814  - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
815  } else {
816  gp_zeta_on_iface_lo = 0.25 * dxInv[2] * (
817  p_arr(i-1,j,k+1) + p_arr(i,j,k+1)
818  - p_arr(i-1,j,k-1) - p_arr(i,j,k-1) );
819  }
820  Real gpx_lo = gp_xi_lo - (met_h_xi_lo/ met_h_zeta_lo) * gp_zeta_on_iface_lo;
821 
822  Real met_h_xi_hi = Compute_h_xi_AtIface (i+1, j, k, dxInv, z_nd);
823  Real met_h_zeta_hi = Compute_h_zeta_AtIface(i+1, j, k, dxInv, z_nd);
824  Real gp_xi_hi = dxInv[0] * (p_arr(i+1,j,k) - p_arr(i,j,k));
825  Real gp_zeta_on_iface_hi;
826  if (k == klo) {
827  gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
828  p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
829  - p_arr(i+1,j,k ) - p_arr(i,j,k ) );
830  } else if (k == khi) {
831  gp_zeta_on_iface_hi = 0.5 * dxInv[2] * (
832  p_arr(i+1,j,k ) + p_arr(i,j,k )
833  - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
834  } else {
835  gp_zeta_on_iface_hi = 0.25 * dxInv[2] * (
836  p_arr(i+1,j,k+1) + p_arr(i,j,k+1)
837  - p_arr(i+1,j,k-1) - p_arr(i,j,k-1) );
838  }
839  Real gpx_hi = gp_xi_hi - (met_h_xi_hi/ met_h_zeta_hi) * gp_zeta_on_iface_hi;
840 
841  derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_lo + gpx_hi);
842  });
843  } // mfi
844  } else {
845  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
846  {
847  const Box& bx = mfi.tilebox();
848  const Array4<Real >& derdat = mf[lev].array(mfi);
849  const Array4<Real const>& p_arr = p_hse.const_array(mfi);
850  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
851  derdat(i ,j ,k, mf_comp) = 0.5 * (p_arr(i+1,j,k) - p_arr(i-1,j,k)) * dxInv[0];
852  });
853  } // mfi
854  }
855  mf_comp += 1;
856  } // pres_hse_x
857 
858  if (containerHasElement(plot_var_names, "pres_hse_y"))
859  {
860  auto dxInv = geom[lev].InvCellSizeArray();
861  if (z_phys_nd[lev]) {
862 #ifdef _OPENMP
863 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
864 #endif
865  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
866  {
867  const Box& bx = mfi.tilebox();
868  const Array4<Real >& derdat = mf[lev].array(mfi);
869  const Array4<Real const>& p_arr = p_hse.const_array(mfi);
870  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
871  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
872  Real met_h_eta_lo = Compute_h_eta_AtJface (i, j, k, dxInv, z_nd);
873  Real met_h_zeta_lo = Compute_h_zeta_AtJface(i, j, k, dxInv, z_nd);
874  Real gp_eta_lo = dxInv[1] * (p_arr(i,j,k) - p_arr(i,j-1,k));
875  Real gp_zeta_on_jface_lo;
876  if (k == klo) {
877  gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
878  p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
879  - p_arr(i,j,k ) - p_arr(i,j-1,k ) );
880  } else if (k == khi) {
881  gp_zeta_on_jface_lo = 0.5 * dxInv[2] * (
882  p_arr(i,j,k ) + p_arr(i,j-1,k )
883  - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
884  } else {
885  gp_zeta_on_jface_lo = 0.25 * dxInv[2] * (
886  p_arr(i,j,k+1) + p_arr(i,j-1,k+1)
887  - p_arr(i,j,k-1) - p_arr(i,j-1,k-1) );
888  }
889  Real gpy_lo = gp_eta_lo - (met_h_eta_lo / met_h_zeta_lo) * gp_zeta_on_jface_lo;
890 
891  Real met_h_eta_hi = Compute_h_eta_AtJface (i, j+1, k, dxInv, z_nd);
892  Real met_h_zeta_hi = Compute_h_zeta_AtJface(i, j+1, k, dxInv, z_nd);
893  Real gp_eta_hi = dxInv[1] * (p_arr(i,j+1,k) - p_arr(i,j,k));
894  Real gp_zeta_on_jface_hi;
895  if (k == klo) {
896  gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
897  p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
898  - p_arr(i,j+1,k ) - p_arr(i,j,k ) );
899  } else if (k == khi) {
900  gp_zeta_on_jface_hi = 0.5 * dxInv[2] * (
901  p_arr(i,j+1,k ) + p_arr(i,j,k )
902  - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
903  } else {
904  gp_zeta_on_jface_hi = 0.25 * dxInv[2] * (
905  p_arr(i,j+1,k+1) + p_arr(i,j,k+1)
906  - p_arr(i,j+1,k-1) - p_arr(i,j,k-1) );
907  }
908  Real gpy_hi = gp_eta_hi - (met_h_eta_hi / met_h_zeta_hi) * gp_zeta_on_jface_hi;
909 
910  derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_lo + gpy_hi);
911  });
912  } // mfi
913  } else {
914  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
915  {
916  const Box& bx = mfi.tilebox();
917  const Array4<Real >& derdat = mf[lev].array(mfi);
918  const Array4<Real const>& p_arr = p_hse.const_array(mfi);
919  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
920  derdat(i ,j ,k, mf_comp) = 0.5 * (p_arr(i,j+1,k) - p_arr(i,j-1,k)) * dxInv[1];
921  });
922  } // mfi
923  }
924  mf_comp += 1;
925  } // pres_hse_y
926 
927  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
928  if (containerHasElement(plot_var_names, "z_phys"))
929  {
930  MultiFab::Copy(mf[lev],*z_phys_cc[lev],0,mf_comp,1,0);
931  mf_comp ++;
932  }
933 
934  if (containerHasElement(plot_var_names, "detJ"))
935  {
936  MultiFab::Copy(mf[lev],*detJ_cc[lev],0,mf_comp,1,0);
937  mf_comp ++;
938  }
939  } // use_terrain
940 
941  if (containerHasElement(plot_var_names, "mapfac")) {
942 #ifdef _OPENMP
943 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
944 #endif
945  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
946  {
947  const Box& bx = mfi.tilebox();
948  const Array4<Real>& derdat = mf[lev].array(mfi);
949  const Array4<Real>& mf_m = mapfac_m[lev]->array(mfi);
950  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
951  derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
952  });
953  }
954  mf_comp ++;
955  }
956 
957 #ifdef ERF_USE_NETCDF
959  if (containerHasElement(plot_var_names, "lat_m")) {
960 #ifdef _OPENMP
961 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
962 #endif
963  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
964  {
965  const Box& bx = mfi.tilebox();
966  const Array4<Real>& derdat = mf[lev].array(mfi);
967  const Array4<Real>& data = lat_m[lev]->array(mfi);
968  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
969  derdat(i, j, k, mf_comp) = data(i,j,0);
970  });
971  }
972  mf_comp ++;
973  } // lat_m
974  if (containerHasElement(plot_var_names, "lon_m")) {
975 #ifdef _OPENMP
976 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
977 #endif
978  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
979  {
980  const Box& bx = mfi.tilebox();
981  const Array4<Real>& derdat = mf[lev].array(mfi);
982  const Array4<Real>& data = lon_m[lev]->array(mfi);
983  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
984  derdat(i, j, k, mf_comp) = data(i,j,0);
985  });
986  }
987  mf_comp ++;
988  } // lon_m
989  } // use_real_bcs
990 #endif
991 
992 
994  if (containerHasElement(plot_var_names, "u_t_avg")) {
995 #ifdef _OPENMP
996 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
997 #endif
998  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
999  {
1000  const Box& bx = mfi.tilebox();
1001  const Array4<Real>& derdat = mf[lev].array(mfi);
1002  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
1003  const Real norm = t_avg_cnt[lev];
1004  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1005  {
1006  derdat(i ,j ,k, mf_comp) = data(i,j,k,0) / norm;
1007  });
1008  }
1009  mf_comp ++;
1010  }
1011 
1012  if (containerHasElement(plot_var_names, "v_t_avg")) {
1013 #ifdef _OPENMP
1014 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1015 #endif
1016  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1017  {
1018  const Box& bx = mfi.tilebox();
1019  const Array4<Real>& derdat = mf[lev].array(mfi);
1020  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
1021  const Real norm = t_avg_cnt[lev];
1022  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1023  {
1024  derdat(i ,j ,k, mf_comp) = data(i,j,k,1) / norm;
1025  });
1026  }
1027  mf_comp ++;
1028  }
1029 
1030  if (containerHasElement(plot_var_names, "w_t_avg")) {
1031 #ifdef _OPENMP
1032 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1033 #endif
1034  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1035  {
1036  const Box& bx = mfi.tilebox();
1037  const Array4<Real>& derdat = mf[lev].array(mfi);
1038  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
1039  const Real norm = t_avg_cnt[lev];
1040  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1041  {
1042  derdat(i ,j ,k, mf_comp) = data(i,j,k,2) / norm;
1043  });
1044  }
1045  mf_comp ++;
1046  }
1047 
1048  if (containerHasElement(plot_var_names, "umag_t_avg")) {
1049 #ifdef _OPENMP
1050 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1051 #endif
1052  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1053  {
1054  const Box& bx = mfi.tilebox();
1055  const Array4<Real>& derdat = mf[lev].array(mfi);
1056  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
1057  const Real norm = t_avg_cnt[lev];
1058  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1059  {
1060  derdat(i ,j ,k, mf_comp) = data(i,j,k,3) / norm;
1061  });
1062  }
1063  mf_comp ++;
1064  }
1065  }
1066 
1067  if (containerHasElement(plot_var_names, "nut")) {
1068  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
1069  MultiFab cmf(vars_new[lev][Vars::cons], make_alias, 0, 1); // to provide rho only
1070 #ifdef _OPENMP
1071 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1072 #endif
1073  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
1074  {
1075  const Box& bx = mfi.tilebox();
1076  auto prim = dmf[mfi].array();
1077  auto const cons = cmf[mfi].const_array();
1078  auto const diff = (*eddyDiffs_lev[lev])[mfi].const_array();
1079  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1080  {
1081  const Real rho = cons(i, j, k, Rho_comp);
1082  const Real Kmv = diff(i, j, k, EddyDiff::Mom_v);
1083  prim(i,j,k) = Kmv / rho;
1084  });
1085  }
1086 
1087  mf_comp++;
1088  }
1089 
1090  if (containerHasElement(plot_var_names, "Kmv")) {
1091  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_v,mf_comp,1,0);
1092  mf_comp ++;
1093  }
1094  if (containerHasElement(plot_var_names, "Kmh")) {
1095  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
1096  mf_comp ++;
1097  }
1098  if (containerHasElement(plot_var_names, "Khv")) {
1099  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_v,mf_comp,1,0);
1100  mf_comp ++;
1101  }
1102  if (containerHasElement(plot_var_names, "Khh")) {
1103  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
1104  mf_comp ++;
1105  }
1106  if (containerHasElement(plot_var_names, "Lturb")) {
1107  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Turb_lengthscale,mf_comp,1,0);
1108  mf_comp ++;
1109  }
1110  if (containerHasElement(plot_var_names, "walldist")) {
1111  MultiFab::Copy(mf[lev],*walldist[lev],0,mf_comp,1,0);
1112  mf_comp ++;
1113  }
1114  if (containerHasElement(plot_var_names, "diss")) {
1115  MultiFab::Copy(mf[lev],*SFS_diss_lev[lev],0,mf_comp,1,0);
1116  mf_comp ++;
1117  }
1118 
1119  // TODO: The size of the q variables can vary with different
1120  // moisture models. Therefore, certain components may
1121  // reside at different indices. For example, Kessler is
1122  // warm but precipitating. This puts qp at index 3.
1123  // However, SAM is cold and precipitating so qp is index 4.
1124  // Need to built an external enum struct or a better pathway.
1125 
1126  // NOTE: Protect against accessing non-existent data
1127  if (use_moisture) {
1128  int n_qstate = micro->Get_Qstate_Size();
1129 
1130  // Moist density
1131  if(containerHasElement(plot_var_names, "moist_density"))
1132  {
1133  int n_start = RhoQ1_comp; // qv
1134  int n_end = RhoQ2_comp; // qc
1135  if (n_qstate > 3) n_end = RhoQ3_comp; // qi
1136  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1137  for (int n_comp(n_start); n_comp <= n_end; ++n_comp) {
1138  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1139  }
1140  mf_comp += 1;
1141  }
1142 
1143  // Non-precipitating components
1144  //--------------------------------------------------------------------------
1145  if(containerHasElement(plot_var_names, "qt"))
1146  {
1147  int n_start = RhoQ1_comp; // qv
1148  int n_end = RhoQ2_comp; // qc
1149  if (n_qstate > 3) n_end = RhoQ3_comp; // qi
1150  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1151  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1152  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1153  }
1154  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1155  mf_comp += 1;
1156  }
1157 
1158  if(containerHasElement(plot_var_names, "qv") && (n_qstate >= 1))
1159  {
1160  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ1_comp, mf_comp, 1, 0);
1161  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1162  mf_comp += 1;
1163  }
1164 
1165  if(containerHasElement(plot_var_names, "qc") && (n_qstate >= 2))
1166  {
1167  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ2_comp, mf_comp, 1, 0);
1168  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1169  mf_comp += 1;
1170  }
1171 
1172  if(containerHasElement(plot_var_names, "qi") && (n_qstate >= 4))
1173  {
1174  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ3_comp, mf_comp, 1, 0);
1175  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1176  mf_comp += 1;
1177  }
1178 
1179  // Precipitating components
1180  //--------------------------------------------------------------------------
1181  if(containerHasElement(plot_var_names, "qp") && (n_qstate >= 3))
1182  {
1183  int n_start = (n_qstate > 3) ? RhoQ4_comp : RhoQ3_comp;
1184  int n_end = ncomp_cons - 1;
1185  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1186  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1187  MultiFab::Add( mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1188  }
1189  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1190  mf_comp += 1;
1191  }
1192 
1193  if(containerHasElement(plot_var_names, "qrain") && (n_qstate >= 3))
1194  {
1195  int n_start = (n_qstate > 3) ? RhoQ4_comp : RhoQ3_comp;
1196  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start , mf_comp, 1, 0);
1197  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1198  mf_comp += 1;
1199  }
1200 
1201  if(containerHasElement(plot_var_names, "qsnow") && (n_qstate >= 5))
1202  {
1203  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ5_comp, mf_comp, 1, 0);
1204  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1205  mf_comp += 1;
1206  }
1207 
1208  if(containerHasElement(plot_var_names, "qgraup") && (n_qstate >= 6))
1209  {
1210  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ6_comp, mf_comp, 1, 0);
1211  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1212  mf_comp += 1;
1213  }
1214 
1215  if (containerHasElement(plot_var_names, "qsat"))
1216  {
1217 #ifdef _OPENMP
1218 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1219 #endif
1220  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1221  {
1222  const Box& bx = mfi.tilebox();
1223  const Array4<Real>& derdat = mf[lev].array(mfi);
1224  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1225  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1226  {
1227  Real qv = S_arr(i,j,k,RhoQ1_comp) / S_arr(i,j,k,Rho_comp);
1228  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
1229  Real pressure = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp), qv) * Real(0.01);
1230  erf_qsatw(T, pressure, derdat(i,j,k,mf_comp));
1231  });
1232  }
1233  mf_comp ++;
1234  }
1235 
1236  if ( (solverChoice.moisture_type == MoistureType::Kessler) ||
1237  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
1238  (solverChoice.moisture_type == MoistureType::SAM_NoIce) )
1239  {
1240  int offset = (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ? 5 : 0;
1241  if (containerHasElement(plot_var_names, "rain_accum"))
1242  {
1243  MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
1244  mf_comp += 1;
1245  }
1246  }
1247  else if ( (solverChoice.moisture_type == MoistureType::SAM) ||
1248  (solverChoice.moisture_type == MoistureType::Morrison) )
1249  {
1250  int offset = (solverChoice.moisture_type == MoistureType::Morrison) ? 5 : 0;
1251  if (containerHasElement(plot_var_names, "rain_accum"))
1252  {
1253  MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
1254  mf_comp += 1;
1255  }
1256  if (containerHasElement(plot_var_names, "snow_accum"))
1257  {
1258  MultiFab::Copy(mf[lev],*(qmoist[lev][offset+1]),0,mf_comp,1,0);
1259  mf_comp += 1;
1260  }
1261  if (containerHasElement(plot_var_names, "graup_accum"))
1262  {
1263  MultiFab::Copy(mf[lev],*(qmoist[lev][offset+2]),0,mf_comp,1,0);
1264  mf_comp += 1;
1265  }
1266  }
1267  } // use_moisture
1268 
1269 #ifdef ERF_USE_PARTICLES
1270  const auto& particles_namelist( particleData.getNames() );
1271  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++) {
1272  if (containerHasElement(plot_var_names, std::string(particles_namelist[i]+"_count"))) {
1273  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
1274  temp_dat.setVal(0);
1275  particleData[particles_namelist[i]]->Increment(temp_dat, lev);
1276  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1277  mf_comp += 1;
1278  }
1279  }
1280 
1281  Vector<std::string> particle_mesh_plot_names(0);
1282  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
1283  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
1284  std::string plot_var_name(particle_mesh_plot_names[i]);
1285  if (containerHasElement(plot_var_names, plot_var_name) ) {
1286  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
1287  temp_dat.setVal(0);
1288  particleData.GetMeshPlotVar(plot_var_name, temp_dat, lev);
1289  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1290  mf_comp += 1;
1291  }
1292  }
1293 #endif
1294 
1295  if (containerHasElement(plot_var_names, "volfrac")) {
1296  if ( solverChoice.terrain_type == TerrainType::EB ||
1297  solverChoice.terrain_type == TerrainType::ImmersedForcing)
1298  {
1299  MultiFab::Copy(mf[lev], EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
1300  } else {
1301  mf[lev].setVal(1.0, mf_comp, 1, 0);
1302  }
1303  mf_comp += 1;
1304  }
1305 
1306 #ifdef ERF_COMPUTE_ERROR
1307  // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
1308  if (containerHasElement(plot_var_names, "xvel_err") ||
1309  containerHasElement(plot_var_names, "yvel_err") ||
1310  containerHasElement(plot_var_names, "zvel_err"))
1311  {
1312  //
1313  // Moving terrain ANALYTICAL
1314  //
1315  Real H = geom[lev].ProbHi()[2];
1316  Real Ampl = 0.16;
1317  Real wavelength = 100.;
1318  Real kp = 2. * PI / wavelength;
1319  Real g = CONST_GRAV;
1320  Real omega = std::sqrt(g * kp);
1321  Real omega_t = omega * t_new[lev];
1322 
1323  const auto dx = geom[lev].CellSizeArray();
1324 
1325 #ifdef _OPENMP
1326 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1327 #endif
1328  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1329  {
1330  const Box& bx = mfi.validbox();
1331  Box xbx(bx); xbx.surroundingNodes(0);
1332  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1333  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1334 
1335  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1336 
1337  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1338  {
1339  Real x = i * dx[0];
1340  Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1341 
1342  Real z_base = Ampl * std::sin(kp * x - omega_t);
1343  z -= z_base;
1344 
1345  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1346 
1347  xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
1348  });
1349 
1350  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1351  {
1352  Real x = (i + 0.5) * dx[0];
1353  Real z = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1354 
1355  Real z_base = Ampl * std::sin(kp * x - omega_t);
1356  z -= z_base;
1357 
1358  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1359 
1360  zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
1361  });
1362  }
1363 
1364  MultiFab temp_mf(mf[lev].boxArray(), mf[lev].DistributionMap(), AMREX_SPACEDIM, 0);
1365  average_face_to_cellcenter(temp_mf,0,
1366  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
1367 
1368  if (containerHasElement(plot_var_names, "xvel_err")) {
1369  MultiFab::Copy(mf[lev],temp_mf,0,mf_comp,1,0);
1370  mf_comp += 1;
1371  }
1372  if (containerHasElement(plot_var_names, "yvel_err")) {
1373  MultiFab::Copy(mf[lev],temp_mf,1,mf_comp,1,0);
1374  mf_comp += 1;
1375  }
1376  if (containerHasElement(plot_var_names, "zvel_err")) {
1377  MultiFab::Copy(mf[lev],temp_mf,2,mf_comp,1,0);
1378  mf_comp += 1;
1379  }
1380 
1381  // Now restore the velocities to what they were
1382 #ifdef _OPENMP
1383 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1384 #endif
1385  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1386  {
1387  const Box& bx = mfi.validbox();
1388  Box xbx(bx); xbx.surroundingNodes(0);
1389 
1390  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1391  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1392 
1393  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1394 
1395  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1396  {
1397  Real x = i * dx[0];
1398  Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1399  Real z_base = Ampl * std::sin(kp * x - omega_t);
1400 
1401  z -= z_base;
1402 
1403  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1404  xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
1405  });
1406  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1407  {
1408  Real x = (i + 0.5) * dx[0];
1409  Real z = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1410  Real z_base = Ampl * std::sin(kp * x - omega_t);
1411 
1412  z -= z_base;
1413  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1414 
1415  zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
1416  });
1417  }
1418  } // end xvel_err, yvel_err, zvel_err
1419 
1420  if (containerHasElement(plot_var_names, "pp_err"))
1421  {
1422  // Moving terrain ANALYTICAL
1423 #ifdef _OPENMP
1424 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1425 #endif
1426  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1427  {
1428  const Box& bx = mfi.tilebox();
1429  const Array4<Real>& derdat = mf[lev].array(mfi);
1430  const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
1431  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1432 
1433  const auto dx = geom[lev].CellSizeArray();
1434  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1435  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
1436 
1437  Real H = geom[lev].ProbHi()[2];
1438  Real Ampl = 0.16;
1439  Real wavelength = 100.;
1440  Real kp = 2. * PI / wavelength;
1441  Real g = CONST_GRAV;
1442  Real omega = std::sqrt(g * kp);
1443  Real omega_t = omega * t_new[lev];
1444 
1445  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1446  {
1447  const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
1448  derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta) - p0_arr(i,j,k);
1449 
1450  Real rho_hse = r0_arr(i,j,k);
1451 
1452  Real x = (i + 0.5) * dx[0];
1453  Real z = 0.125 * ( z_nd(i,j,k ) + z_nd(i+1,j,k ) + z_nd(i,j+1,k ) + z_nd(i+1,j+1,k )
1454  +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
1455  Real z_base = Ampl * std::sin(kp * x - omega_t);
1456 
1457  z -= z_base;
1458  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1459  Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
1460  std::sin(kp * x - omega_t) * r0_arr(i,j,k);
1461 
1462  derdat(i,j,k,mf_comp) -= pprime_exact;
1463  });
1464  }
1465  mf_comp += 1;
1466  }
1467 #endif
1468 
1469 #ifdef ERF_USE_RRTMGP
1470  if (containerHasElement(plot_var_names, "qsrc_sw")) {
1471  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 0, mf_comp, 1, 0);
1472  mf_comp += 1;
1473  }
1474  if (containerHasElement(plot_var_names, "qsrc_lw")) {
1475  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 1, mf_comp, 1, 0);
1476  mf_comp += 1;
1477  }
1478 #endif
1479  }
1480 
1481  if (solverChoice.terrain_type == TerrainType::EB)
1482  {
1483  for (int lev = 0; lev <= finest_level; ++lev) {
1484  EB_set_covered(mf[lev], 0.0);
1485  }
1486  }
1487 
1488  // Fill terrain distortion MF (nu_nd)
1489  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1490  for (int lev(0); lev <= finest_level; ++lev) {
1491  MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
1492  Real dz = Geom()[lev].CellSizeArray()[2];
1493  for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1494  const Box& bx = mfi.tilebox();
1495  Array4<Real> mf_arr = mf_nd[lev].array(mfi);
1496  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1497  {
1498  mf_arr(i,j,k,2) -= k * dz;
1499  });
1500  }
1501  }
1502  }
1503 
1504  std::string plotfilename;
1505  std::string plotfilenameU;
1506  std::string plotfilenameV;
1507  std::string plotfilenameW;
1508  if (which == 1) {
1509  plotfilename = Concatenate(plot_file_1, istep[0], 5);
1510  plotfilenameU = Concatenate(plot_file_1+"U", istep[0], 5);
1511  plotfilenameV = Concatenate(plot_file_1+"V", istep[0], 5);
1512  plotfilenameW = Concatenate(plot_file_1+"W", istep[0], 5);
1513  } else if (which == 2) {
1514  plotfilename = Concatenate(plot_file_2, istep[0], 5);
1515  plotfilenameU = Concatenate(plot_file_2+"U", istep[0], 5);
1516  plotfilenameV = Concatenate(plot_file_2+"V", istep[0], 5);
1517  plotfilenameW = Concatenate(plot_file_2+"W", istep[0], 5);
1518  }
1519 
1520  // LSM writes it's own data
1521  if (which==1 && plot_lsm) {
1522  lsm.Plot_Lsm_Data(t_new[0], istep, refRatio());
1523  }
1524 
1525 #ifdef ERF_USE_RRTMGP
1526  /*
1527  // write additional RRTMGP data
1528  // TODO: currently single level only
1529  if (which==1 && plot_rad) {
1530  rad[0]->writePlotfile(plot_file_1, t_new[0], istep[0]);
1531  }
1532  */
1533 #endif
1534 
1535  // Single level
1536  if (finest_level == 0)
1537  {
1538  if (plotfile_type == PlotFileType::Amrex)
1539  {
1540  Print() << "Writing native plotfile " << plotfilename << "\n";
1541  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1542  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1543  GetVecOfConstPtrs(mf),
1544  GetVecOfConstPtrs(mf_nd),
1545  varnames,
1546  Geom(), t_new[0], istep, refRatio());
1547  } else {
1548  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1549  GetVecOfConstPtrs(mf),
1550  varnames,
1551  Geom(), t_new[0], istep, refRatio());
1552  }
1553  writeJobInfo(plotfilename);
1554 
1555  if (m_plot_face_vels) {
1556  Print() << "Writing face velocities" << std::endl;
1557  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1558  GetVecOfConstPtrs(mf_u),
1559  {"x_velocity_stag"},
1560  Geom(), t_new[0], istep, refRatio());
1561  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1562  GetVecOfConstPtrs(mf_v),
1563  {"y_velocity_stag"},
1564  Geom(), t_new[0], istep, refRatio());
1565  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1566  GetVecOfConstPtrs(mf_w),
1567  {"z_velocity_stag"},
1568  Geom(), t_new[0], istep, refRatio());
1569  }
1570 
1571 #ifdef ERF_USE_PARTICLES
1572  particleData.writePlotFile(plotfilename);
1573 #endif
1574 #ifdef ERF_USE_NETCDF
1575  } else if (plotfile_type == PlotFileType::Netcdf) {
1576  int lev = 0;
1577  int l_which = 0;
1578  writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep, t_new[0]);
1579 #endif
1580  } else {
1581  // Here we assume the plotfile_type is PlotFileType::None
1582  Print() << "Writing no plotfile since plotfile_type is none" << std::endl;
1583  }
1584 
1585  } else { // Multilevel
1586 
1587  if (plotfile_type == PlotFileType::Amrex) {
1588 
1589  int lev0 = 0;
1590  int desired_ratio = std::max(std::max(ref_ratio[lev0][0],ref_ratio[lev0][1]),ref_ratio[lev0][2]);
1591  bool any_ratio_one = ( ( (ref_ratio[lev0][0] == 1) || (ref_ratio[lev0][1] == 1) ) ||
1592  (ref_ratio[lev0][2] == 1) );
1593  for (int lev = 1; lev < finest_level; lev++) {
1594  any_ratio_one = any_ratio_one ||
1595  ( ( (ref_ratio[lev][0] == 1) || (ref_ratio[lev][1] == 1) ) ||
1596  (ref_ratio[lev][2] == 1) );
1597  }
1598 
1599  if (any_ratio_one && m_expand_plotvars_to_unif_rr)
1600  {
1601  Vector<IntVect> r2(finest_level);
1602  Vector<Geometry> g2(finest_level+1);
1603  Vector<MultiFab> mf2(finest_level+1);
1604 
1605  mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
1606 
1607  // Copy level 0 as is
1608  MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
1609 
1610  // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
1611  Array<int,AMREX_SPACEDIM> periodicity =
1612  {Geom()[lev0].isPeriodic(0),Geom()[lev0].isPeriodic(1),Geom()[lev0].isPeriodic(2)};
1613  g2[lev0].define(Geom()[lev0].Domain(),&(Geom()[lev0].ProbDomain()),0,periodicity.data());
1614 
1615  r2[0] = IntVect(desired_ratio/ref_ratio[lev0][0],
1616  desired_ratio/ref_ratio[lev0][1],
1617  desired_ratio/ref_ratio[lev0][2]);
1618 
1619  for (int lev = 1; lev <= finest_level; ++lev) {
1620  if (lev > 1) {
1621  r2[lev-1][0] = r2[lev-2][0] * desired_ratio / ref_ratio[lev-1][0];
1622  r2[lev-1][1] = r2[lev-2][1] * desired_ratio / ref_ratio[lev-1][1];
1623  r2[lev-1][2] = r2[lev-2][2] * desired_ratio / ref_ratio[lev-1][2];
1624  }
1625 
1626  mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
1627 
1628  // Set the new problem domain
1629  Box d2(Geom()[lev].Domain());
1630  d2.refine(r2[lev-1]);
1631 
1632  g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
1633  }
1634 
1635  //
1636  // We need to make a temporary that is the size of ncomp_mf
1637  // in order to not get an out of bounds error
1638  // even though the values will not be used
1639  //
1640  Vector<BCRec> temp_domain_bcs_type;
1641  temp_domain_bcs_type.resize(ncomp_mf);
1642 
1643  //
1644  // Do piecewise constant interpolation of mf into mf2
1645  //
1646  for (int lev = 1; lev <= finest_level; ++lev) {
1647  Interpolater* mapper_c = &pc_interp;
1648  InterpFromCoarseLevel(mf2[lev], t_new[lev], mf[lev],
1649  0, 0, ncomp_mf,
1650  geom[lev], g2[lev],
1652  r2[lev-1], mapper_c, temp_domain_bcs_type, 0);
1653  }
1654 
1655  // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
1656  Vector<IntVect> rr(finest_level);
1657  for (int lev = 0; lev < finest_level; ++lev) {
1658  rr[lev] = IntVect(desired_ratio);
1659  }
1660 
1661  Print() << "Writing plotfile " << plotfilename << "\n";
1662  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1663  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1664  GetVecOfConstPtrs(mf2),
1665  GetVecOfConstPtrs(mf_nd),
1666  varnames,
1667  g2, t_new[0], istep, rr);
1668  } else {
1669  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1670  GetVecOfConstPtrs(mf2), varnames,
1671  g2, t_new[0], istep, rr);
1672  }
1673 
1674  } else {
1675  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1676  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1677  GetVecOfConstPtrs(mf),
1678  GetVecOfConstPtrs(mf_nd),
1679  varnames,
1680  geom, t_new[0], istep, ref_ratio);
1681  } else {
1682  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1683  GetVecOfConstPtrs(mf), varnames,
1684  geom, t_new[0], istep, ref_ratio);
1685  }
1686  if (m_plot_face_vels) {
1687  Print() << "Writing face velocities" << std::endl;
1688  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1689  GetVecOfConstPtrs(mf_u),
1690  {"x_velocity_stag"},
1691  geom, t_new[0], istep, ref_ratio);
1692  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1693  GetVecOfConstPtrs(mf_v),
1694  {"y_velocity_stag"},
1695  geom, t_new[0], istep, ref_ratio);
1696  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1697  GetVecOfConstPtrs(mf_w),
1698  {"z_velocity_stag"},
1699  geom, t_new[0], istep, ref_ratio);
1700  }
1701  } // ref_ratio test
1702 
1703  writeJobInfo(plotfilename);
1704 
1705 #ifdef ERF_USE_PARTICLES
1706  particleData.writePlotFile(plotfilename);
1707 #endif
1708 
1709 #ifdef ERF_USE_NETCDF
1710  } else if (plotfile_type == PlotFileType::Netcdf) {
1711  for (int lev = 0; lev <= finest_level; ++lev) {
1712  for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
1713  writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep, t_new[0]);
1714  }
1715  }
1716 #endif
1717  }
1718  } // end multi-level
1719 }
constexpr amrex::Real PI
Definition: ERF_Constants.H:6
constexpr amrex::Real Cp_l
Definition: ERF_Constants.H:14
#define RhoQ4_comp
Definition: ERF_IndexDefines.H:45
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real erf_esatw(amrex::Real t)
Definition: ERF_MicrophysicsUtils.H:64
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void erf_qsatw(amrex::Real t, amrex::Real p, amrex::Real &qsatw)
Definition: ERF_MicrophysicsUtils.H:158
PhysBCFunctNoOp null_bc_for_fill
Definition: ERF_Plotfile.cpp:10
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_xi_AtIface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:110
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_eta_AtCellCenter(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:77
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_zeta_AtIface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:96
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_zeta_AtJface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:139
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_xi_AtCellCenter(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:62
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real Compute_h_eta_AtJface(const int &i, const int &j, const int &k, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &cellSizeInv, const amrex::Array4< const amrex::Real > &z_nd)
Definition: ERF_TerrainMetrics.H:167
static amrex::Vector< std::string > PlotFileVarNames(amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:219
void WriteMultiLevelPlotfileWithTerrain(const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
Definition: ERF_Plotfile.cpp:1722
void writeJobInfo(const std::string &dir) const
Definition: ERF_WriteJobInfo.cpp:9
void Plot_Lsm_Data(amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &ref_ratio)
Definition: ERF_LandSurface.H:98
@ Turb_lengthscale
Definition: ERF_IndexDefines.H:172
@ Mom_h
Definition: ERF_IndexDefines.H:162
@ Theta_h
Definition: ERF_IndexDefines.H:163
void erf_dervortx(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:200
void erf_dervorty(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:228
void erf_dermagvel(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:319
void erf_dernull(const Box &, FArrayBox &, int, int, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:39
void erf_dertemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:91
void erf_derKE(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:186
void erf_dermoisttemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:113
void erf_dersoundspeed(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:58
real(c_double), parameter g
Definition: ERF_module_model_constants.F90:19
Here is the call graph for this function:

◆ WriteSubvolume()

void ERF::WriteSubvolume ( )
10 {
11  ParmParse pp("erf.subvol");
12 
13  Vector<Real> origin;
14  Vector< int> ncell;
15  Vector<Real> delta;
16 
17  // **************************************************************
18  // Read in the origin, number of cells in each dir, and resolution
19  // **************************************************************
20  pp.getarr("origin",origin,0,AMREX_SPACEDIM);
21  pp.getarr("nxnynz", ncell,0,AMREX_SPACEDIM);
22  pp.getarr("dxdydz", delta,0,AMREX_SPACEDIM);
23 
24  int lev_for_sub = 0;
25 
26  bool found = false;
27  for (int i = 0; i <= finest_level; i++) {
28  if (!found) {
29  if (almostEqual(delta[0],geom[i].CellSize(0)) &&
30  almostEqual(delta[1],geom[i].CellSize(1)) &&
31  almostEqual(delta[2],geom[i].CellSize(2)) ) {
32 
33  // amrex::Print() << "XDIR " << delta[0] << " " << geom[i].CellSize(0) << std::endl;
34  // amrex::Print() << "YDIR " << delta[1] << " " << geom[i].CellSize(1) << std::endl;
35  // amrex::Print() << "ZDIR " << delta[2] << " " << geom[i].CellSize(2) << std::endl;
36  amrex::Print() << "Resolution specified matches that of level " << i << std::endl;
37  found = true;
38  lev_for_sub = i;
39  }
40  }
41  }
42 
43  if (!found) {
44  amrex::Abort("Resolution specified for subvol does not match the resolution of any of the levels.");
45  }
46 
47  // **************************************************************
48  // Now that we know which level we're at, we can figure out which (i,j,k) the origin corresponds to
49  // Note we use 1.0001 as a fudge factor since the division of two reals --> integer will do a floor
50  // **************************************************************
51  int i0 = static_cast<int>((origin[0] - geom[lev_for_sub].ProbLo(0)) * 1.0001 / delta[0]);
52  int j0 = static_cast<int>((origin[1] - geom[lev_for_sub].ProbLo(1)) * 1.0001 / delta[1]);
53  int k0 = static_cast<int>((origin[2] - geom[lev_for_sub].ProbLo(2)) * 1.0001 / delta[2]);
54 
55  found = false;
56  if (almostEqual(geom[lev_for_sub].ProbLo(0)+i0*delta[0],origin[0]) &&
57  almostEqual(geom[lev_for_sub].ProbLo(1)+j0*delta[1],origin[1]) &&
58  almostEqual(geom[lev_for_sub].ProbLo(2)+k0*delta[2],origin[2]) )
59  {
60  amrex::Print() << "Specified origin is the lower left corner of cell " << IntVect(i0,j0,k0) << std::endl;
61  found = true;
62  }
63 
64  if (!found) {
65  amrex::Abort("Origin specified does not correspond to a node at this level.");
66  }
67 
68  Box domain(geom[lev_for_sub].Domain());
69 
70  Box bx(IntVect(i0,j0,k0),IntVect(i0+ncell[0]-1,j0+ncell[1]-1,k0+ncell[2]-1));
71  amrex::Print() << "Box requested is " << bx << std::endl;
72 
73  if (!domain.contains(bx))
74  {
75  amrex::Abort("Box requested is larger than the existing domain");
76  }
77 
78  Vector<int> cs(3);
79  int count = pp.countval("chunk_size");
80  if (count > 0) {
81  pp.queryarr("chunk_size",cs,0,AMREX_SPACEDIM);
82  } else {
83  cs[0] = max_grid_size[0][0];
84  cs[1] = max_grid_size[0][1];
85  cs[2] = max_grid_size[0][2];
86  }
87  IntVect chunk_size(cs[0],cs[1],cs[2]);
88 
89  BoxArray ba(bx);
90  ba.maxSize(chunk_size);
91 
92  amrex::Print() << "BoxArray is " << ba << std::endl;
93 
94  int ncomp_mf = AMREX_SPACEDIM;
95 
96  DistributionMapping dm(ba);
97 
98  MultiFab mf(ba, dm, ncomp_mf, 0);
99 
100  MultiFab mf_cc_vel(grids[lev_for_sub], dmap[lev_for_sub], ncomp_mf, 0);
101  average_face_to_cellcenter(mf_cc_vel,0,
102  Array<const MultiFab*,3>{&vars_new[lev_for_sub][Vars::xvel],
103  &vars_new[lev_for_sub][Vars::yvel],
104  &vars_new[lev_for_sub][Vars::zvel]});
105 
106  mf.ParallelCopy(mf_cc_vel,0,0,AMREX_SPACEDIM,0,0);
107 
108  std::string subvol_filename = Concatenate(subvol_file, istep[0], 5);
109 
110  Vector<std::string> varnames;
111  varnames.push_back("x_velocity");
112  varnames.push_back("y_velocity");
113  varnames.push_back("z_velocity");
114 
115  Real time = t_new[lev_for_sub];
116 
117  amrex::Print() <<"Writing subvolume into " << subvol_filename << std::endl;
118  WriteSingleLevelPlotfile(subvol_filename,mf,varnames,geom[lev_for_sub],time,istep[0]);
119 }
real(c_double), private cs
Definition: ERF_module_mp_morr_two_moment.F90:202
Here is the call graph for this function:

◆ WriteVTKPolyline()

void ERF::WriteVTKPolyline ( const std::string &  filename,
amrex::Vector< std::array< amrex::Real, 2 >> &  points_xy 
)
595 {
596  std::ofstream vtkfile(filename);
597  if (!vtkfile.is_open()) {
598  std::cerr << "Error: Cannot open file " << filename << std::endl;
599  return;
600  }
601 
602  int num_points = points_xy.size();
603  if (num_points == 0) {
604  vtkfile << "# vtk DataFile Version 3.0\n";
605  vtkfile << "Hurricane Track\n";
606  vtkfile << "ASCII\n";
607  vtkfile << "DATASET POLYDATA\n";
608  vtkfile << "POINTS " << num_points << " float\n";
609  vtkfile.close();
610  return;
611  }
612  if (num_points < 2) {
613  points_xy.push_back(points_xy[0]);
614  }
615  num_points = points_xy.size();
616 
617  vtkfile << "# vtk DataFile Version 3.0\n";
618  vtkfile << "Hurricane Track\n";
619  vtkfile << "ASCII\n";
620  vtkfile << "DATASET POLYDATA\n";
621 
622  // Write points (Z=0 assumed)
623  vtkfile << "POINTS " << num_points << " float\n";
624  for (const auto& pt : points_xy) {
625  vtkfile << pt[0] << " " << pt[1] << " 10000.0\n";
626  }
627 
628  // Write polyline connectivity
629  vtkfile << "LINES 1 " << num_points + 1 << "\n";
630  vtkfile << num_points << " ";
631  for (int i = 0; i < num_points; ++i) {
632  vtkfile << i << " ";
633  }
634  vtkfile << "\n";
635 
636  vtkfile.close();
637 }

Member Data Documentation

◆ advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg
private

Referenced by getAdvFluxReg().

◆ ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax
private

◆ ax_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_new
private

◆ ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src
private

◆ ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay
private

◆ ay_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_new
private

◆ ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src
private

◆ az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az
private

◆ az_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_new
private

◆ az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src
private

◆ ba1d

amrex::Vector<amrex::BoxArray> ERF::ba1d
private

◆ ba2d

amrex::Vector<amrex::BoxArray> ERF::ba2d
private

◆ base_state

amrex::Vector<amrex::MultiFab> ERF::base_state
private

◆ base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new
private

◆ bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1
staticprivate

◆ bndry_output_planes_per

Real ERF::bndry_output_planes_per = -1.0
staticprivate

◆ bndry_output_planes_start_time

Real ERF::bndry_output_planes_start_time = 0.0
staticprivate

◆ boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level
private

◆ cf_set_width

int ERF::cf_set_width {0}
private

◆ cf_width

int ERF::cf_width {0}
private

◆ cfl

Real ERF::cfl = 0.8
staticprivate

◆ change_max

Real ERF::change_max = 1.1
staticprivate

◆ check_file

std::string ERF::check_file {"chk"}
private

◆ column_file_name

std::string ERF::column_file_name = "column_data.nc"
staticprivate

◆ column_interval

int ERF::column_interval = -1
staticprivate

◆ column_loc_x

Real ERF::column_loc_x = 0.0
staticprivate

◆ column_loc_y

Real ERF::column_loc_y = 0.0
staticprivate

◆ column_per

Real ERF::column_per = -1.0
staticprivate

◆ cons_names

const amrex::Vector<std::string> ERF::cons_names
private
Initial value:
{"density", "rhotheta", "rhoKE", "rhoadv_0",
"rhoQ1", "rhoQ2", "rhoQ3",
"rhoQ4", "rhoQ5", "rhoQ6"}

◆ cosPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::cosPhi_m
private

◆ d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density
private

◆ d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure
private

◆ d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc
private

◆ d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv
private

◆ d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature
private

◆ d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs
private

◆ d_rhoqt_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhoqt_src
private

◆ d_rhotheta_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhotheta_src
private

◆ d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs
private

◆ d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos
private

◆ d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos
private

◆ d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid
private

◆ data_sampler

std::unique_ptr<SampleData> ERF::data_sampler = nullptr
private

◆ datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog
private

◆ datalogname

amrex::Vector<std::string> ERF::datalogname
private

Referenced by DataLogName().

◆ datetime_format

const std::string ERF::datetime_format = "%Y-%m-%d %H:%M:%S"
private

◆ datprecision

const int ERF::datprecision = 6
private

◆ datwidth

const int ERF::datwidth = 14
private

◆ der_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::der_datalog
private

◆ der_datalogname

amrex::Vector<std::string> ERF::der_datalogname
private

Referenced by DerDataLogName().

◆ derived_names

const amrex::Vector<std::string> ERF::derived_names
private

◆ destag_profiles

bool ERF::destag_profiles = true
private

◆ detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc
private

◆ detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new
private

◆ detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src
private

◆ domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type
private

◆ domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type
private

◆ domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d
private

◆ dt

amrex::Vector<amrex::Real> ERF::dt
private

◆ dt_max

Real ERF::dt_max = 1e9
staticprivate

◆ dt_max_initial

Real ERF::dt_max_initial = 2.0e100
staticprivate

◆ dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio
private

◆ dz_min

amrex::Vector<amrex::Real> ERF::dz_min
private

◆ eb

amrex::Vector<std::unique_ptr<eb_> > ERF::eb
private

Referenced by EBFactory(), and get_eb().

◆ eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev
private

◆ fine_mask

amrex::MultiFab ERF::fine_mask
private

◆ finished_wave

bool ERF::finished_wave = false
private

◆ fixed_dt

amrex::Vector<amrex::Real> ERF::fixed_dt
private

◆ fixed_fast_dt

amrex::Vector<amrex::Real> ERF::fixed_fast_dt
private

◆ fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0
staticprivate

◆ FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c
private

◆ FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u
private

◆ FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v
private

◆ FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w
private

◆ h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density
private

◆ h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure
private

◆ h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc
private

◆ h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv
private

◆ h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature
private

◆ h_rayleigh_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs
private

◆ h_rhoqt_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhoqt_src
private

◆ h_rhotheta_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhotheta_src
private

◆ h_sponge_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs
private

◆ h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos
private

◆ h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos
private

◆ h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid
private

◆ hurricane_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_track_xy

◆ Hwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave
private

◆ Hwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave_onegrid
private

◆ init_shrink

Real ERF::init_shrink = 1.0
staticprivate

◆ init_sounding_ideal

bool ERF::init_sounding_ideal = false
staticprivate

◆ input_bndry_planes

int ERF::input_bndry_planes = 0
staticprivate

◆ input_sounding_data

InputSoundingData ERF::input_sounding_data
private

◆ input_sponge_data

InputSpongeData ERF::input_sponge_data
private

◆ interpolation_type

StateInterpType ERF::interpolation_type
staticprivate

◆ istep

amrex::Vector<int> ERF::istep
private

◆ last_check_file_step

int ERF::last_check_file_step
private

◆ last_plot_file_step_1

int ERF::last_plot_file_step_1
private

◆ last_plot_file_step_2

int ERF::last_plot_file_step_2
private

◆ last_subvol

int ERF::last_subvol
private

◆ lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev
private

◆ lsm

LandSurface ERF::lsm
private

◆ lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data
private

◆ lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux
private

◆ Lwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave
private

◆ Lwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave_onegrid
private

◆ m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_extdir_vals
private

◆ m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_neumann_vals
private

◆ m_check_int

int ERF::m_check_int = -1
private

◆ m_check_per

amrex::Real ERF::m_check_per = -1.0
private

◆ m_expand_plotvars_to_unif_rr

bool ERF::m_expand_plotvars_to_unif_rr = false
private

◆ m_forest_drag

amrex::Vector<std::unique_ptr<ForestDrag> > ERF::m_forest_drag
private

◆ m_plot_face_vels

bool ERF::m_plot_face_vels = false
private

◆ m_plot_int_1

int ERF::m_plot_int_1 = -1
private

◆ m_plot_int_2

int ERF::m_plot_int_2 = -1
private

◆ m_plot_per_1

amrex::Real ERF::m_plot_per_1 = -1.0
private

◆ m_plot_per_2

amrex::Real ERF::m_plot_per_2 = -1.0
private

◆ m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr
private

◆ m_subvol_int

int ERF::m_subvol_int = -1
private

◆ m_subvol_per

amrex::Real ERF::m_subvol_per = -1.0
private

◆ m_SurfaceLayer

std::unique_ptr<SurfaceLayer> ERF::m_SurfaceLayer = nullptr
private

◆ m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr
private

◆ mapfac_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mapfac_m
private

◆ mapfac_u

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mapfac_u
private

◆ mapfac_v

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mapfac_v
private

◆ max_step

int ERF::max_step = std::numeric_limits<int>::max()
private

◆ metgrid_basic_linear

bool ERF::metgrid_basic_linear {false}
private

◆ metgrid_debug_dry

bool ERF::metgrid_debug_dry {false}
private

◆ metgrid_debug_isothermal

bool ERF::metgrid_debug_isothermal {false}
private

◆ metgrid_debug_msf

bool ERF::metgrid_debug_msf {false}
private

◆ metgrid_debug_psfc

bool ERF::metgrid_debug_psfc {false}
private

◆ metgrid_debug_quiescent

bool ERF::metgrid_debug_quiescent {false}
private

◆ metgrid_force_sfc_k

int ERF::metgrid_force_sfc_k {6}
private

◆ metgrid_interp_theta

bool ERF::metgrid_interp_theta {false}
private

◆ metgrid_order

int ERF::metgrid_order {2}
private

◆ metgrid_proximity

amrex::Real ERF::metgrid_proximity {500.0}
private

◆ metgrid_retain_sfc

bool ERF::metgrid_retain_sfc {false}
private

◆ metgrid_use_below_sfc

bool ERF::metgrid_use_below_sfc {true}
private

◆ metgrid_use_sfc

bool ERF::metgrid_use_sfc {true}
private

◆ mf_C1H

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_C1H
private

◆ mf_C2H

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_C2H
private

◆ mf_MUB

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_MUB
private

◆ mg_verbose

int ERF::mg_verbose = 0
staticprivate

◆ micro

std::unique_ptr<Microphysics> ERF::micro
private

◆ mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem
private

◆ nc_bdy_file

std::string ERF::nc_bdy_file
staticprivate

◆ nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}
staticprivate

◆ nc_low_file

std::string ERF::nc_low_file
staticprivate

◆ ng_dens_hse

int ERF::ng_dens_hse
staticprivate

◆ ng_pres_hse

int ERF::ng_pres_hse
staticprivate

◆ nsubsteps

amrex::Vector<int> ERF::nsubsteps
private

◆ num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level
private

◆ num_files_at_level

amrex::Vector<int> ERF::num_files_at_level
private

◆ output_1d_column

int ERF::output_1d_column = 0
staticprivate

◆ output_bndry_planes

int ERF::output_bndry_planes = 0
staticprivate

◆ pert_interval

int ERF::pert_interval = -1
staticprivate

◆ phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type
private

◆ physbcs_base

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_base> > ERF::physbcs_base
private

◆ physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons
private

◆ physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u
private

◆ physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v
private

◆ physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w
private

◆ plot_file_1

std::string ERF::plot_file_1 {"plt_1_"}
private

◆ plot_file_2

std::string ERF::plot_file_2 {"plt_2_"}
private

◆ plot_file_on_restart

int ERF::plot_file_on_restart = 1
private

◆ plot_lsm

bool ERF::plot_lsm = false
private

◆ plot_var_names_1

amrex::Vector<std::string> ERF::plot_var_names_1
private

◆ plot_var_names_2

amrex::Vector<std::string> ERF::plot_var_names_2
private

◆ plotfile_type_1

PlotFileType ERF::plotfile_type_1 = PlotFileType::None
staticprivate

◆ plotfile_type_2

PlotFileType ERF::plotfile_type_2 = PlotFileType::None
staticprivate

◆ pp_inc

amrex::Vector<amrex::MultiFab> ERF::pp_inc
private

◆ pp_prefix

std::string ERF::pp_prefix {"erf"}

◆ previousCPUTimeUsed

Real ERF::previousCPUTimeUsed = 0.0
staticprivate

Referenced by getCPUTime().

◆ prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr
private

◆ profile_int

int ERF::profile_int = -1
private

◆ qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist
private

◆ Qr_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qr_prim
private

◆ Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim
private

◆ real_set_width

int ERF::real_set_width {0}
private

◆ real_width

int ERF::real_width {0}
private

◆ ref_tags

Vector< AMRErrorTag > ERF::ref_tags
staticprivate

◆ regrid_int

int ERF::regrid_int = -1
private

◆ regrid_level_0_on_restart

bool ERF::regrid_level_0_on_restart = false
private

◆ restart_chkfile

std::string ERF::restart_chkfile = ""
private

◆ rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new
private

◆ rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old
private

◆ rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new
private

◆ rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old
private

◆ rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new
private

◆ rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old
private

◆ sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline
private

Referenced by NumSampleLines(), and SampleLine().

◆ samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog
private

◆ samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname
private

Referenced by SampleLineLogName().

◆ samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint
private

Referenced by NumSamplePoints(), and SamplePoint().

◆ sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog
private

◆ sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname
private

Referenced by SamplePointLogName().

◆ sampler_interval

int ERF::sampler_interval = -1
private

◆ sampler_per

amrex::Real ERF::sampler_per = -1.0
private

◆ SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev
private

◆ SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev
private

◆ SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev
private

◆ SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev
private

◆ SFS_q1fx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx1_lev
private

◆ SFS_q1fx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx2_lev
private

◆ SFS_q1fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx3_lev
private

◆ SFS_q2fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q2fx3_lev
private

◆ sinPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sinPhi_m
private

◆ SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev
private

◆ solverChoice

SolverChoice ERF::solverChoice
staticprivate

◆ sponge_type

std::string ERF::sponge_type
staticprivate

◆ sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev
private

◆ start_time

amrex::Real ERF::start_time = 0.0
private

◆ startCPUTime

Real ERF::startCPUTime = 0.0
staticprivate

Referenced by getCPUTime().

◆ stop_time

amrex::Real ERF::stop_time = std::numeric_limits<amrex::Real>::max()
private

◆ stretched_dz_d

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::stretched_dz_d
private

◆ stretched_dz_h

amrex::Vector<amrex::Vector<amrex::Real> > ERF::stretched_dz_h
private

◆ sub_cfl

Real ERF::sub_cfl = 1.0
staticprivate

◆ subvol_file

std::string ERF::subvol_file {"subvol"}
private

◆ sum_interval

int ERF::sum_interval = -1
staticprivate

◆ sum_per

Real ERF::sum_per = -1.0
staticprivate

◆ t_avg_cnt

amrex::Vector<amrex::Real> ERF::t_avg_cnt
private

◆ t_new

amrex::Vector<amrex::Real> ERF::t_new
private

◆ t_old

amrex::Vector<amrex::Real> ERF::t_old
private

◆ Tau

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau
private

◆ terrain_blanking

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking
private

◆ th_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::th_bc_data
private

◆ Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim
private

◆ thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce
private

◆ thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce
private

◆ thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce
private

◆ timeprecision

const int ERF::timeprecision = 13
private

◆ tot_e_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::tot_e_datalog
private

Referenced by setRecordEnergyDataInfo().

◆ tot_e_datalogname

amrex::Vector<std::string> ERF::tot_e_datalogname
private

◆ tsk_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::tsk_lev
private

◆ turbPert

TurbulentPerturbation ERF::turbPert
private

◆ use_datetime

bool ERF::use_datetime = false
private

◆ use_fft

bool ERF::use_fft = false
staticprivate

◆ vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new
private

◆ vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old
private

◆ vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg
private

◆ verbose

int ERF::verbose = 0
staticprivate

◆ walldist

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::walldist
private

◆ xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask
private

◆ xvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::xvel_bc_data
private

◆ yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask
private

◆ yvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::yvel_bc_data
private

◆ z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc
private

◆ z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd
private

◆ z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new
private

◆ z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src
private

◆ z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk
private

◆ zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask
private

◆ zlevels_stag

amrex::Vector<amrex::Vector<amrex::Real> > ERF::zlevels_stag
private

◆ zmom_crse_rhs

amrex::Vector<amrex::MultiFab> ERF::zmom_crse_rhs
private

◆ zvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::zvel_bc_data
private

The documentation for this class was generated from the following files: