ERF
Energy Research and Forecasting: An Atmospheric Modeling Code
ERF Class Reference

#include <ERF.H>

Inheritance diagram for ERF:
Collaboration diagram for ERF:

Public Member Functions

 ERF ()
 
 ~ERF () override
 
void ERF_shared ()
 
 ERF (ERF &&) noexcept=delete
 
ERFoperator= (ERF &&other) noexcept=delete
 
 ERF (const ERF &other)=delete
 
ERFoperator= (const ERF &other)=delete
 
void Evolve ()
 
void ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override
 
void HurricaneTracker (int lev, const amrex::MultiFab &U_new, const amrex::MultiFab &V_new, const amrex::MultiFab &W_new, const amrex::Real velmag_threshold, const bool is_track_io, amrex::TagBoxArray *tags=nullptr)
 
std::string MakeVTKFilename (int nstep)
 
void WriteVTKPolyline (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
 
void InitData ()
 
void InitData_pre ()
 
void InitData_post ()
 
void WriteMyEBSurface ()
 
void compute_divergence (int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
 
void project_velocity (int lev, amrex::Real dt)
 
void project_momenta (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
 
void project_velocity_tb (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
 
void poisson_wall_dist (int lev)
 
void make_subdomains (const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
 
void solve_with_EB_mlmg (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
 
void solve_with_gmres (int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &znd_sub)
 
void solve_with_mlmg (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
 
void ImposeBCsOnPhi (int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
 
amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > get_projection_bc (amrex::Orientation::Side side) const noexcept
 
bool projection_has_dirichlet (amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > bcs) const
 
void init_only (int lev, amrex::Real time)
 
void restart ()
 
bool writeNow (const amrex::Real cur_time, const amrex::Real dt, const int nstep, const int plot_int, const amrex::Real plot_per)
 
void post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev)
 
void sum_integrated_quantities (amrex::Real time)
 
void sum_derived_quantities (amrex::Real time)
 
void sum_energy_quantities (amrex::Real time)
 
void write_1D_profiles (amrex::Real time)
 
void write_1D_profiles_stag (amrex::Real time)
 
amrex::Real cloud_fraction (amrex::Real time)
 
void FillBdyCCVels (amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)
 
void sample_points (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void sample_lines (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void derive_diag_profiles (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_diag_profiles_stag (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
void derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
amrex::Real volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, bool finemask)
 
void MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void ClearLevel (int lev) override
 
void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
amrex::Real estTimeStep (int lev, long &dt_fast_ratio) const
 
void advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
 
void advance_microphysics (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
 
void advance_lsm (int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)
 
void advance_radiation (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance)
 
amrex::MultiFab & build_fine_mask (int lev)
 
void MakeHorizontalAverages ()
 
void MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)
 
void derive_upwp (amrex::Vector< amrex::Real > &h_havg)
 
void Write3DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
 
void Write2DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
 
void WriteSubvolume ()
 
void WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
 
void WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
 
void erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
 
void init_from_input_sounding (int lev)
 
void input_sponge (int lev)
 
void init_from_hse (int lev)
 
void init_thin_body (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void CreateWeatherDataGeomBoxArrayDistMap (const std::string &filename, amrex::Geometry &geom_weather, amrex::BoxArray &nba, amrex::DistributionMapping &dm)
 
void FillWeatherDataMultiFab (const std::string &filename, const amrex::Geometry &geom_weather, const amrex::BoxArray &nba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &weather_forecast_data)
 
void InterpWeatherDataOntoMesh (const amrex::Geometry &geom_weather, amrex::MultiFab &weather_forecast_interp)
 
void WeatherDataInterpolation (const amrex::Real time)
 
void fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
 
void MakeEBGeometry ()
 
void make_eb_box ()
 
void make_eb_regular ()
 
void AverageDownTo (int crse_lev, int scomp, int ncomp)
 
void WriteCheckpointFile () const
 
void ReadCheckpointFile ()
 
void ReadCheckpointFileSurfaceLayer ()
 
void init_zphys (int lev, amrex::Real time)
 
void remake_zphys (int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
 
void update_terrain_arrays (int lev)
 
void writeJobInfo (const std::string &dir) const
 

Static Public Member Functions

static bool is_it_time_for_action (int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
 
static void writeBuildInfo (std::ostream &os)
 
static void print_banner (MPI_Comm, std::ostream &)
 
static void print_usage (MPI_Comm, std::ostream &)
 
static void print_error (MPI_Comm, const std::string &msg)
 
static void print_summary (std::ostream &)
 
static void print_tpls (std::ostream &)
 

Public Attributes

amrex::Vector< std::array< amrex::Real, 2 > > hurricane_track_xy
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > initial_state
 
std::string pp_prefix {"erf"}
 

Private Member Functions

void ReadParameters ()
 
void ParameterSanityChecks ()
 
void AverageDown ()
 
void update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void Construct_ERFFillPatchers (int lev)
 
void Define_ERFFillPatchers (int lev)
 
void init1DArrays ()
 
void init_bcs ()
 
void init_custom (int lev)
 
void init_uniform (int lev)
 
void init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
 
void turbPert_update (const int lev, const amrex::Real dt)
 
void turbPert_amplitude (const int lev)
 
void initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
 
void make_physbcs (int lev)
 
void initializeMicrophysics (const int &)
 
void FillPatchCrseLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
 
void FillPatchFineLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
 
void FillIntermediatePatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
 
void FillCoarsePatch (int lev, amrex::Real time)
 
void timeStep (int lev, amrex::Real time, int iteration)
 
void Advance (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
 
void initHSE ()
 Initialize HSE. More...
 
void initHSE (int lev)
 
void initRayleigh ()
 Initialize Rayleigh damping profiles. More...
 
void initSponge ()
 Initialize sponge profiles. More...
 
void setRayleighRefFromSounding (bool restarting)
 Set Rayleigh mean profiles from input sounding. More...
 
void setSpongeRefFromSounding (bool restarting)
 Set sponge mean profiles from input sounding. More...
 
void ComputeDt (int step=-1)
 
std::string PlotFileName (int lev) const
 
void setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void init_Dirichlet_bc_data (const std::string input_file)
 
void InitializeFromFile ()
 
void InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)
 
void post_update (amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
 
void fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
 
void init_geo_wind_profile (const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
 
void refinement_criteria_setup ()
 
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg (int lev)
 
AMREX_FORCE_INLINE std::ostream & DataLog (int i)
 
AMREX_FORCE_INLINE std::ostream & DerDataLog (int i)
 
AMREX_FORCE_INLINE int NumDataLogs () noexcept
 
AMREX_FORCE_INLINE int NumDerDataLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SamplePointLog (int i)
 
AMREX_FORCE_INLINE int NumSamplePointLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SampleLineLog (int i)
 
AMREX_FORCE_INLINE int NumSampleLineLogs () noexcept
 
amrex::IntVect & SamplePoint (int i)
 
AMREX_FORCE_INLINE int NumSamplePoints () noexcept
 
amrex::IntVect & SampleLine (int i)
 
AMREX_FORCE_INLINE int NumSampleLines () noexcept
 
void setRecordDataInfo (int i, const std::string &filename)
 
void setRecordDerDataInfo (int i, const std::string &filename)
 
void setRecordEnergyDataInfo (int i, const std::string &filename)
 
void setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
void setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
std::string DataLogName (int i) const noexcept
 The filename of the ith datalog file. More...
 
std::string DerDataLogName (int i) const noexcept
 
std::string SamplePointLogName (int i) const noexcept
 The filename of the ith sampleptlog file. More...
 
std::string SampleLineLogName (int i) const noexcept
 The filename of the ith samplelinelog file. More...
 
eb_ const & get_eb (int lev) const noexcept
 
amrex::EBFArrayBoxFactory const & EBFactory (int lev) const noexcept
 

Static Private Member Functions

static amrex::Vector< std::string > PlotFileVarNames (amrex::Vector< std::string > plot_var_names)
 
static void GotoNextLine (std::istream &is)
 
static AMREX_FORCE_INLINE int ComputeGhostCells (const SolverChoice &sc)
 
static amrex::Real getCPUTime ()
 
static int nghost_eb_basic ()
 
static int nghost_eb_volume ()
 
static int nghost_eb_full ()
 

Private Attributes

amrex::Vector< std::unique_ptr< amrex::MultiFab > > lat_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lon_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
 
InputSoundingData input_sounding_data
 
InputSpongeData input_sponge_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
 
std::unique_ptr< ProblemBaseprob = nullptr
 
amrex::Vector< int > num_boxes_at_level
 
amrex::Vector< int > num_files_at_level
 
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
 
amrex::Vector< int > istep
 
amrex::Vector< int > nsubsteps
 
amrex::Vector< amrex::Realt_new
 
amrex::Vector< amrex::Realt_old
 
amrex::Vector< amrex::Realdt
 
amrex::Vector< long > dt_mri_ratio
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > gradp
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
 
amrex::Vector< amrex::Realt_avg_cnt
 
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
 
amrex::Vector< amrex::MultiFab > pp_inc
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
 
amrex::Vector< amrex::MultiFab > rU_old
 
amrex::Vector< amrex::MultiFab > rU_new
 
amrex::Vector< amrex::MultiFab > rV_old
 
amrex::Vector< amrex::MultiFab > rV_new
 
amrex::Vector< amrex::MultiFab > rW_old
 
amrex::Vector< amrex::MultiFab > rW_new
 
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
 
std::unique_ptr< Microphysicsmicro
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
 
LandSurface lsm
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
 
amrex::Vector< std::unique_ptr< IRadiation > > rad
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > qheating_rates
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sw_lw_fluxes
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > solar_zenith
 
bool plot_rad = false
 
int rad_datalog_int = -1
 
int cf_width {0}
 
int cf_set_width {0}
 
amrex::Vector< ERFFillPatcherFPr_c
 
amrex::Vector< ERFFillPatcherFPr_u
 
amrex::Vector< ERFFillPatcherFPr_v
 
amrex::Vector< ERFFillPatcherFPr_w
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
 
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > mapfac
 
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
 
amrex::Vector< amrex::MultiFab > base_state
 
amrex::Vector< amrex::MultiFab > base_state_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
 
bool finished_wave = false
 
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
 
amrex::Vector< amrex::BCRec > domain_bcs_type
 
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
 
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_maxm_bc_extdir_vals
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_maxm_bc_neumann_vals
 
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
 
int last_plot3d_file_step_1
 
int last_plot3d_file_step_2
 
int last_plot2d_file_step_1
 
int last_plot2d_file_step_2
 
int last_subvol
 
int last_check_file_step
 
int plot_file_on_restart = 1
 
const int datwidth = 14
 
const int datprecision = 6
 
const int timeprecision = 13
 
int max_step = std::numeric_limits<int>::max()
 
bool use_datetime = false
 
const std::string datetime_format = "%Y-%m-%d %H:%M:%S"
 
std::string restart_chkfile = ""
 
amrex::Vector< amrex::Realfixed_dt
 
amrex::Vector< amrex::Realfixed_fast_dt
 
int regrid_int = -1
 
bool regrid_level_0_on_restart = false
 
std::string plot3d_file_1 {"plt_1_"}
 
std::string plot3d_file_2 {"plt_2_"}
 
std::string plot2d_file_1 {"plt2d_1_"}
 
std::string plot2d_file_2 {"plt2d_2_"}
 
std::string subvol_file {"subvol"}
 
bool m_expand_plotvars_to_unif_rr = false
 
int m_plot3d_int_1 = -1
 
int m_plot3d_int_2 = -1
 
int m_plot2d_int_1 = -1
 
int m_plot2d_int_2 = -1
 
int m_subvol_int = -1
 
amrex::Real m_plot3d_per_1 = -1.0
 
amrex::Real m_plot3d_per_2 = -1.0
 
amrex::Real m_plot2d_per_1 = -1.0
 
amrex::Real m_plot2d_per_2 = -1.0
 
amrex::Real m_subvol_per = -1.0
 
bool m_plot_face_vels = false
 
bool plot_lsm = false
 
int profile_int = -1
 
bool destag_profiles = true
 
std::string check_file {"chk"}
 
int m_check_int = -1
 
amrex::Real m_check_per = -1.0
 
amrex::Vector< std::string > plot3d_var_names_1
 
amrex::Vector< std::string > plot3d_var_names_2
 
amrex::Vector< std::string > plot2d_var_names_1
 
amrex::Vector< std::string > plot2d_var_names_2
 
const amrex::Vector< std::string > cons_names
 
const amrex::Vector< std::string > derived_names
 
TurbulentPerturbation turbPert
 
int real_width {0}
 
int real_set_width {0}
 
bool real_extrap_w {true}
 
bool metgrid_debug_quiescent {false}
 
bool metgrid_debug_isothermal {false}
 
bool metgrid_debug_dry {false}
 
bool metgrid_debug_psfc {false}
 
bool metgrid_debug_msf {false}
 
bool metgrid_interp_theta {false}
 
bool metgrid_basic_linear {false}
 
bool metgrid_use_below_sfc {true}
 
bool metgrid_use_sfc {true}
 
bool metgrid_retain_sfc {false}
 
amrex::Real metgrid_proximity {500.0}
 
int metgrid_order {2}
 
int metgrid_force_sfc_k {6}
 
amrex::Vector< amrex::BoxArray > ba1d
 
amrex::Vector< amrex::BoxArray > ba2d
 
std::unique_ptr< amrex::MultiFab > mf_C1H
 
std::unique_ptr< amrex::MultiFab > mf_C2H
 
std::unique_ptr< amrex::MultiFab > mf_MUB
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_PSFC
 
amrex::Vector< amrex::Vector< amrex::Real > > h_rhotheta_src
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhotheta_src
 
amrex::Vector< amrex::Vector< amrex::Real > > h_rhoqt_src
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhoqt_src
 
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
 
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
 
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
 
amrex::Vector< amrex::Realh_havg_density
 
amrex::Vector< amrex::Realh_havg_temperature
 
amrex::Vector< amrex::Realh_havg_pressure
 
amrex::Vector< amrex::Realh_havg_qv
 
amrex::Vector< amrex::Realh_havg_qc
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_density
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_temperature
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_pressure
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_qv
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_qc
 
std::unique_ptr< WriteBndryPlanesm_w2d = nullptr
 
std::unique_ptr< ReadBndryPlanesm_r2d = nullptr
 
std::unique_ptr< SurfaceLayerm_SurfaceLayer = nullptr
 
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
 
amrex::Vector< amrex::Vector< amrex::BoxArray > > subdomains
 
amrex::MultiFab fine_mask
 
amrex::Vector< amrex::Realdz_min
 
int sampler_interval = -1
 
amrex::Real sampler_per = -1.0
 
std::unique_ptr< SampleDatadata_sampler = nullptr
 
amrex::Vector< std::unique_ptr< std::fstream > > datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
 
amrex::Vector< std::string > datalogname
 
amrex::Vector< std::string > der_datalogname
 
amrex::Vector< std::string > tot_e_datalogname
 
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
 
amrex::Vector< std::string > sampleptlogname
 
amrex::Vector< amrex::IntVect > samplepoint
 
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
 
amrex::Vector< std::string > samplelinelogname
 
amrex::Vector< amrex::IntVect > sampleline
 
amrex::Vector< std::unique_ptr< eb_ > > eb
 

Static Private Attributes

static amrex::Real start_time = 0.0
 
static amrex::Real stop_time = std::numeric_limits<amrex::Real>::max()
 
static amrex::Real cfl = 0.8
 
static amrex::Real sub_cfl = 1.0
 
static amrex::Real init_shrink = 1.0
 
static amrex::Real change_max = 1.1
 
static amrex::Real dt_max_initial = 2.0e100
 
static amrex::Real dt_max = 1.0e9
 
static int fixed_mri_dt_ratio = 0
 
static SolverChoice solverChoice
 
static int verbose = 0
 
static int mg_verbose = 0
 
static bool use_fft = false
 
static int sum_interval = -1
 
static int pert_interval = -1
 
static amrex::Real sum_per = -1.0
 
static PlotFileType plotfile3d_type_1 = PlotFileType::None
 
static PlotFileType plotfile3d_type_2 = PlotFileType::None
 
static PlotFileType plotfile2d_type_1 = PlotFileType::None
 
static PlotFileType plotfile2d_type_2 = PlotFileType::None
 
static StateInterpType interpolation_type
 
static std::string sponge_type
 
static amrex::Vector< amrex::Vector< std::string > > nc_init_file = {{""}}
 
static std::string nc_bdy_file
 
static std::string nc_low_file
 
static int output_1d_column = 0
 
static int column_interval = -1
 
static amrex::Real column_per = -1.0
 
static amrex::Real column_loc_x = 0.0
 
static amrex::Real column_loc_y = 0.0
 
static std::string column_file_name = "column_data.nc"
 
static int output_bndry_planes = 0
 
static int bndry_output_planes_interval = -1
 
static amrex::Real bndry_output_planes_per = -1.0
 
static amrex::Real bndry_output_planes_start_time = 0.0
 
static int input_bndry_planes = 0
 
static int ng_dens_hse
 
static int ng_pres_hse
 
static amrex::Vector< amrex::AMRErrorTag > ref_tags
 
static amrex::Real startCPUTime = 0.0
 
static amrex::Real previousCPUTimeUsed = 0.0
 

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

◆ ERF() [1/3]

ERF::ERF ( )
105 {
106  int fix_random_seed = 0;
107  ParmParse pp("erf"); pp.query("fix_random_seed", fix_random_seed);
108  // Note that the value of 1024UL is not significant -- the point here is just to set the
109  // same seed for all MPI processes for the purpose of regression testing
110  if (fix_random_seed) {
111  Print() << "Fixing the random seed" << std::endl;
112  InitRandom(1024UL);
113  }
114 
115  ERF_shared();
116 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real pp(amrex::Real y)
Definition: ERF_MicrophysicsUtils.H:230
void ERF_shared()
Definition: ERF.cpp:119
Here is the call graph for this function:

◆ ~ERF()

ERF::~ERF ( )
overridedefault

◆ ERF() [2/3]

ERF::ERF ( ERF &&  )
deletenoexcept

◆ ERF() [3/3]

ERF::ERF ( const ERF other)
delete

Member Function Documentation

◆ Advance()

void ERF::Advance ( int  lev,
amrex::Real  time,
amrex::Real  dt_lev,
int  iteration,
int  ncycle 
)
private
24 {
25  BL_PROFILE("ERF::Advance()");
26 
27  // We must swap the pointers so the previous step's "new" is now this step's "old"
28  std::swap(vars_old[lev], vars_new[lev]);
29 
30  MultiFab& S_old = vars_old[lev][Vars::cons];
31  MultiFab& S_new = vars_new[lev][Vars::cons];
32 
33  MultiFab& U_old = vars_old[lev][Vars::xvel];
34  MultiFab& V_old = vars_old[lev][Vars::yvel];
35  MultiFab& W_old = vars_old[lev][Vars::zvel];
36 
37  MultiFab& U_new = vars_new[lev][Vars::xvel];
38  MultiFab& V_new = vars_new[lev][Vars::yvel];
39  MultiFab& W_new = vars_new[lev][Vars::zvel];
40 
41  // We need to set these because otherwise in the first call to erf_advance we may
42  // read uninitialized data on ghost values in setting the bc's on the velocities
43  U_new.setVal(1.e34,U_new.nGrowVect());
44  V_new.setVal(1.e34,V_new.nGrowVect());
45  W_new.setVal(1.e34,W_new.nGrowVect());
46 
47  // Do error checking for negative (rho theta) here
48  if (solverChoice.anelastic[lev] != 1) {
50  }
51 
52  //
53  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
54  // If lev == 0 we have already FillPatched this in ERF::TimeStep
55  //
56  if (lev > 0) {
57  FillPatchFineLevel(lev, time, {&S_old, &U_old, &V_old, &W_old},
58  {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]},
59  base_state[lev], base_state[lev]);
60  }
61 
62  //
63  // So we must convert the fillpatched to momenta, including the ghost values
64  //
65  VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
66  V_old, rV_old[lev].nGrowVect(),
67  W_old, rW_old[lev].nGrowVect(),
68  S_old, rU_old[lev], rV_old[lev], rW_old[lev],
69  Geom(lev).Domain(),
71 
72  // Update the inflow perturbation update time and amplitude
73  if (solverChoice.pert_type == PerturbationType::Source ||
74  solverChoice.pert_type == PerturbationType::Direct ||
75  solverChoice.pert_type == PerturbationType::CPM)
76  {
77  turbPert.calc_tpi_update(lev, dt_lev, U_old, V_old, S_old);
78  }
79 
80  // If PerturbationType::Direct or CPM is selected, directly add the computed perturbation
81  // on the conserved field
82  if (solverChoice.pert_type == PerturbationType::Direct ||
83  solverChoice.pert_type == PerturbationType::CPM)
84  {
85  auto m_ixtype = S_old.boxArray().ixType(); // Conserved term
86  for (MFIter mfi(S_old,TileNoZ()); mfi.isValid(); ++mfi) {
87  Box bx = mfi.tilebox();
88  const Array4<Real> &cell_data = S_old.array(mfi);
89  const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
90  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cell_data, pert_cell);
91  }
92  }
93 
94  // configure SurfaceLayer params if needed
95  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
96  if (m_SurfaceLayer) {
97  IntVect ng = Theta_prim[lev]->nGrowVect();
98  MultiFab::Copy( *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
99  MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp , 0, 1, ng);
100  if (solverChoice.moisture_type != MoistureType::None) {
101  ng = Qv_prim[lev]->nGrowVect();
102 
103  MultiFab::Copy( *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
104  MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp , 0, 1, ng);
105 
106  if (solverChoice.moisture_indices.qr > -1) {
107  MultiFab::Copy( *Qr_prim[lev], S_old, solverChoice.moisture_indices.qr, 0, 1, ng);
108  MultiFab::Divide(*Qr_prim[lev], S_old, Rho_comp , 0, 1, ng);
109  } else {
110  Qr_prim[lev]->setVal(0.0);
111  }
112  }
113  // NOTE: std::swap above causes the field ptrs to be out of date.
114  // Reassign the field ptrs for MAC avg computation.
115  m_SurfaceLayer->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim, Qr_prim);
116  m_SurfaceLayer->update_pblh(lev, vars_old, z_phys_cc[lev].get(),
118  m_SurfaceLayer->update_fluxes(lev, time, S_old, z_phys_nd[lev]);
119  }
120  }
121 
122 #if defined(ERF_USE_WINDFARM)
123  // **************************************************************************************
124  // Update the windfarm sources
125  // **************************************************************************************
126  if (solverChoice.windfarm_type != WindFarmType::None) {
127  advance_windfarm(Geom(lev), dt_lev, S_old,
128  U_old, V_old, W_old, vars_windfarm[lev],
129  Nturb[lev], SMark[lev], time);
130  }
131 
132 #endif
133 
134  // **************************************************************************************
135  // Update the radiation sources
136  // **************************************************************************************
137  advance_radiation(lev, S_new, dt_lev);
138 
139  const BoxArray& ba = S_old.boxArray();
140  const DistributionMapping& dm = S_old.DistributionMap();
141 
142  int nvars = S_old.nComp();
143 
144  // Source array for conserved cell-centered quantities -- this will be filled
145  // in the call to make_sources in ERF_TI_slow_rhs_pre.H
146  MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0.0);
147 
148  // Source arrays for momenta -- these will be filled
149  // in the call to make_mom_sources in ERF_TI_slow_rhs_pre.H
150  BoxArray ba_x(ba); ba_x.surroundingNodes(0);
151  MultiFab xmom_source(ba_x,dm,1,1); xmom_source.setVal(0.0);
152 
153  BoxArray ba_y(ba); ba_y.surroundingNodes(1);
154  MultiFab ymom_source(ba_y,dm,1,1); ymom_source.setVal(0.0);
155 
156  BoxArray ba_z(ba); ba_z.surroundingNodes(2);
157  MultiFab zmom_source(ba_z,dm,1,1); zmom_source.setVal(0.0);
158  MultiFab buoyancy(ba_z,dm,1,1); buoyancy.setVal(0.0);
159 
160  amrex::Vector<MultiFab> state_old;
161  amrex::Vector<MultiFab> state_new;
162 
163  // **************************************************************************************
164  // Here we define state_old and state_new which are to be advanced
165  // **************************************************************************************
166  // Initial solution
167  // Note that "old" and "new" here are relative to each RK stage.
168  state_old.push_back(MultiFab(S_old , amrex::make_alias, 0, nvars)); // cons
169  state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0, 1)); // xmom
170  state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0, 1)); // ymom
171  state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0, 1)); // zmom
172 
173  // Final solution
174  // state_new at the end of the last RK stage holds the t^{n+1} data
175  state_new.push_back(MultiFab(S_new , amrex::make_alias, 0, nvars)); // cons
176  state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0, 1)); // xmom
177  state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0, 1)); // ymom
178  state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0, 1)); // zmom
179 
180  // **************************************************************************************
181  // Update the dycore
182  // **************************************************************************************
183  advance_dycore(lev, state_old, state_new,
184  U_old, V_old, W_old,
185  U_new, V_new, W_new,
186  cc_source, xmom_source, ymom_source, zmom_source, buoyancy,
187  Geom(lev), dt_lev, time);
188 
189  // **************************************************************************************
190  // Update the microphysics (moisture)
191  // **************************************************************************************
193  advance_microphysics(lev, S_new, dt_lev, iteration, time);
194  }
195 
196  // **************************************************************************************
197  // Update the land surface model
198  // **************************************************************************************
199  advance_lsm(lev, S_new, U_new, V_new, dt_lev);
200 
201 #ifdef ERF_USE_PARTICLES
202  // **************************************************************************************
203  // Update the particle positions
204  // **************************************************************************************
205  evolveTracers( lev, dt_lev, vars_new, z_phys_nd );
206 #endif
207 
208  // ***********************************************************************************************
209  // Impose domain boundary conditions here so that in FillPatching the fine data we won't
210  // need to re-fill these
211  // ***********************************************************************************************
212  if (lev < finest_level) {
213  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
215  0,vars_new[lev][Vars::cons].nComp(),
216  vars_new[lev][Vars::cons].nGrowVect(),time,BCVars::cons_bc,true);
217  (*physbcs_u[lev])(vars_new[lev][Vars::xvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
218  ngvect_vels,time,BCVars::xvel_bc,true);
219  (*physbcs_v[lev])(vars_new[lev][Vars::yvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
220  ngvect_vels,time,BCVars::yvel_bc,true);
221  (*physbcs_w[lev])(vars_new[lev][Vars::zvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
222  ngvect_vels,time,BCVars::zvel_bc,true);
223  }
224 
225  // **************************************************************************************
226  // Register old and new coarse data if we are at a level less than the finest level
227  // **************************************************************************************
228  if (lev < finest_level) {
229  if (cf_width > 0) {
230  // We must fill the ghost cells of these so that the parallel copy works correctly
231  state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
232  state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
233  FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
234  {time, time+dt_lev});
235  }
236 
237  if (cf_width >= 0) {
238  // We must fill the ghost cells of these so that the parallel copy works correctly
239  state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
240  state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
241  FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
242  {time, time+dt_lev});
243 
244  state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
245  state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
246  FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
247  {time, time+dt_lev});
248 
249  state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
250  state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
251  FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
252  {time, time+dt_lev});
253  }
254 
255  //
256  // Now create a MultiFab that holds (S_new - S_old) / dt from the coarse level interpolated
257  // on to the coarse/fine boundary at the fine resolution
258  //
259  Interpolater* mapper_f = &face_cons_linear_interp;
260 
261  // PhysBCFunctNoOp null_bc;
262  // MultiFab tempx(vars_new[lev+1][Vars::xvel].boxArray(),vars_new[lev+1][Vars::xvel].DistributionMap(),1,0);
263  // tempx.setVal(0.0);
264  // xmom_crse_rhs[lev+1].setVal(0.0);
265  // FPr_u[lev].FillSet(tempx , time , null_bc, domain_bcs_type);
266  // FPr_u[lev].FillSet(xmom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
267  // MultiFab::Subtract(xmom_crse_rhs[lev+1],tempx,0,0,1,IntVect{0});
268  // xmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
269 
270  // MultiFab tempy(vars_new[lev+1][Vars::yvel].boxArray(),vars_new[lev+1][Vars::yvel].DistributionMap(),1,0);
271  // tempy.setVal(0.0);
272  // ymom_crse_rhs[lev+1].setVal(0.0);
273  // FPr_v[lev].FillSet(tempy , time , null_bc, domain_bcs_type);
274  // FPr_v[lev].FillSet(ymom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
275  // MultiFab::Subtract(ymom_crse_rhs[lev+1],tempy,0,0,1,IntVect{0});
276  // ymom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
277 
278  MultiFab temp_state(zmom_crse_rhs[lev+1].boxArray(),zmom_crse_rhs[lev+1].DistributionMap(),1,0);
279  InterpFromCoarseLevel(temp_state, IntVect{0}, IntVect{0}, state_old[IntVars::zmom], 0, 0, 1,
280  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
281  InterpFromCoarseLevel(zmom_crse_rhs[lev+1], IntVect{0}, IntVect{0}, state_new[IntVars::zmom], 0, 0, 1,
282  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
283  MultiFab::Subtract(zmom_crse_rhs[lev+1],temp_state,0,0,1,IntVect{0});
284  zmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
285  }
286 
287  // ***********************************************************************************************
288  // Update the time averaged velocities if they are requested
289  // ***********************************************************************************************
291  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
292  }
293 }
void check_for_negative_theta(amrex::MultiFab &S_old)
Definition: ERF_Advance.cpp:296
@ nvars
Definition: ERF_DataStruct.H:91
#define Rho_comp
Definition: ERF_IndexDefines.H:36
#define RhoTheta_comp
Definition: ERF_IndexDefines.H:37
#define RhoQ1_comp
Definition: ERF_IndexDefines.H:42
@ surface_layer
AMREX_FORCE_INLINE amrex::IntVect TileNoZ()
Definition: ERF_TileNoZ.H:11
void Time_Avg_Vel_atCC(const Real &dt, Real &t_avg_cnt, MultiFab *vel_t_avg, MultiFab &xvel, MultiFab &yvel, MultiFab &zvel)
Definition: ERF_TimeAvgVel.cpp:9
void VelocityToMomentum(const amrex::MultiFab &xvel_in, const amrex::IntVect &xvel_ngrow, const amrex::MultiFab &yvel_in, const amrex::IntVect &yvel_ngrow, const amrex::MultiFab &zvel_in, const amrex::IntVect &zvel_ngrow, const amrex::MultiFab &cons_in, amrex::MultiFab &xmom_out, amrex::MultiFab &ymom_out, amrex::MultiFab &zmom_out, const amrex::Box &domain, const amrex::Vector< amrex::BCRec > &domain_bcs_type_h)
amrex::Vector< amrex::MultiFab > rU_new
Definition: ERF.H:805
amrex::Vector< ERFFillPatcher > FPr_u
Definition: ERF.H:857
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
Definition: ERF.H:777
amrex::Vector< ERFFillPatcher > FPr_v
Definition: ERF.H:858
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
Definition: ERF.H:792
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
Definition: ERF.H:882
static SolverChoice solverChoice
Definition: ERF.H:1087
amrex::Vector< ERFFillPatcher > FPr_c
Definition: ERF.H:856
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
Definition: ERF.H:784
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
Definition: ERF.H:795
amrex::Vector< amrex::MultiFab > base_state
Definition: ERF.H:913
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
Definition: ERF.H:800
amrex::Vector< amrex::MultiFab > rV_new
Definition: ERF.H:807
amrex::Vector< amrex::BCRec > domain_bcs_type
Definition: ERF.H:929
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
Definition: ERF.H:801
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
Definition: ERF.H:793
amrex::Vector< amrex::Real > t_avg_cnt
Definition: ERF.H:785
void FillPatchFineLevel(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
Definition: ERF_FillPatch.cpp:20
amrex::Vector< amrex::MultiFab > rU_old
Definition: ERF.H:804
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
Definition: ERF.H:799
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
Definition: ERF.H:794
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
Definition: ERF.H:881
void advance_dycore(int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
Definition: ERF_AdvanceDycore.cpp:38
amrex::Vector< amrex::MultiFab > rW_new
Definition: ERF.H:809
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
Definition: ERF.H:813
void advance_lsm(int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)
Definition: ERF_AdvanceLSM.cpp:5
TurbulentPerturbation turbPert
Definition: ERF.H:1090
amrex::Vector< amrex::MultiFab > rW_old
Definition: ERF.H:808
std::unique_ptr< SurfaceLayer > m_SurfaceLayer
Definition: ERF.H:1250
amrex::Vector< ERFFillPatcher > FPr_w
Definition: ERF.H:859
amrex::Vector< amrex::Real > dt
Definition: ERF.H:771
void advance_radiation(int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance)
Definition: ERF_AdvanceRadiation.cpp:5
void advance_microphysics(int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
Definition: ERF_AdvanceMicrophysics.cpp:5
int cf_width
Definition: ERF.H:854
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
Definition: ERF.H:942
amrex::Vector< amrex::MultiFab > rV_old
Definition: ERF.H:806
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
Definition: ERF.H:778
@ zvel_bc
Definition: ERF_IndexDefines.H:89
@ yvel_bc
Definition: ERF_IndexDefines.H:88
@ cons_bc
Definition: ERF_IndexDefines.H:76
@ xvel_bc
Definition: ERF_IndexDefines.H:87
@ ymom
Definition: ERF_IndexDefines.H:160
@ cons
Definition: ERF_IndexDefines.H:158
@ zmom
Definition: ERF_IndexDefines.H:161
@ xmom
Definition: ERF_IndexDefines.H:159
@ ng
Definition: ERF_Morrison.H:48
@ xvel
Definition: ERF_IndexDefines.H:141
@ cons
Definition: ERF_IndexDefines.H:140
@ zvel
Definition: ERF_IndexDefines.H:143
@ yvel
Definition: ERF_IndexDefines.H:142
int qr
Definition: ERF_DataStruct.H:103
bool moisture_tight_coupling
Definition: ERF_DataStruct.H:894
amrex::Vector< int > anelastic
Definition: ERF_DataStruct.H:793
MoistureType moisture_type
Definition: ERF_DataStruct.H:873
PerturbationType pert_type
Definition: ERF_DataStruct.H:863
WindFarmType windfarm_type
Definition: ERF_DataStruct.H:874
MoistureComponentIndices moisture_indices
Definition: ERF_DataStruct.H:892
bool time_avg_vel
Definition: ERF_DataStruct.H:860
amrex::Vector< amrex::MultiFab > pb_cell
Definition: ERF_TurbPertStruct.H:640
void calc_tpi_update(const int lev, const amrex::Real dt, amrex::MultiFab &mf_xvel, amrex::MultiFab &mf_yvel, amrex::MultiFab &mf_cons)
Definition: ERF_TurbPertStruct.H:223
void apply_tpi(const int &lev, const amrex::Box &vbx, const int &comp, const amrex::IndexType &m_ixtype, const amrex::Array4< amrex::Real > &src_arr, const amrex::Array4< amrex::Real const > &pert_cell)
Definition: ERF_TurbPertStruct.H:324
Here is the call graph for this function:

◆ advance_dycore()

void ERF::advance_dycore ( int  level,
amrex::Vector< amrex::MultiFab > &  state_old,
amrex::Vector< amrex::MultiFab > &  state_new,
amrex::MultiFab &  xvel_old,
amrex::MultiFab &  yvel_old,
amrex::MultiFab &  zvel_old,
amrex::MultiFab &  xvel_new,
amrex::MultiFab &  yvel_new,
amrex::MultiFab &  zvel_new,
amrex::MultiFab &  source,
amrex::MultiFab &  xmom_src,
amrex::MultiFab &  ymom_src,
amrex::MultiFab &  zmom_src,
amrex::MultiFab &  buoyancy,
amrex::Geometry  fine_geom,
amrex::Real  dt,
amrex::Real  time 
)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters
[in]levellevel of refinement (coarsest level is 0)
[in]state_oldold-time conserved variables
[in]state_newnew-time conserved variables
[in]xvel_oldold-time x-component of velocity
[in]yvel_oldold-time y-component of velocity
[in]zvel_oldold-time z-component of velocity
[in]xvel_newnew-time x-component of velocity
[in]yvel_newnew-time y-component of velocity
[in]zvel_newnew-time z-component of velocity
[in]cc_srcsource term for conserved variables
[in]xmom_srcsource term for x-momenta
[in]ymom_srcsource term for y-momenta
[in]zmom_srcsource term for z-momenta
[in]fine_geomcontainer for geometry information at current level
[in]dt_advancetime step for this time advance
[in]old_timeold time for this time advance
48 {
49  BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
50 
51  const Box& domain = fine_geom.Domain();
52 
56 
57  MultiFab r_hse (base_state[level], make_alias, BaseState::r0_comp , 1);
58  MultiFab p_hse (base_state[level], make_alias, BaseState::p0_comp , 1);
59  MultiFab pi_hse(base_state[level], make_alias, BaseState::pi0_comp, 1);
60 
61  // These pointers are used in the MRI utility functions
62  MultiFab* r0 = &r_hse;
63  MultiFab* p0 = &p_hse;
64  MultiFab* pi0 = &pi_hse;
65 
66  Real* dptr_rhotheta_src = solverChoice.custom_rhotheta_forcing ? d_rhotheta_src[level].data() : nullptr;
67  Real* dptr_rhoqt_src = solverChoice.custom_moisture_forcing ? d_rhoqt_src[level].data() : nullptr;
68  Real* dptr_wbar_sub = solverChoice.custom_w_subsidence ? d_w_subsid[level].data() : nullptr;
69 
70  // Turbulent Perturbation Pointer
71  //Real* dptr_rhotheta_src = solverChoice.pert_type ? d_rhotheta_src[level].data() : nullptr;
72 
73  Vector<Real*> d_rayleigh_ptrs_at_lev;
74  d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
75  d_rayleigh_ptrs_at_lev[Rayleigh::ubar] = solverChoice.rayleigh_damp_U ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
76  d_rayleigh_ptrs_at_lev[Rayleigh::vbar] = solverChoice.rayleigh_damp_V ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
77  d_rayleigh_ptrs_at_lev[Rayleigh::wbar] = solverChoice.rayleigh_damp_W ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
78  d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.rayleigh_damp_T ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
79 
80  Vector<Real*> d_sponge_ptrs_at_lev;
81  if(sc.sponge_type=="input_sponge")
82  {
83  d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
84  d_sponge_ptrs_at_lev[Sponge::ubar_sponge] = d_sponge_ptrs[level][Sponge::ubar_sponge].data();
85  d_sponge_ptrs_at_lev[Sponge::vbar_sponge] = d_sponge_ptrs[level][Sponge::vbar_sponge].data();
86  }
87 
88  bool l_use_terrain_fitted_coords = (solverChoice.mesh_type != MeshType::ConstantDz);
89  bool l_use_kturb = tc.use_kturb;
90  bool l_use_diff = ( (dc.molec_diff_type != MolecDiffType::None) ||
91  l_use_kturb );
92  bool l_use_moisture = ( solverChoice.moisture_type != MoistureType::None );
93  bool l_implicit_substepping = ( solverChoice.substepping_type[level] == SubsteppingType::Implicit );
94 
95  const bool use_SurfLayer = (m_SurfaceLayer != nullptr);
96  const MultiFab* z_0 = (use_SurfLayer) ? m_SurfaceLayer->get_z0(level) : nullptr;
97 
98  const BoxArray& ba = state_old[IntVars::cons].boxArray();
99  const BoxArray& ba_z = zvel_old.boxArray();
100  const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
101 
102  int num_prim = state_old[IntVars::cons].nComp() - 1;
103 
104  MultiFab S_prim (ba , dm, num_prim, state_old[IntVars::cons].nGrowVect());
105  MultiFab pi_stage (ba , dm, 1, state_old[IntVars::cons].nGrowVect());
106  MultiFab fast_coeffs(ba_z, dm, 5, 0);
107  MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
108  MultiFab* SmnSmn = SmnSmn_lev[level].get();
109 
110  // **************************************************************************************
111  // Compute strain for use in slow RHS and Smagorinsky model
112  // **************************************************************************************
113  {
114  BL_PROFILE("erf_advance_strain");
115  if (l_use_diff) {
116 
117  const BCRec* bc_ptr_h = domain_bcs_type.data();
118  const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
119 
120 #ifdef _OPENMP
121 #pragma omp parallel if (Gpu::notInLaunchRegion())
122 #endif
123  for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
124  {
125  Box bxcc = mfi.growntilebox(IntVect(1,1,0));
126  Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
127  Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
128  Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
129 
130  if (bxcc.smallEnd(2) != domain.smallEnd(2)) {
131  bxcc.growLo(2,1);
132  tbxxy.growLo(2,1);
133  tbxxz.growLo(2,1);
134  tbxyz.growLo(2,1);
135  }
136 
137  if (bxcc.bigEnd(2) != domain.bigEnd(2)) {
138  bxcc.growHi(2,1);
139  tbxxy.growHi(2,1);
140  tbxxz.growHi(2,1);
141  tbxyz.growHi(2,1);
142  }
143 
144  const Array4<const Real> & u = xvel_old.array(mfi);
145  const Array4<const Real> & v = yvel_old.array(mfi);
146  const Array4<const Real> & w = zvel_old.array(mfi);
147 
148  Array4<Real> tau11 = Tau[level][TauType::tau11].get()->array(mfi);
149  Array4<Real> tau22 = Tau[level][TauType::tau22].get()->array(mfi);
150  Array4<Real> tau33 = Tau[level][TauType::tau33].get()->array(mfi);
151  Array4<Real> tau12 = Tau[level][TauType::tau12].get()->array(mfi);
152  Array4<Real> tau13 = Tau[level][TauType::tau13].get()->array(mfi);
153  Array4<Real> tau23 = Tau[level][TauType::tau23].get()->array(mfi);
154 
155  Array4<Real> tau21 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau21].get()->array(mfi) : Array4<Real>{};
156  Array4<Real> tau31 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau31].get()->array(mfi) : Array4<Real>{};
157  Array4<Real> tau32 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau32].get()->array(mfi) : Array4<Real>{};
158  const Array4<const Real>& z_nd = z_phys_nd[level]->const_array(mfi);
159 
160  const Array4<const Real> mf_mx = mapfac[level][MapFacType::m_x]->const_array(mfi);
161  const Array4<const Real> mf_ux = mapfac[level][MapFacType::u_x]->const_array(mfi);
162  const Array4<const Real> mf_vx = mapfac[level][MapFacType::v_x]->const_array(mfi);
163  const Array4<const Real> mf_my = mapfac[level][MapFacType::m_y]->const_array(mfi);
164  const Array4<const Real> mf_uy = mapfac[level][MapFacType::u_y]->const_array(mfi);
165  const Array4<const Real> mf_vy = mapfac[level][MapFacType::v_y]->const_array(mfi);
166 
167  if (solverChoice.mesh_type == MeshType::StretchedDz) {
168  ComputeStrain_S(bxcc, tbxxy, tbxxz, tbxyz, domain,
169  u, v, w,
170  tau11, tau22, tau33,
171  tau12, tau21,
172  tau13, tau31,
173  tau23, tau32,
174  stretched_dz_d[level], dxInv,
175  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h);
176  } else if (l_use_terrain_fitted_coords) {
177  ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
178  u, v, w,
179  tau11, tau22, tau33,
180  tau12, tau21,
181  tau13, tau31,
182  tau23, tau32,
183  z_nd, detJ_cc[level]->const_array(mfi), dxInv,
184  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h);
185  } else {
186  ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
187  u, v, w,
188  tau11, tau22, tau33,
189  tau12, tau13, tau23,
190  dxInv,
191  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h);
192  }
193  } // mfi
194  } // l_use_diff
195  } // profile
196 
197 #include "ERF_TI_utils.H"
198 
199  // Additional SFS quantities, calculated once per timestep
200  MultiFab* Hfx1 = SFS_hfx1_lev[level].get();
201  MultiFab* Hfx2 = SFS_hfx2_lev[level].get();
202  MultiFab* Hfx3 = SFS_hfx3_lev[level].get();
203  MultiFab* Q1fx1 = SFS_q1fx1_lev[level].get();
204  MultiFab* Q1fx2 = SFS_q1fx2_lev[level].get();
205  MultiFab* Q1fx3 = SFS_q1fx3_lev[level].get();
206  MultiFab* Q2fx3 = SFS_q2fx3_lev[level].get();
207  MultiFab* Diss = SFS_diss_lev[level].get();
208 
209  // *************************************************************************
210  // Calculate cell-centered eddy viscosity & diffusivities
211  //
212  // Notes -- we fill all the data in ghost cells before calling this so
213  // that we can fill the eddy viscosity in the ghost regions and
214  // not have to call a boundary filler on this data itself
215  //
216  // LES - updates both horizontal and vertical eddy viscosity components
217  // PBL - only updates vertical eddy viscosity components so horizontal
218  // components come from the LES model or are left as zero.
219  // *************************************************************************
220  if (l_use_kturb)
221  {
222  // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
223  const BCRec* bc_ptr_h = domain_bcs_type.data();
224  ComputeTurbulentViscosity(xvel_old, yvel_old,Tau[level],
225  state_old[IntVars::cons],
226  *walldist[level].get(),
227  *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
228  fine_geom, mapfac[level],
229  z_phys_nd[level], solverChoice,
230  m_SurfaceLayer, z_0, l_use_terrain_fitted_coords,
231  l_use_moisture, level, bc_ptr_h);
232  }
233 
234  // ***********************************************************************************************
235  // Update user-defined source terms -- these are defined once per time step (not per RK stage)
236  // ***********************************************************************************************
238  prob->update_rhotheta_sources(old_time,
239  h_rhotheta_src[level], d_rhotheta_src[level],
240  fine_geom, z_phys_cc[level]);
241  }
242 
244  prob->update_rhoqt_sources(old_time,
245  h_rhoqt_src[level], d_rhoqt_src[level],
246  fine_geom, z_phys_cc[level]);
247  }
248 
250  prob->update_geostrophic_profile(old_time,
251  h_u_geos[level], d_u_geos[level],
252  h_v_geos[level], d_v_geos[level],
253  fine_geom, z_phys_cc[level]);
254  }
255 
257  prob->update_w_subsidence(old_time,
258  h_w_subsid[level], d_w_subsid[level],
259  fine_geom, z_phys_nd[level]);
260  }
261 
262  // ***********************************************************************************************
263  // Convert old velocity available on faces to old momentum on faces to be used in time integration
264  // ***********************************************************************************************
265  MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
266 
267  //
268  // This is an optimization since we won't need more than one ghost
269  // cell of momentum in the integrator if not using numerical diffusion
270  //
271  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
272  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
273  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
274 
275  VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
276  state_old[IntVars::xmom],
277  state_old[IntVars::ymom],
278  state_old[IntVars::zmom],
279  domain, domain_bcs_type);
280 
281  MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
282  MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
283  MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
284 
285  bool fast_only = false;
286  bool vel_and_mom_synced = true;
287 
288  apply_bcs(state_old, old_time,
289  state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
290  fast_only, vel_and_mom_synced);
291  cons_to_prim(state_old[IntVars::cons], state_old[IntVars::cons].nGrow());
292 
293  // ***********************************************************************************************
294  // Define a new MultiFab that holds q_total and fill it by summing the moisture components --
295  // to be used in buoyancy calculation and as part of the inertial weighting in the
296  // ***********************************************************************************************
297  MultiFab qt(grids[level], dmap[level], 1, 1);
298  qt.setVal(0.0);
299 
300 #include "ERF_TI_no_substep_fun.H"
301 #include "ERF_TI_substep_fun.H"
302 #include "ERF_TI_slow_rhs_pre.H"
303 #include "ERF_TI_slow_rhs_post.H"
304 
305  // ***************************************************************************************
306  // Setup the integrator and integrate for a single timestep
307  // **************************************************************************************
308  MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
309 
310  // Define rhs and 'post update' utility function that is called after calculating
311  // any state data (e.g. at RK stages or at the end of a timestep)
312  mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
313  mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
314 
315  mri_integrator.set_fast_rhs(fast_rhs_fun);
317  mri_integrator.set_no_substep(no_substep_fun);
318 
319  mri_integrator.advance(state_old, state_new, old_time, dt_advance);
320 
321  if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
322 }
void ComputeStrain_N(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau13, Array4< Real > &tau23, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr)
Definition: ERF_ComputeStrain_N.cpp:28
void ComputeStrain_S(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau21, Array4< Real > &tau13, Array4< Real > &tau31, Array4< Real > &tau23, Array4< Real > &tau32, const Gpu::DeviceVector< Real > &stretched_dz_d, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr)
Definition: ERF_ComputeStrain_S.cpp:36
void ComputeStrain_T(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau21, Array4< Real > &tau13, Array4< Real > &tau31, Array4< Real > &tau23, Array4< Real > &tau32, const Array4< const Real > &z_nd, const Array4< const Real > &detJ, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr)
Definition: ERF_ComputeStrain_T.cpp:36
void ComputeTurbulentViscosity(const MultiFab &xvel, const MultiFab &yvel, Vector< std::unique_ptr< MultiFab >> &Tau_lev, const MultiFab &cons_in, const MultiFab &wdist, MultiFab &eddyViscosity, MultiFab &Hfx1, MultiFab &Hfx2, MultiFab &Hfx3, MultiFab &Diss, const Geometry &geom, Vector< std::unique_ptr< MultiFab >> &mapfac, const std::unique_ptr< MultiFab > &z_phys_nd, const SolverChoice &solverChoice, std::unique_ptr< SurfaceLayer > &SurfLayer, const MultiFab *z_0, const bool &use_terrain_fitted_coords, const bool &use_moisture, int level, const BCRec *bc_ptr, bool vert_only)
Definition: ERF_ComputeTurbulentViscosity.cpp:596
@ tau12
Definition: ERF_DataStruct.H:30
@ tau23
Definition: ERF_DataStruct.H:30
@ tau33
Definition: ERF_DataStruct.H:30
@ tau22
Definition: ERF_DataStruct.H:30
@ tau11
Definition: ERF_DataStruct.H:30
@ tau32
Definition: ERF_DataStruct.H:30
@ tau31
Definition: ERF_DataStruct.H:30
@ tau21
Definition: ERF_DataStruct.H:30
@ tau13
Definition: ERF_DataStruct.H:30
@ ubar
Definition: ERF_DataStruct.H:91
@ wbar
Definition: ERF_DataStruct.H:91
@ vbar
Definition: ERF_DataStruct.H:91
@ thetabar
Definition: ERF_DataStruct.H:91
@ nvars_sponge
Definition: ERF_DataStruct.H:96
@ vbar_sponge
Definition: ERF_DataStruct.H:96
@ ubar_sponge
Definition: ERF_DataStruct.H:96
@ v_x
Definition: ERF_DataStruct.H:22
@ u_y
Definition: ERF_DataStruct.H:23
@ v_y
Definition: ERF_DataStruct.H:23
@ m_y
Definition: ERF_DataStruct.H:23
@ u_x
Definition: ERF_DataStruct.H:22
@ m_x
Definition: ERF_DataStruct.H:22
amrex::Real Real
Definition: ERF_ShocInterface.H:16
auto no_substep_fun
Definition: ERF_TI_no_substep_fun.H:4
auto slow_rhs_fun_post
Definition: ERF_TI_slow_rhs_post.H:3
auto slow_rhs_fun_pre
Definition: ERF_TI_slow_rhs_pre.H:6
auto fast_rhs_fun
Definition: ERF_TI_substep_fun.H:6
auto apply_bcs
Definition: ERF_TI_utils.H:71
auto cons_to_prim
Definition: ERF_TI_utils.H:4
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
Definition: ERF.H:905
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > mapfac
Definition: ERF.H:908
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
Definition: ERF.H:787
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhotheta_src
Definition: ERF.H:1199
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
Definition: ERF.H:874
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
Definition: ERF.H:872
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
Definition: ERF.H:1204
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
Definition: ERF.H:872
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
Definition: ERF.H:884
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
Definition: ERF.H:863
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
Definition: ERF.H:1232
amrex::Vector< amrex::Vector< amrex::Real > > h_rhoqt_src
Definition: ERF.H:1201
amrex::Vector< long > dt_mri_ratio
Definition: ERF.H:772
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
Definition: ERF.H:862
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
Definition: ERF.H:875
static int verbose
Definition: ERF.H:1122
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
Definition: ERF.H:874
std::unique_ptr< ProblemBase > prob
Definition: ERF.H:759
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
Definition: ERF.H:911
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
Definition: ERF.H:873
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
Definition: ERF.H:1211
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
Definition: ERF.H:1210
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhoqt_src
Definition: ERF.H:1202
amrex::Vector< amrex::Vector< amrex::Real > > h_rhotheta_src
Definition: ERF.H:1198
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
Definition: ERF.H:1207
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
Definition: ERF.H:864
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
Definition: ERF.H:1208
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
Definition: ERF.H:1205
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
Definition: ERF.H:872
static int fixed_mri_dt_ratio
Definition: ERF.H:997
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
Definition: ERF.H:1229
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
Definition: ERF.H:874
Definition: ERF_MRI.H:16
void set_slow_rhs_pre(std::function< void(T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const int)> F)
Definition: ERF_MRI.H:133
void set_no_substep(std::function< void(T &, T &, T &, amrex::Real, amrex::Real, int)> F)
Definition: ERF_MRI.H:159
void set_fast_rhs(std::function< void(int, int, int, T &, const T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const amrex::Real)> F)
Definition: ERF_MRI.H:142
void set_slow_fast_timestep_ratio(const int timestep_ratio=1)
Definition: ERF_MRI.H:149
amrex::Real advance(T &S_old, T &S_new, amrex::Real time, const amrex::Real time_step)
Definition: ERF_MRI.H:169
void set_slow_rhs_post(std::function< void(T &, T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const int)> F)
Definition: ERF_MRI.H:137
@ pi0_comp
Definition: ERF_IndexDefines.H:65
@ p0_comp
Definition: ERF_IndexDefines.H:64
@ r0_comp
Definition: ERF_IndexDefines.H:63
@ qt
Definition: ERF_Kessler.H:27
real(c_double), parameter p0
Definition: ERF_module_model_constants.F90:40
Definition: ERF_DiffStruct.H:19
MolecDiffType molec_diff_type
Definition: ERF_DiffStruct.H:84
bool rayleigh_damp_T
Definition: ERF_DataStruct.H:812
static MeshType mesh_type
Definition: ERF_DataStruct.H:777
bool rayleigh_damp_V
Definition: ERF_DataStruct.H:810
DiffChoice diffChoice
Definition: ERF_DataStruct.H:786
bool custom_rhotheta_forcing
Definition: ERF_DataStruct.H:843
bool custom_w_subsidence
Definition: ERF_DataStruct.H:845
bool rayleigh_damp_U
Definition: ERF_DataStruct.H:809
bool custom_geostrophic_profile
Definition: ERF_DataStruct.H:846
amrex::Vector< SubsteppingType > substepping_type
Definition: ERF_DataStruct.H:792
bool use_num_diff
Definition: ERF_DataStruct.H:866
bool custom_moisture_forcing
Definition: ERF_DataStruct.H:844
amrex::Vector< TurbChoice > turbChoice
Definition: ERF_DataStruct.H:788
bool rayleigh_damp_W
Definition: ERF_DataStruct.H:811
SpongeChoice spongeChoice
Definition: ERF_DataStruct.H:787
Definition: ERF_SpongeStruct.H:15
std::string sponge_type
Definition: ERF_SpongeStruct.H:58
Definition: ERF_TurbStruct.H:41
bool use_kturb
Definition: ERF_TurbStruct.H:376
Here is the call graph for this function:

◆ advance_lsm()

void ERF::advance_lsm ( int  lev,
amrex::MultiFab &  cons_in,
amrex::MultiFab &  xvel_in,
amrex::MultiFab &  yvel_in,
const amrex::Real dt_advance 
)
10 {
11  if (solverChoice.lsm_type != LandSurfaceType::None) {
12  if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
13  lsm.Advance(lev, cons_in, xvel_in, yvel_in, SFS_hfx3_lev[lev].get(), SFS_q1fx3_lev[lev].get(), dt_advance, istep[lev]);
14  } else {
15  lsm.Advance(lev, dt_advance);
16  }
17  }
18 }
LandSurface lsm
Definition: ERF.H:831
amrex::Vector< int > istep
Definition: ERF.H:765
void Advance(const int &lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, amrex::MultiFab *hfx3_out, amrex::MultiFab *qfx3_out, const amrex::Real &dt_advance, const int &nstep)
Definition: ERF_LandSurface.H:52
LandSurfaceType lsm_type
Definition: ERF_DataStruct.H:876

◆ advance_microphysics()

void ERF::advance_microphysics ( int  lev,
amrex::MultiFab &  cons_in,
const amrex::Real dt_advance,
const int &  iteration,
const amrex::Real time 
)
10 {
11  if (solverChoice.moisture_type != MoistureType::None) {
12  micro->Update_Micro_Vars_Lev(lev, cons);
13  micro->Advance(lev, dt_advance, iteration, time, solverChoice, vars_new, z_phys_nd, phys_bc_type);
14  micro->Update_State_Vars_Lev(lev, cons);
15  }
16 }
std::unique_ptr< Microphysics > micro
Definition: ERF.H:815

◆ advance_radiation()

void ERF::advance_radiation ( int  lev,
amrex::MultiFab &  cons_in,
const amrex::Real dt_advance 
)
8 {
9  if (solverChoice.rad_type != RadiationType::None) {
10 #ifdef ERF_USE_NETCDF
11  MultiFab *lat_ptr = lat_m[lev].get();
12  MultiFab *lon_ptr = lon_m[lev].get();
13 #else
14  MultiFab *lat_ptr = nullptr;
15  MultiFab *lon_ptr = nullptr;
16 #endif
17  // RRTMGP inputs names and pointers
18  Vector<std::string> lsm_input_names = rad[lev]->get_lsm_input_varnames();
19  Vector<MultiFab*> lsm_input_ptrs(lsm_input_names.size(),nullptr);
20  for (int i(0); i<lsm_input_ptrs.size(); ++i) {
21  int varIdx = lsm.Get_VarIdx(lev,lsm_input_names[i]);
22  lsm_input_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
23  }
24 
25  // RRTMGP output names and pointers
26  Vector<std::string> lsm_output_names = rad[lev]->get_lsm_output_varnames();
27  Vector<MultiFab*> lsm_output_ptrs(lsm_output_names.size(),nullptr);
28  for (int i(0); i<lsm_output_ptrs.size(); ++i) {
29  int varIdx = lsm.Get_VarIdx(lev,lsm_output_names[i]);
30  lsm_output_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
31  }
32 
33  // Enter radiation class driver
34  amrex::Real time_for_rad = t_new[lev] + start_time;
35  rad[lev]->Run(lev, istep[lev], time_for_rad, dt_advance,
36  cons.boxArray(), geom[lev], &(cons),
37  sw_lw_fluxes[lev].get(), solar_zenith[lev].get(),
38  lsm_input_ptrs, lsm_output_ptrs,
39  qheating_rates[lev].get(), z_phys_nd[lev].get() ,
40  lat_ptr, lon_ptr);
41  }
42 }
static amrex::Real start_time
Definition: ERF.H:977
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sw_lw_fluxes
Definition: ERF.H:847
amrex::Vector< std::unique_ptr< IRadiation > > rad
Definition: ERF.H:835
amrex::Vector< amrex::Real > t_new
Definition: ERF.H:769
amrex::Vector< std::unique_ptr< amrex::MultiFab > > solar_zenith
Definition: ERF.H:848
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lon_m
Definition: ERF.H:722
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lat_m
Definition: ERF.H:722
amrex::Vector< std::unique_ptr< amrex::MultiFab > > qheating_rates
Definition: ERF.H:836
int Get_VarIdx(const int &lev, std::string &varname)
Definition: ERF_LandSurface.H:98
amrex::MultiFab * Get_Data_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:83
RadiationType rad_type
Definition: ERF_DataStruct.H:877

◆ appendPlotVariables()

void ERF::appendPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
185 {
186  ParmParse pp(pp_prefix);
187 
188  Vector<std::string> plot_var_names(0);
189  if (pp.contains(pp_plot_var_names.c_str())) {
190  std::string nm;
191  int nPltVars = pp.countval(pp_plot_var_names.c_str());
192  for (int i = 0; i < nPltVars; i++) {
193  pp.get(pp_plot_var_names.c_str(), nm, i);
194  // Add the named variable to our list of plot variables
195  // if it is not already in the list
196  if (!containerHasElement(plot_var_names, nm)) {
197  plot_var_names.push_back(nm);
198  }
199  }
200  }
201 
202  Vector<std::string> tmp_plot_names(0);
203 #ifdef ERF_USE_PARTICLES
204  Vector<std::string> particle_mesh_plot_names;
205  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
206  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
207  std::string tmp(particle_mesh_plot_names[i]);
208  if (containerHasElement(plot_var_names, tmp) ) {
209  tmp_plot_names.push_back(tmp);
210  }
211  }
212 #endif
213 
214  {
215  Vector<std::string> microphysics_plot_names;
216  micro->GetPlotVarNames(microphysics_plot_names);
217  if (microphysics_plot_names.size() > 0) {
218  static bool first_call = true;
219  if (first_call) {
220  Print() << getEnumNameString(solverChoice.moisture_type)
221  << ": the following additional variables are available to plot:\n";
222  for (int i = 0; i < microphysics_plot_names.size(); i++) {
223  Print() << " " << microphysics_plot_names[i] << "\n";
224  }
225  first_call = false;
226  }
227  for (auto& plot_name : microphysics_plot_names) {
228  if (containerHasElement(plot_var_names, plot_name)) {
229  tmp_plot_names.push_back(plot_name);
230  }
231  }
232  }
233  }
234 
235  for (int i = 0; i < tmp_plot_names.size(); i++) {
236  a_plot_var_names.push_back( tmp_plot_names[i] );
237  }
238 
239  // Finally, check to see if we found all the requested variables
240  for (const auto& plot_name : plot_var_names) {
241  if (!containerHasElement(a_plot_var_names, plot_name)) {
242  if (amrex::ParallelDescriptor::IOProcessor()) {
243  Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
244  }
245  }
246  }
247 }
bool containerHasElement(const V &iterable, const T &query)
Definition: ERF_Container.H:5
std::string pp_prefix
Definition: ERF.H:495
Here is the call graph for this function:

◆ AverageDown()

void ERF::AverageDown ( )
private
17 {
18  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
19 
20  int src_comp, num_comp;
21  for (int lev = finest_level-1; lev >= 0; --lev)
22  {
23  // If anelastic we don't average down rho because rho == rho0.
24  if (solverChoice.anelastic[lev]) {
25  src_comp = 1;
26  } else {
27  src_comp = 0;
28  }
29  num_comp = vars_new[0][Vars::cons].nComp() - src_comp;
30  AverageDownTo(lev,src_comp,num_comp);
31  }
32 }
void AverageDownTo(int crse_lev, int scomp, int ncomp)
Definition: ERF_AverageDown.cpp:36
CouplingType coupling_type
Definition: ERF_DataStruct.H:872

◆ AverageDownTo()

void ERF::AverageDownTo ( int  crse_lev,
int  scomp,
int  ncomp 
)
37 {
38  if (solverChoice.anelastic[crse_lev]) {
39  AMREX_ALWAYS_ASSERT(scomp == 1);
40  } else {
41  AMREX_ALWAYS_ASSERT(scomp == 0);
42  }
43 
44  AMREX_ALWAYS_ASSERT(ncomp == vars_new[crse_lev][Vars::cons].nComp() - scomp);
45  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
46 
47  // ******************************************************************************************
48  // First do cell-centered quantities
49  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
50  // m is the map scale factor at cell centers
51  // Here we multiply (rho S) by detJ and divide (rho S) by m^2 before average down
52  // ******************************************************************************************
53  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
54  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
55  const Box& bx = mfi.tilebox();
56  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
57  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
58  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
59  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
60  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
61  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
62  {
63  cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
64  });
65  } else {
66  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
67  {
68  cons_arr(i,j,k,scomp+n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
69  });
70  }
71  } // mfi
72  } // lev
73 
74  int fine_lev = crse_lev+1;
75 
76  if (interpolation_type == StateInterpType::Perturbational) {
77  // Make the fine rho and (rho theta) be perturbational
78  MultiFab::Divide(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
79  Rho_comp,RhoTheta_comp,1,IntVect{0});
80  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
81  BaseState::r0_comp,Rho_comp,1,IntVect{0});
82  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
83  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
84 
85  // Make the crse rho and (rho theta) be perturbational
86  MultiFab::Divide(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
87  Rho_comp,RhoTheta_comp,1,IntVect{0});
88  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
89  BaseState::r0_comp,Rho_comp,1,IntVect{0});
90  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
91  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
92  }
93 
94  average_down(vars_new[crse_lev+1][Vars::cons],vars_new[crse_lev ][Vars::cons],
95  scomp, ncomp, refRatio(crse_lev));
96 
97  if (interpolation_type == StateInterpType::Perturbational) {
98  // Restore the fine data to what it was
99  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
100  BaseState::r0_comp,Rho_comp,1,IntVect{0});
101  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
102  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
103  MultiFab::Multiply(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
104  Rho_comp,RhoTheta_comp,1,IntVect{0});
105 
106  // Make the crse data be full values not perturbational
107  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
108  BaseState::r0_comp,Rho_comp,1,IntVect{0});
109  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
110  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
111  MultiFab::Multiply(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
112  Rho_comp,RhoTheta_comp,1,IntVect{0});
113  }
114 
115  vars_new[crse_lev][Vars::cons].FillBoundary(geom[crse_lev].periodicity());
116 
117  // ******************************************************************************************
118  // Here we multiply (rho S) by m^2 and divide by detJ after average down
119  // ******************************************************************************************
120  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
121  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
122  const Box& bx = mfi.tilebox();
123  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
124  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
125  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
126  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
127  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
128  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
129  {
130  cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
131  });
132  } else {
133  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
134  {
135  cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
136  });
137  }
138  } // mfi
139  } // lev
140 
141  // ******************************************************************************************
142  // Now average down momenta.
143  // Note that vars_new holds velocities not momenta, but we want to do conservative
144  // averaging so we first convert to momentum, then average down, then convert
145  // back to velocities -- only on the valid region
146  // ******************************************************************************************
147  for (int lev = crse_lev; lev <= crse_lev+1; lev++)
148  {
149  // FillBoundary for density so we can go back and forth between velocity and momentum
150  vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
151 
152  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
153  vars_new[lev][Vars::yvel], IntVect(0,0,0),
154  vars_new[lev][Vars::zvel], IntVect(0,0,0),
155  vars_new[lev][Vars::cons],
156  rU_new[lev],
157  rV_new[lev],
158  rW_new[lev],
159  Geom(lev).Domain(),
161  }
162 
163  average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
164  average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
165  average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
166 
167  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
169  vars_new[lev][Vars::yvel],
170  vars_new[lev][Vars::zvel],
171  vars_new[lev][Vars::cons],
172  rU_new[lev],
173  rV_new[lev],
174  rW_new[lev],
175  Geom(lev).Domain(),
177  }
178 }
void MomentumToVelocity(MultiFab &xvel, MultiFab &yvel, MultiFab &zvel, const MultiFab &density, const MultiFab &xmom_in, const MultiFab &ymom_in, const MultiFab &zmom_in, const Box &domain, const Vector< BCRec > &domain_bcs_type_h)
Definition: ERF_MomentumToVelocity.cpp:25
static StateInterpType interpolation_type
Definition: ERF.H:1137
@ th0_comp
Definition: ERF_IndexDefines.H:66
Here is the call graph for this function:

◆ build_fine_mask()

MultiFab & ERF::build_fine_mask ( int  level)

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters
levelFine level index which masks underlying coarser data
711 {
712  // Mask for zeroing covered cells
713  AMREX_ASSERT(level > 0);
714 
715  const BoxArray& cba = grids[level-1];
716  const DistributionMapping& cdm = dmap[level-1];
717 
718  // TODO -- we should make a vector of these a member of ERF class
719  fine_mask.define(cba, cdm, 1, 0, MFInfo());
720  fine_mask.setVal(1.0);
721 
722  BoxArray fba = grids[level];
723  iMultiFab ifine_mask = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
724 
725  const auto fma = fine_mask.arrays();
726  const auto ifma = ifine_mask.arrays();
727  ParallelFor(fine_mask, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
728  {
729  fma[bno](i,j,k) = ifma[bno](i,j,k);
730  });
731 
732  return fine_mask;
733 }
amrex::MultiFab fine_mask
Definition: ERF.H:1264

◆ ClearLevel()

void ERF::ClearLevel ( int  lev)
override
667 {
668  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
669  vars_new[lev][var_idx].clear();
670  vars_old[lev][var_idx].clear();
671  }
672 
673  base_state[lev].clear();
674 
675  rU_new[lev].clear();
676  rU_old[lev].clear();
677  rV_new[lev].clear();
678  rV_old[lev].clear();
679  rW_new[lev].clear();
680  rW_old[lev].clear();
681 
682  if (lev > 0) {
683  zmom_crse_rhs[lev].clear();
684  }
685 
687  pp_inc[lev].clear();
688  }
689 
690  // Clears the integrator memory
691  mri_integrator_mem[lev].reset();
692 
693  // Clears the physical boundary condition routines
694  physbcs_cons[lev].reset();
695  physbcs_u[lev].reset();
696  physbcs_v[lev].reset();
697  physbcs_w[lev].reset();
698  physbcs_base[lev].reset();
699 
700  // Clears the flux register array
701  advflux_reg[lev]->reset();
702 }
amrex::Vector< amrex::MultiFab > pp_inc
Definition: ERF.H:789
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
Definition: ERF.H:924
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
Definition: ERF.H:796
@ NumTypes
Definition: ERF_IndexDefines.H:144
bool project_initial_velocity
Definition: ERF_DataStruct.H:835

◆ cloud_fraction()

Real ERF::cloud_fraction ( amrex::Real  time)
452 {
453  BL_PROFILE("ERF::cloud_fraction()");
454 
455  int lev = 0;
456  // This holds all of qc
457  MultiFab qc(vars_new[lev][Vars::cons],make_alias,RhoQ2_comp,1);
458 
459  int direction = 2; // z-direction
460  Box const& domain = geom[lev].Domain();
461 
462  auto const& qc_arr = qc.const_arrays();
463 
464  // qc_2d is an BaseFab<int> holding the max value over the column
465  auto qc_2d = ReduceToPlane<ReduceOpMax,int>(direction, domain, qc,
466  [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) -> int
467  {
468  if (qc_arr[box_no](i,j,k) > 0) {
469  return 1;
470  } else {
471  return 0;
472  }
473  });
474 
475  auto* p = qc_2d.dataPtr();
476 
477  Long numpts = qc_2d.numPts();
478 
479  AMREX_ASSERT(numpts < Long(std::numeric_limits<int>::max));
480 
481 #if 1
482  if (ParallelDescriptor::UseGpuAwareMpi()) {
483  ParallelDescriptor::ReduceIntMax(p,static_cast<int>(numpts));
484  } else {
485  Gpu::PinnedVector<int> hv(numpts);
486  Gpu::copyAsync(Gpu::deviceToHost, p, p+numpts, hv.data());
487  Gpu::streamSynchronize();
488  ParallelDescriptor::ReduceIntMax(hv.data(),static_cast<int>(numpts));
489  Gpu::copyAsync(Gpu::hostToDevice, hv.data(), hv.data()+numpts, p);
490  }
491 
492  // Sum over component 0
493  Long num_cloudy = qc_2d.template sum<RunOn::Device>(0);
494 
495 #else
496  //
497  // We need this if we allow domain decomposition in the vertical
498  // but for now we leave it commented out
499  //
500  Long num_cloudy = Reduce::Sum<Long>(numpts,
501  [=] AMREX_GPU_DEVICE (Long i) -> Long {
502  if (p[i] == 1) {
503  return 1;
504  } else {
505  return 0;
506  }
507  });
508  ParallelDescriptor::ReduceLongSum(num_cloudy);
509 #endif
510 
511  Real num_total = qc_2d.box().d_numPts();
512 
513  Real cloud_frac = num_cloudy / num_total;
514 
515  return cloud_frac;
516 }
#define RhoQ2_comp
Definition: ERF_IndexDefines.H:43
@ qc
Definition: ERF_SatAdj.H:36

◆ compute_divergence()

void ERF::compute_divergence ( int  lev,
amrex::MultiFab &  rhs,
amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM >  rho0_u_const,
amrex::Geometry const &  geom_at_lev 
)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level 0.

11 {
12  BL_PROFILE("ERF::compute_divergence()");
13 
14  auto dxInv = geom_at_lev.InvCellSizeArray();
15 
16  // ****************************************************************************
17  // Compute divergence which will form RHS
18  // Note that we replace "rho0w" with the contravariant momentum, Omega
19  // ****************************************************************************
20  if (solverChoice.terrain_type == TerrainType::EB)
21  {
22  bool already_on_centroids = true;
23  EB_computeDivergence(rhs, rho0_u_const, geom_at_lev, already_on_centroids);
24  }
25  else if (SolverChoice::mesh_type == MeshType::ConstantDz)
26  {
27  computeDivergence(rhs, rho0_u_const, geom_at_lev);
28  }
29  else
30  {
31  for ( MFIter mfi(rhs,TilingIfNotGPU()); mfi.isValid(); ++mfi)
32  {
33  Box bx = mfi.tilebox();
34  const Array4<Real const>& rho0u_arr = rho0_u_const[0]->const_array(mfi);
35  const Array4<Real const>& rho0v_arr = rho0_u_const[1]->const_array(mfi);
36  const Array4<Real const>& rho0w_arr = rho0_u_const[2]->const_array(mfi);
37  const Array4<Real >& rhs_arr = rhs.array(mfi);
38 
39  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
40  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
41 
42  if (SolverChoice::mesh_type == MeshType::StretchedDz) {
43  Real* stretched_dz_d_ptr = stretched_dz_d[lev].data();
44  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
45  Real dz = stretched_dz_d_ptr[k];
46  Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
47  rhs_arr(i,j,k) = mfsq * ( (rho0u_arr(i+1,j ,k ) - rho0u_arr(i,j,k)) * dxInv[0]
48  +(rho0v_arr(i ,j+1,k ) - rho0v_arr(i,j,k)) * dxInv[1]
49  +(rho0w_arr(i ,j ,k+1) - rho0w_arr(i,j,k)) / dz );
50  });
51 
52  } else {
53 
54  //
55  // Note we compute the divergence using "rho0w" == Omega
56  //
57  const Array4<Real const>& ax_arr = ax[lev]->const_array(mfi);
58  const Array4<Real const>& ay_arr = ay[lev]->const_array(mfi);
59  const Array4<Real const>& dJ_arr = detJ_cc[lev]->const_array(mfi);
60  //
61  // az == 1 for terrain-fitted coordinates
62  //
63  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
64  {
65  Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
66  rhs_arr(i,j,k) = mfsq * ( (ax_arr(i+1,j,k)*rho0u_arr(i+1,j,k) - ax_arr(i,j,k)*rho0u_arr(i,j,k)) * dxInv[0]
67  +(ay_arr(i,j+1,k)*rho0v_arr(i,j+1,k) - ay_arr(i,j,k)*rho0v_arr(i,j,k)) * dxInv[1]
68  +( rho0w_arr(i,j,k+1) - rho0w_arr(i,j,k)) * dxInv[2] ) / dJ_arr(i,j,k);
69  });
70  }
71  } // mfi
72  }
73 }
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
Definition: ERF.H:885
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
Definition: ERF.H:886
static TerrainType terrain_type
Definition: ERF_DataStruct.H:768

◆ ComputeDt()

void ERF::ComputeDt ( int  step = -1)
private

Function that calls estTimeStep for each level

12 {
13  Vector<Real> dt_tmp(finest_level+1);
14 
15  for (int lev = 0; lev <= finest_level; ++lev)
16  {
17  dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
18  }
19 
20  ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
21 
22  Real dt_0 = dt_tmp[0];
23  int n_factor = 1;
24  for (int lev = 0; lev <= finest_level; ++lev) {
25  dt_tmp[lev] = amrex::min(dt_tmp[lev], change_max*dt[lev]);
26  n_factor *= nsubsteps[lev];
27  dt_0 = amrex::min(dt_0, n_factor*dt_tmp[lev]);
28  if (step == 0){
29  dt_0 *= init_shrink;
30  if (verbose && init_shrink != 1.0) {
31  Print() << "Timestep 0: shrink initial dt at level " << lev << " by " << init_shrink << std::endl;
32  }
33  }
34  }
35  // Limit dt's by the value of stop_time.
36  const Real eps = 1.e-3*dt_0;
37  if (t_new[0] + dt_0 > stop_time - eps) {
38  dt_0 = stop_time - t_new[0];
39  }
40 
41  dt[0] = dt_0;
42  for (int lev = 1; lev <= finest_level; ++lev) {
43  dt[lev] = dt[lev-1] / nsubsteps[lev];
44  }
45 }
amrex::Real estTimeStep(int lev, long &dt_fast_ratio) const
Definition: ERF_ComputeTimestep.cpp:54
static amrex::Real stop_time
Definition: ERF.H:978
amrex::Vector< int > nsubsteps
Definition: ERF.H:766
static amrex::Real init_shrink
Definition: ERF.H:989
static amrex::Real change_max
Definition: ERF.H:990

◆ ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells ( const SolverChoice sc)
inlinestaticprivate
1271  {
1272  int ngrow = 0;
1273 
1274  if (sc.use_num_diff)
1275  {
1276  ngrow = 3;
1277  } else {
1278  if (
1285  { ngrow = 3; }
1286  else if (
1293  { ngrow = 3; }
1294  else if (
1303  { ngrow = 3; }
1304  else if (
1313  { ngrow = 4; }
1314  else
1315  {
1316  if (sc.terrain_type == TerrainType::EB){
1317  ngrow = 3;
1318  } else {
1319  ngrow = 2;
1320  }
1321  }
1322  }
1323 
1324  return ngrow;
1325  }
@ Centered_6th
AdvType moistscal_horiz_adv_type
Definition: ERF_AdvStruct.H:399
AdvType dycore_vert_adv_type
Definition: ERF_AdvStruct.H:396
AdvType moistscal_vert_adv_type
Definition: ERF_AdvStruct.H:400
AdvType dryscal_horiz_adv_type
Definition: ERF_AdvStruct.H:397
AdvType dycore_horiz_adv_type
Definition: ERF_AdvStruct.H:395
AdvType dryscal_vert_adv_type
Definition: ERF_AdvStruct.H:398
AdvChoice advChoice
Definition: ERF_DataStruct.H:785

◆ Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int  lev)
private
2372 {
2373  auto& fine_new = vars_new[lev];
2374  auto& crse_new = vars_new[lev-1];
2375  auto& ba_fine = fine_new[Vars::cons].boxArray();
2376  auto& ba_crse = crse_new[Vars::cons].boxArray();
2377  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2378  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2379 
2380  int ncomp = vars_new[lev][Vars::cons].nComp();
2381 
2382  FPr_c.emplace_back(ba_fine, dm_fine, geom[lev] ,
2383  ba_crse, dm_crse, geom[lev-1],
2384  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2385  FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2386  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2387  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2388  FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2389  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2390  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2391  FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2392  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2393  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2394 }
int cf_set_width
Definition: ERF.H:855

◆ CreateWeatherDataGeomBoxArrayDistMap()

void ERF::CreateWeatherDataGeomBoxArrayDistMap ( const std::string &  filename,
amrex::Geometry &  geom_weather,
amrex::BoxArray &  nba,
amrex::DistributionMapping &  dm 
)
159 {
160  Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
161  Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
162 
163  ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
164  xvec_h, yvec_h, zvec_h, rho_h,
165  uvel_h, vvel_h, wvel_h,
166  theta_h, qv_h, qc_h, qr_h);
167 
168  const auto prob_lo_erf = geom[0].ProbLoArray();
169  const auto prob_hi_erf = geom[0].ProbHiArray();
170  const auto dx_erf = geom[0].CellSizeArray();
171 
172  if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
173  amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
174  }
175  if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
176  amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
177  }
178  if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
179  amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
180  }
181  if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
182  amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
183  }
184 
185  // Number of cells
186  int nx_cells = xvec_h.size()-1;
187  int ny_cells = yvec_h.size()-1;
188 
189  const amrex::Geometry& geom0 = geom[0]; // or whatever your Geometry vector is called
190  const amrex::Box& domainBox = geom0.Domain();
191  const amrex::IntVect& domainSize = domainBox.size(); // Number of cells in each direction
192  int nz_cells = domainSize[2];
193 
194  IntVect dom_lo(0, 0, 0);
195  IntVect dom_hi(nx_cells-1, ny_cells-1, nz_cells-1);
196  Box domain(dom_lo, dom_hi);
197 
198  const amrex::Real* prob_hi = geom0.ProbHi();
199 
200  // Define the extents of the physical domain box
201  RealBox real_box({xvec_h[0], yvec_h[0], zvec_h[0]}, {xvec_h[nx_cells], yvec_h[ny_cells], prob_hi[2]});
202 
203  int coord = 0; // Cartesian
204  Array<int, AMREX_SPACEDIM> is_periodic{0, 0, 0}; // non-periodic
205 
206  geom_weather.define(domain, real_box, coord, is_periodic);
207 
208  BoxArray ba(domain);
209  ba.maxSize(64);
210  nba = amrex::convert(ba, IntVect::TheNodeVector()); // nodal in all directions
211 
212  // Create DistributionMapping
213  dm = DistributionMapping(nba);
214  }
const auto & dom_hi
Definition: ERF_DiffSetup.H:10
const auto & dom_lo
Definition: ERF_DiffSetup.H:9
void ReadCustomBinaryIC(const std::string filename, amrex::Vector< amrex::Real > &latvec_h, amrex::Vector< amrex::Real > &lonvec_h, amrex::Vector< amrex::Real > &xvec_h, amrex::Vector< amrex::Real > &yvec_h, amrex::Vector< amrex::Real > &zvec_h, amrex::Vector< amrex::Real > &rho_h, amrex::Vector< amrex::Real > &uvel_h, amrex::Vector< amrex::Real > &vvel_h, amrex::Vector< amrex::Real > &wvel_h, amrex::Vector< amrex::Real > &theta_h, amrex::Vector< amrex::Real > &qv_h, amrex::Vector< amrex::Real > &qc_h, amrex::Vector< amrex::Real > &qr_h)
Definition: ERF_ReadCustomBinaryIC.H:12
Here is the call graph for this function:

◆ DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int  i)
inlineprivate
1336  {
1337  return *datalog[i];
1338  }
amrex::Vector< std::unique_ptr< std::fstream > > datalog
Definition: ERF.H:1512

◆ DataLogName()

std::string ERF::DataLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith datalog file.

1528 { return datalogname[i]; }
amrex::Vector< std::string > datalogname
Definition: ERF.H:1515

◆ Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int  lev)
private
2398 {
2399  auto& fine_new = vars_new[lev];
2400  auto& crse_new = vars_new[lev-1];
2401  auto& ba_fine = fine_new[Vars::cons].boxArray();
2402  auto& ba_crse = crse_new[Vars::cons].boxArray();
2403  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2404  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2405 
2406  int ncomp = fine_new[Vars::cons].nComp();
2407 
2408  FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev] ,
2409  ba_crse, dm_crse, geom[lev-1],
2410  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2411  FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2412  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2413  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2414  FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2415  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2416  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2417  FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2418  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2419  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2420 }

◆ DerDataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DerDataLog ( int  i)
inlineprivate
1343  {
1344  return *der_datalog[i];
1345  }
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
Definition: ERF.H:1513

◆ DerDataLogName()

std::string ERF::DerDataLogName ( int  i) const
inlineprivatenoexcept
1529 { return der_datalogname[i]; }
amrex::Vector< std::string > der_datalogname
Definition: ERF.H:1516

◆ derive_diag_profiles()

void ERF::derive_diag_profiles ( amrex::Real  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
205 {
206  // We assume that this is always called at level 0
207  int lev = 0;
208 
209  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
210  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
211  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, uw, vv, vw, ww, uth, vth, wth,
212  // 0 1 2 3 4 5 6 7 8 9 10 11 12 13
213  // thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr,
214  // 14 15 16 17 18 19 20 21 22 23 24 25 26 27
215  // qi, qs, qg, wthv
216  // 28 29 30 31
217  MultiFab mf_out(grids[lev], dmap[lev], 32, 0);
218 
219  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
220 
221  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
222  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
223  MultiFab w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
224 
225  average_face_to_cellcenter(mf_vels,0,
226  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
227 
228  int zdir = 2;
229  auto domain = geom[0].Domain();
230 
231  // Sum in the horizontal plane
232  h_avg_u = sumToLine(mf_vels ,0,1,domain,zdir);
233  h_avg_v = sumToLine(mf_vels ,1,1,domain,zdir);
234  h_avg_w = sumToLine(mf_vels ,2,1,domain,zdir);
235 
236  int hu_size = h_avg_u.size();
237 
238  // Divide by the total number of cells we are averaging over
239  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
240  for (int k = 0; k < hu_size; ++k) {
241  h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
242  }
243 
244  Gpu::DeviceVector<Real> d_avg_u(hu_size, Real(0.0));
245  Gpu::DeviceVector<Real> d_avg_v(hu_size, Real(0.0));
246  Gpu::DeviceVector<Real> d_avg_w(hu_size, Real(0.0));
247 
248 #if 0
249  auto* avg_u_ptr = d_avg_u.data();
250  auto* avg_v_ptr = d_avg_v.data();
251  auto* avg_w_ptr = d_avg_w.data();
252 #endif
253 
254  Gpu::copy(Gpu::hostToDevice, h_avg_u.begin(), h_avg_u.end(), d_avg_u.begin());
255  Gpu::copy(Gpu::hostToDevice, h_avg_v.begin(), h_avg_v.end(), d_avg_v.begin());
256  Gpu::copy(Gpu::hostToDevice, h_avg_w.begin(), h_avg_w.end(), d_avg_w.begin());
257 
258  int nvars = vars_new[lev][Vars::cons].nComp();
259  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
260 
261  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
262 
263  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
264 
265  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
266  {
267  const Box& bx = mfi.tilebox();
268  const Array4<Real>& fab_arr = mf_out.array(mfi);
269  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
270  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
271  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
272  const Array4<Real>& cons_arr = mf_cons.array(mfi);
273  const Array4<Real>& p0_arr = p_hse.array(mfi);
274  const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
275  Array4<const Real>{};
276 
277  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
278  {
279  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
280  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
281  fab_arr(i, j, k, 1) = theta;
282  Real ksgs = 0.0;
283  if (l_use_KE) {
284  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
285  }
286  fab_arr(i, j, k, 2) = ksgs;
287 #if 1
288  if (l_use_kturb) {
289  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
290  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
291  } else {
292  fab_arr(i, j, k, 3) = 0.0;
293  fab_arr(i, j, k, 4) = 0.0;
294  }
295 #else
296  // Here we hijack the "Kturb" variable name to print out the resolved kinetic energy
297  Real upert = u_cc_arr(i,j,k) - avg_u_ptr[k];
298  Real vpert = v_cc_arr(i,j,k) - avg_v_ptr[k];
299  Real wpert = w_cc_arr(i,j,k) - avg_w_ptr[k];
300  fab_arr(i, j, k, 3) = 0.5 * (upert*upert + vpert*vpert + wpert*wpert);
301 #endif
302  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
303  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
304  fab_arr(i, j, k, 7) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k); // u*w
305  fab_arr(i, j, k, 8) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
306  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k); // v*w
307  fab_arr(i, j, k,10) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k); // w*w
308  fab_arr(i, j, k,11) = u_cc_arr(i,j,k) * theta; // u*th
309  fab_arr(i, j, k,12) = v_cc_arr(i,j,k) * theta; // v*th
310  fab_arr(i, j, k,13) = w_cc_arr(i,j,k) * theta; // w*th
311  fab_arr(i, j, k,14) = theta * theta; // th*th
312 
313  // if the number of fields is changed above, then be sure to update
314  // the following def!
315  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,8) + fab_arr(i,j,k,10);
316  fab_arr(i, j, k,15) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
317  fab_arr(i, j, k,16) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
318  fab_arr(i, j, k,17) = uiui * w_cc_arr(i,j,k); // (ui*ui)*w
319 
320  if (!use_moisture) {
321  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
322  p -= p0_arr(i,j,k);
323  fab_arr(i, j, k,18) = p; // p
324  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
325  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
326  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
327  fab_arr(i, j, k,22) = 0.; // qv
328  fab_arr(i, j, k,23) = 0.; // qc
329  fab_arr(i, j, k,24) = 0.; // qr
330  fab_arr(i, j, k,25) = 0.; // w*qv
331  fab_arr(i, j, k,26) = 0.; // w*qc
332  fab_arr(i, j, k,27) = 0.; // w*qr
333  fab_arr(i, j, k,28) = 0.; // qi
334  fab_arr(i, j, k,29) = 0.; // qs
335  fab_arr(i, j, k,30) = 0.; // qg
336  fab_arr(i, j, k,31) = 0.; // w*thv
337  }
338  });
339  } // mfi
340 
341  if (use_moisture)
342  {
343  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
344 
345  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
346  {
347  const Box& bx = mfi.tilebox();
348  const Array4<Real>& fab_arr = mf_out.array(mfi);
349  const Array4<Real>& cons_arr = mf_cons.array(mfi);
350  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
351  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
352  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
353  const Array4<Real>& p0_arr = p_hse.array(mfi);
354 
355  int rhoqr_comp = solverChoice.moisture_indices.qr;
356 
357  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
358  {
359  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
360  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
361  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
362  Real(0.0);
363  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
364 
365  p -= p0_arr(i,j,k);
366  fab_arr(i, j, k,18) = p; // p
367  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
368  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
369  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
370  fab_arr(i, j, k,22) = qv; // qv
371  fab_arr(i, j, k,23) = qc; // qc
372  fab_arr(i, j, k,24) = qr; // qr
373  fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * qv; // w*qv
374  fab_arr(i, j, k,26) = w_cc_arr(i,j,k) * qc; // w*qc
375  fab_arr(i, j, k,27) = w_cc_arr(i,j,k) * qr; // w*qr
376  if (n_qstate_moist > 3) {
377  fab_arr(i, j, k,28) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
378  fab_arr(i, j, k,29) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
379  fab_arr(i, j, k,30) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
380  } else {
381  fab_arr(i, j, k,28) = 0.0; // qi
382  fab_arr(i, j, k,29) = 0.0; // qs
383  fab_arr(i, j, k,30) = 0.0; // qg
384  }
385  Real ql = qc + qr;
386  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
387  Real thv = theta * (1 + 0.61*qv - ql);
388  fab_arr(i, j, k,31) = w_cc_arr(i,j,k) * thv; // w*thv
389  });
390  } // mfi
391  } // use_moisture
392 
393  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
394  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
395  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
396  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
397  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
398  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
399  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
400  h_avg_uw = sumToLine(mf_out, 7,1,domain,zdir);
401  h_avg_vv = sumToLine(mf_out, 8,1,domain,zdir);
402  h_avg_vw = sumToLine(mf_out, 9,1,domain,zdir);
403  h_avg_ww = sumToLine(mf_out,10,1,domain,zdir);
404  h_avg_uth = sumToLine(mf_out,11,1,domain,zdir);
405  h_avg_vth = sumToLine(mf_out,12,1,domain,zdir);
406  h_avg_wth = sumToLine(mf_out,13,1,domain,zdir);
407  h_avg_thth = sumToLine(mf_out,14,1,domain,zdir);
408  h_avg_uiuiu = sumToLine(mf_out,15,1,domain,zdir);
409  h_avg_uiuiv = sumToLine(mf_out,16,1,domain,zdir);
410  h_avg_uiuiw = sumToLine(mf_out,17,1,domain,zdir);
411  h_avg_p = sumToLine(mf_out,18,1,domain,zdir);
412  h_avg_pu = sumToLine(mf_out,19,1,domain,zdir);
413  h_avg_pv = sumToLine(mf_out,20,1,domain,zdir);
414  h_avg_pw = sumToLine(mf_out,21,1,domain,zdir);
415  h_avg_qv = sumToLine(mf_out,22,1,domain,zdir);
416  h_avg_qc = sumToLine(mf_out,23,1,domain,zdir);
417  h_avg_qr = sumToLine(mf_out,24,1,domain,zdir);
418  h_avg_wqv = sumToLine(mf_out,25,1,domain,zdir);
419  h_avg_wqc = sumToLine(mf_out,26,1,domain,zdir);
420  h_avg_wqr = sumToLine(mf_out,27,1,domain,zdir);
421  h_avg_qi = sumToLine(mf_out,28,1,domain,zdir);
422  h_avg_qs = sumToLine(mf_out,29,1,domain,zdir);
423  h_avg_qg = sumToLine(mf_out,30,1,domain,zdir);
424  h_avg_wthv = sumToLine(mf_out,31,1,domain,zdir);
425 
426  // Divide by the total number of cells we are averaging over
427  int h_avg_u_size = static_cast<int>(h_avg_u.size());
428  for (int k = 0; k < h_avg_u_size; ++k) {
429  h_avg_rho[k] /= area_z;
430  h_avg_ksgs[k] /= area_z;
431  h_avg_Kmv[k] /= area_z;
432  h_avg_Khv[k] /= area_z;
433  h_avg_th[k] /= area_z;
434  h_avg_thth[k] /= area_z;
435  h_avg_uu[k] /= area_z;
436  h_avg_uv[k] /= area_z;
437  h_avg_uw[k] /= area_z;
438  h_avg_vv[k] /= area_z;
439  h_avg_vw[k] /= area_z;
440  h_avg_ww[k] /= area_z;
441  h_avg_uth[k] /= area_z;
442  h_avg_vth[k] /= area_z;
443  h_avg_wth[k] /= area_z;
444  h_avg_uiuiu[k] /= area_z;
445  h_avg_uiuiv[k] /= area_z;
446  h_avg_uiuiw[k] /= area_z;
447  h_avg_p[k] /= area_z;
448  h_avg_pu[k] /= area_z;
449  h_avg_pv[k] /= area_z;
450  h_avg_pw[k] /= area_z;
451  h_avg_qv[k] /= area_z;
452  h_avg_qc[k] /= area_z;
453  h_avg_qr[k] /= area_z;
454  h_avg_wqv[k] /= area_z;
455  h_avg_wqc[k] /= area_z;
456  h_avg_wqr[k] /= area_z;
457  h_avg_qi[k] /= area_z;
458  h_avg_qs[k] /= area_z;
459  h_avg_qg[k] /= area_z;
460  h_avg_wthv[k] /= area_z;
461  }
462 
463 #if 0
464  // Here we print the integrated total kinetic energy as computed in the 1D profile above
465  Real sum = 0.;
466  Real dz = geom[0].ProbHi(2) / static_cast<Real>(h_avg_u_size);
467  for (int k = 0; k < h_avg_u_size; ++k) {
468  sum += h_avg_kturb[k] * h_avg_rho[k] * dz;
469  }
470  amrex::Print() << "ITKE " << time << " " << sum << " using " << h_avg_u_size << " " << dz << std::endl;
471 #endif
472 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getPgivenRTh(const amrex::Real rhotheta, const amrex::Real qv=0.)
Definition: ERF_EOS.H:81
#define RhoQ3_comp
Definition: ERF_IndexDefines.H:44
#define RhoQ6_comp
Definition: ERF_IndexDefines.H:47
#define RhoQ5_comp
Definition: ERF_IndexDefines.H:46
#define RhoKE_comp
Definition: ERF_IndexDefines.H:38
@ Theta_v
Definition: ERF_IndexDefines.H:176
@ Mom_v
Definition: ERF_IndexDefines.H:175
@ theta
Definition: ERF_MM5.H:20
@ qv
Definition: ERF_Kessler.H:28
Here is the call graph for this function:

◆ derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag ( amrex::Real  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
311 {
312  // We assume that this is always called at level 0
313  int lev = 0;
314 
315  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
316  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
317  // Note: "uiui" == u_i*u_i = u*u + v*v + w*w
318  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, vv, uth, vth,
319  // indices: 0 1 2 3 4 5 6 7 8 9
320  // thth, uiuiu, uiuiv, p, pu, pv, qv, qc, qr, qi, qs, qg
321  // 10 11 12 13 14 15 16 17 18 19 20 21
322  MultiFab mf_out(grids[lev], dmap[lev], 22, 0);
323 
324  // This will hold uw, vw, ww, wth, uiuiw, pw, wqv, wqc, wqr, wthv
325  // indices: 0 1 2 3 4 5 6 7 8 9
326  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 10, 0);
327 
328  // This is only used to average u and v; w is not averaged to cell centers
329  MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
330 
331  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
332  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
333  MultiFab w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers (staggered)
334 
335  int zdir = 2;
336  auto domain = geom[0].Domain();
337  Box stag_domain = domain;
338  stag_domain.convert(IntVect(0,0,1));
339 
340  int nvars = vars_new[lev][Vars::cons].nComp();
341  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
342 
343  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
344 
345  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
346 
347  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
348  {
349  const Box& bx = mfi.tilebox();
350  const Array4<Real>& fab_arr = mf_out.array(mfi);
351  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
352  const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
353  const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
354  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
355  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
356  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
357  const Array4<Real>& cons_arr = mf_cons.array(mfi);
358  const Array4<Real>& p0_arr = p_hse.array(mfi);
359  const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
360  Array4<const Real>{};
361 
362  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
363  {
364  u_cc_arr(i,j,k) = 0.5 * (u_arr(i,j,k) + u_arr(i+1,j ,k));
365  v_cc_arr(i,j,k) = 0.5 * (v_arr(i,j,k) + v_arr(i ,j+1,k));
366 
367  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
368  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
369  fab_arr(i, j, k, 1) = theta;
370  Real ksgs = 0.0;
371  if (l_use_KE) {
372  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
373  }
374  fab_arr(i, j, k, 2) = ksgs;
375  if (l_use_kturb) {
376  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
377  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
378  } else {
379  fab_arr(i, j, k, 3) = 0.0;
380  fab_arr(i, j, k, 4) = 0.0;
381  }
382  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
383  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
384  fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
385  fab_arr(i, j, k, 8) = u_cc_arr(i,j,k) * theta; // u*th
386  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * theta; // v*th
387  fab_arr(i, j, k,10) = theta * theta; // th*th
388 
389  Real wcc = 0.5 * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
390 
391  // if the number of fields is changed above, then be sure to update
392  // the following def!
393  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,7) + wcc*wcc;
394  fab_arr(i, j, k,11) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
395  fab_arr(i, j, k,12) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
396 
397  if (!use_moisture) {
398  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
399  p -= p0_arr(i,j,k);
400  fab_arr(i, j, k,13) = p; // p
401  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
402  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
403  fab_arr(i, j, k,16) = 0.; // qv
404  fab_arr(i, j, k,17) = 0.; // qc
405  fab_arr(i, j, k,18) = 0.; // qr
406  fab_arr(i, j, k,19) = 0.; // qi
407  fab_arr(i, j, k,20) = 0.; // qs
408  fab_arr(i, j, k,21) = 0.; // qg
409  }
410  });
411 
412  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
413  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
414  {
415  // average to z faces (first to cell centers, then in z)
416  Real uface = 0.25 * ( u_arr(i ,j,k) + u_arr(i ,j,k-1)
417  + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
418  Real vface = 0.25 * ( v_arr(i,j ,k) + v_arr(i,j ,k-1)
419  + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
420  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
421  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
422  Real thface = 0.5*(theta0 + theta1);
423  fab_arr_stag(i,j,k,0) = uface * w_fc_arr(i,j,k); // u*w
424  fab_arr_stag(i,j,k,1) = vface * w_fc_arr(i,j,k); // v*w
425  fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
426  fab_arr_stag(i,j,k,3) = thface * w_fc_arr(i,j,k); // th*w
427  Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
428  fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
429  if (!use_moisture) {
430  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp)) - p0_arr(i,j,k );
431  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
432  Real pface = 0.5 * (p0 + p1);
433  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
434  fab_arr_stag(i,j,k,6) = 0.; // w*qv
435  fab_arr_stag(i,j,k,7) = 0.; // w*qc
436  fab_arr_stag(i,j,k,8) = 0.; // w*qr
437  fab_arr_stag(i,j,k,9) = 0.; // w*thv
438  }
439  });
440 
441  } // mfi
442 
443  if (use_moisture)
444  {
445  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
446 
447  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
448  {
449  const Box& bx = mfi.tilebox();
450  const Array4<Real>& fab_arr = mf_out.array(mfi);
451  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
452  const Array4<Real>& cons_arr = mf_cons.array(mfi);
453  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
454  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
455  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
456  const Array4<Real>& p0_arr = p_hse.array(mfi);
457 
458  int rhoqr_comp = solverChoice.moisture_indices.qr;
459 
460  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
461  {
462  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
463  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
464  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
465  Real(0.0);
466  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
467 
468  p -= p0_arr(i,j,k);
469  fab_arr(i, j, k,13) = p; // p
470  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
471  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
472  fab_arr(i, j, k,16) = qv; // qv
473  fab_arr(i, j, k,17) = qc; // qc
474  fab_arr(i, j, k,18) = qr; // qr
475  if (n_qstate_moist > 3) { // SAM model
476  fab_arr(i, j, k,19) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
477  fab_arr(i, j, k,20) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
478  fab_arr(i, j, k,21) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
479  } else {
480  fab_arr(i, j, k,19) = 0.0; // qi
481  fab_arr(i, j, k,20) = 0.0; // qs
482  fab_arr(i, j, k,21) = 0.0; // qg
483  }
484  });
485 
486  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
487  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
488  {
489  Real qv0 = cons_arr(i,j,k ,RhoQ1_comp) / cons_arr(i,j,k ,Rho_comp);
490  Real qv1 = cons_arr(i,j,k-1,RhoQ1_comp) / cons_arr(i,j,k-1,Rho_comp);
491  Real qc0 = cons_arr(i,j,k ,RhoQ2_comp) / cons_arr(i,j,k ,Rho_comp);
492  Real qc1 = cons_arr(i,j,k-1,RhoQ2_comp) / cons_arr(i,j,k-1,Rho_comp);
493  Real qr0 = (rhoqr_comp > -1) ? cons_arr(i,j,k ,RhoQ3_comp) / cons_arr(i,j,k ,Rho_comp) :
494  Real(0.0);
495  Real qr1 = (rhoqr_comp > -1) ? cons_arr(i,j,k-1,RhoQ3_comp) / cons_arr(i,j,k-1,Rho_comp) :
496  Real(0.0);
497  Real qvface = 0.5 * (qv0 + qv1);
498  Real qcface = 0.5 * (qc0 + qc1);
499  Real qrface = 0.5 * (qr0 + qr1);
500 
501  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp), qv0) - p0_arr(i,j,k );
502  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv1) - p0_arr(i,j,k-1);
503  Real pface = 0.5 * (p0 + p1);
504 
505  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
506  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
507  Real thface = 0.5*(theta0 + theta1);
508  Real ql = qcface + qrface;
509  Real thv = thface * (1 + 0.61*qvface - ql);
510 
511  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
512  fab_arr_stag(i,j,k,6) = qvface * w_fc_arr(i,j,k); // w*qv
513  fab_arr_stag(i,j,k,7) = qcface * w_fc_arr(i,j,k); // w*qc
514  fab_arr_stag(i,j,k,8) = qrface * w_fc_arr(i,j,k); // w*qr
515  fab_arr_stag(i,j,k,9) = thv * w_fc_arr(i,j,k); // w*thv
516  });
517  } // mfi
518  } // use_moisture
519 
520  // Sum in the horizontal plane
521  h_avg_u = sumToLine(u_cc,0,1, domain,zdir);
522  h_avg_v = sumToLine(v_cc,0,1, domain,zdir);
523  h_avg_w = sumToLine(w_fc,0,1,stag_domain,zdir);
524 
525  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
526  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
527  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
528  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
529  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
530  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
531  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
532  h_avg_vv = sumToLine(mf_out, 7,1,domain,zdir);
533  h_avg_uth = sumToLine(mf_out, 8,1,domain,zdir);
534  h_avg_vth = sumToLine(mf_out, 9,1,domain,zdir);
535  h_avg_thth = sumToLine(mf_out,10,1,domain,zdir);
536  h_avg_uiuiu = sumToLine(mf_out,11,1,domain,zdir);
537  h_avg_uiuiv = sumToLine(mf_out,12,1,domain,zdir);
538  h_avg_p = sumToLine(mf_out,13,1,domain,zdir);
539  h_avg_pu = sumToLine(mf_out,14,1,domain,zdir);
540  h_avg_pv = sumToLine(mf_out,15,1,domain,zdir);
541  h_avg_qv = sumToLine(mf_out,16,1,domain,zdir);
542  h_avg_qc = sumToLine(mf_out,17,1,domain,zdir);
543  h_avg_qr = sumToLine(mf_out,18,1,domain,zdir);
544  h_avg_qi = sumToLine(mf_out,19,1,domain,zdir);
545  h_avg_qs = sumToLine(mf_out,20,1,domain,zdir);
546  h_avg_qg = sumToLine(mf_out,21,1,domain,zdir);
547 
548  h_avg_uw = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
549  h_avg_vw = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
550  h_avg_ww = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
551  h_avg_wth = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
552  h_avg_uiuiw = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
553  h_avg_pw = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
554  h_avg_wqv = sumToLine(mf_out_stag,6,1,stag_domain,zdir);
555  h_avg_wqc = sumToLine(mf_out_stag,7,1,stag_domain,zdir);
556  h_avg_wqr = sumToLine(mf_out_stag,8,1,stag_domain,zdir);
557  h_avg_wthv = sumToLine(mf_out_stag,9,1,stag_domain,zdir);
558 
559  // Divide by the total number of cells we are averaging over
560  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
561  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
562  for (int k = 0; k < unstag_size; ++k) {
563  h_avg_u[k] /= area_z;
564  h_avg_v[k] /= area_z;
565  h_avg_rho[k] /= area_z;
566  h_avg_ksgs[k] /= area_z;
567  h_avg_Kmv[k] /= area_z;
568  h_avg_Khv[k] /= area_z;
569  h_avg_th[k] /= area_z;
570  h_avg_thth[k] /= area_z;
571  h_avg_uu[k] /= area_z;
572  h_avg_uv[k] /= area_z;
573  h_avg_vv[k] /= area_z;
574  h_avg_uth[k] /= area_z;
575  h_avg_vth[k] /= area_z;
576  h_avg_uiuiu[k] /= area_z;
577  h_avg_uiuiv[k] /= area_z;
578  h_avg_p[k] /= area_z;
579  h_avg_pu[k] /= area_z;
580  h_avg_pv[k] /= area_z;
581  h_avg_qv[k] /= area_z;
582  h_avg_qc[k] /= area_z;
583  h_avg_qr[k] /= area_z;
584  h_avg_qi[k] /= area_z;
585  h_avg_qs[k] /= area_z;
586  h_avg_qg[k] /= area_z;
587  }
588 
589  for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
590  h_avg_w[k] /= area_z;
591  h_avg_uw[k] /= area_z;
592  h_avg_vw[k] /= area_z;
593  h_avg_ww[k] /= area_z;
594  h_avg_wth[k] /= area_z;
595  h_avg_uiuiw[k] /= area_z;
596  h_avg_pw[k] /= area_z;
597  h_avg_wqv[k] /= area_z;
598  h_avg_wqc[k] /= area_z;
599  h_avg_wqr[k] /= area_z;
600  h_avg_wthv[k] /= area_z;
601  }
602 }
const Box zbx
Definition: ERF_DiffSetup.H:23
Here is the call graph for this function:

◆ derive_stress_profiles()

void ERF::derive_stress_profiles ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
480 {
481  int lev = 0;
482 
483  // This will hold the stress tensor components
484  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
485 
486  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
487 
488  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
489 
490  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
491  {
492  const Box& bx = mfi.tilebox();
493  const Array4<Real>& fab_arr = mf_out.array(mfi);
494 
495  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
496 
497  // NOTE: These are from the last RK stage...
498  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
499  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
500  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
501  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
502  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
503  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
504 
505  // These should be re-calculated during ERF_slow_rhs_post
506  // -- just vertical SFS kinematic heat flux for now
507  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
508  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
509  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
510  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
511  Array4<const Real>{};
512  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
513  Array4<const Real>{};
514  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
515 
516  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
517  {
518  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
519  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
520  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
521  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
522  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
523  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
524  fab_arr(i, j, k, 2) = ( tau13_arr(i,j,k ) + tau13_arr(i+1,j,k )
525  + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) )
526  / ( rho_arr(i,j,k ) + rho_arr(i+1,j,k )
527  + rho_arr(i,j,k+1) + rho_arr(i+1,j,k+1) );
528  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
529  fab_arr(i, j, k, 4) = ( tau23_arr(i,j,k ) + tau23_arr(i,j+1,k )
530  + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) )
531  / ( rho_arr(i,j,k ) + rho_arr(i,j+1,k )
532  + rho_arr(i,j,k+1) + rho_arr(i,j+1,k+1) );
533  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
534  fab_arr(i, j, k, 6) = 0.5 * ( hfx3_arr(i,j,k) + hfx3_arr(i,j,k+1) ) / rho_arr(i,j,k);
535  fab_arr(i, j, k, 7) = (l_use_moist) ? 0.5 * ( q1fx3_arr(i,j,k) + q1fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
536  fab_arr(i, j, k, 8) = (l_use_moist) ? 0.5 * ( q2fx3_arr(i,j,k) + q2fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
537  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
538  });
539  }
540 
541  int zdir = 2;
542  auto domain = geom[0].Domain();
543 
544  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
545  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
546  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
547  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
548  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
549  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
550  h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
551  h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
552  h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
553  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
554 
555  int ht_size = h_avg_tau11.size();
556 
557  // Divide by the total number of cells we are averaging over
558  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
559  for (int k = 0; k < ht_size; ++k) {
560  h_avg_tau11[k] /= area_z;
561  h_avg_tau12[k] /= area_z;
562  h_avg_tau13[k] /= area_z;
563  h_avg_tau22[k] /= area_z;
564  h_avg_tau23[k] /= area_z;
565  h_avg_tau33[k] /= area_z;
566  h_avg_hfx3[k] /= area_z;
567  h_avg_q1fx3[k] /= area_z;
568  h_avg_q2fx3[k] /= area_z;
569  h_avg_diss[k] /= area_z;
570  }
571 }

◆ derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
610 {
611  int lev = 0;
612 
613  // This will hold the stress tensor components
614  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
615 
616  // This will hold Tau13 and Tau23
617  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 5, 0);
618 
619  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
620 
621  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
622 
623  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
624  {
625  const Box& bx = mfi.tilebox();
626  const Array4<Real>& fab_arr = mf_out.array(mfi);
627  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
628 
629  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
630 
631  // NOTE: These are from the last RK stage...
632  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
633  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
634  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
635  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
636  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
637  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
638 
639  // These should be re-calculated during ERF_slow_rhs_post
640  // -- just vertical SFS kinematic heat flux for now
641  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
642  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
643  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
644  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
645  Array4<const Real>{};
646  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
647  Array4<const Real>{};
648  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
649 
650  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
651  {
652  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
653  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
654  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
655  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
656  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
657  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
658  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
659  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
660  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
661  });
662 
663  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
664  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
665  {
666  Real rho_face = 0.5 * (rho_arr(i,j,k-1) + rho_arr(i,j,k));
667  // average from edge to face center
668  fab_arr_stag(i,j,k,0) = 0.5*(tau13_arr(i,j,k) + tau13_arr(i+1,j ,k)) / rho_face;
669  fab_arr_stag(i,j,k,1) = 0.5*(tau23_arr(i,j,k) + tau23_arr(i ,j+1,k)) / rho_face;
670 
671  fab_arr_stag(i,j,k,2) = hfx3_arr(i,j,k) / rho_face;
672  fab_arr_stag(i,j,k,3) = (l_use_moist) ? q1fx3_arr(i,j,k) / rho_face : 0.0;
673  fab_arr_stag(i,j,k,4) = (l_use_moist) ? q2fx3_arr(i,j,k) / rho_face : 0.0;
674  });
675  }
676 
677  int zdir = 2;
678  auto domain = geom[0].Domain();
679  Box stag_domain = domain;
680  stag_domain.convert(IntVect(0,0,1));
681 
682  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
683  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
684 // h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
685  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
686 // h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
687  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
688 // h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
689 // h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
690 // h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
691  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
692 
693  h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
694  h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
695  h_avg_hfx3 = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
696  h_avg_q1fx3 = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
697  h_avg_q2fx3 = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
698 
699  int ht_size = h_avg_tau11.size(); // _un_staggered
700 
701  // Divide by the total number of cells we are averaging over
702  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
703  for (int k = 0; k < ht_size; ++k) {
704  h_avg_tau11[k] /= area_z;
705  h_avg_tau12[k] /= area_z;
706  h_avg_tau13[k] /= area_z;
707  h_avg_tau22[k] /= area_z;
708  h_avg_tau23[k] /= area_z;
709  h_avg_tau33[k] /= area_z;
710  h_avg_hfx3[k] /= area_z;
711  h_avg_q1fx3[k] /= area_z;
712  h_avg_q2fx3[k] /= area_z;
713  h_avg_diss[k] /= area_z;
714  }
715  // staggered heights
716  h_avg_tau13[ht_size] /= area_z;
717  h_avg_tau23[ht_size] /= area_z;
718  h_avg_hfx3[ht_size] /= area_z;
719  h_avg_q1fx3[ht_size] /= area_z;
720  h_avg_q2fx3[ht_size] /= area_z;
721 }

◆ derive_upwp()

void ERF::derive_upwp ( amrex::Vector< amrex::Real > &  h_havg)

◆ EBFactory()

amrex::EBFArrayBoxFactory const& ERF::EBFactory ( int  lev) const
inlineprivatenoexcept
1546  {
1547  return *(eb[lev]->get_const_factory());
1548  }
amrex::Vector< std::unique_ptr< eb_ > > eb
Definition: ERF.H:1538

Referenced by WriteMyEBSurface().

Here is the caller graph for this function:

◆ erf_enforce_hse()

void ERF::erf_enforce_hse ( int  lev,
amrex::MultiFab &  dens,
amrex::MultiFab &  pres,
amrex::MultiFab &  pi,
amrex::MultiFab &  th,
amrex::MultiFab &  qv,
std::unique_ptr< amrex::MultiFab > &  z_cc 
)

Enforces hydrostatic equilibrium when using terrain.

Parameters
[in]levInteger specifying the current level
[out]densMultiFab storing base state density
[out]presMultiFab storing base state pressure
[out]piMultiFab storing base state Exner function
[in]z_ccPointer to MultiFab storing cell centered z-coordinates
163 {
164  Real l_gravity = solverChoice.gravity;
165  bool l_use_terrain = (solverChoice.mesh_type != MeshType::ConstantDz);
166 
167  const auto geomdata = geom[lev].data();
168  const Real dz = geomdata.CellSize(2);
169 
170  for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
171  {
172  // Create a flat box with same horizontal extent but only one cell in vertical
173  const Box& tbz = mfi.nodaltilebox(2);
174  int klo = tbz.smallEnd(2);
175  int khi = tbz.bigEnd(2);
176 
177  // Note we only grow by 1 because that is how big z_cc is.
178  Box b2d = tbz; // Copy constructor
179  b2d.grow(0,1);
180  b2d.grow(1,1);
181  b2d.setRange(2,0);
182 
183  // Intersect this box with the domain
184  Box zdomain = convert(geom[lev].Domain(),tbz.ixType());
185  b2d &= zdomain;
186 
187  // We integrate to the first cell (and below) by using rho in this cell
188  // If gravity == 0 this is constant pressure
189  // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
190  // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
191  // Note ng_pres_hse = 1
192 
193  // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
194  // Note that gravity is positive
195 
196  Array4<Real> rho_arr = dens.array(mfi);
197  Array4<Real> pres_arr = pres.array(mfi);
198  Array4<Real> pi_arr = pi.array(mfi);
199  Array4<Real> th_arr = theta.array(mfi);
200  Array4<Real> zcc_arr;
201  if (l_use_terrain) {
202  zcc_arr = z_cc->array(mfi);
203  }
204 
205  const Real rdOcp = solverChoice.rdOcp;
206 
207  ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
208  {
209  // Set value at surface from Newton iteration for rho
210  if (klo == 0)
211  {
212  // Physical height of the terrain at cell center
213  Real hz;
214  if (l_use_terrain) {
215  hz = zcc_arr(i,j,klo);
216  } else {
217  hz = 0.5*dz;
218  }
219 
220  pres_arr(i,j,klo) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
221  pi_arr(i,j,klo) = getExnergivenP(pres_arr(i,j,klo), rdOcp);
222  th_arr(i,j,klo) = getRhoThetagivenP(pres_arr(i,j,klo)) / rho_arr(i,j,klo);
223 
224  //
225  // Set ghost cell with dz and rho at boundary
226  // (We will set the rest of the ghost cells in the boundary condition routine)
227  //
228  pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
229  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
230  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
231 
232  } else {
233 
234  // If level > 0 and klo > 0, we need to use the value of pres_arr(i,j,klo-1) which was
235  // filled from FillPatch-ing it.
236  Real dz_loc;
237  if (l_use_terrain) {
238  dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
239  } else {
240  dz_loc = dz;
241  }
242 
243  Real dens_interp = 0.5*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
244  pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
245 
246  pi_arr(i,j,klo ) = getExnergivenP(pres_arr(i,j,klo ), rdOcp);
247  th_arr(i,j,klo ) = getRhoThetagivenP(pres_arr(i,j,klo )) / rho_arr(i,j,klo );
248 
249  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
250  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
251  }
252 
253  Real dens_interp;
254  if (l_use_terrain) {
255  for (int k = klo+1; k <= khi; k++) {
256  Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
257  dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
258  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
259  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
260  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
261  }
262  } else {
263  for (int k = klo+1; k <= khi; k++) {
264  dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
265  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
266  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
267  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
268  }
269  }
270  });
271 
272  } // mfi
273 
274  dens.FillBoundary(geom[lev].periodicity());
275  pres.FillBoundary(geom[lev].periodicity());
276  pi.FillBoundary(geom[lev].periodicity());
277  theta.FillBoundary(geom[lev].periodicity());
278  qv.FillBoundary(geom[lev].periodicity());
279 }
constexpr amrex::Real p_0
Definition: ERF_Constants.H:18
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getRhoThetagivenP(const amrex::Real p, const amrex::Real qv=0.0)
Definition: ERF_EOS.H:172
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getExnergivenP(const amrex::Real P, const amrex::Real rdOcp)
Definition: ERF_EOS.H:141
@ pres
Definition: ERF_Kessler.H:25
real(c_double), parameter, private pi
Definition: ERF_module_mp_morr_two_moment.F90:100
amrex::Real rdOcp
Definition: ERF_DataStruct.H:828
amrex::Real gravity
Definition: ERF_DataStruct.H:826
Here is the call graph for this function:

◆ ERF_shared()

void ERF::ERF_shared ( )
120 {
121  if (ParallelDescriptor::IOProcessor()) {
122  const char* erf_hash = buildInfoGetGitHash(1);
123  const char* amrex_hash = buildInfoGetGitHash(2);
124  const char* buildgithash = buildInfoGetBuildGitHash();
125  const char* buildgitname = buildInfoGetBuildGitName();
126 
127  if (strlen(erf_hash) > 0) {
128  Print() << "\n"
129  << "ERF git hash: " << erf_hash << "\n";
130  }
131  if (strlen(amrex_hash) > 0) {
132  Print() << "AMReX git hash: " << amrex_hash << "\n";
133  }
134  if (strlen(buildgithash) > 0) {
135  Print() << buildgitname << " git hash: " << buildgithash << "\n";
136  }
137 
138  Print() << "\n";
139  }
140 
141  int nlevs_max = max_level + 1;
142 
143 #ifdef ERF_USE_WINDFARM
144  Nturb.resize(nlevs_max);
145  vars_windfarm.resize(nlevs_max);
146  SMark.resize(nlevs_max);
147 #endif
148 
149  qheating_rates.resize(nlevs_max);
150  sw_lw_fluxes.resize(nlevs_max);
151  solar_zenith.resize(nlevs_max);
152 
153  // NOTE: size lsm before readparams (chooses the model at all levels)
154  lsm.ReSize(nlevs_max);
155  lsm_data.resize(nlevs_max);
156  lsm_flux.resize(nlevs_max);
157 
158  // NOTE: size canopy model before readparams (if file exists, we construct)
159  m_forest_drag.resize(nlevs_max);
160  for (int lev = 0; lev <= max_level; ++lev) { m_forest_drag[lev] = nullptr;}
161 
162  ReadParameters();
163  initializeMicrophysics(nlevs_max);
164 
165 #ifdef ERF_USE_WINDFARM
166  initializeWindFarm(nlevs_max);
167 #endif
168 
169  rad.resize(nlevs_max);
170  for (int lev = 0; lev <= max_level; ++lev) {
171  if (solverChoice.rad_type == RadiationType::RRTMGP) {
172 #ifdef ERF_USE_RRTMGP
173  rad[lev] = std::make_unique<Radiation>(lev, solverChoice);
174  // pass radiation datalog frequency to model - RRTMGP needs to know when to save data for profiles
175  rad[lev]->setDataLogFrequency(rad_datalog_int);
176 #endif
177  } else if (solverChoice.rad_type != RadiationType::None) {
178  Abort("Don't know this radiation model!");
179  }
180  }
181 
182  const std::string& pv3d_1 = "plot_vars_1" ; setPlotVariables(pv3d_1,plot3d_var_names_1);
183  const std::string& pv3d_2 = "plot_vars_2" ; setPlotVariables(pv3d_2,plot3d_var_names_2);
184  const std::string& pv2d_1 = "plot2d_vars_1"; setPlotVariables(pv2d_1,plot2d_var_names_1);
185  const std::string& pv2d_2 = "plot2d_vars_2"; setPlotVariables(pv2d_2,plot2d_var_names_2);
186 
187  // This is only used when we have mesh_type == MeshType::StretchedDz
188  stretched_dz_h.resize(nlevs_max);
189  stretched_dz_d.resize(nlevs_max);
190 
191  // Initialize staggered vertical levels for grid stretching or terrain, and
192  // to simplify Rayleigh damping layer calculations.
193  zlevels_stag.resize(max_level+1);
197  geom,
198  refRatio(),
201  solverChoice.dz0);
202 
203  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
204  SolverChoice::mesh_type == MeshType::VariableDz) {
205  int nz = geom[0].Domain().length(2) + 1; // staggered
206  if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
207  Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
208  << " does not match highest requested z level " << zlevels_stag[0][nz-1]
209  << std::endl;
210  }
211  if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > 1.0e-4) {
212  Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
213  << " does not match lowest requested level " << zlevels_stag[0][0]
214  << std::endl;
215  }
216 
217  // Redefine the problem domain here?
218  }
219 
220  // Get lo/hi indices for massflux calc
222  if (solverChoice.mesh_type == MeshType::ConstantDz) {
223  const Real massflux_zlo = solverChoice.const_massflux_layer_lo - geom[0].ProbLo(2);
224  const Real massflux_zhi = solverChoice.const_massflux_layer_hi - geom[0].ProbLo(2);
225  const Real dz = geom[0].CellSize(2);
226  if (massflux_zlo == -1e34) {
227  solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
228  } else {
229  solverChoice.massflux_klo = static_cast<int>(std::ceil(massflux_zlo / dz - 0.5));
230  }
231  if (massflux_zhi == 1e34) {
232  solverChoice.massflux_khi = geom[0].Domain().bigEnd(2);
233  } else {
234  solverChoice.massflux_khi = static_cast<int>(std::floor(massflux_zhi / dz - 0.5));
235  }
236  } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
237  const Real massflux_zlo = solverChoice.const_massflux_layer_lo;
238  const Real massflux_zhi = solverChoice.const_massflux_layer_hi;
239  solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
240  solverChoice.massflux_khi = geom[0].Domain().bigEnd(2) + 1;
241  for (int k=0; k <= geom[0].Domain().bigEnd(2)+1; ++k) {
242  if (zlevels_stag[0][k] <= massflux_zlo) solverChoice.massflux_klo = k;
243  if (zlevels_stag[0][k] <= massflux_zhi) solverChoice.massflux_khi = k;
244  }
245  } else { // solverChoice.mesh_type == MeshType::VariableDz
246  Error("Const massflux with variable dz not supported -- planar averages are on k rather than constant-z planes");
247  }
248 
249  Print() << "Constant mass flux based on k in ["
250  << solverChoice.massflux_klo << ", " << solverChoice.massflux_khi << "]" << std::endl;
251  }
252 
253  prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
254 
255  // Geometry on all levels has been defined already.
256 
257  // No valid BoxArray and DistributionMapping have been defined.
258  // But the arrays for them have been resized.
259 
260  istep.resize(nlevs_max, 0);
261  nsubsteps.resize(nlevs_max, 1);
262  for (int lev = 1; lev <= max_level; ++lev) {
263  nsubsteps[lev] = MaxRefRatio(lev-1);
264  }
265 
266  t_new.resize(nlevs_max, 0.0);
267  t_old.resize(nlevs_max, -1.e100);
268  dt.resize(nlevs_max, std::min(1.e100,dt_max_initial));
269  dt_mri_ratio.resize(nlevs_max, 1);
270 
271  vars_new.resize(nlevs_max);
272  vars_old.resize(nlevs_max);
273  gradp.resize(nlevs_max);
274 
275  // We resize this regardless in order to pass it without error
276  pp_inc.resize(nlevs_max);
277 
278  rU_new.resize(nlevs_max);
279  rV_new.resize(nlevs_max);
280  rW_new.resize(nlevs_max);
281 
282  rU_old.resize(nlevs_max);
283  rV_old.resize(nlevs_max);
284  rW_old.resize(nlevs_max);
285 
286  // xmom_crse_rhs.resize(nlevs_max);
287  // ymom_crse_rhs.resize(nlevs_max);
288  zmom_crse_rhs.resize(nlevs_max);
289 
290  for (int lev = 0; lev < nlevs_max; ++lev) {
291  vars_new[lev].resize(Vars::NumTypes);
292  vars_old[lev].resize(Vars::NumTypes);
293  gradp[lev].resize(AMREX_SPACEDIM);
294  }
295 
296  // Time integrator
297  mri_integrator_mem.resize(nlevs_max);
298 
299  // Physical boundary conditions
300  physbcs_cons.resize(nlevs_max);
301  physbcs_u.resize(nlevs_max);
302  physbcs_v.resize(nlevs_max);
303  physbcs_w.resize(nlevs_max);
304  physbcs_base.resize(nlevs_max);
305 
306  // Planes to hold Dirichlet values at boundaries
307  xvel_bc_data.resize(nlevs_max);
308  yvel_bc_data.resize(nlevs_max);
309  zvel_bc_data.resize(nlevs_max);
310  th_bc_data.resize(nlevs_max);
311 
312  advflux_reg.resize(nlevs_max);
313 
314  // Stresses
315  Tau.resize(nlevs_max);
316  SFS_hfx1_lev.resize(nlevs_max); SFS_hfx2_lev.resize(nlevs_max); SFS_hfx3_lev.resize(nlevs_max);
317  SFS_diss_lev.resize(nlevs_max);
318  SFS_q1fx1_lev.resize(nlevs_max); SFS_q1fx2_lev.resize(nlevs_max); SFS_q1fx3_lev.resize(nlevs_max);
319  SFS_q2fx3_lev.resize(nlevs_max);
320  eddyDiffs_lev.resize(nlevs_max);
321  SmnSmn_lev.resize(nlevs_max);
322 
323  // Sea surface temps
324  sst_lev.resize(nlevs_max);
325  tsk_lev.resize(nlevs_max);
326  lmask_lev.resize(nlevs_max);
327 
328  // Metric terms
329  z_phys_nd.resize(nlevs_max);
330  z_phys_cc.resize(nlevs_max);
331  detJ_cc.resize(nlevs_max);
332  ax.resize(nlevs_max);
333  ay.resize(nlevs_max);
334  az.resize(nlevs_max);
335 
336  z_phys_nd_new.resize(nlevs_max);
337  detJ_cc_new.resize(nlevs_max);
338 
339  z_phys_nd_src.resize(nlevs_max);
340  z_phys_cc_src.resize(nlevs_max);
341  detJ_cc_src.resize(nlevs_max);
342  ax_src.resize(nlevs_max);
343  ay_src.resize(nlevs_max);
344  az_src.resize(nlevs_max);
345 
346  z_t_rk.resize(nlevs_max);
347 
348  terrain_blanking.resize(nlevs_max);
349 
350  // Wall distance
351  walldist.resize(nlevs_max);
352 
353  // BoxArrays to make MultiFabs needed to convert WRFBdy data
354  ba1d.resize(nlevs_max);
355  ba2d.resize(nlevs_max);
356 
357  // MultiFabs needed to convert WRFBdy data
358  mf_PSFC.resize(nlevs_max);
359 
360  // Map factors
361  mapfac.resize(nlevs_max);
362 
363  // Thin immersed body
364  xflux_imask.resize(nlevs_max);
365  yflux_imask.resize(nlevs_max);
366  zflux_imask.resize(nlevs_max);
367  //overset_imask.resize(nlevs_max);
368  thin_xforce.resize(nlevs_max);
369  thin_yforce.resize(nlevs_max);
370  thin_zforce.resize(nlevs_max);
371 
372  // Base state
373  base_state.resize(nlevs_max);
374  base_state_new.resize(nlevs_max);
375 
376  // Wave coupling data
377  Hwave.resize(nlevs_max);
378  Lwave.resize(nlevs_max);
379  for (int lev = 0; lev < max_level; ++lev)
380  {
381  Hwave[lev] = nullptr;
382  Lwave[lev] = nullptr;
383  }
384  Hwave_onegrid.resize(nlevs_max);
385  Lwave_onegrid.resize(nlevs_max);
386  for (int lev = 0; lev < max_level; ++lev)
387  {
388  Hwave_onegrid[lev] = nullptr;
389  Lwave_onegrid[lev] = nullptr;
390  }
391 
392  // Theta prim for MOST
393  Theta_prim.resize(nlevs_max);
394 
395  // Qv prim for MOST
396  Qv_prim.resize(nlevs_max);
397 
398  // Qr prim for MOST
399  Qr_prim.resize(nlevs_max);
400 
401  // Time averaged velocity field
402  vel_t_avg.resize(nlevs_max);
403  t_avg_cnt.resize(nlevs_max);
404 
405  // Size lat long arrays and default to null pointers
406  lat_m.resize(nlevs_max);
407  lon_m.resize(nlevs_max);
408  for (int lev = 0; lev < max_level; ++lev) {
409  lat_m[lev] = nullptr;
410  lon_m[lev] = nullptr;
411  }
412 
413  // Variable coriolis
414  sinPhi_m.resize(nlevs_max);
415  cosPhi_m.resize(nlevs_max);
416  for (int lev = 0; lev < max_level; ++lev) {
417  sinPhi_m[lev] = nullptr;
418  cosPhi_m[lev] = nullptr;
419  }
420 
421  // Initialize tagging criteria for mesh refinement
423 
424  for (int lev = 0; lev < max_level; ++lev)
425  {
426  Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
427  ref_ratio[lev][0] << " " << ref_ratio[lev][1] << " " << ref_ratio[lev][2] << std::endl;
428  }
429 
430  // We will create each of these in MakeNewLevelFromScratch
431  eb.resize(max_level+1);
432  for (int lev = 0; lev < max_level + 1; lev++){
433  eb[lev] = std::make_unique<eb_>();
434  }
435 
436  //
437  // Construct the EB data structures and store in a separate class
438  //
439  // This is needed before initializing level MultiFabs
440  if ( solverChoice.terrain_type == TerrainType::EB ||
441  solverChoice.terrain_type == TerrainType::ImmersedForcing)
442  {
443  std::string geometry ="terrain";
444  ParmParse pp("eb2");
445  pp.queryAdd("geometry", geometry);
446 
447  bool build_coarse_level_by_coarsening(false);
448  // Note this just needs to be an integer > number of V-cycles one might use
449  int max_coarsening_level = ( solverChoice.terrain_type == TerrainType::EB &&
451  solverChoice.anelastic[0] == 1) ) ? 100 : 0;
452  if (geometry == "terrain") {
453  Box terrain_bx(surroundingNodes(geom[max_level].Domain())); terrain_bx.grow(3);
454  FArrayBox terrain_fab(makeSlab(terrain_bx,2,0),1);
455  Real dummy_time = 0.0;
456  prob->init_terrain_surface(geom[max_level], terrain_fab, dummy_time);
457  TerrainIF terrain_if(terrain_fab, geom[max_level], stretched_dz_d[max_level]);
458  auto gshop = EB2::makeShop(terrain_if);
459  amrex::EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level, build_coarse_level_by_coarsening);
460  } else if (geometry == "sphere") {
461  auto ProbLoArr = geom[max_level].ProbLoArray();
462  auto ProbHiArr = geom[max_level].ProbHiArray();
463  const Real xcen = 0.5 * (ProbLoArr[0] + ProbHiArr[0]);
464  const Real ycen = 0.5 * (ProbLoArr[1] + ProbHiArr[1]);
465  RealArray sphere_center = {xcen, ycen, 0.0};
466  EB2::SphereIF sphere_if(0.5, sphere_center, false);
467  auto gshop = EB2::makeShop(sphere_if);
468  amrex::EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level, build_coarse_level_by_coarsening);
469  }
470  }
471 }
void init_zlevels(Vector< Vector< Real >> &zlevels_stag, Vector< Vector< Real >> &stretched_dz_h, Vector< Gpu::DeviceVector< Real >> &stretched_dz_d, Vector< Geometry > const &geom, Vector< IntVect > const &ref_ratio, const Real grid_stretching_ratio, const Real zsurf, const Real dz0)
Definition: ERF_InitZLevels.cpp:11
std::unique_ptr< ProblemBase > amrex_probinit(const amrex_real *problo, const amrex_real *probhi) AMREX_ATTRIBUTE_WEAK
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
Definition: ERF.H:919
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
Definition: ERF.H:952
void setPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:23
amrex::Vector< amrex::BoxArray > ba2d
Definition: ERF.H:1170
amrex::Vector< amrex::Vector< amrex::MultiFab > > gradp
Definition: ERF.H:781
void ReadParameters()
Definition: ERF.cpp:1803
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_PSFC
Definition: ERF.H:1175
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
Definition: ERF.H:889
amrex::Vector< amrex::MultiFab > base_state_new
Definition: ERF.H:914
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
Definition: ERF.H:887
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
Definition: ERF.H:869
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
Definition: ERF.H:902
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
Definition: ERF.H:896
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
Definition: ERF.H:953
amrex::Vector< std::string > plot3d_var_names_2
Definition: ERF.H:1038
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
Definition: ERF.H:867
amrex::Vector< std::string > plot2d_var_names_1
Definition: ERF.H:1039
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
Definition: ERF.H:951
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
Definition: ERF.H:736
amrex::Vector< amrex::Real > t_old
Definition: ERF.H:770
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
Definition: ERF.H:899
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
Definition: ERF.H:920
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
Definition: ERF.H:1251
amrex::Vector< amrex::BoxArray > ba1d
Definition: ERF.H:1169
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
Definition: ERF.H:733
int rad_datalog_int
Definition: ERF.H:851
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
Definition: ERF.H:891
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
Definition: ERF.H:893
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
Definition: ERF.H:946
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
Definition: ERF.H:833
amrex::Vector< std::string > plot3d_var_names_1
Definition: ERF.H:1037
void refinement_criteria_setup()
Definition: ERF_Tagging.cpp:227
amrex::Vector< std::string > plot2d_var_names_2
Definition: ERF.H:1040
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
Definition: ERF.H:724
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
Definition: ERF.H:892
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc_src
Definition: ERF.H:890
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
Definition: ERF.H:878
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
Definition: ERF.H:832
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
Definition: ERF.H:910
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
Definition: ERF.H:894
static amrex::Real dt_max_initial
Definition: ERF.H:991
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
Definition: ERF.H:918
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
Definition: ERF.H:724
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
Definition: ERF.H:947
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
Definition: ERF.H:735
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
Definition: ERF.H:897
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
Definition: ERF.H:734
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
Definition: ERF.H:917
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
Definition: ERF.H:945
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
Definition: ERF.H:868
void initializeMicrophysics(const int &)
Definition: ERF.cpp:1594
void ReSize(const int &nlev)
Definition: ERF_LandSurface.H:24
Definition: ERF_EBIFTerrain.H:14
const char * buildInfoGetGitHash(int i)
amrex::Real dz0
Definition: ERF_DataStruct.H:833
amrex::Real const_massflux_layer_lo
Definition: ERF_DataStruct.H:910
amrex::Real const_massflux_v
Definition: ERF_DataStruct.H:908
int massflux_klo
Definition: ERF_DataStruct.H:912
amrex::Real grid_stretching_ratio
Definition: ERF_DataStruct.H:831
amrex::Real const_massflux_u
Definition: ERF_DataStruct.H:907
amrex::Real zsurf
Definition: ERF_DataStruct.H:832
amrex::Real const_massflux_layer_hi
Definition: ERF_DataStruct.H:911
int massflux_khi
Definition: ERF_DataStruct.H:913
Here is the call graph for this function:

◆ ErrorEst()

void ERF::ErrorEst ( int  lev,
amrex::TagBoxArray &  tags,
amrex::Real  time,
int  ngrow 
)
override

Function to tag cells for refinement – this overrides the pure virtual function in AmrCore

Parameters
[in]levclevel of refinement at which we tag cells (0 is coarsest level)
[out]tagsarray of tagged cells
[in]timecurrent time
21 {
22  const int clearval = TagBox::CLEAR;
23  const int tagval = TagBox::SET;
24 
25 
26 #ifdef ERF_USE_NETCDF
27  if (solverChoice.init_type == InitType::WRFInput) {
28  int ratio;
29  Box subdomain;
30  if (!nc_init_file[levc+1].empty()) {
31  amrex::Print() << "WRFIinput file to read: " << nc_init_file[levc+1][0] << std::endl;
32  subdomain = read_subdomain_from_wrfinput(levc, nc_init_file[levc+1][0], ratio);
33  amrex::Print() << " WRFInput subdomain at level " << levc+1 << " is " << subdomain << std::endl;
34  }
35 
36  if ( (ratio != ref_ratio[levc][0]) || (ratio != ref_ratio[levc][1]) ) {
37  amrex::Print() << "File " << nc_init_file[levc+1][0] << " has refinement ratio = " << ratio << std::endl;
38  amrex::Print() << "The inputs file has refinement ratio = " << ref_ratio[levc] << std::endl;
39  amrex::Abort("These must be the same -- please edit your inputs file and try again.");
40  }
41 
42  if ( (ref_ratio[levc][2]) != 1) {
43  amrex::Abort("The ref_ratio specified in the inputs file must have 1 in the z direction; please use ref_ratio_vect rather than ref_ratio");
44  }
45 
46  subdomain.coarsen(IntVect(ratio,ratio,1));
47 
48  // We assume there is only one subdomain at levc; otherwise we don't know
49  // which one is the parent of the fine region we are trying to create
50  AMREX_ALWAYS_ASSERT(subdomains[levc].size() == 1);
51 
52  // We assume there is only one box in the first subdomain at levc; otherwise we don't know
53  // how to compute the offset
54  AMREX_ALWAYS_ASSERT(subdomains[levc][0].size() == 1);
55 
56  Box coarser_level(subdomains[levc][0].minimalBox());
57  subdomain.shift(coarser_level.smallEnd());
58 
59  if (verbose > 0) {
60  amrex::Print() << " Crse subdomain to be tagged is" << subdomain << std::endl;
61  }
62 
63  Box new_fine(subdomain); new_fine.refine(IntVect(ratio,ratio,1));
64  num_boxes_at_level[levc+1] = 1;
65  boxes_at_level[levc+1].push_back(new_fine);
66 
67  for (MFIter mfi(tags); mfi.isValid(); ++mfi) {
68  auto tag_arr = tags.array(mfi); // Get device-accessible array
69 
70  Box bx = mfi.validbox(); bx &= subdomain;
71 
72  if (!bx.isEmpty()) {
73  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
74  tag_arr(i,j,k) = TagBox::SET;
75  });
76  }
77  }
78  return;
79  }
80 #endif
81 
82  //
83  // Make sure the ghost cells of the level we are tagging at are filled
84  // in case we take differences that require them
85  // NOTE: We are Fillpatching only the cell-centered variables here
86  //
87  MultiFab& S_new = vars_new[levc][Vars::cons];
88  MultiFab& U_new = vars_new[levc][Vars::xvel];
89  MultiFab& V_new = vars_new[levc][Vars::yvel];
90  MultiFab& W_new = vars_new[levc][Vars::zvel];
91  //
92  if (levc == 0) {
93  FillPatchCrseLevel(levc, time, {&S_new, &U_new, &V_new, &W_new});
94  } else {
95  FillPatchFineLevel(levc, time, {&S_new, &U_new, &V_new, &W_new},
96  {&S_new, &rU_new[levc], &rV_new[levc], &rW_new[levc]},
97  base_state[levc], base_state[levc],
98  false, true);
99  }
100 
101  for (int j=0; j < ref_tags.size(); ++j)
102  {
103  //
104  // This mf must have ghost cells because we may take differences between adjacent values
105  //
106  std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 1);
107 
108  // This allows dynamic refinement based on the value of the density
109  if (ref_tags[j].Field() == "density")
110  {
111  MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,1);
112 
113  // This allows dynamic refinement based on the value of qv
114  } else if ( ref_tags[j].Field() == "qv" ) {
115  MultiFab::Copy( *mf, vars_new[levc][Vars::cons], RhoQ1_comp, 0, 1, 1);
116  MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
117 
118 
119  // This allows dynamic refinement based on the value of qc
120  } else if (ref_tags[j].Field() == "qc" ) {
121  MultiFab::Copy( *mf, vars_new[levc][Vars::cons], RhoQ2_comp, 0, 1, 1);
122  MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
123 
124  // This allows dynamic refinement based on the value of the z-component of vorticity
125  } else if (ref_tags[j].Field() == "vorticity" ) {
126  Vector<MultiFab> mf_cc_vel(1);
127  mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
128  average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
129 
130  // Impose bc's at domain boundaries at all levels
131  FillBdyCCVels(mf_cc_vel,levc);
132 
133  mf->setVal(0.);
134 
135  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
136  {
137  const Box& bx = mfi.tilebox();
138  auto& dfab = (*mf)[mfi];
139  auto& sfab = mf_cc_vel[0][mfi];
140  derived::erf_dervortz(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
141  }
142 
143  // This allows dynamic refinement based on the value of the scalar/theta
144  } else if ( (ref_tags[j].Field() == "scalar" ) ||
145  (ref_tags[j].Field() == "theta" ) )
146  {
147  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
148  {
149  const Box& bx = mfi.growntilebox();
150  auto& dfab = (*mf)[mfi];
151  auto& sfab = vars_new[levc][Vars::cons][mfi];
152  if (ref_tags[j].Field() == "scalar") {
153  derived::erf_derscalar(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
154  } else if (ref_tags[j].Field() == "theta") {
155  derived::erf_dertheta(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
156  }
157  } // mfi
158  // This allows dynamic refinement based on the value of the density
159  } else if ( (SolverChoice::terrain_type == TerrainType::ImmersedForcing) &&
160  (ref_tags[j].Field() == "terrain_blanking") )
161  {
162  MultiFab::Copy(*mf,*terrain_blanking[levc],0,0,1,1);
163  } else if (ref_tags[j].Field() == "velmag") {
164  mf->setVal(0.0);
165  ParmParse pp(pp_prefix);
166  Vector<std::string> refinement_indicators;
167  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
168  Real velmag_threshold = 1e10;
169  for (int i=0; i<refinement_indicators.size(); ++i)
170  {
171  if(refinement_indicators[i]=="hurricane_tracker"){
172  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
173  ParmParse ppr(ref_prefix);
174  ppr.get("value_greater",velmag_threshold);
175  break;
176  }
177  }
178  HurricaneTracker(levc, U_new, V_new, W_new, velmag_threshold, false, &tags);
179 #ifdef ERF_USE_PARTICLES
180  } else {
181  //
182  // This allows dynamic refinement based on the number of particles per cell
183  //
184  // Note that we must count all the particles in levels both at and above the current,
185  // since otherwise, e.g., if the particles are all at level 1, counting particles at
186  // level 0 will not trigger refinement when regridding so level 1 will disappear,
187  // then come back at the next regridding
188  //
189  const auto& particles_namelist( particleData.getNames() );
190  mf->setVal(0.0);
191  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
192  {
193  std::string tmp_string(particles_namelist[i]+"_count");
194  IntVect rr = IntVect::TheUnitVector();
195  if (ref_tags[j].Field() == tmp_string) {
196  for (int lev = levc; lev <= finest_level; lev++)
197  {
198  MultiFab temp_dat(grids[lev], dmap[lev], 1, 0); temp_dat.setVal(0);
199  particleData[particles_namelist[i]]->IncrementWithTotal(temp_dat, lev);
200 
201  MultiFab temp_dat_crse(grids[levc], dmap[levc], 1, 0); temp_dat_crse.setVal(0);
202 
203  if (lev == levc) {
204  MultiFab::Copy(*mf, temp_dat, 0, 0, 1, 0);
205  } else {
206  for (int d = 0; d < AMREX_SPACEDIM; d++) {
207  rr[d] *= ref_ratio[levc][d];
208  }
209  average_down(temp_dat, temp_dat_crse, 0, 1, rr);
210  MultiFab::Add(*mf, temp_dat_crse, 0, 0, 1, 0);
211  }
212  }
213  }
214  }
215 #endif
216  }
217 
218  ref_tags[j](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
219  } // loop over j
220 }
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
Definition: ERF.H:763
void FillBdyCCVels(amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)
Definition: ERF_FillBdyCCVels.cpp:11
void HurricaneTracker(int lev, const amrex::MultiFab &U_new, const amrex::MultiFab &V_new, const amrex::MultiFab &W_new, const amrex::Real velmag_threshold, const bool is_track_io, amrex::TagBoxArray *tags=nullptr)
Definition: ERF_Tagging.cpp:455
void FillPatchCrseLevel(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
Definition: ERF_FillPatch.cpp:282
static amrex::Vector< amrex::Vector< std::string > > nc_init_file
Definition: ERF.H:1143
amrex::Vector< amrex::Vector< amrex::BoxArray > > subdomains
Definition: ERF.H:1258
static amrex::Vector< amrex::AMRErrorTag > ref_tags
Definition: ERF.H:1256
amrex::Vector< int > num_boxes_at_level
Definition: ERF.H:761
void erf_derscalar(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:165
void erf_dervortz(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:256
void erf_dertheta(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:144
real(c_double), private rr
Definition: ERF_module_mp_morr_two_moment.F90:223
static InitType init_type
Definition: ERF_DataStruct.H:762
Here is the call graph for this function:

◆ estTimeStep()

Real ERF::estTimeStep ( int  level,
long &  dt_fast_ratio 
) const

Function that calls estTimeStep for each level

Parameters
[in]levellevel of refinement (coarsest level i 0)
[out]dt_fast_ratioratio of slow to fast time step
55 {
56  BL_PROFILE("ERF::estTimeStep()");
57 
58  Real estdt_comp = 1.e20;
59  Real estdt_lowM = 1.e20;
60 
61  // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation
62  const int nxc = geom[0].Domain().length(0);
63  const int nyc = geom[0].Domain().length(1);
64 
65  auto const dxinv = geom[level].InvCellSizeArray();
66  auto const dzinv = 1.0 / dz_min[level];
67 
68  MultiFab const& S_new = vars_new[level][Vars::cons];
69 
70  MultiFab ccvel(grids[level],dmap[level],3,0);
71 
72  average_face_to_cellcenter(ccvel,0,
73  Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
74  &vars_new[level][Vars::yvel],
75  &vars_new[level][Vars::zvel]});
76 
77  int l_implicit_substepping = (solverChoice.substepping_type[level] == SubsteppingType::Implicit);
78  int l_anelastic = solverChoice.anelastic[level];
79 
80  Real estdt_comp_inv;
81 
82  if (solverChoice.terrain_type == TerrainType::EB)
83  {
84  const eb_& eb_lev = get_eb(level);
85  const MultiFab& detJ = (eb_lev.get_const_factory())->getVolFrac();
86 
87  estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
88  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
89  Array4<Real const> const& s,
90  Array4<Real const> const& u,
91  Array4<Real const> const& vf) -> Real
92  {
93  Real new_comp_dt = -1.e100;
94  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
95  {
96  if (vf(i,j,k) > 0.)
97  {
98  const Real rho = s(i, j, k, Rho_comp);
99  const Real rhotheta = s(i, j, k, RhoTheta_comp);
100 
101  // NOTE: even when moisture is present,
102  // we only use the partial pressure of the dry air
103  // to compute the soundspeed
104  Real pressure = getPgivenRTh(rhotheta);
105  Real c = std::sqrt(Gamma * pressure / rho);
106 
107  // If we are doing implicit acoustic substepping, then the z-direction does not contribute
108  // to the computation of the time step
109  if (l_implicit_substepping) {
110  if ((nxc > 1) && (nyc==1)) {
111  // 2-D in x-z
112  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
113  } else if ((nyc > 1) && (nxc==1)) {
114  // 2-D in y-z
115  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
116  } else {
117  // 3-D or SCM
118  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
119  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
120  }
121 
122  // If we are not doing implicit acoustic substepping, then the z-direction contributes
123  // to the computation of the time step
124  } else {
125  if (nxc > 1 && nyc > 1) {
126  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
127  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
128  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
129  } else if (nxc > 1) {
130  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
131  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
132  } else if (nyc > 1) {
133  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
134  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
135  } else {
136  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
137  }
138 
139  }
140  }
141  });
142  return new_comp_dt;
143  });
144 
145  } else {
146  estdt_comp_inv = ReduceMax(S_new, ccvel, 0,
147  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
148  Array4<Real const> const& s,
149  Array4<Real const> const& u) -> Real
150  {
151  Real new_comp_dt = -1.e100;
152  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
153  {
154  {
155  const Real rho = s(i, j, k, Rho_comp);
156  const Real rhotheta = s(i, j, k, RhoTheta_comp);
157 
158  // NOTE: even when moisture is present,
159  // we only use the partial pressure of the dry air
160  // to compute the soundspeed
161  Real pressure = getPgivenRTh(rhotheta);
162  Real c = std::sqrt(Gamma * pressure / rho);
163 
164  // If we are doing implicit acoustic substepping, then the z-direction does not contribute
165  // to the computation of the time step
166  if (l_implicit_substepping) {
167  if ((nxc > 1) && (nyc==1)) {
168  // 2-D in x-z
169  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
170  } else if ((nyc > 1) && (nxc==1)) {
171  // 2-D in y-z
172  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
173  } else {
174  // 3-D or SCM
175  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
176  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
177  }
178 
179  // If we are not doing implicit acoustic substepping, then the z-direction contributes
180  // to the computation of the time step
181  } else {
182  if (nxc > 1 && nyc > 1) {
183  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
184  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
185  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
186  } else if (nxc > 1) {
187  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
188  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
189  } else if (nyc > 1) {
190  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
191  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
192  } else {
193  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
194  }
195 
196  }
197  }
198  });
199  return new_comp_dt;
200  });
201  } // not EB
202 
203  ParallelDescriptor::ReduceRealMax(estdt_comp_inv);
204  estdt_comp = cfl / estdt_comp_inv;
205 
206  Real estdt_lowM_inv = ReduceMax(ccvel, 0,
207  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
208  Array4<Real const> const& u) -> Real
209  {
210  Real new_lm_dt = -1.e100;
211  Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
212  {
213  new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
214  ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
215  ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
216  });
217  return new_lm_dt;
218  });
219 
220  ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
221  if (estdt_lowM_inv > 0.0_rt)
222  estdt_lowM = cfl / estdt_lowM_inv;
223 
224  if (verbose) {
225  if (fixed_dt[level] <= 0.0) {
226  Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " <<
227  1.0/dxinv[0] << " " << 1.0/dxinv[1] << " " << dz_min[level] << std::endl;
228  Print() << "Compressible dt at level " << level << ": " << estdt_comp << std::endl;
229  if (estdt_lowM_inv > 0.0_rt) {
230  Print() << "Anelastic dt at level " << level << ": " << estdt_lowM << std::endl;
231  } else {
232  Print() << "Anelastic dt at level " << level << ": undefined " << std::endl;
233  }
234  }
235 
236  if (fixed_dt[level] > 0.0) {
237  Print() << "Based on cfl of 1.0 " << std::endl;
238  Print() << "Compressible dt at level " << level << " would be: " << estdt_comp/cfl << std::endl;
239  if (estdt_lowM_inv > 0.0_rt) {
240  Print() << "Anelastic dt at level " << level << " would be: " << estdt_lowM/cfl << std::endl;
241  } else {
242  Print() << "Anelastic dt at level " << level << " would be undefined " << std::endl;
243  }
244  Print() << "Fixed dt at level " << level << " is: " << fixed_dt[level] << std::endl;
245  if (fixed_fast_dt[level] > 0.0) {
246  Print() << "Fixed fast dt at level " << level << " is: " << fixed_fast_dt[level] << std::endl;
247  }
248  }
249  }
250 
251  if (solverChoice.substepping_type[level] != SubsteppingType::None) {
252  if (fixed_dt[level] > 0. && fixed_fast_dt[level] > 0.) {
253  dt_fast_ratio = static_cast<long>( fixed_dt[level] / fixed_fast_dt[level] );
254  } else if (fixed_dt[level] > 0.) {
255  // Max CFL_c = 1.0 for substeps by default, but we enforce a min of 4 substeps
256  auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
257  dt_fast_ratio = static_cast<long>( std::max(fixed_dt[level]/dt_sub_max,4.) );
258  } else {
259  // auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
260  // dt_fast_ratio = static_cast<long>( std::max(estdt_comp/dt_sub_max,4.) );
261  dt_fast_ratio = static_cast<long>( std::max(cfl / sub_cfl, 4.) );
262  }
263 
264  // Force time step ratio to be an even value
266  if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
267  } else {
268  if ( dt_fast_ratio%6 != 0) {
269  Print() << "mri_dt_ratio = " << dt_fast_ratio
270  << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
271  dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/6.0) * 6);
272  }
273  }
274 
275  if (verbose) {
276  Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
277  }
278  } // if substepping, either explicit or implicit
279 
280  if (fixed_dt[level] > 0.0) {
281  return fixed_dt[level];
282  } else {
283  // Anelastic (substepping is not allowed)
284  if (l_anelastic) {
285 
286  // Make sure that timestep is less than the dt_max
287  estdt_lowM = amrex::min(estdt_lowM, dt_max);
288 
289  // On the first timestep enforce dt_max_initial
290  if(istep[level] == 0){
291  return amrex::min(dt_max_initial, estdt_lowM);
292  }
293  else{
294  return estdt_lowM;
295  }
296 
297 
298  // Compressible with or without substepping
299  } else {
300  return estdt_comp;
301  }
302  }
303 }
constexpr amrex::Real Gamma
Definition: ERF_Constants.H:19
amrex::Vector< amrex::Real > dz_min
Definition: ERF.H:1266
eb_ const & get_eb(int lev) const noexcept
Definition: ERF.H:1540
amrex::Vector< amrex::Real > fixed_dt
Definition: ERF.H:995
static amrex::Real dt_max
Definition: ERF.H:992
amrex::Vector< amrex::Real > fixed_fast_dt
Definition: ERF.H:996
static amrex::Real cfl
Definition: ERF.H:987
static amrex::Real sub_cfl
Definition: ERF.H:988
Definition: ERF_EB.H:13
const std::unique_ptr< amrex::EBFArrayBoxFactory > & get_const_factory() const noexcept
Definition: ERF_EB.H:46
@ rho
Definition: ERF_Kessler.H:22
int force_stage1_single_substep
Definition: ERF_DataStruct.H:790
Here is the call graph for this function:

◆ Evolve()

void ERF::Evolve ( )
478 {
479  BL_PROFILE_VAR("ERF::Evolve()", evolve);
480 
481  Real cur_time = t_new[0];
482 
483  // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
484  // for finer levels (with or without subcycling)
485  for (int step = istep[0]; step < max_step && cur_time < stop_time; ++step)
486  {
487  if (use_datetime) {
488  Print() << "\n" << getTimestamp(start_time+cur_time, datetime_format)
489  << " (" << cur_time << " s elapsed)" << std::endl;
490  }
491  Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
492 
493  ComputeDt(step);
494 
495  // Make sure we have read enough of the boundary plane data to make it through this timestep
496  if (input_bndry_planes)
497  {
498  m_r2d->read_input_files(cur_time,dt[0],m_bc_extdir_vals);
499  }
500 
501 #ifdef ERF_USE_PARTICLES
502  // We call this every time step with the knowledge that the particles may be
503  // initialized at a later time than the simulation start time.
504  // The ParticleContainer carries a "start time" so the initialization will happen
505  // only when a) time > start_time, and b) particles have not yet been initialized
506  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,cur_time);
507 #endif
508 
509  int lev = 0;
510  int iteration = 1;
511  timeStep(lev, cur_time, iteration);
512 
513  cur_time += dt[0];
514 
515  Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
516  << " DT = " << dt[0] << std::endl;
517 
518  post_timestep(step, cur_time, dt[0]);
519 
520  if (writeNow(cur_time, dt[0], step+1, m_plot3d_int_1, m_plot3d_per_1)) {
521  last_plot3d_file_step_1 = step+1;
523  }
524  if (writeNow(cur_time, dt[0], step+1, m_plot3d_int_2, m_plot3d_per_2)) {
525  last_plot3d_file_step_2 = step+1;
527  }
528 
529  if (writeNow(cur_time, dt[0], step+1, m_plot2d_int_1, m_plot2d_per_1)) {
530  last_plot2d_file_step_1 = step+1;
532  }
533 
534  if (writeNow(cur_time, dt[0], step+1, m_plot2d_int_2, m_plot2d_per_2)) {
535  last_plot2d_file_step_2 = step+1;
537  }
538 
539  if (writeNow(cur_time, dt[0], step+1, m_subvol_int, m_subvol_per)) {
540  last_subvol = step+1;
541  WriteSubvolume();
542  }
543 
544  if (writeNow(cur_time, dt[0], step+1, m_check_int, m_check_per)) {
545  last_check_file_step = step+1;
547  }
548 
549 #ifdef AMREX_MEM_PROFILING
550  {
551  std::ostringstream ss;
552  ss << "[STEP " << step+1 << "]";
553  MemProfiler::report(ss.str());
554  }
555 #endif
556 
557  if (cur_time >= stop_time - 1.e-6*dt[0]) break;
558  }
559 
560  // Write plotfiles at final time
561  if ( (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.) && istep[0] > last_plot3d_file_step_1 ) {
563  }
564  if ( (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.) && istep[0] > last_plot3d_file_step_2) {
566  }
567  if ( (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.) && istep[0] > last_plot2d_file_step_1 ) {
569  }
570  if ( (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.) && istep[0] > last_plot2d_file_step_2) {
572  }
573  if ( (m_subvol_int > 0 || m_subvol_per > 0.) && istep[0] > last_subvol) {
574  WriteSubvolume();
575  }
576 
577  if ( (m_check_int > 0 || m_check_per > 0.) && istep[0] > last_check_file_step) {
579  }
580 
581  BL_PROFILE_VAR_STOP(evolve);
582 }
AMREX_FORCE_INLINE std::string getTimestamp(const amrex::Real epoch_real, const std::string &datetime_format)
Definition: ERF_EpochTime.H:72
int last_check_file_step
Definition: ERF.H:961
int max_step
Definition: ERF.H:974
amrex::Real m_plot2d_per_1
Definition: ERF.H:1021
int last_plot2d_file_step_2
Definition: ERF.H:958
int m_subvol_int
Definition: ERF.H:1018
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_extdir_vals
Definition: ERF.H:936
void WriteSubvolume()
Definition: ERF_WriteSubvolume.cpp:9
int m_plot2d_int_2
Definition: ERF.H:1017
int m_plot3d_int_1
Definition: ERF.H:1014
int last_plot3d_file_step_2
Definition: ERF.H:956
void post_timestep(int nstep, amrex::Real time, amrex::Real dt_lev)
Definition: ERF.cpp:586
amrex::Real m_subvol_per
Definition: ERF.H:1023
amrex::Real m_plot2d_per_2
Definition: ERF.H:1022
amrex::Real m_check_per
Definition: ERF.H:1035
int m_check_int
Definition: ERF.H:1034
static int input_bndry_planes
Definition: ERF.H:1192
void Write2DPlotFile(int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:1859
int last_subvol
Definition: ERF.H:959
const std::string datetime_format
Definition: ERF.H:981
bool use_datetime
Definition: ERF.H:980
void ComputeDt(int step=-1)
Definition: ERF_ComputeTimestep.cpp:11
amrex::Real m_plot3d_per_2
Definition: ERF.H:1020
static PlotFileType plotfile3d_type_2
Definition: ERF.H:1133
static PlotFileType plotfile2d_type_2
Definition: ERF.H:1135
int m_plot2d_int_1
Definition: ERF.H:1016
void WriteCheckpointFile() const
Definition: ERF_Checkpoint.cpp:26
void Write3DPlotFile(int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:263
int last_plot2d_file_step_1
Definition: ERF.H:957
amrex::Real m_plot3d_per_1
Definition: ERF.H:1019
std::unique_ptr< ReadBndryPlanes > m_r2d
Definition: ERF.H:1249
int last_plot3d_file_step_1
Definition: ERF.H:955
static PlotFileType plotfile2d_type_1
Definition: ERF.H:1134
bool writeNow(const amrex::Real cur_time, const amrex::Real dt, const int nstep, const int plot_int, const amrex::Real plot_per)
Definition: ERF.cpp:2448
static PlotFileType plotfile3d_type_1
Definition: ERF.H:1132
int m_plot3d_int_2
Definition: ERF.H:1015
void timeStep(int lev, amrex::Real time, int iteration)
Definition: ERF_TimeStep.cpp:17

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fill_from_bndryregs()

void ERF::fill_from_bndryregs ( const amrex::Vector< amrex::MultiFab * > &  mfs,
amrex::Real  time 
)
14 {
15  //
16  // We now assume that if we read in on one face, we read in on all faces
17  //
18  AMREX_ALWAYS_ASSERT(m_r2d);
19 
20  int lev = 0;
21  const Box& domain = geom[lev].Domain();
22 
23  const auto& dom_lo = lbound(domain);
24  const auto& dom_hi = ubound(domain);
25 
26  Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time);
27 
28  const BCRec* bc_ptr = domain_bcs_type_d.data();
29 
30  // xlo: ori = 0
31  // ylo: ori = 1
32  // zlo: ori = 2
33  // xhi: ori = 3
34  // yhi: ori = 4
35  // zhi: ori = 5
36  const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
37  const auto& bdatylo = (*bndry_data[1])[lev].const_array();
38  const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
39  const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
40 
41  int bccomp;
42 
43  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
44  {
45  MultiFab& mf = *mfs[var_idx];
46  const int icomp = 0;
47  const int ncomp = mf.nComp();
48 
49  if (var_idx == Vars::xvel) {
50  bccomp = BCVars::xvel_bc;
51  } else if (var_idx == Vars::yvel) {
52  bccomp = BCVars::yvel_bc;
53  } else if (var_idx == Vars::zvel) {
54  bccomp = BCVars::zvel_bc;
55  } else if (var_idx == Vars::cons) {
56  bccomp = BCVars::cons_bc;
57  }
58 
59 #ifdef AMREX_USE_OMP
60 #pragma omp parallel if (Gpu::notInLaunchRegion())
61 #endif
62  for (MFIter mfi(mf); mfi.isValid(); ++mfi)
63  {
64  const Array4<Real>& dest_arr = mf.array(mfi);
65  Box bx = mfi.growntilebox();
66 
67  // x-faces
68  {
69  Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
70  if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
71 
72  Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
73  if (var_idx == Vars::xvel) bx_xhi.setSmall(0,dom_hi.x);
74 
75  ParallelFor(
76  bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
77  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
78  BCVars::RhoScalar_bc_comp : icomp+n;
79  if (bc_ptr[bc_comp].lo(0) == ERFBCType::ext_dir_ingested) {
80  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
81  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
82  dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
83  }
84  },
85  bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
86  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
87  BCVars::RhoScalar_bc_comp : icomp+n;
88  if (bc_ptr[bc_comp].hi(0) == ERFBCType::ext_dir_ingested) {
89  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
90  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
91  dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
92  }
93  }
94  );
95  } // x-faces
96 
97  // y-faces
98  {
99  Box bx_ylo(bx); bx_ylo.setBig (1,dom_lo.y-1);
100  if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
101 
102  Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
103  if (var_idx == Vars::yvel) bx_yhi.setSmall(1,dom_hi.y);
104 
105  ParallelFor(
106  bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
107  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
108  BCVars::RhoScalar_bc_comp : icomp+n;
109  if (bc_ptr[bc_comp].lo(1) == ERFBCType::ext_dir_ingested) {
110  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
111  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
112  dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
113  }
114  },
115  bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
116  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
117  BCVars::RhoScalar_bc_comp : icomp+n;
118  if (bc_ptr[bc_comp].hi(1) == ERFBCType::ext_dir_ingested) {
119  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
120  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
121  dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
122  }
123  }
124  );
125  } // y-faces
126  } // mf
127  } // var_idx
128 }
#define RhoScalar_comp
Definition: ERF_IndexDefines.H:40
#define NSCALARS
Definition: ERF_IndexDefines.H:16
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
Definition: ERF.H:930
@ RhoScalar_bc_comp
Definition: ERF_IndexDefines.H:80
@ ext_dir_ingested
Definition: ERF_IndexDefines.H:212

◆ fill_rhs()

void ERF::fill_rhs ( amrex::MultiFab &  rhs_mf,
const amrex::MultiFab &  state_mf,
amrex::Real  time,
const amrex::Geometry &  geom 
)
private

◆ FillBdyCCVels()

void ERF::FillBdyCCVels ( amrex::Vector< amrex::MultiFab > &  mf_cc_vel,
int  levc = 0 
)
12 {
13  // Impose bc's at domain boundaries
14  for (int ilev(0); ilev < mf_cc_vel.size(); ++ilev)
15  {
16  int lev = ilev + levc;
17  Box domain(Geom(lev).Domain());
18 
19  int ihi = domain.bigEnd(0);
20  int jhi = domain.bigEnd(1);
21  int khi = domain.bigEnd(2);
22 
23  // Impose periodicity first
24  mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
25 
26  int jper = (Geom(lev).isPeriodic(1));
27  int kper = (Geom(lev).isPeriodic(2));
28 
29  for (MFIter mfi(mf_cc_vel[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
30  {
31  const Box& bx = mfi.tilebox();
32  const Array4<Real>& vel_arr = mf_cc_vel[lev].array(mfi);
33 
34  if (!Geom(lev).isPeriodic(0)) {
35  // Low-x side
36  if (bx.smallEnd(0) <= domain.smallEnd(0)) {
37  Real multn = ( (phys_bc_type[0] == ERF_BC::slip_wall ) ||
39  (phys_bc_type[0] == ERF_BC::symmetry ) ) ? -1. : 1.;
40  Real multt = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -1. : 1.;
41  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
42  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
43  {
44  vel_arr(-1,j,k,0) = multn*vel_arr(0,j,k,0); // u
45  vel_arr(-1,j,k,1) = multt*vel_arr(0,j,k,1); // v
46  vel_arr(-1,j,k,2) = multt*vel_arr(0,j,k,2); // w
47  });
48  }
49 
50  // High-x side
51  if (bx.bigEnd(0) >= domain.bigEnd(0)) {
52  Real multn = ( (phys_bc_type[3] == ERF_BC::slip_wall ) ||
54  (phys_bc_type[3] == ERF_BC::symmetry ) ) ? -1. : 1.;
55  Real multt = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -1. : 1.;
56  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
57  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
58  {
59  vel_arr(ihi+1,j,k,0) = multn*vel_arr(ihi,j,k,0); // u
60  vel_arr(ihi+1,j,k,1) = multt*vel_arr(ihi,j,k,1); // v
61  vel_arr(ihi+1,j,k,2) = multt*vel_arr(ihi,j,k,2); // w
62  });
63  }
64  } // !periodic
65 
66  if (!Geom(lev).isPeriodic(1)) {
67  // Low-y side
68  if (bx.smallEnd(1) <= domain.smallEnd(1)) {
69  Real multn = ( (phys_bc_type[1] == ERF_BC::slip_wall ) ||
71  (phys_bc_type[1] == ERF_BC::symmetry ) ) ? -1. : 1.;
72  Real multt = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -1. : 1.;
73  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
74  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
75  {
76  vel_arr(i,-1,k,0) = multt*vel_arr(i,0,k,0); // u
77  vel_arr(i,-1,k,1) = multn*vel_arr(i,0,k,1); // u
78  vel_arr(i,-1,k,2) = multt*vel_arr(i,0,k,2); // w
79  });
80  }
81 
82  // High-y side
83  if (bx.bigEnd(1) >= domain.bigEnd(1)) {
84  Real multn = ( (phys_bc_type[4] == ERF_BC::slip_wall ) ||
86  (phys_bc_type[4] == ERF_BC::symmetry ) ) ? -1. : 1.;
87  Real multt = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -1. : 1.;
88  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
89  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
90  {
91  vel_arr(i,jhi+1,k,0) = multt*vel_arr(i,jhi,k,0); // u
92  vel_arr(i,jhi+1,k,1) = multn*vel_arr(i,jhi,k,1); // v
93  vel_arr(i,jhi+1,k,2) = multt*vel_arr(i,jhi,k,2); // w
94  });
95  }
96  } // !periodic
97 
98  if (!Geom(lev).isPeriodic(2)) {
99  // Low-z side
100  if (bx.smallEnd(2) <= domain.smallEnd(2)) {
101  Real multn = ( (phys_bc_type[2] == ERF_BC::slip_wall ) ||
103  (phys_bc_type[2] == ERF_BC::symmetry ) ) ? -1. : 1.;
104  Real multt = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -1. : 1.;
105  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
106  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
107  {
108  vel_arr(i,j,-1,0) = multt*vel_arr(i,j,0,0); // u
109  vel_arr(i,j,-1,1) = multt*vel_arr(i,j,0,1); // v
110  vel_arr(i,j,-1,2) = multn*vel_arr(i,j,0,2); // w
111  });
112  }
113 
114  // High-z side
115  if (bx.bigEnd(2) >= domain.bigEnd(2)) {
116  Real multn = ( (phys_bc_type[5] == ERF_BC::slip_wall ) ||
118  (phys_bc_type[5] == ERF_BC::symmetry ) ) ? -1. : 1.;
119  Real multt = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -1. : 1.;
120  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
121  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
122  {
123  vel_arr(i,j,khi+1,0) = multt*vel_arr(i,j,khi,0); // u
124  vel_arr(i,j,khi+1,1) = multt*vel_arr(i,j,khi,1); // v
125  vel_arr(i,j,khi+1,2) = multn*vel_arr(i,j,khi,2); // w
126  });
127  }
128  } // !periodic
129  } // MFIter
130 
131  // Impose periodicity again
132  mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
133  } // lev
134 }
@ no_slip_wall

◆ FillCoarsePatch()

void ERF::FillCoarsePatch ( int  lev,
amrex::Real  time 
)
private
22 {
23  BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
24  AMREX_ASSERT(lev > 0);
25 
26  //
27  //****************************************************************************************************************
28  // First fill velocities and density at the COARSE level so we can convert velocity to momenta at the COARSE level
29  //****************************************************************************************************************
30  //
31  bool cons_only = false;
32  if (lev == 1) {
33  FillPatchCrseLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
34  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
35  cons_only);
36  } else {
37  FillPatchFineLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
38  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
39  {&vars_new[lev-1][Vars::cons],
40  &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
41  base_state[lev-1], base_state[lev-1],
42  false, cons_only);
43  }
44 
45  //
46  // ************************************************
47  // Convert velocity to momentum at the COARSE level
48  // ************************************************
49  //
50  VelocityToMomentum(vars_new[lev-1][Vars::xvel], IntVect{0},
51  vars_new[lev-1][Vars::yvel], IntVect{0},
52  vars_new[lev-1][Vars::zvel], IntVect{0},
53  vars_new[lev-1][Vars::cons],
54  rU_new[lev-1],
55  rV_new[lev-1],
56  rW_new[lev-1],
57  Geom(lev).Domain(),
59  //
60  // *****************************************************************
61  // Interpolate all cell-centered variables from coarse to fine level
62  // *****************************************************************
63  //
64  Interpolater* mapper_c = &cell_cons_interp;
65  Interpolater* mapper_f = &face_cons_linear_interp;
66 
67  //
68  //************************************************************************************************
69  // Interpolate cell-centered data from coarse to fine level
70  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
71  // ************************************************************************************************
72  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
73  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
74 
75  InterpFromCoarseLevel(vars_new[lev ][Vars::cons], ngvect_cons, IntVect(0,0,0),
76  vars_new[lev-1][Vars::cons], 0, 0, ncomp_cons,
77  geom[lev-1], geom[lev],
78  refRatio(lev-1), mapper_c, domain_bcs_type, BCVars::cons_bc);
79 
80  // ***************************************************************************
81  // Physical bc's for cell centered variables at domain boundary
82  // ***************************************************************************
84  0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc,true);
85 
86  //
87  //************************************************************************************************
88  // Interpolate x-momentum from coarse to fine level
89  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
90  // ************************************************************************************************
91  //
92  InterpFromCoarseLevel(rU_new[lev], IntVect{0}, IntVect{0}, rU_new[lev-1], 0, 0, 1,
93  geom[lev-1], geom[lev],
94  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::xvel_bc);
95 
96  //
97  //************************************************************************************************
98  // Interpolate y-momentum from coarse to fine level
99  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
100  // ************************************************************************************************
101  //
102  InterpFromCoarseLevel(rV_new[lev], IntVect{0}, IntVect{0}, rV_new[lev-1], 0, 0, 1,
103  geom[lev-1], geom[lev],
104  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::yvel_bc);
105 
106  //************************************************************************************************
107  // Interpolate z-momentum from coarse to fine level
108  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
109  // ************************************************************************************************
110  InterpFromCoarseLevel(rW_new[lev], IntVect{0}, IntVect{0}, rW_new[lev-1], 0, 0, 1,
111  geom[lev-1], geom[lev],
112  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::zvel_bc);
113  //
114  // *********************************************************
115  // After interpolation of momentum, convert back to velocity
116  // *********************************************************
117  //
118  for (int which_lev = lev-1; which_lev <= lev; which_lev++)
119  {
121  vars_new[which_lev][Vars::yvel],
122  vars_new[which_lev][Vars::zvel],
123  vars_new[which_lev][Vars::cons],
124  rU_new[which_lev],
125  rV_new[which_lev],
126  rW_new[which_lev],
127  Geom(lev).Domain(),
129  }
130 
131  // ***************************************************************************
132  // Physical bc's at domain boundary
133  // ***************************************************************************
134  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
135 
137  ngvect_vels,time,BCVars::xvel_bc,true);
139  ngvect_vels,time,BCVars::yvel_bc,true);
141  ngvect_vels,time,BCVars::zvel_bc,true);
142 
143  // ***************************************************************************
144  // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
145  // ***************************************************************************
146 }
void FillCoarsePatch(int lev, amrex::Real time)
Definition: ERF_FillCoarsePatch.cpp:21
Here is the call graph for this function:

◆ FillIntermediatePatch()

void ERF::FillIntermediatePatch ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
int  ng_cons,
int  ng_vel,
bool  cons_only,
int  icomp_cons,
int  ncomp_cons 
)
private
33 {
34  BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
35  Interpolater* mapper;
36 
37  PhysBCFunctNoOp null_bc;
38 
39  //
40  // ***************************************************************************
41  // The first thing we do is interpolate the momenta on the "valid" faces of
42  // the fine grids (where the interface is coarse/fine not fine/fine) -- this
43  // will not be over-written by interpolation below because the FillPatch
44  // operators see these as valid faces. But we must have these interpolated
45  // values in the fine data before we call FillPatchTwoLevels.
46  //
47  // Also -- note that we might be filling values by interpolation at physical boundaries
48  // here but that's ok because we will overwrite those values when we impose
49  // the physical bc's below
50  // ***************************************************************************
51  if (lev>0) {
52  if (cf_set_width > 0) {
53  // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
54  FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
55  }
56  if ( !cons_only && (cf_set_width >= 0) ) {
57  FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
58  FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
59  FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
60  }
61  }
62 
63  // amrex::Print() << "LEVEL " << lev << " CONS ONLY " << cons_only <<
64  // " ICOMP NCOMP " << icomp_cons << " " << ncomp_cons << " NGHOST " << ng_cons << std::endl;
65 
66  if (!cons_only) {
67  AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
68  AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
69  }
70 
71  // Enforce no penetration for thin immersed body
72  if (!cons_only) {
73  // Enforce no penetration for thin immersed body
74  if (xflux_imask[lev]) {
75  ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
76  }
77  if (yflux_imask[lev]) {
78  ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
79  }
80  if (zflux_imask[lev]) {
81  ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
82  }
83  }
84 
85  //
86  // We now start working on conserved quantities + VELOCITY
87  //
88  if (lev == 0)
89  {
90  // We don't do anything here because we will call the physbcs routines below,
91  // which calls FillBoundary and fills other domain boundary conditions
92  // Physical boundaries will be filled below
93 
94  if (!cons_only)
95  {
96  // ***************************************************************************
97  // We always come in to this call with updated momenta but we need to create updated velocity
98  // in order to impose the rest of the bc's
99  // ***************************************************************************
100  // This only fills VALID region of velocity
101  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
102  *mfs_vel[Vars::cons],
103  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
104  Geom(lev).Domain(), domain_bcs_type);
105  }
106  }
107  else
108  {
109  //
110  // We must fill a temporary then copy it back so we don't double add/subtract
111  //
112  MultiFab mf(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
113  mfs_vel[Vars::cons]->nComp() ,mfs_vel[Vars::cons]->nGrowVect());
114  //
115  // Set all components to 1.789e19, then copy just the density from *mfs_vel[Vars::cons]
116  //
117  mf.setVal(1.789e19);
118  MultiFab::Copy(mf,*mfs_vel[Vars::cons],Rho_comp,Rho_comp,1,mf.nGrowVect());
119 
120  Vector<MultiFab*> fmf = {mfs_vel[Vars::cons],mfs_vel[Vars::cons]};
121  Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
122  Vector<Real> ctime = {t_old[lev-1], t_new[lev-1]};
123  Vector<Real> ftime = {time,time};
124 
125  if (interpolation_type == StateInterpType::Perturbational)
126  {
127  if (icomp_cons+ncomp_cons > 1)
128  {
129  // Divide (rho theta) by rho to get theta
130  MultiFab::Divide(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
131 
132  // Subtract theta_0 from theta
133  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
134 
135  if (!amrex::almostEqual(time,ctime[1])) {
136  MultiFab::Divide(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
137  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
138  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
139  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
140  }
141  if (!amrex::almostEqual(time,ctime[0])) {
142  MultiFab::Divide(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
143  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
144  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
145  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
146  }
147  }
148 
149  // Subtract rho_0 from rho before we interpolate -- note we only subtract
150  // on valid region of mf since the ghost cells will be filled below
151  if (icomp_cons == 0)
152  {
153  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
154 
155  if (!amrex::almostEqual(time,ctime[1])) {
156  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
157  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
158  }
159  if (!amrex::almostEqual(time,ctime[0])) {
160  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
161  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
162  }
163  }
164  } // interpolation_type == StateInterpType::Perturbational
165 
166  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
167  mapper = &cell_cons_interp;
168  FillPatchTwoLevels(mf, IntVect{ng_cons}, IntVect(0,0,0),
169  time, cmf, ctime, fmf, ftime,
170  icomp_cons, icomp_cons, ncomp_cons, geom[lev-1], geom[lev],
171  refRatio(lev-1), mapper, domain_bcs_type,
172  icomp_cons);
173 
174  if (interpolation_type == StateInterpType::Perturbational)
175  {
176  if (icomp_cons == 0)
177  {
178  // Restore the coarse values to what they were
179  if (!amrex::almostEqual(time,ctime[1])) {
180  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
181  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
182  }
183  if (!amrex::almostEqual(time,ctime[0])) {
184  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
185  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
186  }
187 
188  // Set values in the cells outside the domain boundary so that we can do the Add
189  // without worrying about uninitialized values outside the domain -- these
190  // will be filled in the physbcs call
191  mf.setDomainBndry(1.234e20,Rho_comp,1,geom[lev]);
192 
193  // Add rho_0 back to rho after we interpolate -- on all the valid + ghost region
194  MultiFab::Add(mf, base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{ng_cons});
195  }
196 
197  if (icomp_cons+ncomp_cons > 1)
198  {
199  // Add theta_0 to theta
200  if (!amrex::almostEqual(time,ctime[1])) {
201  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
202  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
203  MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
204  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
205  }
206  if (!amrex::almostEqual(time,ctime[0])) {
207  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
208  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
209  MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
210  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
211  }
212 
213  // Multiply theta by rho to get (rho theta)
214  MultiFab::Multiply(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
215 
216  // Add theta_0 to theta
217  MultiFab::Add(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
218 
219  // Add theta_0 back to theta
220  MultiFab::Add(mf,base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{ng_cons});
221 
222  // Multiply (theta) by rho to get (rho theta)
223  MultiFab::Multiply(mf,mf,Rho_comp,RhoTheta_comp,1,IntVect{ng_cons});
224  }
225  } // interpolation_type == StateInterpType::Perturbational
226 
227  // Impose physical bc's on fine data (note time and 0 are not used)
228  // Note that we do this after the FillPatch because imposing physical bc's on fine ghost
229  // cells that need to be filled from coarse requires that we have done the interpolation first
230  bool do_fb = true; bool do_terrain_adjustment = false;
231  (*physbcs_cons[lev])(mf,*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
232  icomp_cons,ncomp_cons,IntVect{ng_cons},time,BCVars::cons_bc,
233  do_fb, do_terrain_adjustment);
234 
235  // Make sure to only copy back the components we worked on
236  MultiFab::Copy(*mfs_vel[Vars::cons],mf,icomp_cons,icomp_cons,ncomp_cons,IntVect{ng_cons});
237 
238  // *****************************************************************************************
239 
240  if (!cons_only)
241  {
242  // ***************************************************************************
243  // We always come in to this call with updated momenta but we need to create updated velocity
244  // in order to impose the rest of the bc's
245  // ***************************************************************************
246  // This only fills VALID region of velocity
247  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
248  *mfs_vel[Vars::cons],
249  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
250  Geom(lev).Domain(), domain_bcs_type);
251 
252  mapper = &face_cons_linear_interp;
253 
254  //
255  // NOTE: All interpolation here happens on velocities not momenta;
256  // note we only do the interpolation and FillBoundary here,
257  // physical bc's are imposed later
258  //
259  // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
260  // unlike the FillSet calls above which filled momenta on the coarse/fine bdy
261  //
262 
263  MultiFab& mfu = *mfs_vel[Vars::xvel];
264 
265  fmf = {&mfu,&mfu};
266  cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
267 
268  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
269  FillPatchTwoLevels(mfu, IntVect{ng_vel}, IntVect(0,0,0),
270  time, cmf, ctime, fmf, ftime,
271  0, 0, 1, geom[lev-1], geom[lev],
272  refRatio(lev-1), mapper, domain_bcs_type,
274 
275  // *****************************************************************************************
276 
277  MultiFab& mfv = *mfs_vel[Vars::yvel];
278 
279  fmf = {&mfv,&mfv};
280  cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
281 
282  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
283  FillPatchTwoLevels(mfv, IntVect{ng_vel}, IntVect(0,0,0),
284  time, cmf, ctime, fmf, ftime,
285  0, 0, 1, geom[lev-1], geom[lev],
286  refRatio(lev-1), mapper, domain_bcs_type,
288 
289  // *****************************************************************************************
290 
291  MultiFab& mfw = *mfs_vel[Vars::zvel];
292 
293  fmf = {&mfw,&mfw};
294  cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
295 
296  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
297  FillPatchTwoLevels(mfw, IntVect{ng_vel}, IntVect(0,0,0),
298  time, cmf, ctime, fmf, ftime,
299  0, 0, 1, geom[lev-1], geom[lev],
300  refRatio(lev-1), mapper, domain_bcs_type,
302  } // !cons_only
303  } // lev > 0
304 
305  // ***************************************************************************
306  // Physical bc's at domain boundary
307  // ***************************************************************************
308  IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
309  IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
310 
311  bool do_fb = true;
312 
313 #ifdef ERF_USE_NETCDF
314  // We call this here because it is an ERF routine
315  if (solverChoice.use_real_bcs && (lev==0)) {
317  fill_from_realbdy_upwind(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
318  } else {
319  fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
320  }
321  do_fb = false;
322  }
323 #endif
324 
325  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
326 
327  // We call this even if use_real_bcs is true because these will fill the vertical bcs
328  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
329  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
330  if (!cons_only) {
331  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
332  ngvect_vels,time,BCVars::xvel_bc, do_fb);
333  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
334  ngvect_vels,time,BCVars::yvel_bc, do_fb);
335  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
336  ngvect_vels,time,BCVars::zvel_bc, do_fb);
337  }
338  // ***************************************************************************
339 
340  // We always come in to this call with momenta so we need to leave with momenta!
341  // We need to make sure we convert back on all ghost cells/faces because this is
342  // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
343  if (!cons_only)
344  {
345  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
346  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
347  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
348 
349  VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
350  *mfs_vel[Vars::yvel], ngv,
351  *mfs_vel[Vars::zvel], ngw,
352  *mfs_vel[Vars::cons],
353  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
354  Geom(lev).Domain(),
356  }
357 
358  // NOTE: There are not FillBoundary calls here for the following reasons:
359  // Removal of the FillBoundary (FB) calls has bee completed for the following reasons:
360  //
361  // 1. physbc_cons is called before VelocityToMomentum and a FB is completed in that functor.
362  // Therefore, the conserved CC vars have their inter-rank ghost cells filled and then their
363  // domain ghost cells filled from the BC operations. We should not call FB on this MF again.
364  //
365  // 2. physbc_u/v/w is also called before VelocityToMomentum and a FB is completed those functors.
366  // Furthermore, VelocityToMomentum operates on a growntilebox so we exit that routine with momentum
367  // filled everywhere---i.e., physbc_u/v/w fills velocity ghost cells (inter-rank and domain)
368  // and then V2M does the conversion to momenta everywhere; so there is again no need to do a FB on momenta.
369 }
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:395
void fill_from_bndryregs(const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
Definition: ERF_BoundaryConditionsBndryReg.cpp:13
void FillIntermediatePatch(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
Definition: ERF_FillIntermediatePatch.cpp:28
@ NumTypes
Definition: ERF_IndexDefines.H:162
static bool upwind_real_bcs
Definition: ERF_DataStruct.H:774
static bool use_real_bcs
Definition: ERF_DataStruct.H:771
Here is the call graph for this function:

◆ FillPatchCrseLevel()

void ERF::FillPatchCrseLevel ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
bool  cons_only = false 
)
private
285 {
286  BL_PROFILE_VAR("ERF::FillPatchCrseLevel()",ERF_FillPatchCrseLevel);
287 
288  AMREX_ALWAYS_ASSERT(lev == 0);
289 
290  IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
291  IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
292 
293  Vector<Real> ftime = {t_old[lev], t_new[lev]};
294 
295  //
296  // Below we call FillPatchSingleLevel which does NOT fill ghost cells outside the domain
297  //
298 
299  Vector<MultiFab*> fmf;
300  Vector<MultiFab*> fmf_u;
301  Vector<MultiFab*> fmf_v;
302  Vector<MultiFab*> fmf_w;
303 
304  if (amrex::almostEqual(time,ftime[0])) {
305  fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
306  } else if (amrex::almostEqual(time,ftime[1])) {
307  fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
308  } else {
309  fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
310  }
311 
312  const int ncomp = mfs_vel[Vars::cons]->nComp();
313 
314  FillPatchSingleLevel(*mfs_vel[Vars::cons], ngvect_cons, time, fmf, IntVect(0,0,0), ftime,
315  0, 0, ncomp, geom[lev]);
316 
317  if (!cons_only) {
318  if (amrex::almostEqual(time,ftime[0])) {
319  fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
320  fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
321  fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
322  } else if (amrex::almostEqual(time,ftime[1])) {
323  fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
324  fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
325  fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
326  } else {
327  fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
328  fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
329  fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
330  }
331  FillPatchSingleLevel(*mfs_vel[Vars::xvel], ngvect_vels, time, fmf_u,
332  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
333 
334  FillPatchSingleLevel(*mfs_vel[Vars::yvel], ngvect_vels, time, fmf_v,
335  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
336 
337  FillPatchSingleLevel(*mfs_vel[Vars::zvel], ngvect_vels, time, fmf_w,
338  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
339  } // !cons_only
340 
341  // ***************************************************************************
342  // Physical bc's at domain boundary
343  // ***************************************************************************
344  int icomp_cons = 0;
345  int ncomp_cons = mfs_vel[Vars::cons]->nComp();
346 
347  bool do_fb = true;
348 
349 #ifdef ERF_USE_NETCDF
350  // We call this here because it is an ERF routine
351  if(solverChoice.use_real_bcs && (lev==0)) {
353  fill_from_realbdy_upwind(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
354  } else {
355  fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
356  }
357  do_fb = false;
358  }
359 #endif
360 
361  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
362 
363  // We call this even if use_real_bcs is true because these will fill the vertical bcs
364  // Note that we call FillBoundary inside the physbcs call
365  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
366  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
367  if (!cons_only) {
368  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
369  ngvect_vels,time,BCVars::xvel_bc, do_fb);
370  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
371  ngvect_vels,time,BCVars::yvel_bc, do_fb);
372  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
373  ngvect_vels,time,BCVars::zvel_bc, do_fb);
374  }
375 }

◆ FillPatchFineLevel()

void ERF::FillPatchFineLevel ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
const amrex::MultiFab &  old_base_state,
const amrex::MultiFab &  new_base_state,
bool  fillset = true,
bool  cons_only = false 
)
private
26 {
27  BL_PROFILE_VAR("ERF::FillPatchFineLevel()",ERF_FillPatchFineLevel);
28 
29  AMREX_ALWAYS_ASSERT(lev > 0);
30 
31  Interpolater* mapper = nullptr;
32 
33  PhysBCFunctNoOp null_bc;
34 
35  //
36  // ***************************************************************************
37  // The first thing we do is interpolate the momenta on the "valid" faces of
38  // the fine grids (where the interface is coarse/fine not fine/fine) -- this
39  // will not be over-written below because the FillPatch operators see these as
40  // valid faces.
41  //
42  // Note that we interpolate momentum not velocity, but all the other boundary
43  // conditions are imposed on velocity, so we convert to momentum here then
44  // convert back.
45  // ***************************************************************************
46  if (fillset) {
47  if (cf_set_width > 0) {
48  FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
49  }
50  if (cf_set_width >= 0 && !cons_only) {
51  VelocityToMomentum(*mfs_vel[Vars::xvel], IntVect{0},
52  *mfs_vel[Vars::yvel], IntVect{0},
53  *mfs_vel[Vars::zvel], IntVect{0},
54  *mfs_vel[Vars::cons],
55  *mfs_mom[IntVars::xmom],
56  *mfs_mom[IntVars::ymom],
57  *mfs_mom[IntVars::zmom],
58  Geom(lev).Domain(),
60 
61  FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
62  FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
63  FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
64 
65  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
66  *mfs_vel[Vars::cons],
67  *mfs_mom[IntVars::xmom],
68  *mfs_mom[IntVars::ymom],
69  *mfs_mom[IntVars::zmom],
70  Geom(lev).Domain(),
72  }
73  }
74 
75  IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
76  IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
77 
78  Vector<Real> ftime = {t_old[lev ], t_new[lev ]};
79  Vector<Real> ctime = {t_old[lev-1], t_new[lev-1]};
80 
81  amrex::Real small_dt = 1.e-8 * (ftime[1] - ftime[0]);
82 
83  Vector<MultiFab*> fmf;
84  if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
85  fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
86  } else if (amrex::almostEqual(time,ftime[1])) {
87  fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
88  } else {
89  fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
90  }
91  Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
92 
93  // We must fill a temporary then copy it back so we don't double add/subtract
94  MultiFab mf_c(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
95  mfs_vel[Vars::cons]->nComp() ,mfs_vel[Vars::cons]->nGrowVect());
96 
97  mapper = &cell_cons_interp;
98 
99  if (interpolation_type == StateInterpType::Perturbational)
100  {
101  // Divide (rho theta) by rho to get theta (before we subtract rho0 from rho!)
102  if (!amrex::almostEqual(time,ctime[1])) {
103  MultiFab::Divide(vars_old[lev-1][Vars::cons],vars_old[lev-1][Vars::cons],
104  Rho_comp,RhoTheta_comp,1,ngvect_cons);
105  MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
106  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
107  MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
108  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
109  }
110  if (!amrex::almostEqual(time,ctime[0])) {
111  MultiFab::Divide(vars_new[lev-1][Vars::cons],vars_new[lev-1][Vars::cons],
112  Rho_comp,RhoTheta_comp,1,ngvect_cons);
113  MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
114  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
115  MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
116  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
117  }
118 
119  if (!amrex::almostEqual(time,ftime[1])) {
120  MultiFab::Divide(vars_old[lev ][Vars::cons],vars_old[lev ][Vars::cons],
121  Rho_comp,RhoTheta_comp,1,IntVect{0});
122  MultiFab::Subtract(vars_old[lev ][Vars::cons],old_base_state,
123  BaseState::r0_comp,Rho_comp,1,IntVect{0});
124  MultiFab::Subtract(vars_old[lev ][Vars::cons],old_base_state,
125  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
126  }
127  if (!amrex::almostEqual(time,ftime[0])) {
128  MultiFab::Divide(vars_new[lev ][Vars::cons],vars_new[lev ][Vars::cons],
129  Rho_comp,RhoTheta_comp,1,IntVect{0});
130  MultiFab::Subtract(vars_new[lev ][Vars::cons],old_base_state,
131  BaseState::r0_comp,Rho_comp,1,IntVect{0});
132  MultiFab::Subtract(vars_new[lev ][Vars::cons],old_base_state,
133  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
134  }
135  }
136 
137  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
138  FillPatchTwoLevels(mf_c, ngvect_cons, IntVect(0,0,0),
139  time, cmf, ctime, fmf, ftime,
140  0, 0, mf_c.nComp(), geom[lev-1], geom[lev],
141  refRatio(lev-1), mapper, domain_bcs_type,
143 
144  if (interpolation_type == StateInterpType::Perturbational)
145  {
146  // Restore the coarse values to what they were
147  if (!amrex::almostEqual(time,ctime[1])) {
148  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
149  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
150  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
151  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
152  MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
153  Rho_comp,RhoTheta_comp,1,ngvect_cons);
154  }
155  if (!amrex::almostEqual(time,ctime[0])) {
156  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
157  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
158  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
159  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
160  MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
161  Rho_comp,RhoTheta_comp,1,ngvect_cons);
162  }
163 
164  if (!amrex::almostEqual(time,ftime[1])) {
165  MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev ],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
166  MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev ],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
167  MultiFab::Multiply(vars_old[lev][Vars::cons], vars_old[lev][Vars::cons],
168  Rho_comp,RhoTheta_comp,1,ngvect_cons);
169  }
170  if (!amrex::almostEqual(time,ftime[0])) {
171  MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
172  MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
173  MultiFab::Multiply(vars_new[lev][Vars::cons], vars_new[lev][Vars::cons],
174  Rho_comp,RhoTheta_comp,1,ngvect_cons);
175  }
176 
177  // Set values in the cells outside the domain boundary so that we can do the Add
178  // without worrying about uninitialized values outside the domain -- these
179  // will be filled in the physbcs call
180  mf_c.setDomainBndry(1.234e20,0,2,geom[lev]); // Do both rho and (rho theta) together
181 
182  // Add rho_0 back to rho and theta_0 back to theta
183  MultiFab::Add(mf_c, new_base_state,BaseState::r0_comp,Rho_comp,1,ngvect_cons);
184  MultiFab::Add(mf_c, new_base_state,BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
185 
186  // Multiply (theta) by rho to get (rho theta)
187  MultiFab::Multiply(mf_c,mf_c,Rho_comp,RhoTheta_comp,1,ngvect_cons);
188  }
189 
190  MultiFab::Copy(*mfs_vel[Vars::cons],mf_c,0,0,mf_c.nComp(),mf_c.nGrowVect());
191 
192  // ***************************************************************************************
193 
194  if (!cons_only)
195  {
196  mapper = &face_cons_linear_interp;
197 
198  MultiFab& mf_u = *mfs_vel[Vars::xvel];
199  MultiFab& mf_v = *mfs_vel[Vars::yvel];
200  MultiFab& mf_w = *mfs_vel[Vars::zvel];
201 
202  Vector<MultiFab*> fmf_u; Vector<MultiFab*> fmf_v; Vector<MultiFab*> fmf_w;
203  Vector<MultiFab*> cmf_u; Vector<MultiFab*> cmf_v; Vector<MultiFab*> cmf_w;
204 
205  // **********************************************************************
206 
207  if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
208  fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
209  fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
210  fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
211  } else if ( amrex::almostEqual(time,ftime[1]) ) {
212  fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
213  fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
214  fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
215  } else {
216  fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
217  fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
218  fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
219  }
220  cmf_u = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
221  cmf_v = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
222  cmf_w = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
223 
224  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
225  FillPatchTwoLevels(mf_u, ngvect_vels, IntVect(0,0,0),
226  time, cmf_u, ctime, fmf_u, ftime,
227  0, 0, 1, geom[lev-1], geom[lev],
228  refRatio(lev-1), mapper, domain_bcs_type,
230 
231  FillPatchTwoLevels(mf_v, ngvect_vels, IntVect(0,0,0),
232  time, cmf_v, ctime, fmf_v, ftime,
233  0, 0, 1, geom[lev-1], geom[lev],
234  refRatio(lev-1), mapper, domain_bcs_type,
236 
237  // We put these here because these may be used in constructing omega outside the
238  // domain when fillpatching w
239  bool do_fb = true;
240  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
241  ngvect_vels,time,BCVars::xvel_bc, do_fb);
242  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
243  ngvect_vels,time,BCVars::yvel_bc, do_fb);
244 
245  // **********************************************************************
246 
247  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
248  FillPatchTwoLevels(mf_w, ngvect_vels, IntVect(0,0,0),
249  time, cmf_w, ctime, fmf_w, ftime,
250  0, 0, 1, geom[lev-1], geom[lev],
251  refRatio(lev-1), mapper, domain_bcs_type,
253  } // !cons_only
254 
255  // ***************************************************************************
256  // Physical bc's at domain boundary
257  // ***************************************************************************
258  int icomp_cons = 0;
259  int ncomp_cons = mfs_vel[Vars::cons]->nComp();
260 
261  bool do_fb = true;
262 
263  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
264 
265  // We call this even if use_real_bcs is true because these will fill the vertical bcs
266  // Note that we call FillBoundary inside the physbcs call
267  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
268  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
269  if (!cons_only) {
270  // Note that we need to fill u and v in the case of terrain because we will use
271  // these in the call of WFromOmega in lateral ghost cells of the fine grid
272  // (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
273  // ngvect_vels,time,BCVars::xvel_bc, do_fb);
274  // (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
275  // ngvect_vels,time,BCVars::yvel_bc, do_fb);
276  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
277  ngvect_vels,time,BCVars::zvel_bc, do_fb);
278  }
279 }
Here is the call graph for this function:

◆ FillWeatherDataMultiFab()

void ERF::FillWeatherDataMultiFab ( const std::string &  filename,
const amrex::Geometry &  geom_weather,
const amrex::BoxArray &  nba,
const amrex::DistributionMapping &  dm,
amrex::Vector< amrex::MultiFab > &  weather_forecast_data 
)
456 {
457 
458  Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
459  Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
460 
461  ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
462  xvec_h, yvec_h, zvec_h, rho_h,
463  uvel_h, vvel_h, wvel_h,
464  theta_h, qv_h, qc_h, qr_h);
465 
466  const auto prob_lo_erf = geom[0].ProbLoArray();
467  const auto prob_hi_erf = geom[0].ProbHiArray();
468  const auto dx_erf = geom[0].CellSizeArray();
469 
470  if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
471  amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
472  }
473  if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
474  amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
475  }
476  if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
477  amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
478  }
479  if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
480  amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
481  }
482 
483  // Number of cells
484  int nx_cells = xvec_h.size()-1;
485  int ny_cells = yvec_h.size()-1;
486 
487  const amrex::Geometry& geom0 = geom[0]; // or whatever your Geometry vector is called
488  const amrex::Box& domainBox = geom0.Domain();
489  const amrex::IntVect& domainSize = domainBox.size(); // Number of cells in each direction
490  int nz_cells = domainSize[2];
491 
492 
493  int ncomp = 10;
494  int ngrow = 0;
495 
496  int n_time = 1; // or however many time slices you want
497  weather_forecast_data.resize(n_time);
498  MultiFab& weather_mf = weather_forecast_data[0];
499  weather_mf.define(nba, dm, ncomp, ngrow);
500 
501  fill_weather_data_multifab(weather_mf, geom_weather, nx_cells+1, ny_cells+1, nz_cells+1,
502  latvec_h, lonvec_h, zvec_h,
503  rho_h,uvel_h, vvel_h, wvel_h,
504  theta_h, qv_h, qc_h, qr_h);
505 
506  PlotMultiFab(weather_mf, geom_weather, "plt_coarse_weather", MultiFabType::NC);
507 }
void PlotMultiFab(const MultiFab &mf, const Geometry &geom_mf, const std::string plotfilename, MultiFabType mftype)
Definition: ERF_WeatherDataInterpolation.cpp:111
void fill_weather_data_multifab(MultiFab &mf, const Geometry &geom_weather, const int nx, const int ny, const int nz, const Vector< Real > &latvec_h, const Vector< Real > &lonvec_h, const Vector< Real > &zvec_h, const Vector< Real > &rho_h, const Vector< Real > &uvel_h, const Vector< Real > &vvel_h, const Vector< Real > &wvel_h, const Vector< Real > &theta_h, const Vector< Real > &qv_h, const Vector< Real > &qc_h, const Vector< Real > &qr_h)
Definition: ERF_WeatherDataInterpolation.cpp:18
Here is the call graph for this function:

◆ get_eb()

eb_ const& ERF::get_eb ( int  lev) const
inlineprivatenoexcept
1540  {
1541  AMREX_ASSERT(lev >= 0 && lev < eb.size() && eb[lev] != nullptr);
1542  return *eb[lev];
1543  }

◆ get_projection_bc()

Array< LinOpBCType, AMREX_SPACEDIM > ERF::get_projection_bc ( amrex::Orientation::Side  side) const
noexcept
18 {
19  amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> r;
20  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
21  if (geom[0].isPeriodic(dir)) {
22  r[dir] = LinOpBCType::Periodic;
23  } else {
24  auto bc_type = domain_bc_type[Orientation(dir,side)];
25  if (bc_type == "Outflow") {
26  r[dir] = LinOpBCType::Dirichlet;
27  } else
28  {
29  r[dir] = LinOpBCType::Neumann;
30  }
31  }
32  }
33  return r;
34 }
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
Definition: ERF.H:933

◆ getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int  lev)
inlineprivate
1329  {
1330  return advflux_reg[lev];
1331  }

◆ getCPUTime()

static amrex::Real ERF::getCPUTime ( )
inlinestaticprivate
1421  {
1422  int numCores = amrex::ParallelDescriptor::NProcs();
1423 #ifdef _OPENMP
1424  numCores = numCores * omp_get_max_threads();
1425 #endif
1426 
1427  amrex::Real T =
1428  numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
1430 
1431  return T;
1432  }
static amrex::Real previousCPUTimeUsed
Definition: ERF.H:1417
static amrex::Real startCPUTime
Definition: ERF.H:1416
@ T
Definition: ERF_IndexDefines.H:110

◆ GotoNextLine()

void ERF::GotoNextLine ( std::istream &  is)
staticprivate

Utility to skip to next line in Header file input stream.

17 {
18  constexpr std::streamsize bl_ignore_max { 100000 };
19  is.ignore(bl_ignore_max, '\n');
20 }

◆ HurricaneTracker()

void ERF::HurricaneTracker ( int  lev,
const amrex::MultiFab &  U_new,
const amrex::MultiFab &  V_new,
const amrex::MultiFab &  W_new,
const amrex::Real  velmag_threshold,
const bool  is_track_io,
amrex::TagBoxArray *  tags = nullptr 
)
462 {
463  const auto dx = geom[levc].CellSizeArray();
464  const auto prob_lo = geom[levc].ProbLoArray();
465 
466  const int ncomp = AMREX_SPACEDIM; // Number of components (3 for 3D)
467 
468  Gpu::DeviceVector<Real> d_coords(3, 0.0); // Initialize to -1
469  Real* d_coords_ptr = d_coords.data(); // Get pointer to device vector
470  Gpu::DeviceVector<int> d_found(1,0);
471  int* d_found_ptr = d_found.data();
472 
473  MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
474  average_face_to_cellcenter(mf_cc_vel,0,{AMREX_D_DECL(&U_new,&V_new,&W_new)},0);
475 
476  // Loop through MultiFab using MFIter
477  for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi) {
478  const Box& box = mfi.validbox(); // Get the valid box for the current MFIter
479  const Array4<const Real>& vel_arr = mf_cc_vel.const_array(mfi); // Get the array for this MFIter
480 
481  // ParallelFor loop to check velocity magnitudes on the GPU
482  amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
483  // Access velocity components using ncomp
484  Real magnitude = 0.0; // Initialize magnitude
485 
486  for (int comp = 0; comp < ncomp; ++comp) {
487  Real vel = vel_arr(i, j, k, comp); // Access the component for each (i, j, k)
488  magnitude += vel * vel; // Sum the square of the components
489  }
490 
491  magnitude = std::sqrt(magnitude)*3.6; // Calculate magnitude
492  Real x = prob_lo[0] + (i + 0.5) * dx[0];
493  Real y = prob_lo[1] + (j + 0.5) * dx[1];
494  Real z = prob_lo[2] + (k + 0.5) * dx[2];
495 
496  // Check if magnitude exceeds threshold
497  if (z < 2.0e3 && magnitude > velmag_threshold) {
498  // Use atomic operations to set found flag and store coordinates
499  Gpu::Atomic::Add(&d_found_ptr[0], 1); // Mark as found
500 
501  // Store coordinates
502  Gpu::Atomic::Add(&d_coords_ptr[0],x); // Store x index
503  Gpu::Atomic::Add(&d_coords_ptr[1],y); // Store x index
504  Gpu::Atomic::Add(&d_coords_ptr[2],z); // Store x index
505  }
506  });
507  }
508 
509  // Synchronize to ensure all threads complete their execution
510  amrex::Gpu::streamSynchronize(); // Wait for all GPU threads to finish
511 
512  Vector<int> h_found(1,0);
513  Gpu::copy(Gpu::deviceToHost, d_found.begin(), d_found.end(), h_found.begin());
514  ParallelAllReduce::Sum(h_found.data(),
515  h_found.size(),
516  ParallelContext::CommunicatorAll());
517 
518  Real eye_x, eye_y;
519  // Broadcast coordinates if found
520  if (h_found[0] > 0) {
521  Vector<Real> h_coords(3,-1e10);
522  Gpu::copy(Gpu::deviceToHost, d_coords.begin(), d_coords.end(), h_coords.begin());
523 
524  ParallelAllReduce::Sum(h_coords.data(),
525  h_coords.size(),
526  ParallelContext::CommunicatorAll());
527 
528  eye_x = h_coords[0]/h_found[0];
529  eye_y = h_coords[1]/h_found[0];
530 
531  // Data structure to hold the hurricane track for I/O
532  if (amrex::ParallelDescriptor::IOProcessor() and is_track_io) {
533  hurricane_track_xy.push_back({eye_x, eye_y});
534  }
535 
536  if(is_track_io) {
537  return;
538  }
539 
540  Real rad_tag = 3e5*std::pow(2, max_level-1-levc);
541 
542  for (MFIter mfi(*tags); mfi.isValid(); ++mfi) {
543  TagBox& tag = (*tags)[mfi];
544  auto tag_arr = tag.array(); // Get device-accessible array
545 
546  const Box& tile_box = mfi.tilebox(); // The box for this tile
547 
548  ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
549  // Compute cell center coordinates
550  Real x = prob_lo[0] + (i + 0.5) * dx[0];
551  Real y = prob_lo[1] + (j + 0.5) * dx[1];
552 
553  Real dist = std::sqrt((x - eye_x)*(x - eye_x) + (y - eye_y)*(y - eye_y));
554 
555  if (dist < rad_tag) {
556  tag_arr(i,j,k) = TagBox::SET;
557  }
558  });
559  }
560  }
561 }
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_track_xy
Definition: ERF.H:153

◆ ImposeBCsOnPhi()

void ERF::ImposeBCsOnPhi ( int  lev,
amrex::MultiFab &  phi,
const amrex::Box &  subdomain 
)

Impose bc's on the pressure that comes out of the solve

13 {
14  BL_PROFILE("ERF::ImposeBCsOnPhi()");
15 
16  auto const sub_lo = lbound(subdomain);
17  auto const sub_hi = ubound(subdomain);
18 
19  auto const dom_lo = lbound(geom[lev].Domain());
20  auto const dom_hi = ubound(geom[lev].Domain());
21 
22  phi.setBndry(1.e25);
23  phi.FillBoundary(geom[lev].periodicity());
24 
25  // ****************************************************************************
26  // Impose bc's on pprime
27  // ****************************************************************************
28 #ifdef _OPENMP
29 #pragma omp parallel if (Gpu::notInLaunchRegion())
30 #endif
31  for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi)
32  {
33  Array4<Real> const& pp_arr = phi.array(mfi);
34  Box const& bx = mfi.tilebox();
35  auto const bx_lo = lbound(bx);
36  auto const bx_hi = ubound(bx);
37 
38  auto bc_type_xlo = domain_bc_type[Orientation(0,Orientation::low)];
39  auto bc_type_xhi = domain_bc_type[Orientation(0,Orientation::high)];
40  auto bc_type_ylo = domain_bc_type[Orientation(1,Orientation::low)];
41  auto bc_type_yhi = domain_bc_type[Orientation(1,Orientation::high)];
42  auto bc_type_zhi = domain_bc_type[Orientation(2,Orientation::high)];
43 
44  if ( (bx_lo.x == dom_lo.x) && (bc_type_xlo == "Outflow" || bc_type_xlo == "Open") ) {
45  ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
46  {
47  pp_arr(i-1,j,k) = -pp_arr(i,j,k);
48  });
49  } else if (bx_lo.x == sub_lo.x) {
50  ParallelFor(makeSlab(bx,0,sub_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
51  {
52  pp_arr(i-1,j,k) = pp_arr(i,j,k);
53  });
54  }
55 
56  if ( (bx_hi.x == dom_hi.x) && (bc_type_xhi == "Outflow" || bc_type_xhi == "Open") ) {
57  ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
58  {
59  pp_arr(i+1,j,k) = -pp_arr(i,j,k);
60  });
61  } else if (bx_hi.x == sub_hi.x) {
62  ParallelFor(makeSlab(bx,0,sub_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
63  {
64  pp_arr(i+1,j,k) = pp_arr(i,j,k);
65  });
66  }
67 
68  if ( (bx_lo.y == dom_lo.y) && (bc_type_ylo == "Outflow" || bc_type_ylo == "Open") ) {
69  ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
70  {
71  pp_arr(i,j-1,k) = -pp_arr(i,j,k);
72  });
73  } else if (bx_lo.y == sub_lo.y) {
74  ParallelFor(makeSlab(bx,1,sub_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
75  {
76  pp_arr(i,j-1,k) = pp_arr(i,j,k);
77  });
78  }
79 
80  if ( (bx_hi.y == dom_hi.y) && (bc_type_yhi == "Outflow" || bc_type_yhi == "Open") ) {
81  ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
82  {
83  pp_arr(i,j+1,k) = -pp_arr(i,j,k);
84  });
85  } else if (bx_hi.y == sub_hi.y) {
86  ParallelFor(makeSlab(bx,1,sub_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
87  {
88  pp_arr(i,j+1,k) = pp_arr(i,j,k);
89  });
90  }
91 
92  // At low z we are always Neumann whether the box touches the bottom boundary or not
93  Box zbx(bx); zbx.grow(0,1); zbx.grow(1,1); // Grow in x-dir and y-dir because we have filled that above
94  if (bx_lo.z == sub_lo.z) {
95  ParallelFor(makeSlab(zbx,2,dom_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
96  {
97  pp_arr(i,j,k-1) = pp_arr(i,j,k);
98  });
99  }
100 
101  if ( (bx_hi.z == dom_hi.z) && (bc_type_zhi == "Outflow" || bc_type_zhi == "Open") ) {
102  ParallelFor(makeSlab(bx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
103  {
104  pp_arr(i,j,k+1) = -pp_arr(i,j,k);
105  });
106  } else if (bx_hi.z == sub_hi.z) {
107  ParallelFor(makeSlab(bx,2,sub_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
108  {
109  pp_arr(i,j,k+1) = pp_arr(i,j,k);
110  });
111  }
112  } // mfi
113 
114  // Now overwrite with periodic fill outside domain and fine-fine fill inside
115  phi.FillBoundary(geom[lev].periodicity());
116 }

◆ init1DArrays()

void ERF::init1DArrays ( )
private

◆ init_bcs()

void ERF::init_bcs ( )
private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, InflowOutflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

21 {
22  bool rho_read = false;
23  bool read_prim_theta = true;
24  Vector<Real> cons_dir_init(NBCVAR_max,0.0);
25  cons_dir_init[BCVars::Rho_bc_comp] = 1.0;
26  cons_dir_init[BCVars::RhoTheta_bc_comp] = -1.0;
27  auto f = [this,&rho_read,&read_prim_theta] (std::string const& bcid, Orientation ori)
28  {
29  // These are simply defaults for Dirichlet faces -- they should be over-written below
31  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = -1.0; // It is important to set this negative
32  // because the sign is tested on below
33  for (int n = BCVars::RhoKE_bc_comp; n < BCVars::xvel_bc; n++) {
34  m_bc_extdir_vals[n][ori] = 0.0;
35  }
36 
37  m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.0; // default
40 
41  // These are simply defaults for Neumann gradients -- they should be over-written below
44 
53 
57 
58  std::string pp_text = pp_prefix + "." + bcid;
59  ParmParse pp(pp_text);
60 
61  std::string bc_type_in;
62  if (pp.query("type", bc_type_in) <= 0)
63  {
64  pp_text = bcid;
65  pp = ParmParse(pp_text);
66  pp.query("type", bc_type_in);
67  }
68 
69  std::string bc_type = amrex::toLower(bc_type_in);
70 
71  if (bc_type == "symmetry")
72  {
73  // Print() << bcid << " set to symmetry.\n";
75  domain_bc_type[ori] = "Symmetry";
76  }
77  else if (bc_type == "outflow")
78  {
79  // Print() << bcid << " set to outflow.\n";
81  domain_bc_type[ori] = "Outflow";
82  }
83  else if (bc_type == "open")
84  {
85  // Print() << bcid << " set to open.\n";
86  AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
88  domain_bc_type[ori] = "Open";
89  }
90  else if (bc_type == "ho_outflow")
91  {
93  domain_bc_type[ori] = "HO_Outflow";
94  }
95 
96  else if (bc_type == "inflow" || bc_type == "inflow_outflow")
97  {
98  if (bc_type == "inflow") {
99  // Print() << bcid << " set to inflow.\n";
101  domain_bc_type[ori] = "Inflow";
102  } else {
103  // Print() << bcid << " set to inflow_outflow.\n";
105  domain_bc_type[ori] = "InflowOutflow";
106  }
107 
108  std::vector<Real> v;
109  if (input_bndry_planes && m_r2d->ingested_velocity()) {
113  } else {
114  // Test for input data file if at xlo face
115  std::string dirichlet_file;
116  auto file_exists = pp.query("dirichlet_file", dirichlet_file);
117  if (file_exists) {
118  pp.query("read_prim_theta", read_prim_theta);
119  init_Dirichlet_bc_data(dirichlet_file);
120  } else {
121  pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
122  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
123  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
124  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
125  }
126  }
127 
128  Real rho_in = 0.;
129  if (input_bndry_planes && m_r2d->ingested_density()) {
131  } else {
132  if (!pp.query("density", rho_in)) {
133  amrex::Print() << "Using interior values to set conserved vars" << std::endl;
134  }
135  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
136  }
137 
138  bool th_read = (th_bc_data[0].data()!=nullptr);
139  Real theta_in = 0.;
140  if (input_bndry_planes && m_r2d->ingested_theta()) {
142  } else if (!th_read) {
143  if (rho_in > 0) {
144  pp.get("theta", theta_in);
145  }
146  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
147  }
148 
149  Real scalar_in = 0.;
150  if (input_bndry_planes && m_r2d->ingested_scalar()) {
152  } else {
153  if (pp.query("scalar", scalar_in))
154  m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
155  }
156 
157  if (solverChoice.moisture_type != MoistureType::None) {
158  Real qv_in = 0.;
159  if (input_bndry_planes && m_r2d->ingested_q1()) {
161  } else {
162  if (pp.query("qv", qv_in))
163  m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
164  }
165  Real qc_in = 0.;
166  if (input_bndry_planes && m_r2d->ingested_q2()) {
168  } else {
169  if (pp.query("qc", qc_in))
170  m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
171  }
172  }
173 
174  Real KE_in = 0.;
175  if (input_bndry_planes && m_r2d->ingested_KE()) {
177  } else {
178  if (pp.query("KE", KE_in))
179  m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
180  }
181  }
182  else if (bc_type == "noslipwall")
183  {
184  // Print() << bcid <<" set to no-slip wall.\n";
186  domain_bc_type[ori] = "NoSlipWall";
187 
188  std::vector<Real> v;
189 
190  // The values of m_bc_extdir_vals default to 0.
191  // But if we find "velocity" in the inputs file, use those values instead.
192  if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
193  {
194  v[ori.coordDir()] = 0.0;
195  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
196  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
197  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
198  }
199 
200  Real rho_in;
201  rho_read = pp.query("density", rho_in);
202  if (rho_read)
203  {
204  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
205  }
206 
207  Real theta_in;
208  if (pp.query("theta", theta_in))
209  {
211  }
212 
213  Real theta_grad_in;
214  if (pp.query("theta_grad", theta_grad_in))
215  {
216  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
217  }
218 
219  Real qv_in;
220  if (pp.query("qv", qv_in))
221  {
223  }
224  }
225  else if (bc_type == "slipwall")
226  {
227  // Print() << bcid <<" set to slip wall.\n";
228 
230  domain_bc_type[ori] = "SlipWall";
231 
232  Real rho_in;
233  rho_read = pp.query("density", rho_in);
234  if (rho_read)
235  {
236  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
237  }
238 
239  Real theta_in;
240  if (pp.query("theta", theta_in))
241  {
243  }
244 
245  Real rho_grad_in;
246  if (pp.query("density_grad", rho_grad_in))
247  {
248  m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = rho_grad_in;
249  }
250 
251  Real theta_grad_in;
252  if (pp.query("theta_grad", theta_grad_in))
253  {
254  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
255  }
256  }
257  else if (bc_type == "surface_layer")
258  {
260  domain_bc_type[ori] = "surface_layer";
261  }
262  else
263  {
265  }
266 
267  if (geom[0].isPeriodic(ori.coordDir())) {
268  domain_bc_type[ori] = "Periodic";
269  if (phys_bc_type[ori] == ERF_BC::undefined)
270  {
272  } else {
273  Abort("Wrong BC type for periodic boundary");
274  }
275  }
276 
277  if (phys_bc_type[ori] == ERF_BC::undefined)
278  {
279  Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
280  Abort("This BC type is unknown");
281  }
282  };
283 
284  f("xlo", Orientation(Direction::x,Orientation::low));
285  f("xhi", Orientation(Direction::x,Orientation::high));
286  f("ylo", Orientation(Direction::y,Orientation::low));
287  f("yhi", Orientation(Direction::y,Orientation::high));
288  f("zlo", Orientation(Direction::z,Orientation::low));
289  f("zhi", Orientation(Direction::z,Orientation::high));
290 
291  // *****************************************************************************
292  //
293  // Here we translate the physical boundary conditions -- one type per face --
294  // into logical boundary conditions for each velocity component
295  //
296  // *****************************************************************************
297  {
298  domain_bcs_type.resize(AMREX_SPACEDIM+NBCVAR_max);
299  domain_bcs_type_d.resize(AMREX_SPACEDIM+NBCVAR_max);
300 
301  for (OrientationIter oit; oit; ++oit) {
302  Orientation ori = oit();
303  int dir = ori.coordDir();
304  Orientation::Side side = ori.faceDir();
305  auto const bct = phys_bc_type[ori];
306  if ( bct == ERF_BC::symmetry )
307  {
308  if (side == Orientation::low) {
309  for (int i = 0; i < AMREX_SPACEDIM; i++) {
311  }
313  } else {
314  for (int i = 0; i < AMREX_SPACEDIM; i++) {
316  }
318  }
319  }
320  else if (bct == ERF_BC::outflow or bct == ERF_BC::ho_outflow )
321  {
322  if (side == Orientation::low) {
323  for (int i = 0; i < AMREX_SPACEDIM; i++) {
325  }
326  if (!solverChoice.anelastic[0]) {
328  }
329  } else {
330  for (int i = 0; i < AMREX_SPACEDIM; i++) {
332  }
333  if (!solverChoice.anelastic[0]) {
335  }
336  }
337  }
338  else if (bct == ERF_BC::open)
339  {
340  if (side == Orientation::low) {
341  for (int i = 0; i < AMREX_SPACEDIM; i++)
343  } else {
344  for (int i = 0; i < AMREX_SPACEDIM; i++)
346  }
347  }
348  else if (bct == ERF_BC::inflow)
349  {
350  if (side == Orientation::low) {
351  for (int i = 0; i < AMREX_SPACEDIM; i++) {
353  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
355  }
356  }
357  } else {
358  for (int i = 0; i < AMREX_SPACEDIM; i++) {
360  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
362  }
363  }
364  }
365  }
366  else if (bct == ERF_BC::inflow_outflow)
367  {
368  if (side == Orientation::low) {
369  for (int i = 0; i < AMREX_SPACEDIM; i++) {
371  }
372  } else {
373  for (int i = 0; i < AMREX_SPACEDIM; i++) {
375  }
376  }
377  }
378  else if (bct == ERF_BC::no_slip_wall)
379  {
380  if (side == Orientation::low) {
381  for (int i = 0; i < AMREX_SPACEDIM; i++) {
383  }
384  } else {
385  for (int i = 0; i < AMREX_SPACEDIM; i++) {
387  }
388  }
389  }
390  else if (bct == ERF_BC::slip_wall)
391  {
392  if (side == Orientation::low) {
393  for (int i = 0; i < AMREX_SPACEDIM; i++) {
395  }
396  // Only normal direction has ext_dir
398 
399  } else {
400  for (int i = 0; i < AMREX_SPACEDIM; i++) {
402  }
403  // Only normal direction has ext_dir
405  }
406  }
407  else if (bct == ERF_BC::periodic)
408  {
409  if (side == Orientation::low) {
410  for (int i = 0; i < AMREX_SPACEDIM; i++) {
412  }
413  } else {
414  for (int i = 0; i < AMREX_SPACEDIM; i++) {
416  }
417  }
418  }
419  else if ( bct == ERF_BC::surface_layer )
420  {
421  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
425  }
426  }
427  }
428 
429  // *****************************************************************************
430  //
431  // Here we translate the physical boundary conditions -- one type per face --
432  // into logical boundary conditions for each cell-centered variable
433  // (including the base state variables)
434  // NOTE: all "scalars" share the same type of boundary condition
435  //
436  // *****************************************************************************
437  {
438  for (OrientationIter oit; oit; ++oit) {
439  Orientation ori = oit();
440  int dir = ori.coordDir();
441  Orientation::Side side = ori.faceDir();
442  auto const bct = phys_bc_type[ori];
443  if ( bct == ERF_BC::symmetry )
444  {
445  if (side == Orientation::low) {
446  for (int i = 0; i < NBCVAR_max; i++) {
448  }
449  } else {
450  for (int i = 0; i < NBCVAR_max; i++) {
452  }
453  }
454  }
455  else if ( bct == ERF_BC::outflow )
456  {
457  if (side == Orientation::low) {
458  for (int i = 0; i < NBCVAR_max; i++) {
460  }
461  } else {
462  for (int i = 0; i < NBCVAR_max; i++) {
464  }
465  }
466  }
467  else if ( bct == ERF_BC::ho_outflow )
468  {
469  if (side == Orientation::low) {
470  for (int i = 0; i < NBCVAR_max; i++) {
472  }
473  } else {
474  for (int i = 0; i < NBCVAR_max; i++) {
476  }
477  }
478  }
479  else if ( bct == ERF_BC::open )
480  {
481  if (side == Orientation::low) {
482  for (int i = 0; i < NBCVAR_max; i++)
484  } else {
485  for (int i = 0; i < NBCVAR_max; i++)
487  }
488  }
489  else if ( bct == ERF_BC::no_slip_wall )
490  {
491  if (side == Orientation::low) {
492  for (int i = 0; i < NBCVAR_max; i++) {
494  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
495  if (rho_read) {
497  } else {
499  }
500  }
501  }
502  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
504  }
505  } else {
506  for (int i = 0; i < NBCVAR_max; i++) {
508  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
509  if (rho_read) {
511  } else {
513  }
514  }
515  }
516  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
518  }
519  }
520  }
521  else if (bct == ERF_BC::slip_wall)
522  {
523  if (side == Orientation::low) {
524  for (int i = 0; i < NBCVAR_max; i++) {
526  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
527  if (rho_read) {
529  } else {
531  }
532  }
533  }
534  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
536  }
537  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
539  }
540  } else {
541  for (int i = 0; i < NBCVAR_max; i++) {
543  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
544  if (rho_read) {
546  } else {
548  }
549  }
550  }
551  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
553  }
554  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
556  }
557  }
558  }
559  else if (bct == ERF_BC::inflow)
560  {
561  if (side == Orientation::low) {
562  for (int i = 0; i < NBCVAR_max; i++) {
564  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
565  (th_bc_data[0].data() != nullptr))
566  {
567  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
568  }
569  else if (input_bndry_planes && dir < 2 && (
570  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
571  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
572  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
573  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
574  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
575  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )) )
576  {
578  }
579  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
581  }
582  }
583  } else {
584  for (int i = 0; i < NBCVAR_max; i++) {
586  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
587  (th_bc_data[0].data() != nullptr))
588  {
589  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
590  }
591  else if (input_bndry_planes && dir < 2 && (
592  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
593  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
594  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
595  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
596  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
597  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )
598  ) )
599  {
601  }
602  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
604  }
605  }
606  }
607  }
608  else if (bct == ERF_BC::inflow_outflow )
609  {
610  if (side == Orientation::low) {
611  for (int i = 0; i < NBCVAR_max; i++) {
613  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
615  }
616  }
617  } else {
618  for (int i = 0; i < NBCVAR_max; i++) {
620  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
622  }
623  }
624  }
625  }
626  else if (bct == ERF_BC::periodic)
627  {
628  if (side == Orientation::low) {
629  for (int i = 0; i < NBCVAR_max; i++) {
631  }
632  } else {
633  for (int i = 0; i < NBCVAR_max; i++) {
635  }
636  }
637  }
638  else if ( bct == ERF_BC::surface_layer )
639  {
640  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
641  for (int i = 0; i < NBCVAR_max; i++) {
643  }
644  }
645  }
646  }
647 
648  // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
649  Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
650 }
#define NBCVAR_max
Definition: ERF_IndexDefines.H:29
@ ho_outflow
@ inflow_outflow
void init_Dirichlet_bc_data(const std::string input_file)
Definition: ERF_InitBCs.cpp:652
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_neumann_vals
Definition: ERF.H:939
@ RhoQ6_bc_comp
Definition: ERF_IndexDefines.H:86
@ RhoQ1_bc_comp
Definition: ERF_IndexDefines.H:81
@ RhoQ4_bc_comp
Definition: ERF_IndexDefines.H:84
@ RhoKE_bc_comp
Definition: ERF_IndexDefines.H:79
@ RhoQ3_bc_comp
Definition: ERF_IndexDefines.H:83
@ RhoTheta_bc_comp
Definition: ERF_IndexDefines.H:78
@ RhoQ2_bc_comp
Definition: ERF_IndexDefines.H:82
@ Rho_bc_comp
Definition: ERF_IndexDefines.H:77
@ RhoQ5_bc_comp
Definition: ERF_IndexDefines.H:85
@ neumann
Definition: ERF_IndexDefines.H:213
@ open
Definition: ERF_IndexDefines.H:215
@ reflect_odd
Definition: ERF_IndexDefines.H:205
@ hoextrap
Definition: ERF_IndexDefines.H:216
@ foextrap
Definition: ERF_IndexDefines.H:208
@ ext_dir
Definition: ERF_IndexDefines.H:209
@ ext_dir_prim
Definition: ERF_IndexDefines.H:211
@ ext_dir_upwind
Definition: ERF_IndexDefines.H:217
@ int_dir
Definition: ERF_IndexDefines.H:206
@ neumann_int
Definition: ERF_IndexDefines.H:214
@ reflect_even
Definition: ERF_IndexDefines.H:207
Here is the call graph for this function:

◆ init_custom()

void ERF::init_custom ( int  lev)
private

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters
levInteger specifying the current level
27 {
28  auto& lev_new = vars_new[lev];
29 
30  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
31  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
32 
33  MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
34  lev_new[Vars::cons].nComp() , lev_new[Vars::cons].nGrow());
35  MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
36  MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
37  MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
38 
39  // Default all perturbations to zero
40  cons_pert.setVal(0.);
41  xvel_pert.setVal(0.);
42  yvel_pert.setVal(0.);
43  zvel_pert.setVal(0.);
44 
45 #ifdef _OPENMP
46 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
47 #endif
48  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
49  {
50  const Box &bx = mfi.tilebox();
51  const Box &xbx = mfi.tilebox(IntVect(1,0,0));
52  const Box &ybx = mfi.tilebox(IntVect(0,1,0));
53  const Box &zbx = mfi.tilebox(IntVect(0,0,1));
54 
55  const auto &cons_pert_arr = cons_pert.array(mfi);
56  const auto &xvel_pert_arr = xvel_pert.array(mfi);
57  const auto &yvel_pert_arr = yvel_pert.array(mfi);
58  const auto &zvel_pert_arr = zvel_pert.array(mfi);
59 
60  Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
61  Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
62  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
63 
64  // Here we arbitrarily choose the x-oriented map factor -- this should be generalized
65  Array4<Real const> mf_m = mapfac[lev][MapFacType::m_x]->const_array(mfi);
66  Array4<Real const> mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
67  Array4<Real const> mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
68 
69  Array4<Real> r_hse_arr = r_hse.array(mfi);
70  Array4<Real> p_hse_arr = p_hse.array(mfi);
71 
72  prob->init_custom_pert(bx, xbx, ybx, zbx, cons_arr, cons_pert_arr,
73  xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
74  r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
75  geom[lev].data(), mf_m, mf_u, mf_v,
76  solverChoice);
77  } //mfi
78 
79  // Add problem-specific perturbation to background flow if not doing anelastic with fixed-in-time density
81  MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp, Rho_comp, 1, cons_pert.nGrow());
82  }
83  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp, 1, cons_pert.nGrow());
84  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,NSCALARS, cons_pert.nGrow());
85 
86  // RhoKE is relevant if using Deardorff with LES, k-equation for RANS, or MYNN with PBL
87  if (solverChoice.turbChoice[lev].use_tke) {
88  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp, RhoKE_comp, 1, cons_pert.nGrow());
89  }
90 
91  if (solverChoice.moisture_type != MoistureType::None) {
92  int qstate_size = micro->Get_Qstate_Size();
93  for (int q_offset(0); q_offset<qstate_size; ++q_offset) {
94  int q_idx = RhoQ1_comp+q_offset;
95  MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
96  }
97  }
98 
99  MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0, 0, 1, xvel_pert.nGrowVect());
100  MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0, 0, 1, yvel_pert.nGrowVect());
101  MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0, 0, 1, zvel_pert.nGrowVect());
102 }
const Box xbx
Definition: ERF_DiffSetup.H:21
const Box ybx
Definition: ERF_DiffSetup.H:22
bool fixed_density
Definition: ERF_DataStruct.H:795
Here is the call graph for this function:

◆ init_Dirichlet_bc_data()

void ERF::init_Dirichlet_bc_data ( const std::string  input_file)
private
653 {
654  const bool use_terrain = (solverChoice.terrain_type != TerrainType::None);
655 
656  // Read the dirichlet_input file
657  Print() << "dirichlet_input file location : " << input_file << std::endl;
658  std::ifstream input_reader(input_file);
659  if (!input_reader.is_open()) {
660  amrex::Abort("Error opening the dirichlet_input file.\n");
661  }
662 
663  Print() << "Successfully opened the dirichlet_input file. Now reading... " << std::endl;
664  std::string line;
665 
666  // Size of Ninp (number of z points in input file)
667  Vector<Real> z_inp_tmp, u_inp_tmp, v_inp_tmp, w_inp_tmp, th_inp_tmp;
668 
669  // Top and bot for domain
670  const int klo = geom[0].Domain().smallEnd()[2];
671  const int khi = geom[0].Domain().bigEnd()[2];
672  const Real zbot = (use_terrain) ? zlevels_stag[0][klo] : geom[0].ProbLo(2);
673  const Real ztop = (use_terrain) ? zlevels_stag[0][khi+1] : geom[0].ProbHi(2);
674 
675  // Flag if theta input
676  Real th_init = -300.0;
677  bool th_read{false};
678 
679  // Add surface
680  z_inp_tmp.push_back(zbot); // height above sea level [m]
681  u_inp_tmp.push_back(0.);
682  v_inp_tmp.push_back(0.);
683  w_inp_tmp.push_back(0.);
684  th_inp_tmp.push_back(th_init);
685 
686  // Read the vertical profile at each given height
687  Real z, u, v, w, th;
688  while(std::getline(input_reader, line)) {
689  std::istringstream iss_z(line);
690 
691  Vector<Real> rval_v;
692  Real rval;
693  while (iss_z >> rval) {
694  rval_v.push_back(rval);
695  }
696  z = rval_v[0];
697  u = rval_v[1];
698  v = rval_v[2];
699  w = rval_v[3];
700 
701  // Format without theta
702  if (rval_v.size() == 4) {
703  if (z == zbot) {
704  u_inp_tmp[0] = u;
705  v_inp_tmp[0] = v;
706  w_inp_tmp[0] = w;
707  } else {
708  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
709  z_inp_tmp.push_back(z);
710  u_inp_tmp.push_back(u);
711  v_inp_tmp.push_back(v);
712  w_inp_tmp.push_back(w);
713  if (z >= ztop) break;
714  }
715  } else if (rval_v.size() == 5) {
716  th_read = true;
717  th = rval_v[4];
718  if (z == zbot) {
719  u_inp_tmp[0] = u;
720  v_inp_tmp[0] = v;
721  w_inp_tmp[0] = w;
722  th_inp_tmp[0] = th;
723  } else {
724  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
725  z_inp_tmp.push_back(z);
726  u_inp_tmp.push_back(u);
727  v_inp_tmp.push_back(v);
728  w_inp_tmp.push_back(w);
729  th_inp_tmp.push_back(th);
730  if (z >= ztop) break;
731  }
732  } else {
733  Abort("Unknown inflow file format!");
734  }
735  }
736 
737  // Ensure we set a reasonable theta surface
738  if (th_read) {
739  if (th_inp_tmp[0] == th_init) {
740  Real slope = (th_inp_tmp[2] - th_inp_tmp[1]) / (z_inp_tmp[2] - z_inp_tmp[1]);
741  Real dz = z_inp_tmp[0] - z_inp_tmp[1];
742  th_inp_tmp[0] = slope * dz + th_inp_tmp[1];
743  }
744  }
745 
746  amrex::Print() << "Successfully read and interpolated the dirichlet_input file..." << std::endl;
747  input_reader.close();
748 
749  for (int lev = 0; lev <= max_level; lev++) {
750 
751  const int Nz = geom[lev].Domain().size()[2];
752  const Real dz = geom[lev].CellSize()[2];
753 
754  // Size of Nz (domain grid)
755  Vector<Real> zcc_inp(Nz );
756  Vector<Real> znd_inp(Nz+1);
757  Vector<Real> u_inp(Nz ); xvel_bc_data[lev].resize(Nz ,0.0);
758  Vector<Real> v_inp(Nz ); yvel_bc_data[lev].resize(Nz ,0.0);
759  Vector<Real> w_inp(Nz+1); zvel_bc_data[lev].resize(Nz+1,0.0);
760  Vector<Real> th_inp;
761  if (th_read) {
762  th_inp.resize(Nz);
763  th_bc_data[lev].resize(Nz, 0.0);
764  }
765 
766  // At this point, we have an input from zbot up to
767  // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights
768  const int Ninp = z_inp_tmp.size();
769  for (int k(0); k<Nz; ++k) {
770  zcc_inp[k] = (use_terrain) ? 0.5 * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1])
771  : zbot + (k + 0.5) * dz;
772  znd_inp[k] = (use_terrain) ? zlevels_stag[lev][k+1] : zbot + (k) * dz;
773  u_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), u_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
774  v_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), v_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
775  w_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), znd_inp[k], Ninp);
776  if (th_read) {
777  th_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), th_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
778  }
779  }
780  znd_inp[Nz] = ztop;
781  w_inp[Nz] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), ztop, Ninp);
782 
783  // Copy host data to the device
784  Gpu::copy(Gpu::hostToDevice, u_inp.begin(), u_inp.end(), xvel_bc_data[lev].begin());
785  Gpu::copy(Gpu::hostToDevice, v_inp.begin(), v_inp.end(), yvel_bc_data[lev].begin());
786  Gpu::copy(Gpu::hostToDevice, w_inp.begin(), w_inp.end(), zvel_bc_data[lev].begin());
787  if (th_read) {
788  Gpu::copy(Gpu::hostToDevice, th_inp.begin(), th_inp.end(), th_bc_data[lev].begin());
789  }
790 
791  // NOTE: These device vectors are passed to the PhysBC constructors when that
792  // class is instantiated in ERF_MakeNewArrays.cpp.
793  } // lev
794 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real interpolate_1d(const amrex::Real *alpha, const amrex::Real *beta, const amrex::Real alpha_interp, const int alpha_size)
Definition: ERF_Interpolation_1D.H:12
Here is the call graph for this function:

◆ init_from_hse()

void ERF::init_from_hse ( int  lev)

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

  • call prob->erf_init_dens_hse(...)
    • call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
    • save r_hse
  • call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse and th_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation
Parameters
levInteger specifying the current level
33 {
34  auto& lev_new = vars_new[lev];
35 
36  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
37  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
38 
39 #ifdef _OPENMP
40 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
41 #endif
42  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
43  {
44  const Box &gbx = mfi.growntilebox(1);
45  const Array4<Real >& cons_arr = lev_new[Vars::cons].array(mfi);
46  const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
47  const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
48 
49  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
50  {
51  cons_arr(i,j,k,Rho_comp) = r_hse_arr(i,j,k);
52  cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
53  });
54  } //mfi
55 }
Here is the call graph for this function:

◆ init_from_input_sounding()

void ERF::init_from_input_sounding ( int  lev)

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters
levInteger specifying the current level
53 {
54  // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
55  if (lev == 0) {
57  Error("input_sounding file name must be provided via input");
58  }
59 
61 
62  // this will interpolate the input profiles to the nominal height levels
63  // (ranging from 0 to the domain top)
64  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
66  }
67 
68  // this will calculate the hydrostatically balanced density and pressure
69  // profiles following WRF ideal.exe
70  if (solverChoice.sounding_type == SoundingType::Ideal) {
72  } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
73  solverChoice.sounding_type == SoundingType::DryIsentropic) {
74  input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
76  }
77 
78  } else {
79  //
80  // We need to do this interp from coarse level in order to set the values of
81  // the base state inside the domain but outside of the fine region
82  //
83  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
84  //
85  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
86  // have been pre-filled - this includes ghost cells both inside and outside
87  // the domain
88  //
89  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
90  IntVect(0,0,0), // do not fill ghost cells outside the domain
91  base_state[lev-1], 0, 0, base_state[lev].nComp(),
92  geom[lev-1], geom[lev],
93  refRatio(lev-1), &cell_cons_interp,
95 
96  // We need to do this here because the interpolation above may leave corners unfilled
97  // when the corners need to be filled by, for example, reflection of the fine ghost
98  // cell outside the fine region but inide the domain.
99  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
100  }
101 
102  auto& lev_new = vars_new[lev];
103 
104  // updated if sounding is ideal (following WRF) or isentropic
105  const bool l_isentropic = (solverChoice.sounding_type == SoundingType::Isentropic ||
106  solverChoice.sounding_type == SoundingType::DryIsentropic);
107  const bool sounding_ideal_or_isentropic = (solverChoice.sounding_type == SoundingType::Ideal ||
108  l_isentropic);
109  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
110  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
111  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
112  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
113  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
114 
115  const Real l_gravity = solverChoice.gravity;
116  const Real l_rdOcp = solverChoice.rdOcp;
117  const bool l_moist = (solverChoice.moisture_type != MoistureType::None);
118 
119 #ifdef _OPENMP
120 #pragma omp parallel if (Gpu::notInLaunchRegion())
121 #endif
122  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
123  const Box &bx = mfi.tilebox();
124  const auto &cons_arr = lev_new[Vars::cons].array(mfi);
125  const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
126  const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
127  const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
128  Array4<Real> r_hse_arr = r_hse.array(mfi);
129  Array4<Real> p_hse_arr = p_hse.array(mfi);
130  Array4<Real> pi_hse_arr = pi_hse.array(mfi);
131  Array4<Real> th_hse_arr = th_hse.array(mfi);
132  Array4<Real> qv_hse_arr = qv_hse.array(mfi);
133 
134  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
135  Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
136 
137  if (sounding_ideal_or_isentropic)
138  {
139  // HSE will be initialized here, interpolated from values previously
140  // calculated by calc_rho_p or calc_rho_p_isentropic
142  bx, cons_arr,
143  r_hse_arr, p_hse_arr, pi_hse_arr, th_hse_arr, qv_hse_arr,
144  geom[lev].data(), z_cc_arr,
145  l_gravity, l_rdOcp, l_moist, input_sounding_data,
146  l_isentropic);
147  }
148  else
149  {
150  // This assumes rho_0 = 1.0
151  // HSE will be calculated later with call to initHSE
153  bx, cons_arr,
154  geom[lev].data(), z_cc_arr,
155  l_moist, input_sounding_data);
156  }
157 
159  bx, xvel_arr, yvel_arr, zvel_arr,
160  geom[lev].data(), z_nd_arr,
162 
163  } //mfi
164 }
void init_bx_scalars_from_input_sounding(const Box &bx, Array4< Real > const &state, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const bool &l_moist, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:176
void init_bx_scalars_from_input_sounding_hse(const Box &bx, Array4< Real > const &state, Array4< Real > const &r_hse_arr, Array4< Real > const &p_hse_arr, Array4< Real > const &pi_hse_arr, Array4< Real > const &th_hse_arr, Array4< Real > const &qv_hse_arr, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const Real &l_gravity, const Real &l_rdOcp, const bool &l_moist, InputSoundingData const &inputSoundingData, const bool &l_isentropic)
Definition: ERF_InitFromInputSounding.cpp:238
void init_bx_velocities_from_input_sounding(const Box &bx, Array4< Real > const &x_vel, Array4< Real > const &y_vel, Array4< Real > const &z_vel, GeometryData const &geomdata, Array4< const Real > const &z_nd_arr, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:374
InputSoundingData input_sounding_data
Definition: ERF.H:727
@ rho0_bc_comp
Definition: ERF_IndexDefines.H:98
@ qv0_comp
Definition: ERF_IndexDefines.H:67
void resize_arrays()
Definition: ERF_InputSoundingData.H:60
int n_sounding_files
Definition: ERF_InputSoundingData.H:398
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag, int itime)
Definition: ERF_InputSoundingData.H:77
amrex::Vector< std::string > input_sounding_file
Definition: ERF_InputSoundingData.H:396
void calc_rho_p(int itime)
Definition: ERF_InputSoundingData.H:176
void calc_rho_p_isentropic(int itime)
Definition: ERF_InputSoundingData.H:262
bool assume_dry
Definition: ERF_InputSoundingData.H:401
static SoundingType sounding_type
Definition: ERF_DataStruct.H:765
Here is the call graph for this function:

◆ init_geo_wind_profile()

void ERF::init_geo_wind_profile ( const std::string  input_file,
amrex::Vector< amrex::Real > &  u_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  u_geos_d,
amrex::Vector< amrex::Real > &  v_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  v_geos_d,
const amrex::Geometry &  lgeom,
const amrex::Vector< amrex::Real > &  zlev_stag 
)
private
17 {
18  const int klo = 0;
19  const int khi = lgeom.Domain().bigEnd()[AMREX_SPACEDIM-1];
20  const amrex::Real dz = lgeom.CellSize()[AMREX_SPACEDIM-1];
21 
22  const bool grid_stretch = (zlev_stag.size() > 0);
23  const Real zbot = (grid_stretch) ? zlev_stag[klo] : lgeom.ProbLo(AMREX_SPACEDIM-1);
24  const Real ztop = (grid_stretch) ? zlev_stag[khi+1] : lgeom.ProbHi(AMREX_SPACEDIM-1);
25 
26  amrex::Print() << "Reading geostrophic wind profile from " << input_file << std::endl;
27  std::ifstream profile_reader(input_file);
28  if(!profile_reader.is_open()) {
29  amrex::Error("Error opening the abl_geo_wind_table\n");
30  }
31 
32  // First, read the input data into temp vectors
33  std::string line;
34  Vector<Real> z_inp, Ug_inp, Vg_inp;
35  Real z, Ug, Vg;
36  amrex::Print() << "z Ug Vg" << std::endl;
37  while(std::getline(profile_reader, line)) {
38  std::istringstream iss(line);
39  iss >> z >> Ug >> Vg;
40  amrex::Print() << z << " " << Ug << " " << Vg << std::endl;
41  z_inp.push_back(z);
42  Ug_inp.push_back(Ug);
43  Vg_inp.push_back(Vg);
44  if (z >= ztop) break;
45  }
46 
47  const int Ninp = z_inp.size();
48  AMREX_ALWAYS_ASSERT(z_inp[0] <= zbot);
49  AMREX_ALWAYS_ASSERT(z_inp[Ninp-1] >= ztop);
50 
51  // Now, interpolate vectors to the cell centers
52  for (int k = 0; k <= khi; k++) {
53  z = (grid_stretch) ? 0.5 * (zlev_stag[k] + zlev_stag[k+1])
54  : zbot + (k + 0.5) * dz;
55  u_geos[k] = interpolate_1d(z_inp.dataPtr(), Ug_inp.dataPtr(), z, Ninp);
56  v_geos[k] = interpolate_1d(z_inp.dataPtr(), Vg_inp.dataPtr(), z, Ninp);
57  }
58 
59  // Copy from host version to device version
60  Gpu::copy(Gpu::hostToDevice, u_geos.begin(), u_geos.end(), u_geos_d.begin());
61  Gpu::copy(Gpu::hostToDevice, v_geos.begin(), v_geos.end(), v_geos_d.begin());
62 
63  profile_reader.close();
64 }
Here is the call graph for this function:

◆ init_only()

void ERF::init_only ( int  lev,
amrex::Real  time 
)
1673 {
1674  t_new[lev] = time;
1675  t_old[lev] = time - 1.e200;
1676 
1677  auto& lev_new = vars_new[lev];
1678  auto& lev_old = vars_old[lev];
1679 
1680  // Loop over grids at this level to initialize our grid data
1681  lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
1682  lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
1683  lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
1684  lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
1685 
1686  // Initialize background flow (optional)
1687  if (solverChoice.init_type == InitType::Input_Sounding) {
1688  // The physbc's need the terrain but are needed for initHSE
1689  // We have already made the terrain in the call to init_zphys
1690  // in MakeNewLevelFromScratch
1691  make_physbcs(lev);
1692 
1693  // Now init the base state and the data itself
1695 
1696  // The base state has been initialized by integrating vertically
1697  // through the sounding for ideal (like WRF) or isentropic approaches
1698  if (solverChoice.sounding_type == SoundingType::Ideal ||
1699  solverChoice.sounding_type == SoundingType::Isentropic ||
1700  solverChoice.sounding_type == SoundingType::DryIsentropic) {
1701  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
1702  "Gravity should be on to be consistent with sounding initialization.");
1703  } else { // SoundingType::ConstantDensity
1704  AMREX_ASSERT_WITH_MESSAGE(!solverChoice.use_gravity,
1705  "Constant density probably doesn't make sense with gravity");
1706  initHSE();
1707  }
1708 
1709 #ifdef ERF_USE_NETCDF
1710  }
1711  else if (solverChoice.init_type == InitType::WRFInput)
1712  {
1713  // The base state is initialized from WRF wrfinput data, output by
1714  // ideal.exe or real.exe
1715 
1716  init_from_wrfinput(lev, *mf_C1H, *mf_C2H, *mf_MUB, *mf_PSFC[lev]);
1717 
1718  if (lev==0) {
1719  if ((start_time > 0) && (start_time != start_bdy_time)) {
1720  Print() << "Ignoring specified start_time="
1721  << std::setprecision(timeprecision) << start_time
1722  << std::endl;
1723  }
1724  }
1725 
1726  start_time = start_bdy_time;
1727 
1728  use_datetime = true;
1729 
1730  // The physbc's need the terrain but are needed for initHSE
1731  if (!solverChoice.use_real_bcs) {
1732  make_physbcs(lev);
1733  }
1734  }
1735  else if (solverChoice.init_type == InitType::NCFile)
1736  {
1737  // The state is initialized by reading from a Netcdf file
1738  init_from_ncfile(lev);
1739 
1740  // The physbc's need the terrain but are needed for initHSE
1741  make_physbcs(lev);
1742  }
1743  else if (solverChoice.init_type == InitType::Metgrid)
1744  {
1745  // The base state is initialized from data output by WPS metgrid;
1746  // we will rebalance after interpolation
1747  init_from_metgrid(lev);
1748 #endif
1749  } else if (solverChoice.init_type == InitType::Uniform) {
1750  // Initialize a uniform background field and base state based on the
1751  // problem-specified reference density and temperature
1752 
1753  // The physbc's need the terrain but are needed for initHSE
1754  make_physbcs(lev);
1755 
1756  init_uniform(lev);
1757  initHSE(lev);
1758  } else {
1759  // No background flow initialization specified, initialize the
1760  // background field to be equal to the base state, calculated from the
1761  // problem-specific erf_init_dens_hse
1762 
1763  // The bc's need the terrain but are needed for initHSE
1764  make_physbcs(lev);
1765 
1766  // We will initialize the state from the background state so must set that first
1767  initHSE(lev);
1768  init_from_hse(lev);
1769  }
1770 
1771  // Add problem-specific flow features
1772  //
1773  // Notes:
1774  // - This calls init_custom_pert that is defined for each problem
1775  // - This may modify the base state
1776  // - The fields set by init_custom_pert are **perturbations** to the
1777  // background flow set based on init_type
1778  if (solverChoice.init_type != InitType::NCFile) {
1779  init_custom(lev);
1780  }
1781 
1782  // Ensure that the face-based data are the same on both sides of a periodic domain.
1783  // The data associated with the lower grid ID is considered the correct value.
1784  lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
1785  lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
1786  lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
1787 
1788  if(solverChoice.spongeChoice.sponge_type == "input_sponge"){
1789  input_sponge(lev);
1790  }
1791 
1792  // Initialize turbulent perturbation
1793  if (solverChoice.pert_type == PerturbationType::Source ||
1794  solverChoice.pert_type == PerturbationType::Direct ||
1795  solverChoice.pert_type == PerturbationType::CPM) {
1796  turbPert_update(lev, 0.);
1797  turbPert_amplitude(lev);
1798  }
1799 }
const int timeprecision
Definition: ERF.H:967
void init_from_input_sounding(int lev)
Definition: ERF_InitFromInputSounding.cpp:52
std::unique_ptr< amrex::MultiFab > mf_MUB
Definition: ERF.H:1173
std::unique_ptr< amrex::MultiFab > mf_C2H
Definition: ERF.H:1172
void init_custom(int lev)
Definition: ERF_InitCustom.cpp:26
void init_from_hse(int lev)
Definition: ERF_InitFromHSE.cpp:32
void initHSE()
Initialize HSE.
Definition: ERF_Init1D.cpp:142
void turbPert_update(const int lev, const amrex::Real dt)
Definition: ERF_InitTurbPert.cpp:12
void input_sponge(int lev)
Definition: ERF_InitSponge.cpp:17
void make_physbcs(int lev)
Definition: ERF_MakeNewArrays.cpp:656
void init_uniform(int lev)
Definition: ERF_InitUniform.cpp:17
std::unique_ptr< amrex::MultiFab > mf_C1H
Definition: ERF.H:1171
void turbPert_amplitude(const int lev)
Definition: ERF_InitTurbPert.cpp:32
bool use_gravity
Definition: ERF_DataStruct.H:805

◆ init_stuff()

void ERF::init_stuff ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm,
amrex::Vector< amrex::MultiFab > &  lev_new,
amrex::Vector< amrex::MultiFab > &  lev_old,
amrex::MultiFab &  tmp_base_state,
std::unique_ptr< amrex::MultiFab > &  tmp_zphys_nd 
)
private
28 {
29  // ********************************************************************************************
30  // Base state holds r_0, pres_0, pi_0, th_0 (in that order)
31  //
32  // Here is where we set 3 ghost cells for the base state!
33  //
34  // ********************************************************************************************
35  tmp_base_state.define(ba,dm,BaseState::num_comps,3);
36  tmp_base_state.setVal(0.);
37 
38  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
39  base_state_new[lev].define(ba,dm,BaseState::num_comps,base_state[lev].nGrowVect());
40  base_state_new[lev].setVal(0.);
41  }
42 
43  // ********************************************************************************************
44  // Allocate terrain arrays
45  // ********************************************************************************************
46 
47  BoxArray ba_nd(ba);
48  ba_nd.surroundingNodes();
49 
50  // NOTE: this is where we actually allocate z_phys_nd -- but here it's called "tmp_zphys_nd"
51  // We need this to be one greater than the ghost cells to handle levels > 0
52 
53  int ngrow = ComputeGhostCells(solverChoice) + 2;
54  tmp_zphys_nd = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
55 
56  z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
57  init_default_zphys(lev, geom[lev], *tmp_zphys_nd, *z_phys_cc[lev]);
58 
59  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh)
60  {
61  detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
62  detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
63 
64  ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
65  ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
66  az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
67 
68  z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
69 
70  z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
71  z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
72  z_phys_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
73  }
74  else
75  {
76  z_phys_nd_new[lev] = nullptr;
77  detJ_cc_new[lev] = nullptr;
78 
79  z_phys_nd_src[lev] = nullptr;
80  z_phys_cc_src[lev] = nullptr;
81  detJ_cc_src[lev] = nullptr;
82 
83  z_t_rk[lev] = nullptr;
84  }
85 
86  if (SolverChoice::terrain_type == TerrainType::ImmersedForcing)
87  {
88  terrain_blanking[lev] = std::make_unique<MultiFab>(ba,dm,1,ngrow);
89  terrain_blanking[lev]->setVal(1.0);
90  }
91 
92  // We use these area arrays regardless of terrain, EB or none of the above
93  detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
94  ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
95  ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
96  az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
97 
98  detJ_cc[lev]->setVal(1.0);
99  ax[lev]->setVal(1.0);
100  ay[lev]->setVal(1.0);
101  az[lev]->setVal(1.0);
102 
103  // ********************************************************************************************
104  // Create wall distance array for RANS modeling
105  // ********************************************************************************************
106  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
107  walldist[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
108  walldist[lev]->setVal(1e23);
109  } else {
110  walldist[lev] = nullptr;
111  }
112 
113  // ********************************************************************************************
114  // These are the persistent containers for the old and new data
115  // ********************************************************************************************
116  int ncomp;
117  if (lev > 0) {
118  ncomp = vars_new[lev-1][Vars::cons].nComp();
119  } else {
120  int n_qstate = micro->Get_Qstate_Size();
121  ncomp = NDRY + NSCALARS + n_qstate;
122  }
123 
124  // ********************************************************************************************
125  // The number of ghost cells for density must be 1 greater than that for velocity
126  // so that we can go back in forth between velocity and momentum on all faces
127  // ********************************************************************************************
128  int ngrow_state = ComputeGhostCells(solverChoice) + 1;
129  int ngrow_vels = ComputeGhostCells(solverChoice);
130 
131  // ********************************************************************************************
132  // New solution data containers
133  // ********************************************************************************************
134  lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
135  lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
136 
137  lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
138  lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
139 
140  lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
141  lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
142 
143  gradp[lev][GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp[lev][GpVars::gpx].setVal(0.);
144  gradp[lev][GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp[lev][GpVars::gpy].setVal(0.);
145  gradp[lev][GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp[lev][GpVars::gpz].setVal(0.);
146 
147  // Note that we need the ghost cells in the z-direction if we are doing any
148  // kind of domain decomposition in the vertical (at level 0 or above)
149  lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
150  lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
151 
153  pp_inc[lev].define(ba, dm, 1, 1);
154  pp_inc[lev].setVal(0.0);
155  }
156 
157  // ********************************************************************************************
158  // These are just used for scratch in the time integrator but we might as well define them here
159  // ********************************************************************************************
160  rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
161  rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
162 
163  rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
164  rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
165 
166  rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
167  rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
168 
169  if (lev > 0) {
170  //xmom_crse_rhs[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, IntVect{0});
171  //ymom_crse_rhs[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, IntVect{0});
172  zmom_crse_rhs[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect{0});
173  }
174 
175  // We do this here just so they won't be undefined in the initial FillPatch
176  rU_old[lev].setVal(1.2e21);
177  rV_old[lev].setVal(3.4e22);
178  rW_old[lev].setVal(5.6e23);
179  rU_new[lev].setVal(1.2e21);
180  rV_new[lev].setVal(3.4e22);
181  rW_new[lev].setVal(5.6e23);
182 
183  // ********************************************************************************************
184  // These are just time averaged fields for diagnostics
185  // ********************************************************************************************
186 
187  // NOTE: We are not completing a fillpach call on the time averaged data;
188  // which would copy on intersection and interpolate from coarse.
189  // Therefore, we are restarting the averaging when the ba changes,
190  // this may give poor statistics for dynamic mesh refinement.
191  vel_t_avg[lev] = nullptr;
193  vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
194  vel_t_avg[lev]->setVal(0.0);
195  t_avg_cnt[lev] = 0.0;
196  }
197 
198  // ********************************************************************************************
199  // Initialize flux registers whenever we create/re-create a level
200  // ********************************************************************************************
201  if (solverChoice.coupling_type == CouplingType::TwoWay) {
202  if (lev == 0) {
203  advflux_reg[0] = nullptr;
204  } else {
205  int ncomp_reflux = vars_new[0][Vars::cons].nComp();
206  advflux_reg[lev] = new YAFluxRegister(ba , grids[lev-1],
207  dm , dmap[lev-1],
208  geom[lev], geom[lev-1],
209  ref_ratio[lev-1], lev, ncomp_reflux);
210  }
211  }
212 
213  // ********************************************************************************************
214  // Define Theta_prim storage if using surface_layer BC
215  // ********************************************************************************************
216  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
217  Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
218  if (solverChoice.moisture_type != MoistureType::None) {
219  Qv_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
220  Qr_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
221  } else {
222  Qv_prim[lev] = nullptr;
223  Qr_prim[lev] = nullptr;
224  }
225  } else {
226  Theta_prim[lev] = nullptr;
227  Qv_prim[lev] = nullptr;
228  Qr_prim[lev] = nullptr;
229  }
230 
231  // ********************************************************************************************
232  // Map factors
233  // ********************************************************************************************
234  BoxList bl2d_mf = ba.boxList();
235  for (auto& b : bl2d_mf) {
236  b.setRange(2,0);
237  }
238  BoxArray ba2d_mf(std::move(bl2d_mf));
239 
240  mapfac[lev].resize(MapFacType::num);
241  mapfac[lev][MapFacType::m_x] = std::make_unique<MultiFab>(ba2d_mf,dm,1,3);
242  mapfac[lev][MapFacType::u_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,3);
243  mapfac[lev][MapFacType::v_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,3);
244 
245 #if 0
246  // For now we comment this out to avoid CI failures but we will need to re-enable
247  // this if using non-conformal mappings
249  mapfac[lev][MapFacType::m_y] = std::make_unique<MultiFab>(ba2d_mf,dm,1,3);
250  }
252  mapfac[lev][MapFacType::u_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,3);
253  }
255  mapfac[lev][MapFacType::v_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,3);
256  }
257 #endif
258 
260  for (int i = 0; i < 3; i++) {
261  mapfac[lev][i]->setVal(0.5);
262  }
263  for (int i = 3; i < mapfac[lev].size(); i++) {
264  mapfac[lev][i]->setVal(0.25);
265  }
266  } else {
267  for (int i = 0; i < mapfac[lev].size(); i++) {
268  mapfac[lev][i]->setVal(1.0);
269  }
270  }
271 
272  // ********************************************************************************************
273  // Build 1D BA and 2D BA
274  // ********************************************************************************************
275  BoxList bl1d = ba.boxList();
276  for (auto& b : bl1d) {
277  b.setRange(0,0);
278  b.setRange(1,0);
279  }
280  ba1d[lev] = BoxArray(std::move(bl1d));
281 
282  // Build 2D BA
283  BoxList bl2d = ba.boxList();
284  for (auto& b : bl2d) {
285  b.setRange(2,0);
286  }
287  ba2d[lev] = BoxArray(std::move(bl2d));
288 
289  IntVect ng = vars_new[lev][Vars::cons].nGrowVect();
290 
291  if (lev == 0) {
292  mf_C1H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
293  mf_C2H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
294  mf_MUB = std::make_unique<MultiFab>(ba2d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
295  }
296 
297  mf_PSFC[lev] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ng);
298 
299  //*********************************************************
300  // Variables for Fitch model for windfarm parametrization
301  //*********************************************************
302 #if defined(ERF_USE_WINDFARM)
303  if (solverChoice.windfarm_type == WindFarmType::Fitch){
304  vars_windfarm[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
305  }
306  if (solverChoice.windfarm_type == WindFarmType::EWP){
307  vars_windfarm[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
308  }
309  if (solverChoice.windfarm_type == WindFarmType::SimpleAD) {
310  vars_windfarm[lev].define(ba, dm, 2, ngrow_state);// dudt, dvdt
311  }
312  if (solverChoice.windfarm_type == WindFarmType::GeneralAD) {
313  vars_windfarm[lev].define(ba, dm, 3, ngrow_state);// dudt, dvdt, dwdt
314  }
315  Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
316  SMark[lev].define(ba, dm, 2, 1); // Free stream velocity/source term
317  // sampling marker in a cell - 2 components
318 #endif
319 
320 
321 #ifdef ERF_USE_WW3_COUPLING
322  // create a new BoxArray and DistributionMapping for a MultiFab with 1 box
323  BoxArray ba_onegrid(geom[lev].Domain());
324  BoxList bl2d_onegrid = ba_onegrid.boxList();
325  for (auto& b : bl2d_onegrid) {
326  b.setRange(2,0);
327  }
328  BoxArray ba2d_onegrid(std::move(bl2d_onegrid));
329  Vector<int> pmap;
330  pmap.resize(1);
331  pmap[0]=0;
332  DistributionMapping dm_onegrid(ba2d_onegrid);
333  dm_onegrid.define(pmap);
334 
335  Hwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
336  Lwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
337 
338  BoxList bl2d_wave = ba.boxList();
339  for (auto& b : bl2d_wave) {
340  b.setRange(2,0);
341  }
342  BoxArray ba2d_wave(std::move(bl2d_wave));
343 
344  Hwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
345  Lwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
346 
347  std::cout<<ba_onegrid<<std::endl;
348  std::cout<<ba2d_onegrid<<std::endl;
349  std::cout<<dm_onegrid<<std::endl;
350 #endif
351 
352 
353  //*********************************************************
354  // Radiation heating source terms
355  //*********************************************************
356  if (solverChoice.rad_type != RadiationType::None || solverChoice.lsm_type != LandSurfaceType::None)
357  {
358  qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, ngrow_state);
359  qheating_rates[lev]->setVal(0.);
360  }
361 
362  //*********************************************************
363  // Radiation fluxes for coupling to LSM
364  //*********************************************************
365 
366  // NOTE: Finer levels do not need to coincide with the bottom domain boundary
367  // at k=0. We make slabs here with the kmin for a given box. Therefore,
368  // care must be taken before applying these fluxes to an LSM model. For
369 
370  // Radiative fluxes for LSM
371  if (solverChoice.lsm_type != LandSurfaceType::None &&
372  solverChoice.rad_type != RadiationType::None)
373  {
374  BoxList m_bl = ba.boxList();
375  for (auto& b : m_bl) {
376  int kmin = b.smallEnd(2);
377  b.setRange(2,kmin);
378  }
379  BoxArray m_ba(std::move(m_bl));
380 
381  sw_lw_fluxes[lev] = std::make_unique<MultiFab>(m_ba, dm, 5, ngrow_state); // SW direct (2), SW diffuse (2), LW
382  solar_zenith[lev] = std::make_unique<MultiFab>(m_ba, dm, 2, ngrow_state);
383 
384  sw_lw_fluxes[lev]->setVal(0.);
385  solar_zenith[lev]->setVal(0.);
386  }
387 
388  //*********************************************************
389  // Turbulent perturbation region initialization
390  //*********************************************************
391  if (solverChoice.pert_type == PerturbationType::Source ||
392  solverChoice.pert_type == PerturbationType::Direct ||
393  solverChoice.pert_type == PerturbationType::CPM)
394  {
395  amrex::Box bnd_bx = ba.minimalBox();
397  turbPert.init_tpi(lev, bnd_bx.smallEnd(), bnd_bx.bigEnd(), geom[lev].CellSizeArray(),
398  ba, dm, ngrow_state, pp_prefix, refRatio(), max_level);
399  }
400 
401  //
402  // Define the land mask here and set it to all land by default
403  // NOTE: the logic below will BREAK if we have any grids not touching the bottom boundary
404  //
405  {
406  lmask_lev[lev].resize(1);
407  auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0;
408  BoxList bl2d_mask = ba.boxList();
409  for (auto& b : bl2d_mask) {
410  b.setRange(2,0);
411  }
412  BoxArray ba2d_mask(std::move(bl2d_mask));
413  lmask_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
414  lmask_lev[lev][0]->setVal(1);
415  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
416  }
417 
418  // Read in tables needed for windfarm simulations
419  // fill in Nturb multifab - number of turbines in each mesh cell
420  // write out the vtk files for wind turbine location and/or
421  // actuator disks
422  #ifdef ERF_USE_WINDFARM
423  //init_windfarm(lev);
424  #endif
425 }
@ num
Definition: ERF_DataStruct.H:22
#define NDRY
Definition: ERF_IndexDefines.H:13
void init_default_zphys(int, const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc)
Definition: ERF_TerrainMetrics.cpp:15
static AMREX_FORCE_INLINE int ComputeGhostCells(const SolverChoice &sc)
Definition: ERF.H:1270
@ num_comps
Definition: ERF_IndexDefines.H:68
@ gpz
Definition: ERF_IndexDefines.H:152
@ gpy
Definition: ERF_IndexDefines.H:151
@ gpx
Definition: ERF_IndexDefines.H:150
bool test_mapfactor
Definition: ERF_DataStruct.H:800
void init_tpi_type(const PerturbationType &pert_type)
Definition: ERF_TurbPertStruct.H:28
void init_tpi(const int lev, const amrex::IntVect &valid_box_lo, const amrex::IntVect &valid_box_hi, const amrex::GpuArray< amrex::Real, 3 > dx, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, const int ngrow_state, std::string pp_prefix, const amrex::Vector< amrex::IntVect > refRatio, const int max_level)
Definition: ERF_TurbPertStruct.H:45
Here is the call graph for this function:

◆ init_thin_body()

void ERF::init_thin_body ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
706 {
707  //********************************************************************************************
708  // Thin immersed body
709  // *******************************************************************************************
710 #if 0
711  if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
712  (solverChoice.advChoice.zero_yflux.size() > 0) ||
713  (solverChoice.advChoice.zero_zflux.size() > 0))
714  {
715  overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
716  overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
717  }
718 #endif
719 
720  if (solverChoice.advChoice.zero_xflux.size() > 0) {
721  amrex::Print() << "Setting up thin immersed body for "
722  << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
723  BoxArray ba_xf(ba);
724  ba_xf.surroundingNodes(0);
725  thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
726  thin_xforce[lev]->setVal(0.0);
727  xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
728  xflux_imask[lev]->setVal(1);
729  for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
730  {
731  Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
732  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
733  Box xbx = mfi.nodaltilebox(0);
734  for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
735  const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
736  if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
737  (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
738  (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
739  {
740  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
741  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
742  //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
743  amrex::AllPrint() << " mask xface at " << faceidx << std::endl;
744  }
745  }
746  }
747  } else {
748  thin_xforce[lev] = nullptr;
749  xflux_imask[lev] = nullptr;
750  }
751 
752  if (solverChoice.advChoice.zero_yflux.size() > 0) {
753  amrex::Print() << "Setting up thin immersed body for "
754  << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
755  BoxArray ba_yf(ba);
756  ba_yf.surroundingNodes(1);
757  thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
758  thin_yforce[lev]->setVal(0.0);
759  yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
760  yflux_imask[lev]->setVal(1);
761  for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
762  {
763  Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
764  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
765  Box ybx = mfi.nodaltilebox(1);
766  for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
767  const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
768  if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
769  (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
770  (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
771  {
772  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
773  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
774  //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
775  amrex::AllPrint() << " mask yface at " << faceidx << std::endl;
776  }
777  }
778  }
779  } else {
780  thin_yforce[lev] = nullptr;
781  yflux_imask[lev] = nullptr;
782  }
783 
784  if (solverChoice.advChoice.zero_zflux.size() > 0) {
785  amrex::Print() << "Setting up thin immersed body for "
786  << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
787  BoxArray ba_zf(ba);
788  ba_zf.surroundingNodes(2);
789  thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
790  thin_zforce[lev]->setVal(0.0);
791  zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
792  zflux_imask[lev]->setVal(1);
793  for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
794  {
795  Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
796  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
797  Box zbx = mfi.nodaltilebox(2);
798  for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
799  const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
800  if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
801  (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
802  (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
803  {
804  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
805  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
806  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
807  amrex::AllPrint() << " mask zface at " << faceidx << std::endl;
808  }
809  }
810  }
811  } else {
812  thin_zforce[lev] = nullptr;
813  zflux_imask[lev] = nullptr;
814  }
815 }
amrex::Vector< amrex::IntVect > zero_yflux
Definition: ERF_AdvStruct.H:414
amrex::Vector< amrex::IntVect > zero_xflux
Definition: ERF_AdvStruct.H:413
amrex::Vector< amrex::IntVect > zero_zflux
Definition: ERF_AdvStruct.H:415

◆ init_uniform()

void ERF::init_uniform ( int  lev)
private

Use problem-specific reference density and temperature to set the background state to a uniform value.

Parameters
levInteger specifying the current level
18 {
19  auto& lev_new = vars_new[lev];
20  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
21  const Box &gbx = mfi.growntilebox(1);
22  const auto &cons_arr = lev_new[Vars::cons].array(mfi);
23  prob->init_uniform(gbx, cons_arr);
24  }
25 }

◆ init_zphys()

void ERF::init_zphys ( int  lev,
amrex::Real  time 
)
519 {
520  if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
521  {
522  if (lev > 0) {
523  //
524  // First interpolate from coarser level if there is one
525  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
526  // have been pre-filled - this includes ghost cells both inside and outside
527  // the domain
528  //
529  InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
530  IntVect(0,0,0), // do not fill ghost cells outside the domain
531  *z_phys_nd[lev-1], 0, 0, 1,
532  geom[lev-1], geom[lev],
533  refRatio(lev-1), &node_bilinear_interp,
535  }
536 
537  int ngrow = ComputeGhostCells(solverChoice) + 2;
538  Box bx(surroundingNodes(Geom(lev).Domain())); bx.grow(ngrow);
539  FArrayBox terrain_fab(makeSlab(bx,2,0),1);
540 
541  //
542  // If we are using fitted mesh then we use the surface as defined above
543  // If we are not using fitted mesh but are using z_levels, we still need z_phys (for now)
544  // but we need to use a flat terrain for the mesh itself (the EB data has already been made
545  // from the correct terrain)
546  //
547  if (solverChoice.terrain_type != TerrainType::StaticFittedMesh &&
548  solverChoice.terrain_type != TerrainType::MovingFittedMesh) {
549  terrain_fab.template setVal<RunOn::Device>(0.0);
550  } else {
551  //
552  // Fill the values of the terrain height at k=0 only
553  //
554  prob->init_terrain_surface(geom[lev],terrain_fab,time);
555  }
556 
557  for (MFIter mfi(*z_phys_nd[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
558  {
559  Box isect = terrain_fab.box() & (*z_phys_nd[lev])[mfi].box();
560  if (!isect.isEmpty()) {
561  (*z_phys_nd[lev])[mfi].template copy<RunOn::Device>(terrain_fab,isect,0,isect,0,1);
562  }
563  }
564 
566 
567  z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
568 
569  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
570  terrain_blanking[lev]->setVal(1.0);
571  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
572  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
573  }
574 
575  if (lev == 0) {
576  Real zmax = z_phys_nd[0]->max(0,0,false);
577  Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
578  if (rel_diff < 1.e-8) {
579  amrex::Print() << "max of zphys_nd " << zmax << std::endl;
580  amrex::Print() << "max of zlevels " << zlevels_stag[0][zlevels_stag[0].size()-1] << std::endl;
581  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < 1.e-8, "Terrain is taller than domain top!");
582  }
583  } // lev == 0
584 
585  } // init_type
586 }
void make_terrain_fitted_coords(int lev, const Geometry &geom, MultiFab &z_phys_nd, Vector< Real > const &z_levels_h, GpuArray< ERF_BC, AMREX_SPACEDIM *2 > &phys_bc_type)
Definition: ERF_TerrainMetrics.cpp:46
amrex::EBFArrayBoxFactory const & EBFactory(int lev) const noexcept
Definition: ERF.H:1546
Here is the call graph for this function:

◆ InitData()

void ERF::InitData ( )
773 {
774  BL_PROFILE_VAR("ERF::InitData()", InitData);
775  InitData_pre();
776  InitData_post();
777  BL_PROFILE_VAR_STOP(InitData);
778 }
void InitData_pre()
Definition: ERF.cpp:781
void InitData_post()
Definition: ERF.cpp:855
void InitData()
Definition: ERF.cpp:772

Referenced by main().

Here is the caller graph for this function:

◆ InitData_post()

void ERF::InitData_post ( )
856 {
858  {
859  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
860  "Thin immersed body with refinement not currently supported.");
861  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
862  amrex::Print() << "NOTE: Thin immersed body with non-constant dz has not been tested." << std::endl;
863  }
864  }
865 
866  if (!restart_chkfile.empty()) {
867  restart();
868  }
869 
870  //
871  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
872  //
873  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
874  for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
875  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
876  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
877  detJ_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
878  z_phys_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
879  }
880  }
881 
882  if (restart_chkfile.empty()) {
883  if (solverChoice.coupling_type == CouplingType::TwoWay) {
884  AverageDown();
885  }
886  }
887 
888 #ifdef ERF_USE_PARTICLES
889  if (restart_chkfile.empty()) {
890  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
892  Warning("Tight coupling has not been tested with Lagrangian microphysics");
893  }
894 
895  for (int lev = 0; lev <= finest_level; lev++) {
896  dynamic_cast<LagrangianMicrophysics&>(*micro).initParticles(z_phys_nd[lev]);
897  }
898  }
899  }
900 #endif
901 
902  if (!restart_chkfile.empty()) { // Restart from a checkpoint
903 
904  // Create the physbc objects for {cons, u, v, w, base state}
905  // We fill the additional base state ghost cells just in case we have read the old format
906  for (int lev(0); lev <= finest_level; ++lev) {
907  make_physbcs(lev);
908  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
909  }
910 
912  for (int lev(0); lev <= finest_level; ++lev) {
913  m_forest_drag[lev]->define_drag_field(grids[lev], dmap[lev], geom[lev],
914  z_phys_cc[lev].get(), z_phys_nd[lev].get());
915  }
916  }
917 
918 #ifdef ERF_USE_NETCDF
919  //
920  // Create the needed bdy_data_xlo etc ... since we don't read it in from checkpoint any more
921  //
923 
924  bdy_time_interval = read_times_from_wrfbdy(nc_bdy_file,
925  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
926  start_bdy_time);
927  Real dT = bdy_time_interval;
928 
929  int n_time_old = static_cast<int>(t_new[0] / dT);
930 
931  int lev = 0;
932  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
933 
934  int ntimes = std::min(n_time_old+3, static_cast<int>(bdy_data_xlo.size()));
935 
936  for (int itime = n_time_old; itime < ntimes; itime++)
937  {
938  read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
939  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
940  real_width);
941  convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
942  *mf_MUB, *mf_C1H, *mf_C2H,
944  geom[lev], use_moist);
945  } // itime
946  } // use_real_bcs && lev == 0
947 #endif
948  } // end restart
949 
950 #ifdef ERF_USE_PARTICLES
951  /* If using a Lagrangian microphysics model, its particle container has now been
952  constructed and initialized (calls to micro->Init). So, add its pointer to
953  ERF::particleData and remove its name from list of unallocated particle containers. */
954  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
955  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
956  const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
957  particleData.pushBack(pc_name, pc_ptr);
958  particleData.getNamesUnalloc().remove(pc_name);
959  }
960 #endif
961 
962  if (input_bndry_planes) {
963  // Read the "time.dat" file to know what data is available
964  m_r2d->read_time_file();
965 
966  // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
967  Real dt_dummy = 0.0;
968  m_r2d->read_input_files(t_new[0],dt_dummy,m_bc_extdir_vals);
969  }
970 
972  {
973  h_rhotheta_src.resize(max_level+1, Vector<Real>(0));
974  d_rhotheta_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
975  for (int lev = 0; lev <= finest_level; lev++) {
976  const int domlen = geom[lev].Domain().length(2);
977  h_rhotheta_src[lev].resize(domlen, 0.0_rt);
978  d_rhotheta_src[lev].resize(domlen, 0.0_rt);
979  prob->update_rhotheta_sources(t_new[0],
980  h_rhotheta_src[lev], d_rhotheta_src[lev],
981  geom[lev], z_phys_cc[lev]);
982  }
983  }
984 
986  {
987  h_u_geos.resize(max_level+1, Vector<Real>(0));
988  d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
989  h_v_geos.resize(max_level+1, Vector<Real>(0));
990  d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
991  for (int lev = 0; lev <= finest_level; lev++) {
992  const int domlen = geom[lev].Domain().length(2);
993  h_u_geos[lev].resize(domlen, 0.0_rt);
994  d_u_geos[lev].resize(domlen, 0.0_rt);
995  h_v_geos[lev].resize(domlen, 0.0_rt);
996  d_v_geos[lev].resize(domlen, 0.0_rt);
998  prob->update_geostrophic_profile(t_new[0],
999  h_u_geos[lev], d_u_geos[lev],
1000  h_v_geos[lev], d_v_geos[lev],
1001  geom[lev], z_phys_cc[lev]);
1002  } else {
1003  if (SolverChoice::mesh_type == MeshType::VariableDz) {
1004  amrex::Print() << "Note: 1-D geostrophic wind profile input is not defined for real terrain" << std::endl;
1005  }
1007  h_u_geos[lev], d_u_geos[lev],
1008  h_v_geos[lev], d_v_geos[lev],
1009  geom[lev],
1010  zlevels_stag[0]);
1011  }
1012  }
1013  }
1014 
1016  {
1017  h_rhoqt_src.resize(max_level+1, Vector<Real>(0));
1018  d_rhoqt_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1019  for (int lev = 0; lev <= finest_level; lev++) {
1020  const int domlen = geom[lev].Domain().length(2);
1021  h_rhoqt_src[lev].resize(domlen, 0.0_rt);
1022  d_rhoqt_src[lev].resize(domlen, 0.0_rt);
1023  prob->update_rhoqt_sources(t_new[0],
1024  h_rhoqt_src[lev], d_rhoqt_src[lev],
1025  geom[lev], z_phys_cc[lev]);
1026  }
1027  }
1028 
1030  {
1031  h_w_subsid.resize(max_level+1, Vector<Real>(0));
1032  d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1033  for (int lev = 0; lev <= finest_level; lev++) {
1034  const int domlen = geom[lev].Domain().length(2) + 1; // lives on z-faces
1035  h_w_subsid[lev].resize(domlen, 0.0_rt);
1036  d_w_subsid[lev].resize(domlen, 0.0_rt);
1037  prob->update_w_subsidence(t_new[0],
1038  h_w_subsid[lev], d_w_subsid[lev],
1039  geom[lev], z_phys_nd[lev]);
1040  }
1041  }
1042 
1045  {
1046  initRayleigh();
1047  if (solverChoice.init_type == InitType::Input_Sounding)
1048  {
1049  // Overwrite ubar, vbar, and thetabar with input profiles;
1050  // wbar is assumed to be 0. Note: the tau coefficient set by
1051  // prob->erf_init_rayleigh() is still used
1052  bool restarting = (!restart_chkfile.empty());
1053  setRayleighRefFromSounding(restarting);
1054  }
1055  }
1056 
1057  // Read in sponge data from input file
1058  if(solverChoice.spongeChoice.sponge_type == "input_sponge")
1059  {
1060  initSponge();
1061  bool restarting = (!restart_chkfile.empty());
1062  setSpongeRefFromSounding(restarting);
1063  }
1064 
1065  if (solverChoice.pert_type == PerturbationType::Source ||
1066  solverChoice.pert_type == PerturbationType::Direct ||
1067  solverChoice.pert_type == PerturbationType::CPM) {
1068  if (is_it_time_for_action(istep[0], t_new[0], dt[0], pert_interval, -1.)) {
1069  turbPert.debug(t_new[0]);
1070  }
1071  }
1072 
1073  // We only write the file at level 0 for now
1074  if (output_bndry_planes)
1075  {
1076  // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
1077  m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
1078 
1079  Real time = 0.;
1080  if (time >= bndry_output_planes_start_time) {
1081  bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
1082  m_w2d->write_planes(0, time, vars_new, is_moist);
1083  }
1084  }
1085 
1086  //
1087  // If we are starting from scratch, we have the option to project the initial velocity field
1088  // regardless of how we initialized.
1089  // pp_inc is used as scratch space here; we zero it out after the projection
1090  //
1091  if (restart_chkfile == "")
1092  {
1094  Real dummy_dt = 1.0;
1095  if (verbose > 0) {
1096  amrex::Print() << "Projecting initial velocity field" << std::endl;
1097  }
1098  for (int lev = 0; lev <= finest_level; ++lev)
1099  {
1100  project_velocity(lev, dummy_dt);
1101  pp_inc[lev].setVal(0.);
1102  gradp[lev][GpVars::gpx].setVal(0.);
1103  gradp[lev][GpVars::gpy].setVal(0.);
1104  gradp[lev][GpVars::gpz].setVal(0.);
1105  }
1106  }
1107  }
1108 
1109  // Copy from new into old just in case
1110  for (int lev = 0; lev <= finest_level; ++lev)
1111  {
1112  auto& lev_new = vars_new[lev];
1113  auto& lev_old = vars_old[lev];
1114 
1115  // ***************************************************************************
1116  // Physical bc's at domain boundary
1117  // ***************************************************************************
1118  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
1119  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
1120 
1121  int ncomp_cons = lev_new[Vars::cons].nComp();
1122  bool do_fb = true;
1123 
1124 #ifdef ERF_USE_NETCDF
1125  // We call this here because it is an ERF routine
1126  if (solverChoice.use_real_bcs && (lev==0)) {
1127  int icomp_cons = 0;
1128  bool cons_only = false;
1129  Vector<MultiFab*> mfs_vec = {&lev_new[Vars::cons],&lev_new[Vars::xvel],
1130  &lev_new[Vars::yvel],&lev_new[Vars::zvel]};
1132  fill_from_realbdy_upwind(mfs_vec,t_new[lev],cons_only,icomp_cons,
1133  ncomp_cons,ngvect_cons,ngvect_vels);
1134  } else {
1135  fill_from_realbdy(mfs_vec,t_new[lev],cons_only,icomp_cons,
1136  ncomp_cons,ngvect_cons,ngvect_vels);
1137  }
1138  do_fb = false;
1139  }
1140 #endif
1141 
1142  (*physbcs_cons[lev])(lev_new[Vars::cons],lev_new[Vars::xvel],lev_new[Vars::yvel],0,ncomp_cons,
1143  ngvect_cons,t_new[lev],BCVars::cons_bc,do_fb);
1144  ( *physbcs_u[lev])(lev_new[Vars::xvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1145  ngvect_vels,t_new[lev],BCVars::xvel_bc,do_fb);
1146  ( *physbcs_v[lev])(lev_new[Vars::yvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1147  ngvect_vels,t_new[lev],BCVars::yvel_bc,do_fb);
1148  ( *physbcs_w[lev])(lev_new[Vars::zvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1149  ngvect_vels,t_new[lev],BCVars::zvel_bc,do_fb);
1150 
1151  MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,ncomp_cons,lev_new[Vars::cons].nGrowVect());
1152  MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0, 1,lev_new[Vars::xvel].nGrowVect());
1153  MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0, 1,lev_new[Vars::yvel].nGrowVect());
1154  MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0, 1,lev_new[Vars::zvel].nGrowVect());
1155  }
1156 
1157  // Compute the minimum dz in the domain at each level (to be used for setting the timestep)
1158  dz_min.resize(max_level+1);
1159  for (int lev = 0; lev <= finest_level; ++lev)
1160  {
1161  dz_min[lev] = geom[lev].CellSize(2);
1162  if ( SolverChoice::mesh_type != MeshType::ConstantDz ) {
1163  dz_min[lev] *= (*detJ_cc[lev]).min(0);
1164  }
1165  }
1166 
1167  // We don't need to recompute dt[lev] on restart because we read it in from the checkpoint file.
1168  if (restart_chkfile.empty()) {
1169  ComputeDt();
1170  }
1171 
1172  // Check the viscous limit
1176  Real delta = std::min({geom[finest_level].CellSize(0),
1177  geom[finest_level].CellSize(1),
1178  dz_min[finest_level]});
1179  if (dc.dynamic_viscosity == 0) {
1180  Print() << "Note: Molecular diffusion specified but dynamic_viscosity has not been specified" << std::endl;
1181  } else {
1182  Real nu = dc.dynamic_viscosity / dc.rho0_trans;
1183  Real viscous_limit = 0.5 * delta*delta / nu;
1184  Print() << "Viscous CFL is " << dt[finest_level] / viscous_limit << std::endl;
1185  if (fixed_dt[finest_level] >= viscous_limit) {
1186  Warning("Specified fixed_dt is above the viscous limit");
1187  } else if (dt[finest_level] >= viscous_limit) {
1188  Warning("Adaptive dt based on convective CFL only is above the viscous limit");
1189  }
1190  }
1191  }
1192 
1193  // Fill ghost cells/faces
1194  for (int lev = 0; lev <= finest_level; ++lev)
1195  {
1196  if (lev > 0 && cf_width >= 0) {
1198  }
1199 
1200  auto& lev_new = vars_new[lev];
1201 
1202  //
1203  // Fill boundary conditions -- not sure why we need this here
1204  //
1205  bool fillset = false;
1206  if (lev == 0) {
1207  FillPatchCrseLevel(lev, t_new[lev],
1208  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]});
1209  } else {
1210  FillPatchFineLevel(lev, t_new[lev],
1211  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
1212  {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
1213  base_state[lev], base_state[lev],
1214  fillset);
1215  }
1216 
1217  //
1218  // We do this here to make sure level (lev-1) boundary conditions are filled
1219  // before we interpolate to level (lev) ghost cells
1220  //
1221  if (lev < finest_level) {
1222  auto& lev_old = vars_old[lev];
1223  MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
1224  MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
1225  MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
1226  MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
1227  }
1228 
1229  //
1230  // We fill the ghost cell values of the base state in case it wasn't done in the initialization
1231  //
1232  base_state[lev].FillBoundary(geom[lev].periodicity());
1233 
1234  // For moving terrain only
1235  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
1236  MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
1237  base_state_new[lev].FillBoundary(geom[lev].periodicity());
1238  }
1239 
1240  }
1241 
1242  // Allow idealized cases over water, used to set lmask
1243  ParmParse pp("erf");
1244  int is_land;
1245  for (int lev = 0; lev <= finest_level; ++lev)
1246  {
1247  if (pp.query("is_land", is_land, lev)) {
1248  if (is_land == 1) {
1249  amrex::Print() << "Level " << lev << " is land" << std::endl;
1250  } else if (is_land == 0) {
1251  amrex::Print() << "Level " << lev << " is water" << std::endl;
1252  } else {
1253  Error("is_land should be 0 or 1");
1254  }
1255  lmask_lev[lev][0]->setVal(is_land);
1256  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
1257  }
1258  }
1259 
1260  // If lev > 0, we need to fill bc's by interpolation from coarser grid
1261  for (int lev = 1; lev <= finest_level; ++lev)
1262  {
1263  Real time_for_fp = 0.; // This is not actually used
1264  Vector<Real> ftime = {time_for_fp, time_for_fp};
1265  Vector<Real> ctime = {time_for_fp, time_for_fp};
1266  if (lat_m[lev]) {
1267  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1268  Vector<MultiFab*> fmf = {lat_m[lev ].get(), lat_m[lev ].get()};
1269  Vector<MultiFab*> cmf = {lat_m[lev-1].get(), lat_m[lev-1].get()};
1270  IntVect ngv = lat_m[lev]->nGrowVect(); ngv[2] = 0;
1271  Interpolater* mapper = &cell_cons_interp;
1272  FillPatchTwoLevels(*lat_m[lev].get(), ngv, IntVect(0,0,0),
1273  time_for_fp, cmf, ctime, fmf, ftime,
1274  0, 0, 1, geom[lev-1], geom[lev],
1275  refRatio(lev-1), mapper, domain_bcs_type,
1276  BCVars::cons_bc);
1277  }
1278  if (lon_m[lev]) {
1279  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1280  Vector<MultiFab*> fmf = {lon_m[lev ].get(), lon_m[lev ].get()};
1281  Vector<MultiFab*> cmf = {lon_m[lev-1].get(), lon_m[lev-1].get()};
1282  IntVect ngv = lon_m[lev]->nGrowVect(); ngv[2] = 0;
1283  Interpolater* mapper = &cell_cons_interp;
1284  FillPatchTwoLevels(*lon_m[lev].get(), ngv, IntVect(0,0,0),
1285  time_for_fp, cmf, ctime, fmf, ftime,
1286  0, 0, 1, geom[lev-1], geom[lev],
1287  refRatio(lev-1), mapper, domain_bcs_type,
1288  BCVars::cons_bc);
1289  } // lon_m
1290  if (sinPhi_m[lev]) {
1291  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1292  Vector<MultiFab*> fmf = {sinPhi_m[lev ].get(), sinPhi_m[lev ].get()};
1293  Vector<MultiFab*> cmf = {sinPhi_m[lev-1].get(), sinPhi_m[lev-1].get()};
1294  IntVect ngv = sinPhi_m[lev]->nGrowVect(); ngv[2] = 0;
1295  Interpolater* mapper = &cell_cons_interp;
1296  FillPatchTwoLevels(*sinPhi_m[lev].get(), ngv, IntVect(0,0,0),
1297  time_for_fp, cmf, ctime, fmf, ftime,
1298  0, 0, 1, geom[lev-1], geom[lev],
1299  refRatio(lev-1), mapper, domain_bcs_type,
1300  BCVars::cons_bc);
1301  } // sinPhi
1302  if (cosPhi_m[lev]) {
1303  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1304  Vector<MultiFab*> fmf = {cosPhi_m[lev ].get(), cosPhi_m[lev ].get()};
1305  Vector<MultiFab*> cmf = {cosPhi_m[lev-1].get(), cosPhi_m[lev-1].get()};
1306  IntVect ngv = cosPhi_m[lev]->nGrowVect(); ngv[2] = 0;
1307  Interpolater* mapper = &cell_cons_interp;
1308  FillPatchTwoLevels(*cosPhi_m[lev].get(), ngv, IntVect(0,0,0),
1309  time_for_fp, cmf, ctime, fmf, ftime,
1310  0, 0, 1, geom[lev-1], geom[lev],
1311  refRatio(lev-1), mapper, domain_bcs_type,
1312  BCVars::cons_bc);
1313  } // cosPhi
1314  } // lev
1315 
1316 #ifdef ERF_USE_WW3_COUPLING
1317  int my_lev = 0;
1318  amrex::Print() << " About to call send_to_ww3 from ERF.cpp" << std::endl;
1319  send_to_ww3(my_lev);
1320  amrex::Print() << " About to call read_waves from ERF.cpp" << std::endl;
1321  read_waves(my_lev);
1322  // send_to_ww3(my_lev);
1323 #endif
1324 
1325  // Configure SurfaceLayer params if used
1326  // NOTE: we must set up the MOST routine after calling FillPatch
1327  // in order to have lateral ghost cells filled (MOST + terrain interp).
1328  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
1329  {
1330  bool rotate = solverChoice.use_rotate_surface_flux;
1331  if (rotate) {
1332  Print() << "Using surface layer model with stress rotations" << std::endl;
1333  }
1334 
1335  //
1336  // This constructor will make the ABLMost object but not allocate the arrays at each level.
1337  //
1338  m_SurfaceLayer = std::make_unique<SurfaceLayer>(geom, rotate, pp_prefix, Qv_prim,
1340 #ifdef ERF_USE_NETCDF
1341  , bdy_time_interval
1342 #endif
1343  );
1344  // This call will allocate the arrays at each level. If we regrid later, either changing
1345  // the number of level sor just the grids at each existing level, we will call an update routine
1346  // to redefine the internal arrays in m_SurfaceLayer.
1347  int nlevs = geom.size();
1348  for (int lev = 0; lev < nlevs; lev++)
1349  {
1350  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
1351  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
1352  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
1353  mfv_old, Theta_prim[lev], Qv_prim[lev],
1354  Qr_prim[lev], z_phys_nd[lev],
1355  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
1356  lsm_data[lev], lsm_flux[lev], sst_lev[lev],
1357  tsk_lev[lev], lmask_lev[lev]);
1358  }
1359 
1360 
1361  if (restart_chkfile != "") {
1362  // Update surface fields if needed (and available)
1364  }
1365 
1366  // We now configure ABLMost params here so that we can print the averages at t=0
1367  // Note we don't fill ghost cells here because this is just for diagnostics
1368  for (int lev = 0; lev <= finest_level; ++lev)
1369  {
1370  Real time = t_new[lev];
1371  IntVect ng = Theta_prim[lev]->nGrowVect();
1372 
1373  MultiFab::Copy( *Theta_prim[lev], vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, ng);
1374  MultiFab::Divide(*Theta_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1375 
1376  if (solverChoice.moisture_type != MoistureType::None) {
1377  ng = Qv_prim[lev]->nGrowVect();
1378 
1379  MultiFab::Copy( *Qv_prim[lev], vars_new[lev][Vars::cons], RhoQ1_comp, 0, 1, ng);
1380  MultiFab::Divide(*Qv_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1381 
1382  int rhoqr_comp = solverChoice.moisture_indices.qr;
1383  if (rhoqr_comp > -1) {
1384  MultiFab::Copy( *Qr_prim[lev], vars_new[lev][Vars::cons], rhoqr_comp, 0, 1, ng);
1385  MultiFab::Divide(*Qr_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1386  } else {
1387  Qr_prim[lev]->setVal(0.0);
1388  }
1389  }
1390  m_SurfaceLayer->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim, Qr_prim);
1391 
1392  if (restart_chkfile == "") {
1393  // Only do this if starting from scratch; if restarting, then
1394  // we don't want to call update_fluxes multiple times because
1395  // it will change u* and theta* from their previous values
1396  m_SurfaceLayer->update_pblh(lev, vars_new, z_phys_cc[lev].get(),
1398  m_SurfaceLayer->update_fluxes(lev, time, vars_new[lev][Vars::cons], z_phys_nd[lev]);
1399  }
1400  }
1401  } // end if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
1402 
1403  // Update micro vars before first plot file
1404  if (solverChoice.moisture_type != MoistureType::None) {
1405  for (int lev = 0; lev <= finest_level; ++lev) micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons]);
1406  }
1407 
1408  // Fill time averaged velocities before first plot file
1409  if (solverChoice.time_avg_vel) {
1410  for (int lev = 0; lev <= finest_level; ++lev) {
1411  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
1412  vars_new[lev][Vars::xvel],
1413  vars_new[lev][Vars::yvel],
1414  vars_new[lev][Vars::zvel]);
1415  }
1416  }
1417 
1418  // check for additional plotting variables that are available after particle containers
1419  // are setup.
1420  const std::string& pv3d_1 = "plot_vars_1" ; appendPlotVariables(pv3d_1,plot3d_var_names_1);
1421  const std::string& pv3d_2 = "plot_vars_2" ; appendPlotVariables(pv3d_2,plot3d_var_names_2);
1422  const std::string& pv2d_1 = "plot2d_vars_1"; appendPlotVariables(pv2d_1,plot2d_var_names_1);
1423  const std::string& pv2d_2 = "plot2d_vars_2"; appendPlotVariables(pv2d_2,plot2d_var_names_2);
1424 
1425  if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > 0.) )
1426  {
1429  }
1430 
1431  if ( (restart_chkfile.empty()) ||
1432  (!restart_chkfile.empty() && plot_file_on_restart) )
1433  {
1434  if (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.)
1435  {
1438  }
1439  if (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.)
1440  {
1443  }
1444  if (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.)
1445  {
1448  }
1449  if (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.)
1450  {
1453  }
1454  if (m_subvol_int > 0 || m_subvol_per > 0.) {
1455  WriteSubvolume();
1456  last_subvol = istep[0];
1457  }
1458  }
1459 
1460  // Set these up here because we need to know which MPI rank "cell" is on...
1461  if (pp.contains("data_log"))
1462  {
1463  int num_datalogs = pp.countval("data_log");
1464  datalog.resize(num_datalogs);
1465  datalogname.resize(num_datalogs);
1466  pp.queryarr("data_log",datalogname,0,num_datalogs);
1467  for (int i = 0; i < num_datalogs; i++) {
1469  }
1470  }
1471 
1472  if (pp.contains("der_data_log"))
1473  {
1474  int num_der_datalogs = pp.countval("der_data_log");
1475  der_datalog.resize(num_der_datalogs);
1476  der_datalogname.resize(num_der_datalogs);
1477  pp.queryarr("der_data_log",der_datalogname,0,num_der_datalogs);
1478  for (int i = 0; i < num_der_datalogs; i++) {
1480  }
1481  }
1482 
1483  if (pp.contains("energy_data_log"))
1484  {
1485  int num_energy_datalogs = pp.countval("energy_data_log");
1486  tot_e_datalog.resize(num_energy_datalogs);
1487  tot_e_datalogname.resize(num_energy_datalogs);
1488  pp.queryarr("energy_data_log",tot_e_datalogname,0,num_energy_datalogs);
1489  for (int i = 0; i < num_energy_datalogs; i++) {
1491  }
1492  }
1493 
1494  if (solverChoice.rad_type != RadiationType::None)
1495  {
1496  // Create data log for radiation model if requested
1497  rad[0]->setupDataLog();
1498  }
1499 
1500 
1501  if (restart_chkfile.empty() && profile_int > 0) {
1502  if (destag_profiles) {
1503  // all variables cell-centered
1505  } else {
1506  // some variables staggered
1508  }
1509  }
1510 
1511  if (pp.contains("sample_point_log") && pp.contains("sample_point"))
1512  {
1513  int lev = 0;
1514 
1515  int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
1516  if (num_samplepts > 0) {
1517  Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
1518  samplepoint.resize(num_samplepts);
1519 
1520  pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
1521  for (int i = 0; i < num_samplepts; i++) {
1522  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1523  samplepoint[i] = iv;
1524  }
1525  }
1526 
1527  int num_sampleptlogs = pp.countval("sample_point_log");
1528  AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
1529  if (num_sampleptlogs > 0) {
1530  sampleptlog.resize(num_sampleptlogs);
1531  sampleptlogname.resize(num_sampleptlogs);
1532  pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
1533 
1534  for (int i = 0; i < num_sampleptlogs; i++) {
1536  }
1537  }
1538 
1539  }
1540 
1541  if (pp.contains("sample_line_log") && pp.contains("sample_line"))
1542  {
1543  int lev = 0;
1544 
1545  int num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
1546  if (num_samplelines > 0) {
1547  Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
1548  sampleline.resize(num_samplelines);
1549 
1550  pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
1551  for (int i = 0; i < num_samplelines; i++) {
1552  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1553  sampleline[i] = iv;
1554  }
1555  }
1556 
1557  int num_samplelinelogs = pp.countval("sample_line_log");
1558  AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
1559  if (num_samplelinelogs > 0) {
1560  samplelinelog.resize(num_samplelinelogs);
1561  samplelinelogname.resize(num_samplelinelogs);
1562  pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
1563 
1564  for (int i = 0; i < num_samplelinelogs; i++) {
1566  }
1567  }
1568 
1569  }
1570 
1575  }
1576 
1577  // Create object to do line and plane sampling if needed
1578  bool do_line = false; bool do_plane = false;
1579  pp.query("do_line_sampling",do_line); pp.query("do_plane_sampling",do_plane);
1580  if (do_line || do_plane) { data_sampler = std::make_unique<SampleData>(do_line, do_plane); }
1581 
1582  if ( solverChoice.terrain_type == TerrainType::EB ||
1583  solverChoice.terrain_type == TerrainType::ImmersedForcing)
1584  {
1585  bool write_eb_surface = false;
1586  pp.query("write_eb_surface", write_eb_surface);
1587  if (write_eb_surface) WriteMyEBSurface();
1588  }
1589 
1590 }
void initRayleigh()
Initialize Rayleigh damping profiles.
Definition: ERF_InitRayleigh.cpp:14
amrex::Vector< std::string > samplelinelogname
Definition: ERF.H:1524
void setRayleighRefFromSounding(bool restarting)
Set Rayleigh mean profiles from input sounding.
Definition: ERF_InitRayleigh.cpp:55
amrex::Vector< amrex::IntVect > sampleline
Definition: ERF.H:1525
static amrex::Real sum_per
Definition: ERF.H:1129
void setRecordDataInfo(int i, const std::string &filename)
Definition: ERF.H:1434
void WriteMyEBSurface()
Definition: ERF_EBWriteSurface.cpp:5
void write_1D_profiles_stag(amrex::Real time)
Definition: ERF_Write1DProfiles_stag.cpp:25
void sum_energy_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:318
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
Definition: ERF.H:1523
static int sum_interval
Definition: ERF.H:1127
static int pert_interval
Definition: ERF.H:1128
void restart()
Definition: ERF.cpp:1629
void write_1D_profiles(amrex::Real time)
Definition: ERF_Write1DProfiles.cpp:17
int profile_int
Definition: ERF.H:1029
bool destag_profiles
Definition: ERF.H:1030
void appendPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:184
amrex::Vector< std::string > tot_e_datalogname
Definition: ERF.H:1517
static int output_bndry_planes
Definition: ERF.H:1186
static std::string nc_bdy_file
Definition: ERF.H:1146
void AverageDown()
Definition: ERF_AverageDown.cpp:16
std::unique_ptr< SampleData > data_sampler
Definition: ERF.H:1510
static amrex::Real bndry_output_planes_start_time
Definition: ERF.H:1189
void project_velocity(int lev, amrex::Real dt)
Definition: ERF_PoissonSolve.cpp:10
std::string restart_chkfile
Definition: ERF.H:984
amrex::Vector< std::string > sampleptlogname
Definition: ERF.H:1520
void sum_derived_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:187
void sum_integrated_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:15
void setRecordDerDataInfo(int i, const std::string &filename)
Definition: ERF.H:1447
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
Definition: ERF.H:1519
std::unique_ptr< WriteBndryPlanes > m_w2d
Definition: ERF.H:1248
void init_geo_wind_profile(const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
Definition: ERF_InitGeowind.cpp:10
void initSponge()
Initialize sponge profiles.
Definition: ERF_InitSponge.cpp:35
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
Definition: ERF.H:1514
int real_width
Definition: ERF.H:1147
void setRecordEnergyDataInfo(int i, const std::string &filename)
Definition: ERF.H:1460
static bool is_it_time_for_action(int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
Definition: ERF_WriteScalarProfiles.cpp:747
int plot_file_on_restart
Definition: ERF.H:962
void Construct_ERFFillPatchers(int lev)
Definition: ERF.cpp:2371
void setRecordSampleLineInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1490
void setSpongeRefFromSounding(bool restarting)
Set sponge mean profiles from input sounding.
Definition: ERF_InitSponge.cpp:65
amrex::Vector< amrex::IntVect > samplepoint
Definition: ERF.H:1521
void setRecordSamplePointInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1473
void ReadCheckpointFileSurfaceLayer()
Definition: ERF_Checkpoint.cpp:1003
static MoistureModelType modelType(const MoistureType a_moisture_type)
query if a specified moisture model is Eulerian or Lagrangian
Definition: ERF_Microphysics.H:86
bool have_zero_flux_faces
Definition: ERF_AdvStruct.H:416
amrex::Real rho0_trans
Definition: ERF_DiffStruct.H:91
amrex::Real dynamic_viscosity
Definition: ERF_DiffStruct.H:96
bool have_geo_wind_profile
Definition: ERF_DataStruct.H:883
std::string abl_geo_wind_table
Definition: ERF_DataStruct.H:882
bool use_rotate_surface_flux
Definition: ERF_DataStruct.H:854
bool do_forest_drag
Definition: ERF_DataStruct.H:904
void debug(amrex::Real)
Definition: ERF_TurbPertStruct.H:613
Here is the call graph for this function:

◆ InitData_pre()

void ERF::InitData_pre ( )
782 {
783  // Initialize the start time for our CPU-time tracker
784  startCPUTime = ParallelDescriptor::second();
785 
786  // Create the ReadBndryPlanes object so we can read boundary plane data
787  // m_r2d is used by init_bcs so we must instantiate this class before
788  if (input_bndry_planes) {
789  Print() << "Defining r2d for the first time " << std::endl;
790  m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
791  }
792 
798 
799  if (restart_chkfile.empty()) {
800 
801  // Start simulation from the beginning
802  const Real time = start_time;
803  InitFromScratch(time);
804 
805  } else {
806  // For initialization this is done in init_only; it is done here for restart
807  init_bcs();
808  }
809 
810  // Verify solver choices
811  for (int lev(0); lev <= max_level; ++lev) {
812  // BC compatibility
813  if ( ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
814  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ||
815  (solverChoice.turbChoice[lev].pbl_type == PBLType::YSU) ||
816  (solverChoice.turbChoice[lev].pbl_type == PBLType::MRF)
817  ) &&
818  phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::surface_layer ) {
819  Abort("MYNN2.5/MYNNEDMF/YSU/MRF PBL Model requires MOST at lower boundary");
820  }
821  if ( (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) &&
822  (solverChoice.turbChoice[lev].Ce_wall > 0) &&
823  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::surface_layer) &&
824  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::slip_wall) &&
825  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::no_slip_wall) )
826  {
827  Warning("Deardorff LES assumes wall at zlo when applying Ce_wall");
828  }
829 
830  if ( (solverChoice.const_massflux_u != 0) &&
831  (phys_bc_type[Orientation(Direction::x,Orientation::low)] != ERF_BC::periodic ) )
832  {
833  Abort("Constant mass flux (in x) should be used with periodic boundaries");
834  }
835  if ( (solverChoice.const_massflux_v != 0) &&
836  (phys_bc_type[Orientation(Direction::y,Orientation::low)] != ERF_BC::periodic ) )
837  {
838  Abort("Constant mass flux (in y) should be used with periodic boundaries");
839  }
840 
841  // mesoscale diffusion
842  if ((geom[lev].CellSize(0) > 2000.) || (geom[lev].CellSize(1) > 2000.))
843  {
844  if ( (solverChoice.turbChoice[lev].les_type == LESType::Smagorinsky) &&
845  (!solverChoice.turbChoice[lev].smag2d)) {
846  Warning("Should use 2-D Smagorinsky for mesoscale resolution");
847  } else if (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) {
848  Warning("Should not use Deardorff LES for mesoscale resolution");
849  }
850  }
851  }
852 }
void init_bcs()
Definition: ERF_InitBCs.cpp:20

◆ initHSE() [1/2]

void ERF::initHSE ( )
private

Initialize HSE.

143 {
144  for (int lev = 0; lev <= finest_level; lev++)
145  {
146  initHSE(lev);
147  }
148 }

◆ initHSE() [2/2]

void ERF::initHSE ( int  lev)
private

Initialize density and pressure base state in hydrostatic equilibrium.

21 {
22  // This integrates up through column to update p_hse, pi_hse, th_hse;
23  // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
24 
25  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
26  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
27  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
28  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
29  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
30 
31  bool all_boxes_touch_bottom = true;
32  Box domain(geom[lev].Domain());
33 
34  int icomp = 0; int ncomp = BaseState::num_comps;
35 
36  if (lev == 0) {
37  BoxArray ba(base_state[lev].boxArray());
38  for (int i = 0; i < ba.size(); i++) {
39  if (ba[i].smallEnd(2) != domain.smallEnd(2)) {
40  all_boxes_touch_bottom = false;
41  }
42  }
43  }
44  else
45  {
46  //
47  // We need to do this interp from coarse level in order to set the values of
48  // the base state inside the domain but outside of the fine region
49  //
50  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
51  //
52  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
53  // have been pre-filled - this includes ghost cells both inside and outside
54  // the domain
55  //
56  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
57  IntVect(0,0,0), // do not fill ghost cells outside the domain
58  base_state[lev-1], icomp, icomp, ncomp,
59  geom[lev-1], geom[lev],
60  refRatio(lev-1), &cell_cons_interp,
62 
63  // We need to do this here because the interpolation above may leave corners unfilled
64  // when the corners need to be filled by, for example, reflection of the fine ghost
65  // cell outside the fine region but inide the domain.
66  (*physbcs_base[lev])(base_state[lev],icomp,ncomp,base_state[lev].nGrowVect());
67  }
68 
69  if (all_boxes_touch_bottom || lev > 0) {
70 
71  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
73  prob->erf_init_dens_hse_moist(r_hse, z_phys_nd[lev], geom[lev]);
74  } else {
75  prob->erf_init_dens_hse(r_hse, z_phys_nd[lev], z_phys_cc[lev], geom[lev]);
76  }
77 
78  erf_enforce_hse(lev, r_hse, p_hse, pi_hse, th_hse, qv_hse, z_phys_cc[lev]);
79 
80  //
81  // Impose physical bc's on the base state
82  //
83  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
84 
85  } else {
86 
87  BoxArray ba_new(domain);
88 
89  ChopGrids2D(ba_new, domain, ParallelDescriptor::NProcs());
90 
91  DistributionMapping dm_new(ba_new);
92 
93  MultiFab new_base_state(ba_new, dm_new, BaseState::num_comps, base_state[lev].nGrowVect());
94  new_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
95  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
96 
97  MultiFab new_r_hse (new_base_state, make_alias, BaseState::r0_comp, 1);
98  MultiFab new_p_hse (new_base_state, make_alias, BaseState::p0_comp, 1);
99  MultiFab new_pi_hse(new_base_state, make_alias, BaseState::pi0_comp, 1);
100  MultiFab new_th_hse(new_base_state, make_alias, BaseState::th0_comp, 1);
101  MultiFab new_qv_hse(new_base_state, make_alias, BaseState::qv0_comp, 1);
102 
103  std::unique_ptr<MultiFab> new_z_phys_cc;
104  std::unique_ptr<MultiFab> new_z_phys_nd;
105  if (solverChoice.mesh_type != MeshType::ConstantDz) {
106  new_z_phys_cc = std::make_unique<MultiFab>(ba_new,dm_new,1,1);
107  new_z_phys_cc->ParallelCopy(*z_phys_cc[lev],0,0,1,1,1);
108 
109  BoxArray ba_new_nd(ba_new);
110  ba_new_nd.surroundingNodes();
111  new_z_phys_nd = std::make_unique<MultiFab>(ba_new_nd,dm_new,1,1);
112  new_z_phys_nd->ParallelCopy(*z_phys_nd[lev],0,0,1,1,1);
113  }
114 
115  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
117  prob->erf_init_dens_hse_moist(new_r_hse, new_z_phys_nd, geom[lev]);
118  } else {
119  prob->erf_init_dens_hse(new_r_hse, new_z_phys_nd, new_z_phys_cc, geom[lev]);
120  }
121 
122  erf_enforce_hse(lev, new_r_hse, new_p_hse, new_pi_hse, new_th_hse, new_qv_hse, new_z_phys_cc);
123 
124  //
125  // Impose physical bc's on the base state
126  //
127  (*physbcs_base[lev])(new_base_state,0,new_base_state.nComp(),new_base_state.nGrowVect());
128 
129  // Now copy back into the original arrays
130  base_state[lev].ParallelCopy(new_base_state,0,0,base_state[lev].nComp(),
131  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
132  }
133 
134  //
135  // Impose physical bc's on the base state -- the values outside the fine region
136  // but inside the domain have already been filled in the call above to InterpFromCoarseLevel
137  //
138  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
139 }
void ChopGrids2D(BoxArray &ba, const Box &domain, int target_size)
Definition: ERF_ChopGrids.cpp:21
void erf_enforce_hse(int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
Definition: ERF_Init1D.cpp:160
bool use_moist_background
Definition: ERF_DataStruct.H:891
Here is the call graph for this function:

◆ initialize_integrator()

void ERF::initialize_integrator ( int  lev,
amrex::MultiFab &  cons_mf,
amrex::MultiFab &  vel_mf 
)
private
635 {
636  const BoxArray& ba(cons_mf.boxArray());
637  const DistributionMapping& dm(cons_mf.DistributionMap());
638 
639  int ncomp_cons = cons_mf.nComp();
640 
641  // Initialize the integrator memory
642  Vector<MultiFab> int_state; // integration state data structure example
643  int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons)); // cons
644  int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
645  int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
646  int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
647 
648  mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
649  mri_integrator_mem[lev]->setNoSubstepping((solverChoice.substepping_type[lev] == SubsteppingType::None));
650  mri_integrator_mem[lev]->setAnelastic(solverChoice.anelastic[lev]);
651  mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
652  mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
653 }

◆ InitializeFromFile()

void ERF::InitializeFromFile ( )
private

◆ InitializeLevelFromData()

void ERF::InitializeLevelFromData ( int  lev,
const amrex::MultiFab &  initial_data 
)
private

◆ initializeMicrophysics()

void ERF::initializeMicrophysics ( const int &  a_nlevsmax)
private
Parameters
a_nlevsmaxnumber of AMR levels
1595 {
1596  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
1597 
1598  micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1599 
1600  } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
1601 #ifdef ERF_USE_PARTICLES
1602 
1603  micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1604  /* Lagrangian microphysics models will have a particle container; it needs to be added
1605  to ERF::particleData */
1606  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
1607  /* The particle container has not yet been constructed and initialized, so just add
1608  its name here for now (so that functions to set plotting variables can see it). */
1609  particleData.addName( pc_name );
1610 
1611 #else
1612  Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
1613 #endif
1614  }
1615 
1616  qmoist.resize(a_nlevsmax);
1617  return;
1618 }
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
Definition: ERF.H:816
Here is the call graph for this function:

◆ initRayleigh()

void ERF::initRayleigh ( )
private

Initialize Rayleigh damping profiles.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

15 {
16  const int khi = geom[0].Domain().bigEnd(2);
17  solverChoice.rayleigh_ztop = (solverChoice.terrain_type == TerrainType::None) ? geom[0].ProbHi(2) : zlevels_stag[0][khi+1];
18 
19  h_rayleigh_ptrs.resize(max_level+1);
20  d_rayleigh_ptrs.resize(max_level+1);
21 
22  for (int lev = 0; lev <= finest_level; lev++)
23  {
24  // These have 4 components: ubar, vbar, wbar, thetabar
25  h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
26  d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
27 
28  const int zlen_rayleigh = geom[lev].Domain().length(2);
29 
30  // Allocate space for these 1D vectors
31  for (int n = 0; n < Rayleigh::nvars; n++) {
32  h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
33  d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
34  }
35 
36  // Init the host vectors
37  prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_nd[lev], solverChoice.rayleigh_zdamp);
38 
39  // Copy from host vectors to device vectors
40  for (int n = 0; n < Rayleigh::nvars; n++) {
41  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
42  d_rayleigh_ptrs[lev][n].begin());
43  }
44  }
45 }
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
Definition: ERF.H:1223
amrex::Real rayleigh_zdamp
Definition: ERF_DataStruct.H:814
amrex::Real rayleigh_ztop
Definition: ERF_DataStruct.H:815

◆ initSponge()

void ERF::initSponge ( )
private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

36 {
37  h_sponge_ptrs.resize(max_level+1);
38  d_sponge_ptrs.resize(max_level+1);
39 
40  for (int lev = 0; lev <= finest_level; lev++)
41  {
42  // These have 2 components: ubar, vbar
45 
46  const int zlen_sponge = geom[lev].Domain().length(2);
47 
48  // Allocate space for these 1D vectors
49  for (int n = 0; n < Sponge::nvars_sponge; n++) {
50  h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
51  d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
52  }
53 
54  }
55 }
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
Definition: ERF.H:1226

◆ input_sponge()

void ERF::input_sponge ( int  lev)

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters
levInteger specifying the current level
18 {
19  // We only want to read the file once
20  if (lev == 0) {
22  Error("input_sounding file name must be provided via input");
23 
24  // this will interpolate the input profiles to the nominal height levels
25  // (ranging from 0 to the domain top)
27  }
28 }
InputSpongeData input_sponge_data
Definition: ERF.H:730
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag)
Definition: ERF_InputSpongeData.H:28
std::string input_sponge_file
Definition: ERF_InputSpongeData.H:111

◆ InterpWeatherDataOntoMesh()

void ERF::InterpWeatherDataOntoMesh ( const amrex::Geometry &  geom_weather,
amrex::MultiFab &  weather_forecast_interp 
)
251 {
252 
253  ParmParse pp_erf("erf");
254  bool is_lateral_sponges_hurricanes = false;
255  if (pp_erf.query("is_lateral_sponges_hurricanes", is_lateral_sponges_hurricanes)) {
256  initial_state.resize(max_level+1);
257  for (int lev = 0; lev < max_level+1; ++lev) {
258  initial_state[lev].resize(vars_new[lev].size()+1);
259  for (int comp = 0; comp < vars_new[lev].size(); ++comp) {
260  const MultiFab& src = vars_new[lev][comp];
261  initial_state[lev][comp].define(src.boxArray(), src.DistributionMap(),
262  src.nComp(), src.nGrow());
263  }
264  int comp = vars_new[lev].size();
265  const MultiFab& src = vars_new[lev][0];
266  initial_state[lev][comp].define(src.boxArray(), src.DistributionMap(),
267  2, src.nGrow());
268  }
269  }
270 
271  MultiFab& weather_mf = weather_forecast_interp;
272  MultiFab& erf_mf_cons = initial_state[0][Vars::cons];
273  MultiFab& erf_mf_xvel = initial_state[0][Vars::xvel];
274  MultiFab& erf_mf_yvel = initial_state[0][Vars::yvel];
275  MultiFab& erf_mf_zvel = initial_state[0][Vars::zvel];
276  MultiFab& erf_mf_latlon = initial_state[0][4];
277 
278  erf_mf_cons.setVal(0.0);
279  erf_mf_xvel.setVal(0.0);
280  erf_mf_yvel.setVal(0.0);
281  erf_mf_zvel.setVal(0.0);
282  erf_mf_latlon.setVal(0.0);
283 
284  BoxList bl_erf = erf_mf_cons.boxArray().boxList();
285  BoxList bl_weather = weather_mf.boxArray().boxList();
286 
287  const auto prob_lo_erf = geom[0].ProbLoArray();
288  const auto dx_erf = geom[0].CellSizeArray();
289 
290  for (auto& b : bl_erf) {
291  // You look at the lo corner of b, and find out the lowest cell in
292  // coarse weather data you need for the interpolation. That gives
293  // you the lo corner of the new b. Similarly, you can find out the
294  // hi corner of the new b. For cells outside the coarse_weath_data's
295  // bounding data, it's up to you. You probably want to use a biased
296  // interpolation stencil.
297 
298  // Get the cell indices of the bottom corner and top corner
299  const IntVect& lo_erf = b.smallEnd(); // Lower corner (inclusive)
300  const IntVect& hi_erf = b.bigEnd(); // Upper corner (inclusive)
301 
302  Real x = prob_lo_erf[0] + lo_erf[0] * dx_erf[0];
303  Real y = prob_lo_erf[1] + lo_erf[1] * dx_erf[1];
304  Real z = prob_lo_erf[2] + lo_erf[2] * dx_erf[2];
305 
306  auto idx_lo = find_bound_idx(x, y, z, bl_weather, geom_weather, BoundType::Lo);
307 
308  x = prob_lo_erf[0] + (hi_erf[0]+1) * dx_erf[0];
309  y = prob_lo_erf[1] + (hi_erf[1]+1) * dx_erf[1];
310  z = prob_lo_erf[2] + (hi_erf[2]+1) * dx_erf[2];
311 
312  auto idx_hi = find_bound_idx(x, y, z, bl_weather, geom_weather, BoundType::Hi);
313 
314  b.setSmall(idx_lo);
315  b.setBig(idx_hi);
316  }
317 
318  BoxArray cba(std::move(bl_erf));
319  cba.convert(IndexType::TheNodeType()); // <-- Make it nodal in all directions
320  MultiFab tmp_coarse_data(cba, erf_mf_cons.DistributionMap(), weather_mf.nComp(), 0);
321  tmp_coarse_data.ParallelCopy(weather_mf);
322 
323  PlotMultiFab(weather_mf, geom_weather, "plt_coarse_weather_par_copy",MultiFabType::NC);
324 
325  const auto prob_lo_weather = geom_weather.ProbLoArray();
326  const auto dx_weather = geom_weather.CellSizeArray();
327 
328  for (MFIter mfi(erf_mf_cons); mfi.isValid(); ++mfi) {
329  const Array4<Real> &fine_cons_arr = erf_mf_cons.array(mfi);
330  const Array4<Real> &fine_xvel_arr = erf_mf_xvel.array(mfi);
331  const Array4<Real> &fine_yvel_arr = erf_mf_yvel.array(mfi);
332  const Array4<Real> &fine_zvel_arr = erf_mf_zvel.array(mfi);
333  const Array4<Real> &fine_latlon_arr = erf_mf_latlon.array(mfi);
334 
335  const Array4<Real> &crse_arr = tmp_coarse_data.array(mfi);
336 
337  const Box& gbx = mfi.growntilebox(); // tilebox + ghost cells
338 
339  const Box &gtbx = mfi.tilebox(IntVect(1,0,0));
340  const Box &gtby = mfi.tilebox(IntVect(0,1,0));
341  const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
342 
343  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
344  // Physical location of the fine node
345  Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
346  Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
347  Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
348 
349  Real rho = interpolate_from_coarse(crse_arr, 0, x, y, z, prob_lo_weather.data(), dx_weather.data());
350  Real theta = interpolate_from_coarse(crse_arr, 4, x, y, z, prob_lo_weather.data(), dx_weather.data());
351  Real qv = interpolate_from_coarse(crse_arr, 5, x, y, z, prob_lo_weather.data(), dx_weather.data());
352  Real qc = interpolate_from_coarse(crse_arr, 6, x, y, z, prob_lo_weather.data(), dx_weather.data());
353  Real qr = interpolate_from_coarse(crse_arr, 7, x, y, z, prob_lo_weather.data(), dx_weather.data());
354  Real lat = interpolate_from_coarse(crse_arr, 8, x, y, z, prob_lo_weather.data(), dx_weather.data());
355  Real lon = interpolate_from_coarse(crse_arr, 9, x, y, z, prob_lo_weather.data(), dx_weather.data());
356 
357  fine_cons_arr(i,j,k,Rho_comp) = rho;
358  fine_cons_arr(i,j,k,RhoTheta_comp) = rho*theta;
359  fine_cons_arr(i,j,k,RhoQ1_comp) = rho*qv;
360  fine_cons_arr(i,j,k,RhoQ2_comp) = rho*qc;
361  fine_cons_arr(i,j,k,RhoQ3_comp) = rho*qr;
362 
363  fine_latlon_arr(i,j,k,0) = lat;
364  fine_latlon_arr(i,j,k,1) = lon;
365  });
366 
367  ParallelFor(gtbx, gtby, gtbz,
368  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
369  // Physical location of the fine node
370  Real x = prob_lo_erf[0] + i * dx_erf[0];
371  Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
372  Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
373  fine_xvel_arr(i, j, k, 0) = interpolate_from_coarse(crse_arr, 1, x, y, z, prob_lo_weather.data(), dx_weather.data());
374  },
375  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
376  // Physical location of the fine node
377  Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
378  Real y = prob_lo_erf[1] + j * dx_erf[1];
379  Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
380  fine_yvel_arr(i, j, k, 0) = interpolate_from_coarse(crse_arr, 2, x, y, z, prob_lo_weather.data(), dx_weather.data());
381  },
382  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
383  // Physical location of the fine node
384  Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
385  Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
386  Real z = prob_lo_erf[2] + k * dx_erf[2];
387  fine_zvel_arr(i, j, k, 0) = interpolate_from_coarse(crse_arr, 3, x, y, z, prob_lo_weather.data(), dx_weather.data());
388  });
389  }
390 
391  Vector<std::string> varnames = {
392  "rho", "uvel", "vvel", "wvel", "theta", "qv", "qc", "qr"
393  }; // Customize variable names
394 
395  Vector<std::string> varnames_cons = {
396  "rho", "rhotheta", "ke", "sc", "rhoqv", "rhoqc", "rhoqr"
397  }; // Customize variable names
398 
399  Vector<std::string> varnames_plot_mf = {
400  "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
401  }; // Customize variable names
402 
403 
404  const Real time = 0.0;
405 
406  std::string pltname = "plt_interp";
407 
408  MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
409  10, 0);
410 
411  plot_mf.setVal(0.0);
412 
413  for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
414  const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
415  const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
416  const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
417  const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
418  const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
419  const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
420 
421  const Box& bx = mfi.validbox();
422 
423  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
424  plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
425  plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
426  plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
427  plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
428  plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
429 
430  plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/2.0;
431  plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/2.0;
432  plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/2.0;
433 
434  plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
435  plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
436  });
437  }
438 
439 
440  WriteSingleLevelPlotfile(
441  pltname,
442  plot_mf,
443  varnames_plot_mf,
444  geom[0],
445  time,
446  0 // level
447  );
448 }
AMREX_GPU_DEVICE amrex::Real interpolate_from_coarse(const amrex::Array4< const amrex::Real > &crse, int n, amrex::Real x, amrex::Real y, amrex::Real z, const amrex::Real *prob_lo_crse, const amrex::Real *dx_crse)
Definition: ERF_Interpolation_Bilinear.H:77
IntVect find_bound_idx(const Real &x, const Real &y, const Real &z, const BoxList &bl_weather, const Geometry &geom_weather, BoundType bound_type)
Definition: ERF_WeatherDataInterpolation.cpp:217
amrex::Vector< amrex::Vector< amrex::MultiFab > > initial_state
Definition: ERF.H:154
Here is the call graph for this function:

◆ is_it_time_for_action()

bool ERF::is_it_time_for_action ( int  nstep,
amrex::Real  time,
amrex::Real  dt,
int  action_interval,
amrex::Real  action_per 
)
static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters
nstepTimestep number
timeCurrent time
dtlevTimestep for the current level
action_intervalInterval in number of timesteps for taking action
action_perInterval in simulation time for taking action
748 {
749  bool int_test = (action_interval > 0 && nstep % action_interval == 0);
750 
751  bool per_test = false;
752  if (action_per > 0.0) {
753  const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
754  const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
755 
756  if (num_per_old != num_per_new) {
757  per_test = true;
758  }
759  }
760 
761  return int_test || per_test;
762 }

◆ make_eb_box()

void ERF::make_eb_box ( )

◆ make_eb_regular()

void ERF::make_eb_regular ( )

◆ make_physbcs()

void ERF::make_physbcs ( int  lev)
private
657 {
658  if (SolverChoice::mesh_type == MeshType::VariableDz) {
659  AMREX_ALWAYS_ASSERT(z_phys_nd[lev] != nullptr);
660  }
661 
662  physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
664  z_phys_nd[lev], solverChoice.use_real_bcs, th_bc_data[lev].data());
665  physbcs_u[lev] = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
667  z_phys_nd[lev], solverChoice.use_real_bcs, xvel_bc_data[lev].data());
668  physbcs_v[lev] = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
670  z_phys_nd[lev], solverChoice.use_real_bcs, yvel_bc_data[lev].data());
671  physbcs_w[lev] = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
674  solverChoice.use_real_bcs, zvel_bc_data[lev].data());
675  physbcs_base[lev] = std::make_unique<ERFPhysBCFunct_base> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d, z_phys_nd[lev],
676  (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
677 }

◆ make_subdomains()

void ERF::make_subdomains ( const amrex::BoxList &  ba,
amrex::Vector< amrex::BoxArray > &  bins 
)
7 {
8  Vector<BoxList> bins_bl;
9 
10  // Clear out any old bins
11  bins.clear();
12 
13  // Iterate over boxes
14  for (auto bx : bl)
15  {
16  bool added = false;
17 
18  // Try to add box to existing bin
19  for (int j = 0; j < bins_bl.size(); ++j) {
20  BoxList& bin = bins_bl[j];
21  bool touches = false;
22 
23  for (auto& b : bin)
24  {
25  Box gbx(bx); gbx.grow(1);
26  if (gbx.intersects(b)) {
27  touches = true;
28  break;
29  }
30  }
31 
32  if (touches) {
33  bin.push_back(bx);
34  added = true;
35  break;
36  }
37  }
38 
39  // If box couldn't be added to existing bin, create new bin
40  if (!added) {
41  BoxList new_bin;
42  new_bin.push_back(bx);
43  bins_bl.push_back(new_bin);
44  }
45  }
46 
47  // Convert the BoxLists to BoxArrays
48  for (int i = 0; i < bins_bl.size(); ++i) {
49  bins.push_back(BoxArray(bins_bl[i]));
50  }
51 }

◆ MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage ( amrex::Vector< amrex::Real > &  h_havg,
amrex::MultiFab &  S,
int  n 
)
2326 {
2327  // Get the number of cells in z at level 0
2328  int dir_z = AMREX_SPACEDIM-1;
2329  auto domain = geom[0].Domain();
2330  int size_z = domain.length(dir_z);
2331  int start_z = domain.smallEnd()[dir_z];
2332  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
2333 
2334  // resize the level 0 horizontal average vectors
2335  h_havg.resize(size_z, 0.0_rt);
2336 
2337  // Get the cell centered data and construct sums
2338 #ifdef _OPENMP
2339 #pragma omp parallel if (Gpu::notInLaunchRegion())
2340 #endif
2341  for (MFIter mfi(S); mfi.isValid(); ++mfi) {
2342  const Box& box = mfi.validbox();
2343  const IntVect& se = box.smallEnd();
2344  const IntVect& be = box.bigEnd();
2345 
2346  auto fab_arr = S[mfi].array();
2347 
2348  FArrayBox fab_reduce(box, 1, The_Async_Arena());
2349  auto arr_reduce = fab_reduce.array();
2350 
2351  ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2352  arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
2353  });
2354 
2355  for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
2356  Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
2357  h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
2358  }
2359  }
2360 
2361  // combine sums from different MPI ranks
2362  ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), h_havg.size());
2363 
2364  // divide by the total number of cells we are averaging over
2365  for (int k = 0; k < size_z; ++k) {
2366  h_havg[k] /= area_z;
2367  }
2368 }

◆ MakeEBGeometry()

void ERF::MakeEBGeometry ( )

◆ MakeHorizontalAverages()

void ERF::MakeHorizontalAverages ( )
2220 {
2221  int lev = 0;
2222 
2223  // First, average down all levels (if doing two-way coupling)
2224  if (solverChoice.coupling_type == CouplingType::TwoWay) {
2225  AverageDown();
2226  }
2227 
2228  MultiFab mf(grids[lev], dmap[lev], 5, 0);
2229 
2230  int zdir = 2;
2231  auto domain = geom[0].Domain();
2232 
2233  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
2234  bool is_anelastic = (solverChoice.anelastic[lev] == 1);
2235 
2236  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
2237  const Box& bx = mfi.validbox();
2238  auto fab_arr = mf.array(mfi);
2239  auto const hse_arr = base_state[lev].const_array(mfi);
2240  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
2241  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2242  Real dens = cons_arr(i, j, k, Rho_comp);
2243  fab_arr(i, j, k, 0) = dens;
2244  fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
2245  if (!use_moisture) {
2246  if (is_anelastic) {
2247  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
2248  } else {
2249  fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp));
2250  }
2251  }
2252  });
2253  }
2254 
2255  if (use_moisture)
2256  {
2257  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
2258  const Box& bx = mfi.validbox();
2259  auto fab_arr = mf.array(mfi);
2260  auto const hse_arr = base_state[lev].const_array(mfi);
2261  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
2262  int ncomp = vars_new[lev][Vars::cons].nComp();
2263 
2264  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2265  Real dens = cons_arr(i, j, k, Rho_comp);
2266  if (is_anelastic) {
2267  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
2268  } else {
2269  Real qv = cons_arr(i, j, k, RhoQ1_comp) / dens;
2270  fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
2271  }
2272  fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : 0.0);
2273  fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : 0.0);
2274  });
2275  }
2276 
2277  Gpu::HostVector<Real> h_avg_qv = sumToLine(mf,3,1,domain,zdir);
2278  Gpu::HostVector<Real> h_avg_qc = sumToLine(mf,4,1,domain,zdir);
2279  }
2280 
2281  // Sum in the horizontal plane
2282  Gpu::HostVector<Real> h_avg_density = sumToLine(mf,0,1,domain,zdir);
2283  Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
2284  Gpu::HostVector<Real> h_avg_pressure = sumToLine(mf,2,1,domain,zdir);
2285 
2286  // Divide by the total number of cells we are averaging over
2287  int size_z = domain.length(zdir);
2288  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
2289  int klen = static_cast<int>(h_avg_density.size());
2290 
2291  for (int k = 0; k < klen; ++k) {
2292  h_havg_density[k] /= area_z;
2293  h_havg_temperature[k] /= area_z;
2294  h_havg_pressure[k] /= area_z;
2295  if (solverChoice.moisture_type != MoistureType::None)
2296  {
2297  h_havg_qc[k] /= area_z;
2298  h_havg_qv[k] /= area_z;
2299  }
2300  } // k
2301 
2302  // resize device vectors
2303  d_havg_density.resize(size_z, 0.0_rt);
2304  d_havg_temperature.resize(size_z, 0.0_rt);
2305  d_havg_pressure.resize(size_z, 0.0_rt);
2306 
2307  // copy host vectors to device vectors
2308  Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
2309  Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
2310  Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
2311 
2312  if (solverChoice.moisture_type != MoistureType::None)
2313  {
2314  d_havg_qv.resize(size_z, 0.0_rt);
2315  d_havg_qc.resize(size_z, 0.0_rt);
2316  Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
2317  Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
2318  }
2319 }
amrex::Gpu::DeviceVector< amrex::Real > d_havg_temperature
Definition: ERF.H:1241
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qv
Definition: ERF.H:1243
amrex::Vector< amrex::Real > h_havg_pressure
Definition: ERF.H:1236
amrex::Vector< amrex::Real > h_havg_qc
Definition: ERF.H:1238
amrex::Vector< amrex::Real > h_havg_density
Definition: ERF.H:1234
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qc
Definition: ERF.H:1244
amrex::Gpu::DeviceVector< amrex::Real > d_havg_density
Definition: ERF.H:1240
amrex::Vector< amrex::Real > h_havg_temperature
Definition: ERF.H:1235
amrex::Gpu::DeviceVector< amrex::Real > d_havg_pressure
Definition: ERF.H:1242
amrex::Vector< amrex::Real > h_havg_qv
Definition: ERF.H:1237
Here is the call graph for this function:

◆ MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
282 {
283  AMREX_ALWAYS_ASSERT(lev > 0);
284 
285  if (verbose) {
286  amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
287  }
288 
289  //
290  // Grow the subdomains vector and build the subdomains vector at this level
291  //
292  subdomains.resize(lev+1);
293  //
294  // Create subdomains at each level within the domain such that
295  // 1) all boxes in a given subdomain are "connected"
296  // 2) no boxes in a subdomain touch any boxes in any other subdomain
297  //
299  BoxArray dom(geom[lev].Domain());
300  subdomains[lev].push_back(dom);
301  } else {
302  make_subdomains(ba.simplified_list(), subdomains[lev]);
303  }
304 
305  if (lev == 0) init_bcs();
306 
307  //********************************************************************************************
308  // This allocates all kinds of things, including but not limited to: solution arrays,
309  // terrain arrays, metric terms and base state.
310  // *******************************************************************************************
311  init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev], base_state[lev], z_phys_nd[lev]);
312 
313  t_new[lev] = time;
314  t_old[lev] = time - 1.e200;
315 
316  // ********************************************************************************************
317  // Build the data structures for metric quantities used with terrain-fitted coordinates
318  // ********************************************************************************************
319  if ( solverChoice.terrain_type == TerrainType::EB ||
320  solverChoice.terrain_type == TerrainType::ImmersedForcing)
321  {
322  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
323  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
324  if (solverChoice.terrain_type == TerrainType::EB) {
325  eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
326  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
327  eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
328  }
329  }
330  init_zphys(lev, time);
332 
333  //
334  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
335  //
336  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
337  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
338  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
339  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
340  }
341  }
342 
343  // ********************************************************************************************
344  // Build the data structures for canopy model (depends upon z_phys)
345  // ********************************************************************************************
347  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
348  }
349 
350  //********************************************************************************************
351  // Microphysics
352  // *******************************************************************************************
353  int q_size = micro->Get_Qmoist_Size(lev);
354  qmoist[lev].resize(q_size);
355  micro->Define(lev, solverChoice);
356  if (solverChoice.moisture_type != MoistureType::None)
357  {
358  micro->Init(lev, vars_new[lev][Vars::cons],
359  grids[lev], Geom(lev), 0.0,
360  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
361  }
362  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
363  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
364  }
365 
366  //********************************************************************************************
367  // Radiation
368  // *******************************************************************************************
369  if (solverChoice.rad_type != RadiationType::None)
370  {
371  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
372  }
373 
374  // *****************************************************************************************************
375  // Initialize the boundary conditions (after initializing the terrain but before calling
376  // initHSE or FillCoarsePatch)
377  // *****************************************************************************************************
378  make_physbcs(lev);
379 
380  // ********************************************************************************************
381  // Update the base state at this level by interpolation from coarser level
382  // ********************************************************************************************
383  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
384  IntVect(0,0,0), // do not fill ghost cells outside the domain
385  base_state[lev-1], 0, 0, base_state[lev].nComp(),
386  geom[lev-1], geom[lev],
387  refRatio(lev-1), &cell_cons_interp,
389 
390  // Impose bc's outside the domain
391  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
392 
393  // ********************************************************************************************
394  // Build the data structures for calculating diffusive/turbulent terms
395  // ********************************************************************************************
396  update_diffusive_arrays(lev, ba, dm);
397 
398  // ********************************************************************************************
399  // Fill data at the new level by interpolation from the coarser level
400  // Note that internal to FillCoarsePatch we will convert velocity to momentum,
401  // then interpolate momentum, then convert momentum back to velocity
402  // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
403  // ********************************************************************************************
404  FillCoarsePatch(lev, time);
405 
406  // ********************************************************************************************
407  // Initialize the integrator class
408  // ********************************************************************************************
409  dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
411 
412  // ********************************************************************************************
413  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
414  // ********************************************************************************************
415  if (lev > 0 && cf_width >= 0) {
418  }
419 
420  // ********************************************************************************************
421  // Create the SurfaceLayer arrays at this (new) level
422  // ********************************************************************************************
423  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
424  int nlevs = finest_level+1;
425  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
426  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
427  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
428  mfv_old, Theta_prim[lev], Qv_prim[lev],
429  Qr_prim[lev], z_phys_nd[lev],
430  Hwave[lev].get(), Lwave[lev].get(), eddyDiffs_lev[lev].get(),
431  lsm_data[lev], lsm_flux[lev], sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
432  }
433 
434 #ifdef ERF_USE_PARTICLES
435  // particleData.Redistribute();
436 #endif
437 }
void update_diffusive_arrays(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewArrays.cpp:428
void initialize_integrator(int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
Definition: ERF_MakeNewArrays.cpp:634
void make_subdomains(const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
Definition: ERF_MakeSubdomains.cpp:6
void update_terrain_arrays(int lev)
Definition: ERF_MakeNewArrays.cpp:623
void init_zphys(int lev, amrex::Real time)
Definition: ERF_MakeNewArrays.cpp:518
void init_stuff(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:24
void Define_ERFFillPatchers(int lev)
Definition: ERF.cpp:2397

◆ MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
27 {
28  BoxArray ba;
29  DistributionMapping dm;
30  Box domain(Geom(0).Domain());
31  if (lev == 0 && restart_chkfile.empty() &&
32  (max_grid_size[0][0] >= domain.length(0)) &&
33  (max_grid_size[0][1] >= domain.length(1)) &&
34  ba_in.size() != ParallelDescriptor::NProcs())
35  {
36  // We only decompose in z if max_grid_size_z indicates we should
37  bool decompose_in_z = (max_grid_size[0][2] < domain.length(2));
38 
39  ba = ERFPostProcessBaseGrids(Geom(0).Domain(),decompose_in_z);
40  dm = DistributionMapping(ba);
41  } else {
42  ba = ba_in;
43  dm = dm_in;
44  }
45 
46  // ********************************************************************************************
47  // Define grids[lev] to be ba
48  // ********************************************************************************************
49  SetBoxArray(lev, ba);
50 
51  // ********************************************************************************************
52  // Define dmap[lev] to be dm
53  // ********************************************************************************************
54  SetDistributionMap(lev, dm);
55 
56  if (verbose) {
57  amrex::Print() << "BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
58  // amrex::Print() <<" SIMPLIFIED BA FROM SCRATCH AT LEVEL " << lev << " " << ba.simplified_list() << std::endl;
59  }
60 
61  subdomains.resize(lev+1);
62  if ( (lev == 0) || (
64  (solverChoice.init_type != InitType::WRFInput) && (solverChoice.init_type != InitType::Metgrid) ) ) {
65  BoxArray dom(geom[lev].Domain());
66  subdomains[lev].push_back(dom);
67  } else {
68  //
69  // Create subdomains at each level within the domain such that
70  // 1) all boxes in a given subdomain are "connected"
71  // 2) no boxes in a subdomain touch any boxes in any other subdomain
72  //
73  make_subdomains(ba.simplified_list(), subdomains[lev]);
74  }
75 
76  if (lev == 0) init_bcs();
77 
78  if ( solverChoice.terrain_type == TerrainType::EB ||
79  solverChoice.terrain_type == TerrainType::ImmersedForcing)
80  {
81  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
82  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
83  if (solverChoice.terrain_type == TerrainType::EB) {
84  eb[lev]->make_all_factories(lev, geom[lev], grids[lev], dmap[lev], eb_level);
85  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
86  eb[lev]->make_cc_factory(lev, geom[lev], grids[lev], dmap[lev], eb_level);
87  }
88  } else {
89  // m_factory[lev] = std::make_unique<FabFactory<FArrayBox>>();
90  }
91 
92  auto& lev_new = vars_new[lev];
93  auto& lev_old = vars_old[lev];
94 
95  //********************************************************************************************
96  // This allocates all kinds of things, including but not limited to: solution arrays,
97  // terrain arrays, metric terms and base state.
98  // *******************************************************************************************
99  init_stuff(lev, ba, dm, lev_new, lev_old, base_state[lev], z_phys_nd[lev]);
100 
101  //********************************************************************************************
102  // Land Surface Model
103  // *******************************************************************************************
104  int lsm_size = lsm.Get_Data_Size();
105  lsm_data[lev].resize(lsm_size);
106  lsm_flux[lev].resize(lsm_size);
107  lsm.Define(lev, solverChoice);
108  if (solverChoice.lsm_type != LandSurfaceType::None)
109  {
110  lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
111  }
112  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
113  lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
114  lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
115  }
116 
117  // ********************************************************************************************
118  // Build the data structures for calculating diffusive/turbulent terms
119  // ********************************************************************************************
120  update_diffusive_arrays(lev, ba, dm);
121 
122  // ********************************************************************************************
123  // Build the data structures for holding sea surface temps and skin temps
124  // ********************************************************************************************
125  sst_lev[lev].resize(1); sst_lev[lev][0] = nullptr;
126  tsk_lev[lev].resize(1); tsk_lev[lev][0] = nullptr;
127 
128  // ********************************************************************************************
129  // Thin immersed body
130  // *******************************************************************************************
131  init_thin_body(lev, ba, dm);
132 
133  // ********************************************************************************************
134  // Initialize the integrator class
135  // ********************************************************************************************
136  initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
137 
138  // ********************************************************************************************
139  // Initialize the data itself
140  // If (init_type == InitType::WRFInput) then we are initializing terrain and the initial data in
141  // the same call so we must call init_only before update_terrain_arrays
142  // If (init_type != InitType::WRFInput) then we want to initialize the terrain before the initial data
143  // since we may need to use the grid information before constructing
144  // initial idealized data
145  // ********************************************************************************************
146  if (restart_chkfile.empty()) {
147  if ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) )
148  {
149  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
150  init_only(lev, start_time);
151  init_zphys(lev, time);
153  make_physbcs(lev);
154  } else {
155  init_zphys(lev, time);
157  // Note that for init_type != InitType::WRFInput and != InitType::Metgrid,
158  // make_physbcs is called inside init_only
159  init_only(lev, start_time);
160  }
161  } else {
162  // if restarting and nudging from input sounding, load the input sounding files
163  if (lev == 0 && solverChoice.init_type == InitType::Input_Sounding && solverChoice.nudging_from_input_sounding)
164  {
166  Error("input_sounding file name must be provided via input");
167  }
168 
170 
171  // this will interpolate the input profiles to the nominal height levels
172  // (ranging from 0 to the domain top)
173  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
174  input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n);
175  }
176 
177  // this will calculate the hydrostatically balanced density and pressure
178  // profiles following WRF ideal.exe
179  if (solverChoice.sounding_type == SoundingType::Ideal) {
181  } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
182  solverChoice.sounding_type == SoundingType::DryIsentropic) {
183  input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
185  }
186  }
187 
188  // We re-create terrain_blanking on restart rather than storing it in the checkpoint
189  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
190  int ngrow = ComputeGhostCells(solverChoice) + 2;
191  terrain_blanking[lev]->setVal(1.0);
192  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
193  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
194  }
195  }
196 
197  // Read in tables needed for windfarm simulations
198  // fill in Nturb multifab - number of turbines in each mesh cell
199  // write out the vtk files for wind turbine location and/or
200  // actuator disks
201  #ifdef ERF_USE_WINDFARM
202  init_windfarm(lev);
203  #endif
204 
205  // ********************************************************************************************
206  // Build the data structures for canopy model (depends upon z_phys)
207  // ********************************************************************************************
208  if (restart_chkfile.empty()) {
210  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
211  }
212  }
213 
214  //********************************************************************************************
215  // Create wall distance field for RANS model (depends upon z_phys)
216  // *******************************************************************************************
217  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
218  // Handle bottom boundary
219  poisson_wall_dist(lev);
220 
221  // Correct the wall distance for immersed bodies
227  geom[lev],
228  z_phys_cc[lev]);
229  }
230  }
231 
232  //********************************************************************************************
233  // Microphysics
234  // *******************************************************************************************
235  int q_size = micro->Get_Qmoist_Size(lev);
236  qmoist[lev].resize(q_size);
237  micro->Define(lev, solverChoice);
238  if (solverChoice.moisture_type != MoistureType::None)
239  {
240  micro->Init(lev, vars_new[lev][Vars::cons],
241  grids[lev], Geom(lev), 0.0,
242  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
243  }
244  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
245  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
246  }
247 
248  //********************************************************************************************
249  // Radiation
250  // *******************************************************************************************
251  if (solverChoice.rad_type != RadiationType::None)
252  {
253  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
254  }
255 
256  // ********************************************************************************************
257  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
258  // ********************************************************************************************
259  if (lev > 0 && cf_width >= 0) {
262  }
263 
264 #ifdef ERF_USE_PARTICLES
265  if (restart_chkfile.empty()) {
266  if (lev == 0) {
267  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,time);
268  } else {
269  particleData.Redistribute();
270  }
271  }
272 #endif
273 }
BoxArray ERFPostProcessBaseGrids(const Box &domain, bool decompose_in_z)
Definition: ERF_ChopGrids.cpp:6
void thinbody_wall_dist(std::unique_ptr< MultiFab > &wdist, Vector< IntVect > &xfaces, Vector< IntVect > &yfaces, Vector< IntVect > &zfaces, const Geometry &geomdata, std::unique_ptr< MultiFab > &z_phys_cc)
Definition: ERF_ThinBodyWallDist.cpp:12
void init_only(int lev, amrex::Real time)
Definition: ERF.cpp:1672
void poisson_wall_dist(int lev)
Definition: ERF_PoissonWallDist.cpp:20
void init_thin_body(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewLevel.cpp:705
int Get_Data_Size()
Definition: ERF_LandSurface.H:92
amrex::MultiFab * Get_Flux_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:86
void Init(const int &lev, const amrex::MultiFab &cons_in, const amrex::Geometry &geom, const amrex::Real &dt_advance)
Definition: ERF_LandSurface.H:43
void Define(const int &lev, SolverChoice &sc)
Definition: ERF_LandSurface.H:36
bool nudging_from_input_sounding
Definition: ERF_DataStruct.H:851
Here is the call graph for this function:

◆ MakeVTKFilename()

std::string ERF::MakeVTKFilename ( int  nstep)
574  {
575  // Ensure output directory exists
576  const std::string dir = "Output_HurricaneTracker";
577  if (!fs::exists(dir)) {
578  fs::create_directory(dir);
579  }
580 
581  // Construct filename with zero-padded step
582  std::ostringstream oss;
583  if(nstep==0){
584  oss << dir << "/hurricane_track_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
585  } else {
586  oss << dir << "/hurricane_track_" << std::setw(7) << std::setfill('0') << nstep+1 << ".vtk";
587  }
588 
589  return oss.str();
590 }

◆ nghost_eb_basic()

static int ERF::nghost_eb_basic ( )
inlinestaticprivate
1551  { return 5; }

◆ nghost_eb_full()

static int ERF::nghost_eb_full ( )
inlinestaticprivate
1558  { return 4; }

◆ nghost_eb_volume()

static int ERF::nghost_eb_volume ( )
inlinestaticprivate
1555  { return 5; }

◆ NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )
inlineprivatenoexcept
1350  {
1351  return datalog.size();
1352  }

◆ NumDerDataLogs()

AMREX_FORCE_INLINE int ERF::NumDerDataLogs ( )
inlineprivatenoexcept
1357  {
1358  return der_datalog.size();
1359  }

◆ NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )
inlineprivatenoexcept
1386  {
1387  return samplelinelog.size();
1388  }

◆ NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )
inlineprivatenoexcept
1412  {
1413  return sampleline.size();
1414  }

◆ NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )
inlineprivatenoexcept
1372  {
1373  return sampleptlog.size();
1374  }

◆ NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )
inlineprivatenoexcept
1399  {
1400  return samplepoint.size();
1401  }

◆ operator=() [1/2]

ERF& ERF::operator= ( const ERF other)
delete

◆ operator=() [2/2]

ERF& ERF::operator= ( ERF &&  other)
deletenoexcept

◆ ParameterSanityChecks()

void ERF::ParameterSanityChecks ( )
private
2153 {
2154  AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt[0] > 0.);
2155 
2156  // We don't allow use_real_bcs to be true if init_type is not either InitType::WRFInput or InitType::Metgrid
2157  AMREX_ALWAYS_ASSERT( !solverChoice.use_real_bcs ||
2158  ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) );
2159 
2160  AMREX_ALWAYS_ASSERT(real_width >= 0);
2161  AMREX_ALWAYS_ASSERT(real_set_width >= 0);
2162  AMREX_ALWAYS_ASSERT(real_width >= real_set_width);
2163 
2164  if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
2165  Abort("You must set cf_width >= cf_set_width >= 0");
2166  }
2167  if (max_level > 0 && cf_set_width > 0) {
2168  for (int lev = 1; lev <= max_level; lev++) {
2169  if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
2170  cf_set_width%ref_ratio[lev-1][1] != 0 ||
2171  cf_set_width%ref_ratio[lev-1][2] != 0 ) {
2172  Abort("You must set cf_width to be a multiple of ref_ratio");
2173  }
2174  }
2175  }
2176 
2177  // If fixed_mri_dt_ratio is set, it must be even
2178  if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
2179  {
2180  Abort("If you specify fixed_mri_dt_ratio, it must be even");
2181  }
2182 
2183  for (int lev = 0; lev <= max_level; lev++)
2184  {
2185  // We ignore fixed_fast_dt if not substepping
2186  if (solverChoice.substepping_type[lev] == SubsteppingType::None) {
2187  fixed_fast_dt[lev] = -1.0;
2188  }
2189 
2190  // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
2191  if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio <= 0)
2192  {
2193  Real eps = 1.e-12;
2194  int ratio = static_cast<int>( ( (1.0+eps) * fixed_dt[lev] ) / fixed_fast_dt[lev] );
2195  if (fixed_dt[lev] / fixed_fast_dt[lev] != ratio)
2196  {
2197  Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
2198  }
2199  }
2200 
2201  // If all three are specified, they must be consistent
2202  if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio > 0)
2203  {
2204  if (fixed_dt[lev] / fixed_fast_dt[lev] != fixed_mri_dt_ratio)
2205  {
2206  Abort("Dt is over-specfied");
2207  }
2208  }
2209  } // lev
2210 
2211  if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
2212  Abort("For two-way coupling you must set cf_width = 0");
2213  }
2214 }
int real_set_width
Definition: ERF.H:1148

◆ PlotFileName()

std::string ERF::PlotFileName ( int  lev) const
private

◆ PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string >  plot_var_names)
staticprivate
252 {
253  Vector<std::string> names;
254 
255  names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
256 
257  return names;
258 
259 }

◆ poisson_wall_dist()

void ERF::poisson_wall_dist ( int  lev)

Calculate wall distances using the Poisson equation

The zlo boundary is assumed to correspond to the land surface. If there are no boundary walls, then the other use case is to calculate wall distances for immersed boundaries (embedded or thin body).

See Tucker, P. G. (2003). Differential equation-based wall distance computation for DES and RANS. Journal of Computational Physics, 190(1), 229–248. https://doi.org/10.1016/S0021-9991(03)00272-9

21 {
22  BL_PROFILE("ERF::poisson_wall_dist()");
23 
24  bool havewall{false};
25  Orientation zlo(Direction::z, Orientation::low);
26  if ( ( phys_bc_type[zlo] == ERF_BC::surface_layer ) ||
27  ( phys_bc_type[zlo] == ERF_BC::no_slip_wall ) )/*||
28  ((phys_bc_type[zlo] == ERF_BC::slip_wall) && (dom_hi.z > dom_lo.z)) )*/
29  {
30  havewall = true;
31  }
32 
33  auto const& geomdata = geom[lev];
34 
35  if (havewall) {
36  if (solverChoice.mesh_type == MeshType::ConstantDz) {
37 // Comment this out to test the wall dist calc in the trivial case:
38 //#if 0
39  Print() << "Directly calculating direct wall distance for constant dz" << std::endl;
40  const Real* prob_lo = geomdata.ProbLo();
41  const Real* dx = geomdata.CellSize();
42  for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
43  const Box& bx = mfi.validbox();
44  auto dist_arr = walldist[lev]->array(mfi);
45  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
46  dist_arr(i, j, k) = prob_lo[2] + (k + 0.5) * dx[2];
47  });
48  }
49  return;
50 //#endif
51  } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
52  // TODO: Handle this trivial case
53  Error("Wall dist calc not implemented with grid stretching yet");
54  } else {
55  // TODO
56  Error("Wall dist calc not implemented over terrain yet");
57  }
58  }
59 
60  Print() << "Calculating Poisson wall distance" << std::endl;
61 
62  // Make sure the solver only sees the levels over which we are solving
63  BoxArray nba = walldist[lev]->boxArray();
64  nba.surroundingNodes();
65  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
66  Vector<BoxArray> ba_tmp; ba_tmp.push_back(nba);
67  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(walldist[lev]->DistributionMap());
68 
69  Vector<MultiFab> rhs;
70  Vector<MultiFab> phi;
71 
72  if (solverChoice.terrain_type == TerrainType::EB) {
73  amrex::Error("Wall dist calc not implemented for EB");
74  } else {
75  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
76  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
77  }
78 
79  rhs[0].setVal(-1.0);
80 
81  // Define an overset mask to set dirichlet nodes on walls
82  iMultiFab mask(ba_tmp[0], dm_tmp[0], 1, 0);
83  Vector<const iMultiFab*> overset_mask = {&mask};
84 
85  auto const dom_lo = lbound(geom[lev].Domain());
86  auto const dom_hi = ubound(geom[lev].Domain());
87 
88  // ****************************************************************************
89  // Initialize phi
90  // (It is essential that we do this in order to fill the corners; this is
91  // used if we include blanking.)
92  // ****************************************************************************
93  phi[0].setVal(0.0);
94 
95  // ****************************************************************************
96  // Interior boundaries are marked with phi=0
97  // ****************************************************************************
98  // Overset mask is 0/1: 1 means the node is an unknown. 0 means it's known.
99  mask.setVal(1);
101  Warning("Poisson distance is inaccurate for bodies in open domains that are small compared to the domain size, skipping");
102  return;
103 #if 0
104  Gpu::DeviceVector<IntVect> xfacelist, yfacelist, zfacelist;
105 
106  xfacelist.resize(solverChoice.advChoice.zero_xflux.size());
107  yfacelist.resize(solverChoice.advChoice.zero_yflux.size());
108  zfacelist.resize(solverChoice.advChoice.zero_zflux.size());
109 
110  if (xfacelist.size() > 0) {
111  Gpu::copy(amrex::Gpu::hostToDevice,
114  xfacelist.begin());
115  Print() << " masking interior xfaces" << std::endl;
116  }
117  if (yfacelist.size() > 0) {
118  Gpu::copy(amrex::Gpu::hostToDevice,
121  yfacelist.begin());
122  Print() << " masking interior yfaces" << std::endl;
123  }
124  if (zfacelist.size() > 0) {
125  Gpu::copy(amrex::Gpu::hostToDevice,
128  zfacelist.begin());
129  Print() << " masking interior zfaces" << std::endl;
130  }
131 
132  for (MFIter mfi(phi[0]); mfi.isValid(); ++mfi) {
133  const Box& bx = mfi.validbox();
134 
135  auto phi_arr = phi[0].array(mfi);
136  auto mask_arr = mask.array(mfi);
137 
138  if (xfacelist.size() > 0) {
139  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
140  for (int iface=0; iface < xfacelist.size(); ++iface) {
141  if ((i == xfacelist[iface][0]) &&
142  (j == xfacelist[iface][1]) &&
143  (k == xfacelist[iface][2]))
144  {
145  mask_arr(i, j , k ) = 0;
146  mask_arr(i, j , k+1) = 0;
147  mask_arr(i, j+1, k ) = 0;
148  mask_arr(i, j+1, k+1) = 0;
149  }
150  }
151  });
152  }
153 
154  if (yfacelist.size() > 0) {
155  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
156  for (int iface=0; iface < yfacelist.size(); ++iface) {
157  if ((i == yfacelist[iface][0]) &&
158  (j == yfacelist[iface][1]) &&
159  (k == yfacelist[iface][2]))
160  {
161  mask_arr(i , j, k ) = 0;
162  mask_arr(i , j, k+1) = 0;
163  mask_arr(i+1, j, k ) = 0;
164  mask_arr(i+1, j, k+1) = 0;
165  }
166  }
167  });
168  }
169 
170  if (zfacelist.size() > 0) {
171  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
172  for (int iface=0; iface < zfacelist.size(); ++iface) {
173  if ((i == xfacelist[iface][0]) &&
174  (j == xfacelist[iface][1]) &&
175  (k == xfacelist[iface][2]))
176  {
177  mask_arr(i , j , k) = 0;
178  mask_arr(i , j+1, k) = 0;
179  mask_arr(i+1, j , k) = 0;
180  mask_arr(i+1, j+1, k) = 0;
181  }
182  }
183  });
184  }
185  }
186 #endif
187  }
188 
189  // ****************************************************************************
190  // Setup BCs, with solid domain boundaries being dirichlet
191  // ****************************************************************************
192  amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bc3d_lo, bc3d_hi;
193  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
194  if (geom[0].isPeriodic(dir)) {
195  bc3d_lo[dir] = LinOpBCType::Periodic;
196  bc3d_hi[dir] = LinOpBCType::Periodic;
197  } else {
198  bc3d_lo[dir] = LinOpBCType::Neumann;
199  bc3d_hi[dir] = LinOpBCType::Neumann;
200  }
201  }
202  if (havewall) {
203  Print() << " Poisson zlo BC is dirichlet" << std::endl;
204  bc3d_lo[2] = LinOpBCType::Dirichlet;
205  }
206  Print() << " bc lo : " << bc3d_lo << std::endl;
207  Print() << " bc hi : " << bc3d_hi << std::endl;
208 
209  if (!solverChoice.advChoice.have_zero_flux_faces && !havewall) {
210  Error("No solid boundaries in the computational domain");
211  }
212 
213  LPInfo info;
214 /* Nodal solver cannot have hidden dimensions */
215 #if 0
216  // Allow a hidden direction if the domain is one cell wide
217  if (dom_lo.x == dom_hi.x) {
218  info.setHiddenDirection(0);
219  Print() << " domain is 2D in yz" << std::endl;
220  } else if (dom_lo.y == dom_hi.y) {
221  info.setHiddenDirection(1);
222  Print() << " domain is 2D in xz" << std::endl;
223  } else if (dom_lo.z == dom_hi.z) {
224  info.setHiddenDirection(2);
225  Print() << " domain is 2D in xy" << std::endl;
226  }
227 #endif
228 
229  // ****************************************************************************
230  // Solve nodal masked Poisson problem with MLMG
231  // TODO: different solver for terrain?
232  // ****************************************************************************
233  const Real reltol = solverChoice.poisson_reltol;
234  const Real abstol = solverChoice.poisson_abstol;
235 
236  Real sigma = 1.0;
237  Vector<EBFArrayBoxFactory const*> factory_vec = { &EBFactory(lev) };
238  MLNodeLaplacian mlpoisson(geom_tmp, ba_tmp, dm_tmp, info, factory_vec, sigma);
239 
240  mlpoisson.setDomainBC(bc3d_lo, bc3d_hi);
241 
242  if (lev > 0) {
243  mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
244  }
245 
246  mlpoisson.setLevelBC(0, nullptr);
247 
248  mlpoisson.setOversetMask(0, mask);
249 
250  // Solve
251  MLMG mlmg(mlpoisson);
252  int max_iter = 100;
253  mlmg.setMaxIter(max_iter);
254 
255  mlmg.setVerbose(mg_verbose);
256  mlmg.setBottomVerbose(0);
257 
258  mlmg.solve(GetVecOfPtrs(phi),
259  GetVecOfConstPtrs(rhs),
260  reltol, abstol);
261 
262  // Now overwrite with periodic fill outside domain and fine-fine fill inside
263  phi[0].FillBoundary(geom[lev].periodicity());
264 
265  // ****************************************************************************
266  // Compute grad(phi) to get distances
267  // - Note that phi is nodal and walldist is cell-centered
268  // - TODO: include terrain metrics for dphi/dz
269  // ****************************************************************************
270  for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
271  const Box& bx = mfi.validbox();
272 
273  const auto invCellSize = geomdata.InvCellSizeArray();
274 
275  auto const& phi_arr = phi[0].const_array(mfi);
276  auto dist_arr = walldist[lev]->array(mfi);
277 
278  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
279  Real dpdx{0}, dpdy{0}, dpdz{0};
280 
281  // dphi/dx
282  if (dom_lo.x != dom_hi.x) {
283  dpdx = 0.25 * invCellSize[0] * (
284  (phi_arr(i+1, j , k ) - phi_arr(i, j , k ))
285  + (phi_arr(i+1, j , k+1) - phi_arr(i, j , k+1))
286  + (phi_arr(i+1, j+1, k ) - phi_arr(i, j+1, k ))
287  + (phi_arr(i+1, j+1, k+1) - phi_arr(i, j+1, k+1)) );
288  }
289 
290  // dphi/dy
291  if (dom_lo.y != dom_hi.y) {
292  dpdy = 0.25 * invCellSize[1] * (
293  (phi_arr(i , j+1, k ) - phi_arr(i , j, k ))
294  + (phi_arr(i , j+1, k+1) - phi_arr(i , j, k+1))
295  + (phi_arr(i+1, j+1, k ) - phi_arr(i+1, j, k ))
296  + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j, k+1)) );
297  }
298 
299  // dphi/dz
300  if (dom_lo.z != dom_hi.z) {
301  dpdz = 0.25 * invCellSize[2] * (
302  (phi_arr(i , j , k+1) - phi_arr(i , j , k))
303  + (phi_arr(i , j+1, k+1) - phi_arr(i , j+1, k))
304  + (phi_arr(i+1, j , k+1) - phi_arr(i+1, j , k))
305  + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j+1, k)) );
306  }
307 
308  Real dp_dot_dp = dpdx*dpdx + dpdy*dpdy + dpdz*dpdz;
309  Real phi_avg = 0.125 * (
310  phi_arr(i , j , k ) + phi_arr(i , j , k+1) + phi_arr(i , j+1, k ) + phi_arr(i , j+1, k+1)
311  + phi_arr(i+1, j , k ) + phi_arr(i+1, j , k+1) + phi_arr(i+1, j+1, k ) + phi_arr(i+1, j+1, k+1) );
312  dist_arr(i, j, k) = -std::sqrt(dp_dot_dp) + std::sqrt(dp_dot_dp + 2*phi_avg);
313 
314  // DEBUG: output phi instead
315  //dist_arr(i, j, k) = phi_arr(i, j, k);
316  });
317  }
318 }
if(l_use_mynn &&start_comp<=RhoKE_comp &&end_comp >=RhoKE_comp)
Definition: ERF_DiffQKEAdjustment.H:2
static int mg_verbose
Definition: ERF.H:1123
amrex::Real poisson_reltol
Definition: ERF_DataStruct.H:798
amrex::Real poisson_abstol
Definition: ERF_DataStruct.H:797
Here is the call graph for this function:

◆ post_timestep()

void ERF::post_timestep ( int  nstep,
amrex::Real  time,
amrex::Real  dt_lev 
)
587 {
588  BL_PROFILE("ERF::post_timestep()");
589 
590 #ifdef ERF_USE_PARTICLES
591  particleData.Redistribute();
592 #endif
593 
594  if (solverChoice.coupling_type == CouplingType::TwoWay)
595  {
596  int ncomp = vars_new[0][Vars::cons].nComp();
597  for (int lev = finest_level-1; lev >= 0; lev--)
598  {
599  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
600  // m is the map scale factor at cell centers
601  // Here we pre-divide (rho S) by m^2 before refluxing
602  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
603  const Box& bx = mfi.tilebox();
604  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
605  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
606  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
607  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
608  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
609  {
610  cons_arr(i,j,k,n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
611  });
612  } else {
613  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
614  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
615  {
616  cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
617  });
618  }
619  } // mfi
620 
621  // This call refluxes all "slow" cell-centered variables
622  // (i.e. not density or (rho theta) or velocities) from the lev/lev+1 interface onto lev
623  getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 2, 2, ncomp-2);
624 
625  // Here we multiply (rho S) by m^2 after refluxing
626  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
627  const Box& bx = mfi.tilebox();
628  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
629  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
630  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
631  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
632  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
633  {
634  cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
635  });
636  } else {
637  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
638  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
639  {
640  cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
641  });
642  }
643  } // mfi
644 
645  // We need to do this before anything else because refluxing changes the
646  // values of coarse cells underneath fine grids with the assumption they'll
647  // be over-written by averaging down
648  int src_comp;
649  if (solverChoice.anelastic[lev]) {
650  src_comp = 1;
651  } else {
652  src_comp = 0;
653  }
654  int num_comp = ncomp - src_comp;
655  AverageDownTo(lev,src_comp,num_comp);
656  }
657  }
658 
659  if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
662  sum_energy_quantities(time);
663  }
664 
665  if (solverChoice.pert_type == PerturbationType::Source ||
666  solverChoice.pert_type == PerturbationType::Direct ||
667  solverChoice.pert_type == PerturbationType::CPM) {
668  if (is_it_time_for_action(nstep, time, dt_lev0, pert_interval, -1.)) {
669  turbPert.debug(time);
670  }
671  }
672 
673  if (profile_int > 0 && (nstep+1) % profile_int == 0) {
674  if (destag_profiles) {
675  // all variables cell-centered
676  write_1D_profiles(time);
677  } else {
678  // some variables staggered
680  }
681  }
682 
683  if (solverChoice.rad_type != RadiationType::None)
684  {
685  if ( rad_datalog_int > 0 &&
686  (((nstep+1) % rad_datalog_int == 0) || (nstep==0)) ) {
687  if (rad[0]->hasDatalog()) {
688  rad[0]->WriteDataLog(time+start_time);
689  }
690  }
691  }
692 
693  if (output_1d_column) {
694 #ifdef ERF_USE_NETCDF
695  if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
696  {
697  int lev_column = 0;
698  for (int lev = finest_level; lev >= 0; lev--)
699  {
700  Real dx_lev = geom[lev].CellSize(0);
701  Real dy_lev = geom[lev].CellSize(1);
702  int i_lev = static_cast<int>(std::floor(column_loc_x / dx_lev));
703  int j_lev = static_cast<int>(std::floor(column_loc_y / dy_lev));
704  if (grids[lev].contains(IntVect(i_lev,j_lev,0))) lev_column = lev;
705  }
706  writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
707  }
708 #else
709  Abort("To output 1D column files ERF must be compiled with NetCDF");
710 #endif
711  }
712 
714  {
717  {
718  bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
719  m_w2d->write_planes(istep[0], time, vars_new, is_moist);
720  }
721  }
722 
723  // Write plane/line sampler data
724  if (is_it_time_for_action(nstep+1, time, dt_lev0, sampler_interval, sampler_per) && (data_sampler) ) {
725  data_sampler->get_sample_data(geom, vars_new);
726  data_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
727  }
728 
729  // Moving terrain
730  if ( solverChoice.terrain_type == TerrainType::MovingFittedMesh )
731  {
732  for (int lev = finest_level; lev >= 0; lev--)
733  {
734  // Copy z_phs_nd and detJ_cc at end of timestep
735  MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
736  MultiFab::Copy( *detJ_cc[lev], *detJ_cc_new[lev], 0, 0, 1, detJ_cc[lev]->nGrowVect());
737  MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
738 
739  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
740  }
741  }
742 
743  bool is_hurricane_tracker_io=false;
744  ParmParse pp("erf");
745  pp.query("is_hurricane_tracker_io", is_hurricane_tracker_io);
746 
747  if (is_hurricane_tracker_io) {
748  if(nstep == 0 or (nstep+1)%m_plot3d_int_1 == 0){
749  std::string filename = MakeVTKFilename(nstep);
750  Real velmag_threshold = 1e10;
751  pp.query("hurr_track_io_velmag_greater_than", velmag_threshold);
752  if(velmag_threshold==1e10) {
753  Abort("As hurricane tracking IO is active using erf.is_hurricane_tracker_io = true"
754  " there needs to be an input erf.hurr_track_io_velmag_greater_than which specifies the"
755  " magnitude of velocity above which cells will be tagged for refinement.");
756  }
757  int levc=finest_level;
758  MultiFab& U_new = vars_new[levc][Vars::xvel];
759  MultiFab& V_new = vars_new[levc][Vars::yvel];
760  MultiFab& W_new = vars_new[levc][Vars::zvel];
761 
762  HurricaneTracker(levc, U_new, V_new, W_new, velmag_threshold, true);
763  if (ParallelDescriptor::IOProcessor()) {
765  }
766  }
767  }
768 } // post_timestep
void make_zcc(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc)
Definition: ERF_TerrainMetrics.cpp:623
static amrex::Real column_loc_y
Definition: ERF.H:1182
static std::string column_file_name
Definition: ERF.H:1183
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg(int lev)
Definition: ERF.H:1328
static amrex::Real bndry_output_planes_per
Definition: ERF.H:1188
std::string MakeVTKFilename(int nstep)
Definition: ERF_Write1DProfiles.cpp:574
static amrex::Real column_per
Definition: ERF.H:1180
amrex::Real sampler_per
Definition: ERF.H:1509
static amrex::Real column_loc_x
Definition: ERF.H:1181
static int bndry_output_planes_interval
Definition: ERF.H:1187
int sampler_interval
Definition: ERF.H:1508
static int output_1d_column
Definition: ERF.H:1178
void WriteVTKPolyline(const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
Definition: ERF_Write1DProfiles.cpp:593
static int column_interval
Definition: ERF.H:1179
Here is the call graph for this function:

◆ post_update()

void ERF::post_update ( amrex::MultiFab &  state_mf,
amrex::Real  time,
const amrex::Geometry &  geom 
)
private

◆ print_banner()

void ERF::print_banner ( MPI_Comm  comm,
std::ostream &  out 
)
static
65  : " << msg << std::endl;
66 }
67 
68 void ERF::print_banner (MPI_Comm comm, std::ostream& out)
69 {
70 #ifdef AMREX_USE_MPI
71  int irank = 0;
72  int num_ranks = 1;
73  MPI_Comm_size(comm, &num_ranks);
74  MPI_Comm_rank(comm, &irank);
75 
76  // Only root process does the printing
77  if (irank != 0) return;
78 #else
79  amrex::ignore_unused(comm);
80 #endif
81 
82  auto etime = std::chrono::system_clock::now();
83  auto etimet = std::chrono::system_clock::to_time_t(etime);
84 #ifndef _WIN32
85  char time_buf[64];
86  ctime_r(&etimet, time_buf);
87  const std::string tstamp(time_buf);
88 #else
89  char* time_buf = new char[64];
90  ctime_s(time_buf, 64, &etimet);
91  const std::string tstamp(time_buf);
92 #endif
93 
94  const char* githash1 = amrex::buildInfoGetGitHash(1);
95  const char* githash2 = amrex::buildInfoGetGitHash(2);
96 
97  // clang-format off
98  out << dbl_line
99  << " ERF (https://github.com/erf-model/ERF)"
100  << std::endl << std::endl
101  << " ERF Git SHA :: " << githash1 << std::endl
102  << " AMReX Git SHA :: " << githash2 << std::endl
103  << " AMReX version :: " << amrex::Version() << std::endl << std::endl
104  << " Exec. time :: " << tstamp
105  << " Build time :: " << amrex::buildInfoGetBuildDate() << std::endl
106  << " C++ compiler :: " << amrex::buildInfoGetComp()
107  << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
108  << " MPI :: "
109 #ifdef AMREX_USE_MPI
110  << "ON (Num. ranks = " << num_ranks << ")" << std::endl
111 #else
112  << "OFF " << std::endl
113 #endif
114  << " GPU :: "
115 #ifdef AMREX_USE_GPU
116  << "ON "
117 #if defined(AMREX_USE_CUDA)
118  << "(Backend: CUDA)"
119 #elif defined(AMREX_USE_HIP)
120  << "(Backend: HIP)"
121 #elif defined(AMREX_USE_SYCL)
122  << "(Backend: SYCL)"
123 #endif
124  << std::endl
125 #else
126  << "OFF" << std::endl
127 #endif
128  << " OpenMP :: "
129 #ifdef AMREX_USE_OMP
130  << "ON (Num. threads = " << omp_get_max_threads() << ")" << std::endl
131 #else
132  << "OFF" << std::endl
133 #endif
134  << std::endl;
135 
ERF()
Definition: ERF.cpp:104
const char * buildInfoGetBuildDate()
const char * buildInfoGetComp()
const char * buildInfoGetCompVersion()

Referenced by main().

Here is the caller graph for this function:

◆ print_error()

void ERF::print_error ( MPI_Comm  comm,
const std::string &  msg 
)
static
43  :
44  input_file : Input file with simulation settings
45 
46 Optional:
47  param=value : Overrides for parameters during runtime
48 )doc" << std::endl;
49 }
50 
51 void ERF::print_error (MPI_Comm comm, const std::string& msg)
52 {
53 #ifdef AMREX_USE_MPI
54  int irank = 0;
55  int num_ranks = 1;
56  MPI_Comm_size(comm, &num_ranks);
57  MPI_Comm_rank(comm, &irank);
58 

Referenced by main().

Here is the caller graph for this function:

◆ print_summary()

static void ERF::print_summary ( std::ostream &  )
static

◆ print_tpls()

void ERF::print_tpls ( std::ostream &  out)
static
140  ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
141  << dash_line << std::endl;
142  // clang-format on
143 }
144 
145 void ERF::print_tpls (std::ostream& out)
146 {
147  amrex::Vector<std::string> tpls;
148 
149 #ifdef ERF_USE_NETCDF
150  tpls.push_back(std::string("NetCDF ") + NC_VERSION);
151 #endif
152 #ifdef AMREX_USE_SUNDIALS
153  tpls.push_back(std::string("SUNDIALS ") + SUNDIALS_VERSION);
154 #endif
155 
156  if (!tpls.empty()) {
157  out << " Enabled third-party libraries: ";
158  for (const auto& val : tpls) {
static void print_tpls(std::ostream &)
Definition: ERF_ConsoleIO.cpp:137

◆ print_usage()

void ERF::print_usage ( MPI_Comm  comm,
std::ostream &  out 
)
static
27 {
28 #ifdef AMREX_USE_MPI
29  int irank = 0;
30  int num_ranks = 1;
31  MPI_Comm_size(comm, &num_ranks);
32  MPI_Comm_rank(comm, &irank);
33 
34  // Only root process does the printing
35  if (irank != 0) return;
36 #else
37  amrex::ignore_unused(comm);
38 #endif
39 
40  out << R"doc(Usage:
41  ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

◆ project_momenta()

void ERF::project_momenta ( int  lev,
amrex::Real  dt,
amrex::Vector< amrex::MultiFab > &  vars 
)

Project the single-level momenta to enforce the anelastic constraint Note that the level may or may not be level 0.

42 {
43  BL_PROFILE("ERF::project_momenta()");
44 
45  // Make sure the solver only sees the levels over which we are solving
46  Vector<BoxArray> ba_tmp; ba_tmp.push_back(mom_mf[Vars::cons].boxArray());
47  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap());
48  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
49 
50  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
51 
52  Vector<MultiFab> rhs;
53  Vector<MultiFab> phi;
54 
55  if (solverChoice.terrain_type == TerrainType::EB)
56  {
57  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
58  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1, MFInfo(), EBFactory(lev));
59  } else {
60  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
61  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
62  }
63 
64  MultiFab rhs_lev(rhs[0], make_alias, 0, 1);
65 
66  auto dxInv = geom[lev].InvCellSizeArray();
67 
68  // Inflow on an x-face -- note only the normal velocity is used in the projection
69  if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow") {
71  IntVect{1,0,0},t_new[lev],BCVars::xvel_bc,false);
72  }
73 
74  // Inflow on a y-face -- note only the normal velocity is used in the projection
75  if (domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
77  IntVect{0,1,0},t_new[lev],BCVars::yvel_bc,false);
78  }
79 
80  if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow" ||
81  domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
82  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
83  vars_new[lev][Vars::yvel], IntVect{0},
84  vars_new[lev][Vars::zvel], IntVect{0},
85  vars_new[lev][Vars::cons],
86  mom_mf[IntVars::xmom],
87  mom_mf[IntVars::ymom],
88  mom_mf[IntVars::zmom],
89  Geom(lev).Domain(),
91  }
92 
93  // If !fixed_density, we must convert (rho u) which came in
94  // to (rho0 u) which is what we will project
96  ConvertForProjection(mom_mf[Vars::cons], r_hse,
97  mom_mf[IntVars::xmom],
98  mom_mf[IntVars::ymom],
99  mom_mf[IntVars::zmom],
100  Geom(lev).Domain(),
102  }
103 
104  //
105  // ****************************************************************************
106  // Now convert the rho0w MultiFab to hold Omega rather than rhow
107  // ****************************************************************************
108  //
109  if (solverChoice.mesh_type == MeshType::VariableDz)
110  {
111  for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
112  {
113  const Array4<Real const>& rho0u_arr = mom_mf[IntVars::xmom].const_array(mfi);
114  const Array4<Real const>& rho0v_arr = mom_mf[IntVars::ymom].const_array(mfi);
115  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
116 
117  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
118  const Array4<Real const>& mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
119  const Array4<Real const>& mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
120 
121  //
122  // Define Omega from (rho0 W) but store it in the same array
123  //
124  Box tbz = mfi.nodaltilebox(2);
125  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
126  if (k == 0) {
127  rho0w_arr(i,j,k) = Real(0.0);
128  } else {
129  Real rho0w = rho0w_arr(i,j,k);
130  rho0w_arr(i,j,k) = OmegaFromW(i,j,k,rho0w,
131  rho0u_arr,rho0v_arr,
132  mf_u,mf_v,z_nd,dxInv);
133  }
134  });
135  } // mfi
136  }
137 
138  // ****************************************************************************
139  // Compute divergence which will form RHS
140  // Note that we replace "rho0w" with the contravariant momentum, Omega
141  // ****************************************************************************
142  Array<MultiFab const*, AMREX_SPACEDIM> rho0_u_const;
143  rho0_u_const[0] = &mom_mf[IntVars::xmom];
144  rho0_u_const[1] = &mom_mf[IntVars::ymom];
145  rho0_u_const[2] = &mom_mf[IntVars::zmom];
146 
147  compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
148 
149  if (solverChoice.mesh_type == MeshType::VariableDz) {
150  MultiFab::Multiply(rhs_lev, *detJ_cc[lev], 0, 0, 1, 0);
151  }
152 
153  // Max norm over the entire MultiFab
154  Real rhsnorm = rhs_lev.norm0();
155 
156  if (mg_verbose > 0) {
157  Print() << "Max/L2 norm of divergence before solve at level " << lev << " : " << rhsnorm << " " <<
158  rhs_lev.norm2() << " and sum " << rhs_lev.sum() << std::endl;
159  }
160 
161 
162  if (lev > 0)
163  {
164  Vector<Real> sum; sum.resize(subdomains[lev].size(),Real(0.));
165 
166  for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
167  {
168  Box bx = mfi.validbox();
169  for (int i = 0; i < subdomains[lev].size(); ++i) {
170  if (subdomains[lev][i].intersects(bx)) {
171  sum[i] += rhs_lev[mfi.index()].template sum<RunOn::Device>(0);
172  }
173  }
174  }
175  ParallelDescriptor::ReduceRealSum(sum.data(), sum.size());
176 
177  for (int i = 0; i < subdomains[lev].size(); ++i) {
178  sum[i] /= static_cast<Real>(subdomains[lev][i].numPts());
179  }
180 
181  for ( MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
182  {
183  Box bx = mfi.validbox();
184  for (int i = 0; i < subdomains[lev].size(); ++i) {
185  if (subdomains[lev][i].intersects(bx)) {
186  rhs_lev[mfi.index()].template minus<RunOn::Device>(sum[i]);
187  if (mg_verbose > 1) {
188  amrex::Print() << " Subtracting " << sum[i] << " in " << rhs_lev[mfi.index()].box() << std::endl;
189  }
190  }
191  }
192  }
193  }
194 
195  // ****************************************************************************
196  //
197  // No need to build the solver if RHS == 0
198  //
199  if (rhsnorm <= solverChoice.poisson_abstol) return;
200  // ****************************************************************************
201 
202  // ****************************************************************************
203  // Initialize phi to 0
204  // (It is essential that we do this in order to fill the corners; these are never
205  // used but the Saxpy requires the values to be initialized.)
206  // ****************************************************************************
207  phi[0].setVal(0.0);
208 
209  Real start_step = static_cast<Real>(ParallelDescriptor::second());
210 
211  // ****************************************************************************
212  // Allocate fluxes
213  // ****************************************************************************
214  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
215  fluxes.resize(1);
216  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
217  if (solverChoice.terrain_type == TerrainType::EB) {
218  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
219  } else {
220  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
221  }
222  }
223 
224  // ****************************************************************************
225  // Choose the solver and solve
226  // ****************************************************************************
227 
228  std::map<int,int> index_map;
229 
230  BoxArray ba(grids[lev]);
231 
232  Vector<MultiFab> rhs_sub; rhs_sub.resize(1);
233  Vector<MultiFab> phi_sub; phi_sub.resize(1);
234  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes_sub; fluxes_sub.resize(1);
235 
236  MultiFab ax_sub, ay_sub, znd_sub;
237 
238  for (int i = 0; i < subdomains[lev].size(); ++i)
239  {
240  if (mg_verbose > 0) {
241  amrex::Print() << " Solving in subdomain " << i << " of " << subdomains[lev].size() << " bins at level " << lev << std::endl;
242  }
243 
244  BoxList bl_sub;
245  Vector<int> dm_sub;
246 
247  for (int j = 0; j < ba.size(); j++)
248  {
249  if (subdomains[lev][i].intersects(ba[j]))
250  {
251  // amrex::Print() <<" INTERSECTS I " << i << " " << j << " " << grids[lev][j] << std::endl;
252  //
253  // Note that bl_sub.size() is effectively a counter which is
254  // incremented above
255  //
256  // if (ParallelDescriptor::MyProc() == j) {
257  // }
258  index_map[bl_sub.size()] = j;
259 
260  // amrex::Print() <<" PUSHING BACK " << j << " " << index_map[bl_sub.size()] << std::endl;
261  bl_sub.push_back(grids[lev][j]);
262  dm_sub.push_back(dmap[lev][j]);
263  } // intersects
264 
265  } // loop over ba (j)
266 
267  BoxArray ba_sub(bl_sub);
268 
269  // Define MultiFabs that hold only the data in this particular subdomain
270  rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs[0].nGrowVect(), MFInfo{}.SetAlloc(false));
271  phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi[0].nGrowVect(), MFInfo{}.SetAlloc(false));
272 
273  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
274  fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
275  IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false));
276  }
277 
278  // Link the new MultiFabs to the FABs in the original MultiFabs (no copy required)
279  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
280  int orig_index = index_map[mfi.index()];
281  // amrex::Print() << " INDEX " << orig_index << " TO " << mfi.index() << std::endl;
282  rhs_sub[0].setFab(mfi, FArrayBox(rhs[0][orig_index], amrex::make_alias, 0, 1));
283  phi_sub[0].setFab(mfi, FArrayBox(phi[0][orig_index], amrex::make_alias, 0, 1));
284  fluxes_sub[0][0].setFab(mfi,FArrayBox(fluxes[0][0][orig_index], amrex::make_alias, 0, 1));
285  fluxes_sub[0][1].setFab(mfi,FArrayBox(fluxes[0][1][orig_index], amrex::make_alias, 0, 1));
286  fluxes_sub[0][2].setFab(mfi,FArrayBox(fluxes[0][2][orig_index], amrex::make_alias, 0, 1));
287  }
288 
289  if (solverChoice.mesh_type == MeshType::VariableDz) {
290  ax_sub.define(convert(ba_sub,IntVect(1,0,0)), DistributionMapping(dm_sub), 1,
291  ax[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
292  ay_sub.define(convert(ba_sub,IntVect(0,1,0)), DistributionMapping(dm_sub), 1,
293  ay[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
294  znd_sub.define(convert(ba_sub,IntVect(1,1,1)), DistributionMapping(dm_sub), 1,
295  z_phys_nd[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
296 
297  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
298  int orig_index = index_map[mfi.index()];
299  ax_sub.setFab(mfi, FArrayBox((*ax[lev])[orig_index], amrex::make_alias, 0, 1));
300  ay_sub.setFab(mfi, FArrayBox((*ay[lev])[orig_index], amrex::make_alias, 0, 1));
301  znd_sub.setFab(mfi, FArrayBox((*z_phys_nd[lev])[orig_index], amrex::make_alias, 0, 1));
302  }
303  }
304 
305  if (lev > 0) {
306  amrex::Print() << "RHSSUB BA " << rhs_sub[0].boxArray() << std::endl;
307  }
308 
309  // ****************************************************************************
310  // EB
311  // ****************************************************************************
312  if (solverChoice.terrain_type == TerrainType::EB) {
313  solve_with_EB_mlmg(lev, rhs_sub, phi_sub, fluxes_sub);
314  } else {
315 
316  // ****************************************************************************
317  // No terrain or grid stretching
318  // ****************************************************************************
319  if (solverChoice.mesh_type == MeshType::ConstantDz) {
320 #ifdef ERF_USE_FFT
321  if (use_fft) {
322  Box my_region(subdomains[lev][i].minimalBox());
323  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][i].numPts());
324  if (boxes_make_rectangle) {
325  solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
326  } else {
327  amrex::Warning("FFT won't work unless the union of boxes is rectangular: defaulting to MLMG");
328  solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub);
329  }
330  } else {
331  solve_with_mlmg(lev, rhs, phi, fluxes);
332  }
333 #else
334  if (use_fft) {
335  amrex::Warning("You set use_fft=true but didn't build with USE_FFT = TRUE; defaulting to MLMG");
336  }
337  solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub);
338 #endif
339  } // No terrain or grid stretching
340 
341  // ****************************************************************************
342  // Grid stretching (flat terrain)
343  // ****************************************************************************
344  else if (solverChoice.mesh_type == MeshType::StretchedDz) {
345 #ifndef ERF_USE_FFT
346  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT solver");
347 #else
348  Box my_region(subdomains[lev][i].minimalBox());
349  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][i].numPts());
350  if (!boxes_make_rectangle) {
351  amrex::Abort("FFT won't work unless the union of boxes is rectangular");
352  } else {
353  if (!use_fft) {
354  amrex::Warning("Using FFT even though you didn't set use_fft to true; it's the best choice");
355  }
356  solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
357  }
358 #endif
359  } // grid stretching
360 
361  // ****************************************************************************
362  // General terrain
363  // ****************************************************************************
364  else if (solverChoice.mesh_type == MeshType::VariableDz) {
365 #ifdef ERF_USE_FFT
366  Box my_region(subdomains[lev][i].minimalBox());
367  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][i].numPts());
368  if (!boxes_make_rectangle) {
369  amrex::Abort("FFT preconditioner for GMRES won't work unless the union of boxes is rectangular");
370  } else {
371  solve_with_gmres(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0], ax_sub, ay_sub, znd_sub);
372  }
373 #else
374  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT preconditioner for GMRES");
375 #endif
376  } // general terrain
377 
378  } // not EB
379 
380  } // loop over subdomains (i)
381 
382  // ****************************************************************************
383  // Print time in solve
384  // ****************************************************************************
385  Real end_step = static_cast<Real>(ParallelDescriptor::second());
386  if (mg_verbose > 0) {
387  amrex::Print() << "Time in solve " << end_step - start_step << std::endl;
388  }
389 
390  // ****************************************************************************
391  // Subtract dt grad(phi) from the momenta (rho0u, rho0v, Omega)
392  // ****************************************************************************
393  MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0);
394  MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0);
395  MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0);
396 
397  // ****************************************************************************
398  // Define gradp from fluxes -- note that fluxes is dt * change in Gp
399  // ****************************************************************************
400  MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
401  MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
402  MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
403 
404  gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
405  gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
406  gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
407 
408  //
409  // This call is only to verify the divergence after the solve
410  // It is important we do this before computing the rho0w_arr from Omega back to rho0w
411  //
412  // ****************************************************************************
413  // THIS IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE
414  // ****************************************************************************
415  //
416  if (mg_verbose > 0)
417  {
418  compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
419 
420  if (solverChoice.mesh_type == MeshType::VariableDz) {
421  MultiFab::Multiply(rhs_lev, *detJ_cc[lev], 0, 0, 1, 0);
422  }
423 
424  Print() << "Max/L2 norm of divergence after solve at level " << lev << " : " << rhs_lev.norm0() << " " <<
425  rhs_lev.norm2() << " and sum " << rhs_lev.sum() << std::endl;
426 
427 #if 0
428  // FOR DEBUGGING ONLY
429  for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
430  {
431  const Array4<Real const>& rhs_arr = rhs_lev.const_array(mfi);
432  Box bx = mfi.validbox();
433  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
434  if (std::abs(rhs_arr(i,j,k)) > 1.e-10) {
435  amrex::AllPrint() << "RHS AFTER SOLVE AT " <<
436  IntVect(i,j,k) << " " << rhs_arr(i,j,k) << std::endl;
437  }
438  });
439  } // mfi
440 #endif
441 
442  } // mg_verbose
443 
444  //
445  // ****************************************************************************
446  // Now convert the rho0w MultiFab back to holding (rho0w) rather than Omega
447  // ****************************************************************************
448  //
449  if (solverChoice.mesh_type == MeshType::VariableDz)
450  {
451  for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi)
452  {
453  Box tbz = mfi.nodaltilebox(2);
454  const Array4<Real >& rho0u_arr = mom_mf[IntVars::xmom].array(mfi);
455  const Array4<Real >& rho0v_arr = mom_mf[IntVars::ymom].array(mfi);
456  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
457  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
458  const Array4<Real const>& mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
459  const Array4<Real const>& mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
460  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
461  Real omega = rho0w_arr(i,j,k);
462  rho0w_arr(i,j,k) = WFromOmega(i,j,k,omega,
463  rho0u_arr,rho0v_arr,
464  mf_u,mf_v,z_nd,dxInv);
465  });
466  } // mfi
467  }
468 
469  // If !fixed_density, we must convert (rho0 u) back
470  // to (rho0 u) which is what we will pass back out
472  ConvertForProjection(r_hse, mom_mf[Vars::cons],
473  mom_mf[IntVars::xmom],
474  mom_mf[IntVars::ymom],
475  mom_mf[IntVars::zmom],
476  Geom(lev).Domain(),
478  }
479 
480  // ****************************************************************************
481  // Update pressure variable with phi -- note that phi is dt * change in pressure
482  // ****************************************************************************
483  MultiFab::Saxpy(pp_inc[lev], 1.0/l_dt, phi[0],0,0,1,1);
484 }
void ConvertForProjection(const MultiFab &den_div, const MultiFab &den_mlt, MultiFab &xmom, MultiFab &ymom, MultiFab &zmom, const Box &domain, const Vector< BCRec > &domain_bcs_type_h)
Definition: ERF_ConvertForProjection.cpp:25
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real OmegaFromW(int &i, int &j, int &k, amrex::Real w, const amrex::Array4< const amrex::Real > &u_arr, const amrex::Array4< const amrex::Real > &v_arr, const amrex::Array4< const amrex::Real > &mf_u, const amrex::Array4< const amrex::Real > &mf_v, const amrex::Array4< const amrex::Real > &z_nd, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &dxInv)
Definition: ERF_TerrainMetrics.H:415
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real WFromOmega(int &i, int &j, int &k, amrex::Real omega, const amrex::Array4< const amrex::Real > &u_arr, const amrex::Array4< const amrex::Real > &v_arr, const amrex::Array4< const amrex::Real > &mf_u, const amrex::Array4< const amrex::Real > &mf_v, const amrex::Array4< const amrex::Real > &z_nd, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &dxInv)
Definition: ERF_TerrainMetrics.H:465
static bool use_fft
Definition: ERF.H:1124
void compute_divergence(int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
Definition: ERF_ComputeDivergence.cpp:10
void solve_with_mlmg(int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
Definition: ERF_SolveWithMLMG.cpp:40
void solve_with_gmres(int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &znd_sub)
Definition: ERF_SolveWithGMRES.cpp:12
void solve_with_EB_mlmg(int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
Definition: ERF_SolveWithEBMLMG.cpp:19
@ omega
Definition: ERF_Morrison.H:53
Here is the call graph for this function:

◆ project_velocity()

void ERF::project_velocity ( int  lev,
amrex::Real  dt 
)

Project the single-level velocity field to enforce the anelastic constraint Note that the level may or may not be level 0.

11 {
12  BL_PROFILE("ERF::project_velocity()");
13  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
14  vars_new[lev][Vars::yvel], IntVect{0},
15  vars_new[lev][Vars::zvel], IntVect{0},
16  vars_new[lev][Vars::cons],
17  rU_new[lev], rV_new[lev], rW_new[lev],
18  Geom(lev).Domain(), domain_bcs_type);
19 
20  Vector<MultiFab> tmp_mom;
21 
22  tmp_mom.push_back(MultiFab(vars_new[lev][Vars::cons],make_alias,0,1));
23  tmp_mom.push_back(MultiFab(rU_new[lev],make_alias,0,1));
24  tmp_mom.push_back(MultiFab(rV_new[lev],make_alias,0,1));
25  tmp_mom.push_back(MultiFab(rW_new[lev],make_alias,0,1));
26 
27  project_momenta(lev, l_dt, tmp_mom);
28 
30  vars_new[lev][Vars::yvel],
31  vars_new[lev][Vars::zvel],
32  vars_new[lev][Vars::cons],
33  rU_new[lev], rV_new[lev], rW_new[lev],
34  Geom(lev).Domain(), domain_bcs_type);
35  }
void project_momenta(int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
Definition: ERF_PoissonSolve.cpp:41
Here is the call graph for this function:

◆ project_velocity_tb()

void ERF::project_velocity_tb ( int  lev,
amrex::Real  dt,
amrex::Vector< amrex::MultiFab > &  vars 
)

Project the single-level velocity field to enforce incompressibility with a thin body

21 {
22  BL_PROFILE("ERF::project_velocity_tb()");
23  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
24 
25  // Make sure the solver only sees the levels over which we are solving
26  Vector<BoxArray> ba_tmp; ba_tmp.push_back(vmf[Vars::cons].boxArray());
27  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(vmf[Vars::cons].DistributionMap());
28  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
29 
30  // Use the default settings
31  LPInfo info;
32  std::unique_ptr<MLPoisson> p_mlpoisson;
33 #if 0
34  if (overset_imask[0]) {
35  // Add overset mask around thin body
36  p_mlpoisson = std::make_unique<MLPoisson>(geom, grids, dmap, GetVecOfConstPtrs(overset_imask), info);
37  }
38  else
39 #endif
40  {
41  // Use the default settings
42  p_mlpoisson = std::make_unique<MLPoisson>(geom_tmp, ba_tmp, dm_tmp, info);
43  }
44 
45  auto bclo = get_projection_bc(Orientation::low);
46  auto bchi = get_projection_bc(Orientation::high);
47  bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true;
48  p_mlpoisson->setDomainBC(bclo, bchi);
49 
50  if (lev > 0) {
51  p_mlpoisson->setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
52  }
53 
54  p_mlpoisson->setLevelBC(0, nullptr);
55 
56  Vector<MultiFab> rhs;
57  Vector<MultiFab> phi;
58  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
59  Vector<Array<MultiFab,AMREX_SPACEDIM> > deltaf; // f^* - f^{n-1}
60  Vector<Array<MultiFab,AMREX_SPACEDIM> > u_plus_dtdf; // u + dt*deltaf
61 
62  // Used to pass array of const MFs to ComputeDivergence
63  Array<MultiFab const*, AMREX_SPACEDIM> u;
64 
65  rhs.resize(1);
66  phi.resize(1);
67  fluxes.resize(1);
68  deltaf.resize(1);
69  u_plus_dtdf.resize(1);
70 
71  rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
72  phi[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
73  rhs[0].setVal(0.0);
74  phi[0].setVal(0.0);
75 
76  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
77  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
78  u_plus_dtdf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
79 
80  deltaf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
81  deltaf[0][idim].setVal(0.0); // start with f^* == f^{n-1}
82  }
83 
84 #if 0
85  // DEBUG
86  u[0] = &(vmf[Vars::xvel]);
87  u[1] = &(vmf[Vars::yvel]);
88  u[2] = &(vmf[Vars::zvel]);
89  computeDivergence(rhs[0], u, geom[0]);
90  Print() << "Max norm of divergence before solve at level 0 : " << rhs[0].norm0() << std::endl;
91 #endif
92 
93  for (int itp = 0; itp < solverChoice.ncorr; ++itp)
94  {
95  // Calculate u + dt*deltaf
96  for (int idim = 0; idim < 3; ++idim) {
97  MultiFab::Copy(u_plus_dtdf[0][idim], deltaf[0][idim], 0, 0, 1, 0);
98  u_plus_dtdf[0][0].mult(-l_dt,0,1,0);
99  }
100  MultiFab::Add(u_plus_dtdf[0][0], vmf[Vars::xvel], 0, 0, 1, 0);
101  MultiFab::Add(u_plus_dtdf[0][1], vmf[Vars::yvel], 0, 0, 1, 0);
102  MultiFab::Add(u_plus_dtdf[0][2], vmf[Vars::zvel], 0, 0, 1, 0);
103 
104  u[0] = &(u_plus_dtdf[0][0]);
105  u[1] = &(u_plus_dtdf[0][1]);
106  u[2] = &(u_plus_dtdf[0][2]);
107  computeDivergence(rhs[0], u, geom_tmp[0]);
108 
109 #if 0
110  // DEBUG
111  if (itp==0) {
112  for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
113  {
114  const Box& bx = mfi.tilebox();
115  const Array4<Real const>& divU = rhs[0].const_array(mfi);
116  const Array4<Real const>& uarr = vmf[Vars::xvel].const_array(mfi);
117  const Array4<Real const>& varr = vmf[Vars::yvel].const_array(mfi);
118  const Array4<Real const>& warr = vmf[Vars::zvel].const_array(mfi);
119  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
120  {
121  if ((i>=120) && (i<=139) && (j==0) && ((k>=127)&&(k<=128))) {
122  amrex::AllPrint() << "before project div"<<IntVect(i,j,k)<<" = "<< divU(i,j,k)
123  << " u: " << uarr(i,j,k) << " " << uarr(i+1,j,k)
124  << " v: " << varr(i,j,k) << " " << varr(i,j+1,k)
125  << " w: " << warr(i,j,k) << " " << warr(i,j,k+1)
126  << std::endl;
127  }
128  });
129  }
130  }
131 #endif
132 
133  // If all Neumann BCs, adjust RHS to make sure we can converge
134  if (need_adjust_rhs) {
135  Real offset = volWgtSumMF(lev, rhs[0], 0, false);
136  // amrex::Print() << "Poisson solvability offset = " << offset << std::endl;
137  rhs[0].plus(-offset, 0, 1);
138  }
139 
140  // Initialize phi to 0
141  phi[0].setVal(0.0);
142 
143  MLMG mlmg(*p_mlpoisson);
144  int max_iter = 100;
145  mlmg.setMaxIter(max_iter);
146 
147  mlmg.setVerbose(mg_verbose);
148  //mlmg.setBottomVerbose(mg_verbose);
149 
150  // solve for dt*p
151  mlmg.solve(GetVecOfPtrs(phi),
152  GetVecOfConstPtrs(rhs),
155 
156  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
157 
158  // Calculate new intermediate body force with updated gradp
159  if (thin_xforce[lev]) {
160  MultiFab::Copy( deltaf[0][0], fluxes[0][0], 0, 0, 1, 0);
161  ApplyInvertedMask(deltaf[0][0], *xflux_imask[0]);
162  }
163  if (thin_yforce[lev]) {
164  MultiFab::Copy( deltaf[0][1], fluxes[0][1], 0, 0, 1, 0);
165  ApplyInvertedMask(deltaf[0][1], *yflux_imask[0]);
166  }
167  if (thin_zforce[lev]) {
168  MultiFab::Copy( deltaf[0][2], fluxes[0][2], 0, 0, 1, 0);
169  ApplyInvertedMask(deltaf[0][2], *zflux_imask[0]);
170  }
171 
172  // DEBUG
173  // for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
174  // {
175  // const Box& bx = mfi.tilebox();
176  // const Array4<Real const>& dfz_arr = deltaf[0][2].const_array(mfi);
177  // ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
178  // {
179  // if ((i>=120) && (i<=139) && (j==0) && (k==128)) {
180  // amrex::AllPrint()
181  // << " piter" << itp
182  // << " dfz"<<IntVect(i,j,k)<<" = "<< dfz_arr(i,j,k)
183  // << std::endl;
184  // }
185  // });
186  // }
187 
188  // Update pressure variable with phi -- note that phi is change in pressure, not the full pressure
189  MultiFab::Saxpy(pp_inc[lev], 1.0, phi[0],0,0,1,0);
190 
191  // Subtract grad(phi) from the velocity components
192  Real beta = 1.0;
193  MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
194  MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
195  MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
196  if (thin_xforce[lev]) {
197  ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
198  }
199  if (thin_yforce[lev]) {
200  ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
201  }
202  if (thin_zforce[lev]) {
203  ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
204  }
205  } // itp: pressure-force iterations
206 
207  // ****************************************************************************
208  // Define gradp from fluxes -- note that fluxes is dt * change in Gp
209  // ****************************************************************************
210  MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
211  MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
212  MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
213 
214  gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
215  gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
216  gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
217 
218  // Subtract grad(phi) from the velocity components
219 // Real beta = 1.0;
220 // for (int ilev = lev_min; ilev <= lev_max; ++ilev) {
221 // MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
222 // MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
223 // MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
224 // if (thin_xforce[lev]) {
225 // ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
226 // }
227 // if (thin_yforce[lev]) {
228 // ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
229 // }
230 // if (thin_zforce[lev]) {
231 // ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
232 // }
233 // }
234 
235 #if 0
236  // Confirm that the velocity is now divergence free
237  u[0] = &(vmf[Vars::xvel]);
238  u[1] = &(vmf[Vars::yvel]);
239  u[2] = &(vmf[Vars::zvel]);
240  computeDivergence(rhs[0], u, geom_tmp[0]);
241  Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
242 
243 #endif
244 }
AMREX_FORCE_INLINE IntVect offset(const int face_dir, const int normal)
Definition: ERF_ReadBndryPlanes.cpp:28
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyInvertedMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:417
amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > get_projection_bc(amrex::Orientation::Side side) const noexcept
Definition: ERF_SolveWithMLMG.cpp:17
bool projection_has_dirichlet(amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > bcs) const
Definition: ERF_PoissonSolve_tb.cpp:8
amrex::Real volWgtSumMF(int lev, const amrex::MultiFab &mf, int comp, bool finemask)
Definition: ERF_WriteScalarProfiles.cpp:651
int ncorr
Definition: ERF_DataStruct.H:796
Here is the call graph for this function:

◆ projection_has_dirichlet()

bool ERF::projection_has_dirichlet ( amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM >  bcs) const
9 {
10  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
11  if (bcs[dir] == LinOpBCType::Dirichlet) return true;
12  }
13  return false;
14 }

◆ ReadCheckpointFile()

void ERF::ReadCheckpointFile ( )

ERF function for reading data from a checkpoint file during restart.

440 {
441  Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
442 
443  // Header
444  std::string File(restart_chkfile + "/Header");
445 
446  VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
447 
448  Vector<char> fileCharPtr;
449  ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
450  std::string fileCharPtrString(fileCharPtr.dataPtr());
451  std::istringstream is(fileCharPtrString, std::istringstream::in);
452 
453  std::string line, word;
454 
455  int chk_ncomp_cons, chk_ncomp;
456 
457  // read in title line
458  std::getline(is, line);
459 
460  // read in finest_level
461  is >> finest_level;
462  GotoNextLine(is);
463 
464  // read the number of components
465  // for each variable we store
466 
467  // conservative, cell-centered vars
468  is >> chk_ncomp_cons;
469  GotoNextLine(is);
470 
471  // x-velocity on faces
472  is >> chk_ncomp;
473  GotoNextLine(is);
474  AMREX_ASSERT(chk_ncomp == 1);
475 
476  // y-velocity on faces
477  is >> chk_ncomp;
478  GotoNextLine(is);
479  AMREX_ASSERT(chk_ncomp == 1);
480 
481  // z-velocity on faces
482  is >> chk_ncomp;
483  GotoNextLine(is);
484  AMREX_ASSERT(chk_ncomp == 1);
485 
486  // read in array of istep
487  std::getline(is, line);
488  {
489  std::istringstream lis(line);
490  int i = 0;
491  while (lis >> word) {
492  istep[i++] = std::stoi(word);
493  }
494  }
495 
496  // read in array of dt
497  std::getline(is, line);
498  {
499  std::istringstream lis(line);
500  int i = 0;
501  while (lis >> word) {
502  dt[i++] = std::stod(word);
503  }
504  }
505 
506  // read in array of t_new
507  std::getline(is, line);
508  {
509  std::istringstream lis(line);
510  int i = 0;
511  while (lis >> word) {
512  t_new[i++] = std::stod(word);
513  }
514  }
515 
516  for (int lev = 0; lev <= finest_level; ++lev) {
517  // read in level 'lev' BoxArray from Header
518  BoxArray ba;
519  ba.readFrom(is);
520  GotoNextLine(is);
521 
522  // create a distribution mapping
523  DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
524 
525  MakeNewLevelFromScratch (lev, t_new[lev], ba, dm);
526  }
527 
528  // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
529  // NOTE: Data is written over ncomp, so check that we match the header file
530  int ncomp_cons = vars_new[0][Vars::cons].nComp();
531 
532  // NOTE: QKE was removed so this is for backward compatibility
533  AMREX_ASSERT((chk_ncomp_cons==ncomp_cons) || ((chk_ncomp_cons-1)==ncomp_cons));
534  //
535  // See if we have a written separate file that tells how many components and how many ghost cells
536  // we have of the base state
537  //
538  // If we can't find the file, then set the number of components to the original number = 3
539  //
540  int ncomp_base_to_read = 3;
541  IntVect ng_base = IntVect{1};
542  {
543  std::string BaseStateFile(restart_chkfile + "/num_base_state_comps");
544 
545  if (amrex::FileExists(BaseStateFile))
546  {
547  Vector<char> BaseStatefileCharPtr;
548  ParallelDescriptor::ReadAndBcastFile(BaseStateFile, BaseStatefileCharPtr);
549  std::string BaseStatefileCharPtrString(BaseStatefileCharPtr.dataPtr());
550 
551  // We set this to the default value of 3 but allow it be larger if th0 and qv0 were written
552  std::istringstream isb(BaseStatefileCharPtrString, std::istringstream::in);
553  isb >> ncomp_base_to_read;
554  isb >> ng_base;
555  }
556  }
557 
558  // read in the MultiFab data
559  for (int lev = 0; lev <= finest_level; ++lev)
560  {
561  // NOTE: For backward compatibility (chk file has QKE)
562  if ((chk_ncomp_cons-1)==ncomp_cons) {
563  MultiFab cons(grids[lev],dmap[lev],chk_ncomp_cons,0);
564  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
565 
566  // Copy up to RhoKE_comp
567  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,(RhoKE_comp+1),0);
568 
569  // Only if we have a PBL model do we need to copy QKE is src to KE in dst
570  if ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
571  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ) {
572  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+1),RhoKE_comp,1,0);
573  vars_new[lev][Vars::cons].mult(0.5,RhoKE_comp,1,0);
574  }
575 
576  // Copy other components
577  int ncomp_remainder = ncomp_cons - (RhoKE_comp + 1);
578  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+2),(RhoKE_comp+1),ncomp_remainder,0);
579 
580  vars_new[lev][Vars::cons].setBndry(1.0e34);
581  } else {
582  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
583  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
584  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
585  vars_new[lev][Vars::cons].setBndry(1.0e34);
586  }
587 
588  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
589  VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
590  MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
591  vars_new[lev][Vars::xvel].setBndry(1.0e34);
592 
593  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
594  VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
595  MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
596  vars_new[lev][Vars::yvel].setBndry(1.0e34);
597 
598  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
599  VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
600  MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
601  vars_new[lev][Vars::zvel].setBndry(1.0e34);
602 
603  if (solverChoice.anelastic[lev] == 1) {
604  MultiFab ppinc(grids[lev],dmap[lev],1,0);
605  VisMF::Read(ppinc, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PP_Inc"));
606  MultiFab::Copy(pp_inc[lev],ppinc,0,0,1,0);
607  pp_inc[lev].FillBoundary(geom[lev].periodicity());
608 
609  MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
610  VisMF::Read(gpx, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpx"));
611  MultiFab::Copy(gradp[lev][GpVars::gpx],gpx,0,0,1,0);
612  gradp[lev][GpVars::gpx].FillBoundary(geom[lev].periodicity());
613 
614  MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
615  VisMF::Read(gpy, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpy"));
616  MultiFab::Copy(gradp[lev][GpVars::gpy],gpy,0,0,1,0);
617  gradp[lev][GpVars::gpy].FillBoundary(geom[lev].periodicity());
618 
619  MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
620  VisMF::Read(gpz, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpz"));
621  MultiFab::Copy(gradp[lev][GpVars::gpz],gpz,0,0,1,0);
622  gradp[lev][GpVars::gpz].FillBoundary(geom[lev].periodicity());
623  }
624 
625  // Note that we read the ghost cells of the base state (unlike above)
626 
627  // The original base state only had 3 components and 1 ghost cell -- we read this
628  // here to be consistent with the old style
629  MultiFab base(grids[lev],dmap[lev],ncomp_base_to_read,ng_base);
630  VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
631 
632  MultiFab::Copy(base_state[lev],base,0,0,ncomp_base_to_read,ng_base);
633 
634  // Create theta0 from p0, rh0
635  if (ncomp_base_to_read < 4) {
636  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
637  {
638  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
639  const Box& bx = mfi.tilebox();
640  Array4<Real> const& fab = base_state[lev].array(mfi);
641  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
642  {
644  / fab(i,j,k,BaseState::r0_comp);
645  });
646  }
647  }
648  // Default theta0 to 0
649  if (ncomp_base_to_read < 5) {
650  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
651  {
652  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
653  const Box& bx = mfi.tilebox();
654  Array4<Real> const& fab = base_state[lev].array(mfi);
655  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
656  {
657  fab(i,j,k,BaseState::qv0_comp) = 0.0;
658  });
659  }
660  }
661  base_state[lev].FillBoundary(geom[lev].periodicity());
662 
663  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
664  // Note that we also read the ghost cells of z_phys_nd
665  IntVect ng = z_phys_nd[lev]->nGrowVect();
666  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
667  VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
668  MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
670  }
671 
672  // Read in the moisture model restart variables
673  std::vector<int> qmoist_indices;
674  std::vector<std::string> qmoist_names;
675  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
676  int qmoist_nvar = qmoist_indices.size();
677  for (int var = 0; var < qmoist_nvar; var++) {
678  const int ncomp = 1;
679  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
680  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
681  VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", qmoist_names[var]));
682  MultiFab::Copy(*(qmoist[lev][qmoist_indices[var]]),moist_vars,0,0,ncomp,ng_moist);
683  }
684 
685 #if defined(ERF_USE_WINDFARM)
686  if(solverChoice.windfarm_type == WindFarmType::Fitch or
687  solverChoice.windfarm_type == WindFarmType::EWP or
688  solverChoice.windfarm_type == WindFarmType::SimpleAD){
689  IntVect ng = Nturb[lev].nGrowVect();
690  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
691  VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
692  MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
693  }
694 #endif
695 
696  if (solverChoice.lsm_type != LandSurfaceType::None) {
697  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
698  BoxArray ba = lsm_data[lev][mvar]->boxArray();
699  DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
700  IntVect ng = lsm_data[lev][mvar]->nGrowVect();
701  int nvar = lsm_data[lev][mvar]->nComp();
702  MultiFab lsm_vars(ba,dm,nvar,ng);
703  VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmVars"));
704  MultiFab::Copy(*(lsm_data[lev][mvar]),lsm_vars,0,0,nvar,ng);
705  }
706  }
707 
708 
709  IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
710  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
711 
712  std::string MapFacMFileName(restart_chkfile + "/Level_0/MapFactor_mx_H");
713  if (amrex::FileExists(MapFacMFileName)) {
714  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_mx"));
715  } else {
716  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
717  }
718  MultiFab::Copy(*mapfac[lev][MapFacType::m_x],mf_m,0,0,1,ng);
719 
720 #if 0
722  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_my"));
723  MultiFab::Copy(*mapfac[lev][MapFacType::m_y],mf_m,0,0,1,ng);
724  }
725 #endif
726 
727  ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
728  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
729 
730  std::string MapFacUFileName(restart_chkfile + "/Level_0/MapFactor_ux_H");
731  if (amrex::FileExists(MapFacUFileName)) {
732  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_ux"));
733  } else {
734  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
735  }
736  MultiFab::Copy(*mapfac[lev][MapFacType::u_x],mf_u,0,0,1,ng);
737 
738 #if 0
740  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_uy"));
741  MultiFab::Copy(*mapfac[lev][MapFacType::u_y],mf_u,0,0,1,ng);
742  }
743 #endif
744 
745  ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
746  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
747 
748  std::string MapFacVFileName(restart_chkfile + "/Level_0/MapFactor_vx_H");
749  if (amrex::FileExists(MapFacVFileName)) {
750  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vx"));
751  } else {
752  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
753  }
754  MultiFab::Copy(*mapfac[lev][MapFacType::v_x],mf_v,0,0,1,ng);
755 
756 #if 0
758  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vy"));
759  MultiFab::Copy(*mapfac[lev][MapFacType::v_y],mf_v,0,0,1,ng);
760  }
761 #endif
762 
763 
764  // NOTE: We read MOST data in ReadCheckpointFileMOST (see below)!
765 
766  // See if we wrote out SST data
767  std::string FirstSSTFileName(restart_chkfile + "/Level_0/SST_0_H");
768  if (amrex::FileExists(FirstSSTFileName))
769  {
770  amrex::Print() << "Reading SST data" << std::endl;
771  int ntimes = 1;
772  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
773  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
774  sst_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
775  for (int nt(0); nt<ntimes; ++nt) {
776  VisMF::Read(sst_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
777  "SST_" + std::to_string(nt)));
778  MultiFab::Copy(*sst_lev[lev][nt],sst_at_t,0,0,1,ng);
779  }
780  }
781 
782  // See if we wrote out TSK data
783  std::string FirstTSKFileName(restart_chkfile + "/Level_0/TSK_0_H");
784  if (amrex::FileExists(FirstTSKFileName))
785  {
786  amrex::Print() << "Reading TSK data" << std::endl;
787  int ntimes = 1;
788  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
789  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
790  tsk_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
791  for (int nt(0); nt<ntimes; ++nt) {
792  VisMF::Read(tsk_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
793  "TSK_" + std::to_string(nt)));
794  MultiFab::Copy(*tsk_lev[lev][nt],tsk_at_t,0,0,1,ng);
795  }
796  }
797 
798  std::string LMaskFileName(restart_chkfile + "/Level_0/LMASK_0_H");
799  if (amrex::FileExists(LMaskFileName))
800  {
801  amrex::Print() << "Reading LMASK data" << std::endl;
802  int ntimes = 1;
803  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
804  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
805  for (int nt(0); nt<ntimes; ++nt) {
806  VisMF::Read(lmask_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
807  "LMASK_" + std::to_string(nt)));
808  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
809  const Box& bx = mfi.growntilebox();
810  Array4<int> const& dst_arr = lmask_lev[lev][nt]->array(mfi);
811  Array4<Real> const& src_arr = lmask_at_t.array(mfi);
812  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
813  {
814  dst_arr(i,j,k) = int(src_arr(i,j,k));
815  });
816  }
817  }
818  } else {
819  // Allow idealized cases over water, used to set lmask
820  ParmParse pp("erf");
821  int is_land;
822  if (pp.query("is_land", is_land, lev)) {
823  if (is_land == 1) {
824  amrex::Print() << "Level " << lev << " is land" << std::endl;
825  } else if (is_land == 0) {
826  amrex::Print() << "Level " << lev << " is water" << std::endl;
827  } else {
828  Error("is_land should be 0 or 1");
829  }
830  lmask_lev[lev][0]->setVal(is_land);
831  } else {
832  // Default to land everywhere if not specified
833  lmask_lev[lev][0]->setVal(1);
834  }
835  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
836  }
837 
838  IntVect ngv = ng; ngv[2] = 0;
839 
840  // Read lat/lon if it exists
842  amrex::Print() << "Reading Lat/Lon variables" << std::endl;
843  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
844  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
845  VisMF::Read(lat, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LAT"));
846  VisMF::Read(lon, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LON"));
847  lat_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
848  lon_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
849  MultiFab::Copy(*lat_m[lev],lat,0,0,1,ngv);
850  MultiFab::Copy(*lon_m[lev],lon,0,0,1,ngv);
851  }
852 
853 #ifdef ERF_USE_NETCDF
854  // Read sinPhi and cosPhi if it exists
856  amrex::Print() << "Reading Coriolis factors" << std::endl;
857  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
858  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
859  VisMF::Read(sphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SinPhi"));
860  VisMF::Read(cphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "CosPhi"));
861  sinPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
862  cosPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
863  MultiFab::Copy(*sinPhi_m[lev],sphi,0,0,1,ngv);
864  MultiFab::Copy(*cosPhi_m[lev],cphi,0,0,1,ngv);
865  }
866 
867  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
868 
869  if (lev == 0) {
870  MultiFab tmp1d(ba1d[0],dmap[0],1,0);
871 
872  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C1H"));
873  MultiFab::Copy(*mf_C1H,tmp1d,0,0,1,0);
874 
875  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C2H"));
876  MultiFab::Copy(*mf_C2H,tmp1d,0,0,1,0);
877 
878  MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
879 
880  VisMF::Read(tmp2d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MUB"));
881  MultiFab::Copy(*mf_MUB,tmp2d,0,0,1,mf_MUB->nGrowVect());
882  }
883  }
884 #endif
885 
886  } // for lev
887 
888 #ifdef ERF_USE_PARTICLES
889  restartTracers((ParGDBBase*)GetParGDB(),restart_chkfile);
890  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
891  dynamic_cast<LagrangianMicrophysics&>(*micro).restartParticles((ParGDBBase*)GetParGDB(),restart_chkfile);
892  }
893 #endif
894 
895 #if 0
896 #ifdef ERF_USE_NETCDF
897  // Read bdy_data files
898  if ( ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
900  {
901  int ioproc = ParallelDescriptor::IOProcessorNumber(); // I/O rank
902  int num_time;
903  int num_var;
904  Vector<Box> bx_v;
905  if (ParallelDescriptor::IOProcessor()) {
906  // Open header file and read from it
907  std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
908  bdy_h_file >> num_time;
909  bdy_h_file >> num_var;
910  bdy_h_file >> start_bdy_time;
911  bdy_h_file >> bdy_time_interval;
912  bdy_h_file >> real_width;
913  bx_v.resize(4*num_var);
914  for (int ivar(0); ivar<num_var; ++ivar) {
915  bdy_h_file >> bx_v[4*ivar ];
916  bdy_h_file >> bx_v[4*ivar+1];
917  bdy_h_file >> bx_v[4*ivar+2];
918  bdy_h_file >> bx_v[4*ivar+3];
919  }
920 
921  // IO size the FABs
922  bdy_data_xlo.resize(num_time);
923  bdy_data_xhi.resize(num_time);
924  bdy_data_ylo.resize(num_time);
925  bdy_data_yhi.resize(num_time);
926  for (int itime(0); itime<num_time; ++itime) {
927  bdy_data_xlo[itime].resize(num_var);
928  bdy_data_xhi[itime].resize(num_var);
929  bdy_data_ylo[itime].resize(num_var);
930  bdy_data_yhi[itime].resize(num_var);
931  for (int ivar(0); ivar<num_var; ++ivar) {
932  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
933  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
934  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
935  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
936  }
937  }
938 
939  // Open data file and read from it
940  std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
941  for (int itime(0); itime<num_time; ++itime) {
942  for (int ivar(0); ivar<num_var; ++ivar) {
943  bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
944  bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
945  bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
946  bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
947  }
948  }
949  } // IO
950 
951  // Broadcast the data
952  ParallelDescriptor::Barrier();
953  ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
954  ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
955  ParallelDescriptor::Bcast(&real_width,1,ioproc);
956  ParallelDescriptor::Bcast(&num_time,1,ioproc);
957  ParallelDescriptor::Bcast(&num_var,1,ioproc);
958 
959  // Everyone size their boxes
960  bx_v.resize(4*num_var);
961 
962  ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
963 
964  // Everyone but IO size their FABs
965  if (!ParallelDescriptor::IOProcessor()) {
966  bdy_data_xlo.resize(num_time);
967  bdy_data_xhi.resize(num_time);
968  bdy_data_ylo.resize(num_time);
969  bdy_data_yhi.resize(num_time);
970  for (int itime(0); itime<num_time; ++itime) {
971  bdy_data_xlo[itime].resize(num_var);
972  bdy_data_xhi[itime].resize(num_var);
973  bdy_data_ylo[itime].resize(num_var);
974  bdy_data_yhi[itime].resize(num_var);
975  for (int ivar(0); ivar<num_var; ++ivar) {
976  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
977  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
978  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
979  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
980  }
981  }
982  }
983 
984  for (int itime(0); itime<num_time; ++itime) {
985  for (int ivar(0); ivar<num_var; ++ivar) {
986  ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
987  ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
988  ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
989  ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
990  }
991  }
992  } // init_type == WRFInput or Metgrid
993 #endif
994 #endif
995 }
static void GotoNextLine(std::istream &is)
Definition: ERF_Checkpoint.cpp:16
void MakeNewLevelFromScratch(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:25
bool variable_coriolis
Definition: ERF_DataStruct.H:886
bool has_lat_lon
Definition: ERF_DataStruct.H:885
Here is the call graph for this function:

◆ ReadCheckpointFileSurfaceLayer()

void ERF::ReadCheckpointFileSurfaceLayer ( )

ERF function for reading additional data for MOST from a checkpoint file during restart.

This is called after the ABLMost object is instantiated.

1004 {
1005  for (int lev = 0; lev <= finest_level; ++lev)
1006  {
1007  amrex::Print() << "Reading MOST variables" << std::endl;
1008 
1009  IntVect ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
1010  MultiFab m_var(ba2d[lev],dmap[lev],1,ng);
1011  MultiFab* dst = nullptr;
1012 
1013  // U*
1014  std::string UstarFileName(restart_chkfile + "/Level_0/Ustar_H");
1015  if (amrex::FileExists(UstarFileName)) {
1016  dst = m_SurfaceLayer->get_u_star(lev);
1017  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Ustar"));
1018  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1019  }
1020 
1021  // W*
1022  std::string WstarFileName(restart_chkfile + "/Level_0/Wstar_H");
1023  if (amrex::FileExists(WstarFileName)) {
1024  dst = m_SurfaceLayer->get_w_star(lev);
1025  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Wstar"));
1026  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1027  }
1028 
1029  // T*
1030  std::string TstarFileName(restart_chkfile + "/Level_0/Tstar_H");
1031  if (amrex::FileExists(TstarFileName)) {
1032  dst = m_SurfaceLayer->get_t_star(lev);
1033  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Tstar"));
1034  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1035  }
1036 
1037  // Q*
1038  std::string QstarFileName(restart_chkfile + "/Level_0/Qstar_H");
1039  if (amrex::FileExists(QstarFileName)) {
1040  dst = m_SurfaceLayer->get_q_star(lev);
1041  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qstar"));
1042  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1043  }
1044 
1045  // Olen
1046  std::string OlenFileName(restart_chkfile + "/Level_0/Olen_H");
1047  if (amrex::FileExists(OlenFileName)) {
1048  dst = m_SurfaceLayer->get_olen(lev);
1049  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Olen"));
1050  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1051  }
1052 
1053  // Qsurf
1054  std::string QsurfFileName(restart_chkfile + "/Level_0/Qsurf_H");
1055  if (amrex::FileExists(QsurfFileName)) {
1056  dst = m_SurfaceLayer->get_q_surf(lev);
1057  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qsurf"));
1058  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1059  }
1060 
1061  // PBLH
1062  std::string PBLHFileName(restart_chkfile + "/Level_0/PBLH_H");
1063  if (amrex::FileExists(PBLHFileName)) {
1064  dst = m_SurfaceLayer->get_pblh(lev);
1065  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PBLH"));
1066  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1067  }
1068 
1069  // Z0
1070  std::string Z0FileName(restart_chkfile + "/Level_0/Z0_H");
1071  if (amrex::FileExists(Z0FileName)) {
1072  dst = m_SurfaceLayer->get_z0(lev);
1073  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z0"));
1074  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1075  }
1076  }
1077 }

◆ ReadParameters()

void ERF::ReadParameters ( )
private
1804 {
1805  {
1806  ParmParse pp; // Traditionally, max_step and stop_time do not have prefix.
1807  pp.query("max_step", max_step);
1808 
1809  std::string start_datetime, stop_datetime;
1810  if (pp.query("start_datetime", start_datetime)) {
1811  if (start_datetime.length() == 16) { // YYYY-MM-DD HH:MM
1812  start_datetime += ":00"; // add seconds
1813  }
1814  start_time = getEpochTime(start_datetime, datetime_format);
1815 
1816  if (pp.query("stop_datetime", stop_datetime)) {
1817  if (stop_datetime.length() == 16) { // YYYY-MM-DD HH:MM
1818  stop_datetime += ":00"; // add seconds
1819  }
1820  stop_time = getEpochTime(stop_datetime, datetime_format);
1821  }
1822 
1823  use_datetime = true;
1824 
1825  } else {
1826  pp.query("stop_time", stop_time);
1827  pp.query("start_time", start_time); // This is optional, it defaults to 0
1828  }
1829  }
1830 
1831  ParmParse pp(pp_prefix);
1832  ParmParse pp_amr("amr");
1833  {
1834  pp.query("regrid_level_0_on_restart", regrid_level_0_on_restart);
1835  pp.query("regrid_int", regrid_int);
1836  pp.query("check_file", check_file);
1837 
1838  // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
1839  // for those or "erf.restart" / "erf.m_check_int" with the former taking
1840  // precedence if both are specified
1841  pp.query("check_int", m_check_int);
1842  pp.query("check_per", m_check_per);
1843  pp_amr.query("check_int", m_check_int);
1844  pp_amr.query("check_per", m_check_per);
1845 
1846  pp.query("restart", restart_chkfile);
1847  pp_amr.query("restart", restart_chkfile);
1848 
1849  // Verbosity
1850  pp.query("v", verbose);
1851  pp.query("mg_v", mg_verbose);
1852  pp.query("use_fft", use_fft);
1853 #ifndef ERF_USE_FFT
1854  if (use_fft) {
1855  amrex::Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
1856  }
1857 #endif
1858 
1859  // Frequency of diagnostic output
1860  pp.query("sum_interval", sum_interval);
1861  pp.query("sum_period" , sum_per);
1862 
1863  pp.query("pert_interval", pert_interval);
1864 
1865  // Time step controls
1866  pp.query("cfl", cfl);
1867  pp.query("substepping_cfl", sub_cfl);
1868  pp.query("init_shrink", init_shrink);
1869  pp.query("change_max", change_max);
1870  pp.query("dt_max_initial", dt_max_initial);
1871  pp.query("dt_max", dt_max);
1872 
1873  fixed_dt.resize(max_level+1,-1.);
1874  fixed_fast_dt.resize(max_level+1,-1.);
1875 
1876  pp.query("fixed_dt", fixed_dt[0]);
1877  pp.query("fixed_fast_dt", fixed_fast_dt[0]);
1878 
1879  for (int lev = 1; lev <= max_level; lev++)
1880  {
1881  fixed_dt[lev] = fixed_dt[lev-1] / static_cast<Real>(MaxRefRatio(lev-1));
1882  fixed_fast_dt[lev] = fixed_fast_dt[lev-1] / static_cast<Real>(MaxRefRatio(lev-1));
1883  }
1884 
1885  pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
1886 
1887  // We use this to keep track of how many boxes we read in from WRF initialization
1888  num_files_at_level.resize(max_level+1,0);
1889 
1890  // We use this to keep track of how many boxes are specified thru the refinement indicators
1891  num_boxes_at_level.resize(max_level+1,0);
1892  boxes_at_level.resize(max_level+1);
1893 
1894  // We always have exactly one file at level 0
1895  num_boxes_at_level[0] = 1;
1896  boxes_at_level[0].resize(1);
1897  boxes_at_level[0][0] = geom[0].Domain();
1898 
1899 #ifdef ERF_USE_NETCDF
1900  nc_init_file.resize(max_level+1);
1901 
1902  // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
1903  // but we always have exactly one file at level 0
1904  for (int lev = 0; lev <= max_level; lev++) {
1905  const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
1906  if (pp.contains(nc_file_names.c_str())) {
1907  int num_files = pp.countval(nc_file_names.c_str());
1908  num_files_at_level[lev] = num_files;
1909  nc_init_file[lev].resize(num_files);
1910  pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
1911  for (int j = 0; j < num_files; j++) {
1912  Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
1913  } // j
1914  } // if pp.contains
1915  } // lev
1916 
1917  // NetCDF wrfbdy lateral boundary file
1918  if (pp.query("nc_bdy_file", nc_bdy_file)) {
1919  Print() << "Reading NC bdy file name " << nc_bdy_file << std::endl;
1920  }
1921 
1922  // NetCDF wrflow lateral boundary file
1923  if (pp.query("nc_low_file", nc_low_file)) {
1924  Print() << "Reading NC low file name " << nc_low_file << std::endl;
1925  }
1926 
1927 #endif
1928 
1929  // Options for vertical interpolation of met_em*.nc data.
1930  pp.query("metgrid_debug_quiescent", metgrid_debug_quiescent);
1931  pp.query("metgrid_debug_isothermal", metgrid_debug_isothermal);
1932  pp.query("metgrid_debug_dry", metgrid_debug_dry);
1933  pp.query("metgrid_debug_psfc", metgrid_debug_psfc);
1934  pp.query("metgrid_debug_msf", metgrid_debug_msf);
1935  pp.query("metgrid_interp_theta", metgrid_interp_theta);
1936  pp.query("metgrid_basic_linear", metgrid_basic_linear);
1937  pp.query("metgrid_use_below_sfc", metgrid_use_below_sfc);
1938  pp.query("metgrid_use_sfc", metgrid_use_sfc);
1939  pp.query("metgrid_retain_sfc", metgrid_retain_sfc);
1940  pp.query("metgrid_proximity", metgrid_proximity);
1941  pp.query("metgrid_order", metgrid_order);
1942  pp.query("metgrid_force_sfc_k", metgrid_force_sfc_k);
1943 
1944  // Set default to FullState for now ... later we will try Perturbation
1945  interpolation_type = StateInterpType::FullState;
1946  pp.query_enum_case_insensitive("interpolation_type" ,interpolation_type);
1947 
1948  PlotFileType plotfile3d_type_temp = PlotFileType::None;
1949  pp.query_enum_case_insensitive("plotfile_type" ,plotfile3d_type_temp);
1950  pp.query_enum_case_insensitive("plotfile_type_1",plotfile3d_type_1);
1951  pp.query_enum_case_insensitive("plotfile_type_2",plotfile3d_type_2);
1952 
1953  PlotFileType plotfile2d_type_temp = PlotFileType::None;
1954  pp.query_enum_case_insensitive("plotfile2d_type" ,plotfile2d_type_temp);
1955  pp.query_enum_case_insensitive("plotfile2d_type_1",plotfile2d_type_1);
1956  pp.query_enum_case_insensitive("plotfile2d_type_2",plotfile2d_type_2);
1957  //
1958  // This option is for backward consistency -- if only plotfile_type is set,
1959  // then it will be used for both 1 and 2 if and only if they are not set
1960  //
1961  // Default is native amrex if no type is specified
1962  //
1963  if (plotfile3d_type_temp == PlotFileType::None) {
1964  if (plotfile3d_type_1 == PlotFileType::None) {
1965  plotfile3d_type_1 = PlotFileType::Amrex;
1966  }
1967  if (plotfile3d_type_2 == PlotFileType::None) {
1968  plotfile3d_type_2 = PlotFileType::Amrex;
1969  }
1970  } else {
1971  if (plotfile3d_type_1 == PlotFileType::None) {
1972  plotfile3d_type_1 = plotfile3d_type_temp;
1973  } else {
1974  amrex::Abort("You must set either plotfile_type or plotfile_type_1, not both");
1975  }
1976  if (plotfile3d_type_2 == PlotFileType::None) {
1977  plotfile3d_type_2 = plotfile3d_type_temp;
1978  } else {
1979  amrex::Abort("You must set either plotfile_type or plotfile_type_2, not both");
1980  }
1981  }
1982  if (plotfile2d_type_temp == PlotFileType::None) {
1983  if (plotfile2d_type_1 == PlotFileType::None) {
1984  plotfile2d_type_1 = PlotFileType::Amrex;
1985  }
1986  if (plotfile2d_type_2 == PlotFileType::None) {
1987  plotfile2d_type_2 = PlotFileType::Amrex;
1988  }
1989  } else {
1990  if (plotfile2d_type_1 == PlotFileType::None) {
1991  plotfile2d_type_1 = plotfile2d_type_temp;
1992  } else {
1993  amrex::Abort("You must set either plotfile2d_type or plotfile2d_type_1, not both");
1994  }
1995  if (plotfile2d_type_2 == PlotFileType::None) {
1996  plotfile2d_type_2 = plotfile2d_type_temp;
1997  } else {
1998  amrex::Abort("You must set either plotfile2d_type or plotfile2d_type_2, not both");
1999  }
2000  }
2001 #ifndef ERF_USE_NETCDF
2002  if (plotfile3d_type_1 == PlotFileType::Netcdf ||
2003  plotfile3d_type_2 == PlotFileType::Netcdf ||
2004  plotfile2d_type_1 == PlotFileType::Netcdf ||
2005  plotfile2d_type_2 == PlotFileType::Netcdf) {
2006  amrex::Abort("Plotfile type = Netcdf is not allowed without USE_NETCDF = TRUE");
2007  }
2008 #endif
2009 
2010  pp.query("plot_file_1" , plot3d_file_1);
2011  pp.query("plot_file_2" , plot3d_file_2);
2012  pp.query("plot2d_file_1", plot2d_file_1);
2013  pp.query("plot2d_file_2", plot2d_file_2);
2014 
2015  pp.query("plot_int_1" , m_plot3d_int_1);
2016  pp.query("plot_int_2" , m_plot3d_int_2);
2017  pp.query("plot_per_1" , m_plot3d_per_1);
2018  pp.query("plot_per_2" , m_plot3d_per_2);
2019 
2020  pp.query("plot2d_int_1" , m_plot2d_int_1);
2021  pp.query("plot2d_int_2" , m_plot2d_int_2);
2022  pp.query("plot2d_per_1", m_plot2d_per_1);
2023  pp.query("plot2d_per_2", m_plot2d_per_2);
2024 
2025  pp.query("subvol_file", subvol_file);
2026  pp.query("subvol_int" , m_subvol_int);
2027  pp.query("subvol_per" , m_subvol_per);
2028 
2029  pp.query("expand_plotvars_to_unif_rr",m_expand_plotvars_to_unif_rr);
2030 
2031  pp.query("plot_face_vels",m_plot_face_vels);
2032 
2033  if ( (m_plot3d_int_1 > 0 && m_plot3d_per_1 > 0) ||
2034  (m_plot3d_int_2 > 0 && m_plot3d_per_2 > 0.) ) {
2035  Abort("Must choose only one of plot_int or plot_per");
2036  }
2037  if ( (m_plot2d_int_1 > 0 && m_plot2d_per_1 > 0) ||
2038  (m_plot2d_int_2 > 0 && m_plot2d_per_2 > 0.) ) {
2039  Abort("Must choose only one of plot_int or plot_per");
2040  }
2041 
2042  pp.query("profile_int", profile_int);
2043  pp.query("destag_profiles", destag_profiles);
2044 
2045  pp.query("plot_lsm", plot_lsm);
2046 #ifdef ERF_USE_RRTMGP
2047  pp.query("plot_rad", plot_rad);
2048 #endif
2049  pp.query("profile_rad_int", rad_datalog_int);
2050 
2051  pp.query("output_1d_column", output_1d_column);
2052  pp.query("column_per", column_per);
2053  pp.query("column_interval", column_interval);
2054  pp.query("column_loc_x", column_loc_x);
2055  pp.query("column_loc_y", column_loc_y);
2056  pp.query("column_file_name", column_file_name);
2057 
2058  // Sampler output frequency
2059  pp.query("sampler_per", sampler_per);
2060  pp.query("sampler_interval", sampler_interval);
2061 
2062  // Specify information about outputting planes of data
2063  pp.query("output_bndry_planes", output_bndry_planes);
2064  pp.query("bndry_output_planes_interval", bndry_output_planes_interval);
2065  pp.query("bndry_output_planes_per", bndry_output_planes_per);
2066  pp.query("bndry_output_start_time", bndry_output_planes_start_time);
2067 
2068  // Specify whether ingest boundary planes of data
2069  pp.query("input_bndry_planes", input_bndry_planes);
2070 
2071  // Query the set and total widths for wrfbdy interior ghost cells
2072  pp.query("real_width", real_width);
2073  pp.query("real_set_width", real_set_width);
2074 
2075  // If using real boundaries, do we extrapolate w (or set to 0)
2076  pp.query("real_extrap_w", real_extrap_w);
2077 
2078  // Query the set and total widths for crse-fine interior ghost cells
2079  pp.query("cf_width", cf_width);
2080  pp.query("cf_set_width", cf_set_width);
2081 
2082  // AmrMesh iterate on grids?
2083  bool iterate(true);
2084  pp_amr.query("iterate_grids",iterate);
2085  if (!iterate) SetIterateToFalse();
2086  }
2087 
2088 #ifdef ERF_USE_PARTICLES
2089  readTracersParams();
2090 #endif
2091 
2092  solverChoice.init_params(max_level,pp_prefix);
2093 
2094 #ifndef ERF_USE_NETCDF
2095  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(( (solverChoice.init_type != InitType::WRFInput) &&
2096  (solverChoice.init_type != InitType::Metgrid ) &&
2097  (solverChoice.init_type != InitType::NCFile ) ),
2098  "init_type cannot be 'WRFInput', 'MetGrid' or 'NCFile' if we don't build with netcdf!");
2099 #endif
2100 
2101  // Query the canopy model file name
2102  std::string forestfile;
2103  solverChoice.do_forest_drag = pp.query("forest_file", forestfile);
2105  for (int lev = 0; lev <= max_level; ++lev) {
2106  m_forest_drag[lev] = std::make_unique<ForestDrag>(forestfile);
2107  }
2108  }
2109 
2110  // If init from WRFInput or Metgrid make sure a valid file name is present
2111  if ((solverChoice.init_type == InitType::WRFInput) ||
2112  (solverChoice.init_type == InitType::Metgrid) ||
2113  (solverChoice.init_type == InitType::NCFile) ) {
2114  for (int lev = 0; lev <= max_level; lev++) {
2115  int num_files = nc_init_file[lev].size();
2116  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(num_files>0, "A file name must be present for init type WRFInput, Metgrid or NCFile.");
2117  for (int j = 0; j < num_files; j++) {
2118  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!nc_init_file[lev][j].empty(), "Valid file name must be present for init type WRFInput, Metgrid or NCFile.");
2119  } //j
2120  } // lev
2121  } // InitType
2122 
2123  // What type of land surface model to use
2124  // NOTE: Must be checked after init_params
2125  if (solverChoice.lsm_type == LandSurfaceType::SLM) {
2126  lsm.SetModel<SLM>();
2127  Print() << "SLM land surface model!\n";
2128  } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
2129  lsm.SetModel<MM5>();
2130  Print() << "MM5 land surface model!\n";
2131 #ifdef ERF_USE_NOAHMP
2132  } else if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
2133  lsm.SetModel<NOAHMP>();
2134  Print() << "Noah-MP land surface model!\n";
2135 #endif
2136  } else if (solverChoice.lsm_type == LandSurfaceType::None) {
2137  lsm.SetModel<NullSurf>();
2138  Print() << "Null land surface model!\n";
2139  } else {
2140  Abort("Dont know this LandSurfaceType!") ;
2141  }
2142 
2143  if (verbose > 0) {
2144  solverChoice.display(max_level,pp_prefix);
2145  }
2146 
2148 }
AMREX_GPU_HOST AMREX_FORCE_INLINE std::time_t getEpochTime(const std::string &dateTime, const std::string &dateTimeFormat)
Definition: ERF_EpochTime.H:15
bool metgrid_basic_linear
Definition: ERF.H:1161
bool metgrid_debug_msf
Definition: ERF.H:1159
std::string plot2d_file_2
Definition: ERF.H:1011
std::string plot3d_file_1
Definition: ERF.H:1008
bool plot_rad
Definition: ERF.H:850
bool m_plot_face_vels
Definition: ERF.H:1024
std::string plot3d_file_2
Definition: ERF.H:1009
int regrid_int
Definition: ERF.H:1001
bool metgrid_retain_sfc
Definition: ERF.H:1164
bool metgrid_use_sfc
Definition: ERF.H:1163
amrex::Vector< int > num_files_at_level
Definition: ERF.H:762
bool metgrid_debug_quiescent
Definition: ERF.H:1155
bool metgrid_interp_theta
Definition: ERF.H:1160
bool regrid_level_0_on_restart
Definition: ERF.H:1005
int metgrid_force_sfc_k
Definition: ERF.H:1167
bool real_extrap_w
Definition: ERF.H:1149
bool metgrid_use_below_sfc
Definition: ERF.H:1162
std::string subvol_file
Definition: ERF.H:1012
amrex::Real metgrid_proximity
Definition: ERF.H:1165
std::string plot2d_file_1
Definition: ERF.H:1010
bool metgrid_debug_dry
Definition: ERF.H:1157
bool metgrid_debug_isothermal
Definition: ERF.H:1156
bool metgrid_debug_psfc
Definition: ERF.H:1158
static std::string nc_low_file
Definition: ERF.H:1152
void ParameterSanityChecks()
Definition: ERF.cpp:2152
bool m_expand_plotvars_to_unif_rr
Definition: ERF.H:1013
std::string check_file
Definition: ERF.H:1033
int metgrid_order
Definition: ERF.H:1166
bool plot_lsm
Definition: ERF.H:1026
void SetModel()
Definition: ERF_LandSurface.H:28
Definition: ERF_MM5.H:26
Definition: ERF_NOAHMP.H:37
Definition: ERF_NullSurf.H:8
Definition: ERF_SLM.H:26
void display(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:601
void init_params(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:125
Here is the call graph for this function:

◆ refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )
private

Function to define the refinement criteria based on user input

228 {
229  if (max_level > 0)
230  {
231  ParmParse pp(pp_prefix);
232  Vector<std::string> refinement_indicators;
233  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
234 
235  for (int i=0; i<refinement_indicators.size(); ++i)
236  {
237  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
238 
239  ParmParse ppr(ref_prefix);
240  RealBox realbox;
241  int lev_for_box;
242 
243  int num_real_lo = ppr.countval("in_box_lo");
244  int num_indx_lo = ppr.countval("in_box_lo_indices");
245  int num_real_hi = ppr.countval("in_box_hi");
246  int num_indx_hi = ppr.countval("in_box_hi_indices");
247 
248  AMREX_ALWAYS_ASSERT(num_real_lo == num_real_hi);
249  AMREX_ALWAYS_ASSERT(num_indx_lo == num_indx_hi);
250 
251  if ( !((num_real_lo >= AMREX_SPACEDIM-1 && num_indx_lo == 0) ||
252  (num_indx_lo >= AMREX_SPACEDIM-1 && num_real_lo == 0) ||
253  (num_indx_lo == 0 && num_real_lo == 0)) )
254  {
255  amrex::Abort("Must only specify box for refinement using real OR index space");
256  }
257 
258  if (num_real_lo > 0) {
259  std::vector<Real> rbox_lo(3), rbox_hi(3);
260  ppr.get("max_level",lev_for_box);
261  if (lev_for_box <= max_level)
262  {
263  if (n_error_buf[0] != IntVect::TheZeroVector()) {
264  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
265  }
266 
267  const Real* plo = geom[lev_for_box].ProbLo();
268  const Real* phi = geom[lev_for_box].ProbHi();
269 
270  ppr.getarr("in_box_lo",rbox_lo,0,num_real_lo);
271  ppr.getarr("in_box_hi",rbox_hi,0,num_real_hi);
272 
273  if (rbox_lo[0] < plo[0]) rbox_lo[0] = plo[0];
274  if (rbox_lo[1] < plo[1]) rbox_lo[1] = plo[1];
275  if (rbox_hi[0] > phi[0]) rbox_hi[0] = phi[0];
276  if (rbox_hi[1] > phi[1]) rbox_hi[1] = phi[1];
277  if (num_real_lo < AMREX_SPACEDIM) {
278  rbox_lo[2] = plo[2];
279  rbox_hi[2] = phi[2];
280  }
281 
282  realbox = RealBox(&(rbox_lo[0]),&(rbox_hi[0]));
283 
284  Print() << "Realbox read in and intersected laterally with domain is " << realbox << std::endl;
285 
286  num_boxes_at_level[lev_for_box] += 1;
287 
288  int ilo, jlo, klo;
289  int ihi, jhi, khi;
290  const auto* dx = geom[lev_for_box].CellSize();
291  ilo = static_cast<int>((rbox_lo[0] - plo[0])/dx[0]);
292  jlo = static_cast<int>((rbox_lo[1] - plo[1])/dx[1]);
293  ihi = static_cast<int>((rbox_hi[0] - plo[0])/dx[0]-1);
294  jhi = static_cast<int>((rbox_hi[1] - plo[1])/dx[1]-1);
295  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
296  // Search for k indices corresponding to nominal grid
297  // AGL heights
298  const Box& domain = geom[lev_for_box].Domain();
299  klo = domain.smallEnd(2) - 1;
300  khi = domain.smallEnd(2) - 1;
301 
302  if (rbox_lo[2] <= zlevels_stag[lev_for_box][domain.smallEnd(2)])
303  {
304  klo = domain.smallEnd(2);
305  }
306  else
307  {
308  for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
309  if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
310  klo = k-1;
311  break;
312  }
313  }
314  }
315  AMREX_ASSERT(klo >= domain.smallEnd(2));
316 
317  if (rbox_hi[2] >= zlevels_stag[lev_for_box][domain.bigEnd(2)+1])
318  {
319  khi = domain.bigEnd(2);
320  }
321  else
322  {
323  for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
324  if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
325  khi = k-1;
326  break;
327  }
328  }
329  }
330  AMREX_ASSERT((khi <= domain.bigEnd(2)) && (khi > klo));
331 
332  // Need to update realbox because tagging is based on
333  // the initial _un_deformed grid
334  realbox = RealBox(plo[0]+ ilo *dx[0], plo[1]+ jlo *dx[1], plo[2]+ klo *dx[2],
335  plo[0]+(ihi+1)*dx[0], plo[1]+(jhi+1)*dx[1], plo[2]+(khi+1)*dx[2]);
336  } else {
337  klo = static_cast<int>((rbox_lo[2] - plo[2])/dx[2]);
338  khi = static_cast<int>((rbox_hi[2] - plo[2])/dx[2]-1);
339  }
340 
341  Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
342  if ( (ilo%ref_ratio[lev_for_box-1][0] != 0) || ((ihi+1)%ref_ratio[lev_for_box-1][0] != 0) ||
343  (jlo%ref_ratio[lev_for_box-1][1] != 0) || ((jhi+1)%ref_ratio[lev_for_box-1][1] != 0) ||
344  (klo%ref_ratio[lev_for_box-1][2] != 0) || ((khi+1)%ref_ratio[lev_for_box-1][2] != 0) )
345  {
346  amrex::Print() << "Box : " << bx << std::endl;
347  amrex::Print() << "RealBox : " << realbox << std::endl;
348  amrex::Print() << "ilo, ihi+1, jlo, jhi+1, klo, khi+1 by ref_ratio : "
349  << ilo%ref_ratio[lev_for_box-1][0] << " " << (ihi+1)%ref_ratio[lev_for_box-1][0] << " "
350  << jlo%ref_ratio[lev_for_box-1][1] << " " << (jhi+1)%ref_ratio[lev_for_box-1][1] << " "
351  << klo%ref_ratio[lev_for_box-1][2] << " " << (khi+1)%ref_ratio[lev_for_box-1][2] << std::endl;
352  amrex::Error("Fine box is not legit with this ref_ratio");
353  }
354  boxes_at_level[lev_for_box].push_back(bx);
355  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
356  } // lev
357 
358  if (solverChoice.init_type == InitType::WRFInput) {
359  if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
360  amrex::Error("Number of boxes doesn't match number of input files");
361 
362  }
363  }
364 
365  } else if (num_indx_lo > 0) {
366 
367  std::vector<int> box_lo(3), box_hi(3);
368  ppr.get("max_level",lev_for_box);
369  if (lev_for_box <= max_level)
370  {
371  if (n_error_buf[0] != IntVect::TheZeroVector()) {
372  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
373  }
374 
375  ppr.getarr("in_box_lo_indices",box_lo,0,AMREX_SPACEDIM);
376  ppr.getarr("in_box_hi_indices",box_hi,0,AMREX_SPACEDIM);
377 
378  Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
379  amrex::Print() << "BOX " << bx << std::endl;
380 
381  const auto* dx = geom[lev_for_box].CellSize();
382  const Real* plo = geom[lev_for_box].ProbLo();
383  realbox = RealBox(plo[0]+ box_lo[0] *dx[0], plo[1]+ box_lo[1] *dx[1], plo[2]+ box_lo[2] *dx[2],
384  plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
385 
386  Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
387  num_boxes_at_level[lev_for_box] += 1;
388 
389  if ( (box_lo[0]%ref_ratio[lev_for_box-1][0] != 0) || ((box_hi[0]+1)%ref_ratio[lev_for_box-1][0] != 0) ||
390  (box_lo[1]%ref_ratio[lev_for_box-1][1] != 0) || ((box_hi[1]+1)%ref_ratio[lev_for_box-1][1] != 0) ||
391  (box_lo[2]%ref_ratio[lev_for_box-1][2] != 0) || ((box_hi[2]+1)%ref_ratio[lev_for_box-1][2] != 0) )
392  amrex::Error("Fine box is not legit with this ref_ratio");
393  boxes_at_level[lev_for_box].push_back(bx);
394  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
395  } // lev
396 
397  if (solverChoice.init_type == InitType::WRFInput) {
398  if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
399  amrex::Error("Number of boxes doesn't match number of input files");
400 
401  }
402  }
403  }
404 
405  AMRErrorTagInfo info;
406 
407  if (realbox.ok()) {
408  info.SetRealBox(realbox);
409  }
410  if (ppr.countval("start_time") > 0) {
411  Real ref_min_time; ppr.get("start_time",ref_min_time);
412  info.SetMinTime(ref_min_time);
413  }
414  if (ppr.countval("end_time") > 0) {
415  Real ref_max_time; ppr.get("end_time",ref_max_time);
416  info.SetMaxTime(ref_max_time);
417  }
418  if (ppr.countval("max_level") > 0) {
419  int ref_max_level; ppr.get("max_level",ref_max_level);
420  info.SetMaxLevel(ref_max_level);
421  }
422 
423  if (ppr.countval("value_greater")) {
424  int num_val = ppr.countval("value_greater");
425  Vector<Real> value(num_val);
426  ppr.getarr("value_greater",value,0,num_val);
427  std::string field; ppr.get("field_name",field);
428  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
429  }
430  else if (ppr.countval("value_less")) {
431  int num_val = ppr.countval("value_less");
432  Vector<Real> value(num_val);
433  ppr.getarr("value_less",value,0,num_val);
434  std::string field; ppr.get("field_name",field);
435  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
436  }
437  else if (ppr.countval("adjacent_difference_greater")) {
438  int num_val = ppr.countval("adjacent_difference_greater");
439  Vector<Real> value(num_val);
440  ppr.getarr("adjacent_difference_greater",value,0,num_val);
441  std::string field; ppr.get("field_name",field);
442  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
443  }
444  else if (realbox.ok())
445  {
446  ref_tags.push_back(AMRErrorTag(info));
447  } else {
448  Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
449  }
450  } // loop over criteria
451  } // if max_level > 0
452 }
Here is the call graph for this function:

◆ remake_zphys()

void ERF::remake_zphys ( int  lev,
amrex::Real  time,
std::unique_ptr< amrex::MultiFab > &  temp_zphys_nd 
)
590 {
591  if (lev > 0)
592  {
593  //
594  // First interpolate from coarser level
595  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
596  // have been pre-filled - this includes ghost cells both inside and outside
597  // the domain
598  //
599  InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
600  IntVect(0,0,0), // do not fill ghost cells outside the domain
601  *z_phys_nd[lev-1], 0, 0, 1,
602  geom[lev-1], geom[lev],
603  refRatio(lev-1), &node_bilinear_interp,
605 
606  // This recomputes the fine values using the bottom terrain at the fine resolution,
607  // and also fills values of z_phys_nd outside the domain
608  make_terrain_fitted_coords(lev,geom[lev],*temp_zphys_nd,zlevels_stag[lev],phys_bc_type);
609 
610  std::swap(temp_zphys_nd, z_phys_nd[lev]);
611 
612  } // lev > 0
613 
614  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
615  //
616  // This assumes we have already remade the EBGeometry
617  //
618  terrain_blanking[lev]->setVal(1.0);
619  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, z_phys_nd[lev]->nGrowVect());
620  }
621 }
Here is the call graph for this function:

◆ RemakeLevel()

void ERF::RemakeLevel ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
445 {
446  if (verbose) {
447  amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
448  }
449 
450  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::MovingFittedMesh);
451 
452  BoxArray ba_old(vars_new[lev][Vars::cons].boxArray());
453  DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
454 
455  if (verbose) {
456  amrex::Print() <<" OLD BA AT LEVEL " << lev << " " << ba_old << std::endl;
457  }
458 
459  //
460  // Re-define subdomain at this level within the domain such that
461  // 1) all boxes in a given subdomain are "connected"
462  // 2) no boxes in a subdomain touch any boxes in any other subdomain
463  //
464  if (solverChoice.anelastic[lev] == 1) {
465  make_subdomains(ba.simplified_list(), subdomains[lev]);
466  }
467 
468  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
469  IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
470 
471  int ngrow_vels = ComputeGhostCells(solverChoice);
472 
473  Vector<MultiFab> temp_lev_new(Vars::NumTypes);
474  Vector<MultiFab> temp_lev_old(Vars::NumTypes);
475  MultiFab temp_base_state;
476 
477  std::unique_ptr<MultiFab> temp_zphys_nd;
478 
479  //********************************************************************************************
480  // This allocates all kinds of things, including but not limited to: solution arrays,
481  // terrain arrays and metrics, and base state.
482  // *******************************************************************************************
483  init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old, temp_base_state, temp_zphys_nd);
484 
485  // ********************************************************************************************
486  // Build the data structures for terrain-related quantities
487  // ********************************************************************************************
488  if ( solverChoice.terrain_type == TerrainType::EB ||
489  solverChoice.terrain_type == TerrainType::ImmersedForcing)
490  {
491  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
492  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
493  if (solverChoice.terrain_type == TerrainType::EB) {
494  eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
495  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
496  eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
497  }
498  }
499  remake_zphys(lev, time, temp_zphys_nd);
501 
502  // ********************************************************************************************
503  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
504  // ********************************************************************************************
505  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
506  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
507  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
508  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
509  }
510  }
511 
512  // ********************************************************************************************
513  // Build the data structures for canopy model (depends upon z_phys)
514  // ********************************************************************************************
516  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
517  }
518 
519  // *****************************************************************************************************
520  // Create the physbcs objects (after initializing the terrain but before calling FillCoarsePatch
521  // *****************************************************************************************************
522  make_physbcs(lev);
523 
524  // ********************************************************************************************
525  // Update the base state at this level by interpolation from coarser level AND copy
526  // from previous (pre-regrid) base_state array
527  // ********************************************************************************************
528  if (lev > 0) {
529  Interpolater* mapper = &cell_cons_interp;
530 
531  Vector<MultiFab*> fmf = {&base_state[lev ], &base_state[lev ]};
532  Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
533  Vector<Real> ftime = {time, time};
534  Vector<Real> ctime = {time, time};
535 
536  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
537  FillPatchTwoLevels(temp_base_state, temp_base_state.nGrowVect(), IntVect(0,0,0),
538  time, cmf, ctime, fmf, ftime,
539  0, 0, temp_base_state.nComp(), geom[lev-1], geom[lev],
540  refRatio(lev-1), mapper, domain_bcs_type,
542 
543  // Impose bc's outside the domain
544  (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
545 
546  // *************************************************************************************************
547  // This will fill the temporary MultiFabs with data from vars_new
548  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
549  // NOTE: we must create the new base state before calling FillPatch because we will
550  // interpolate perturbational quantities
551  // *************************************************************************************************
552  FillPatchFineLevel(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
553  &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
554  {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
555  base_state[lev], temp_base_state, false);
556  } else {
557  temp_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
558  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
559  temp_lev_new[Vars::cons].ParallelCopy(vars_new[lev][Vars::cons],0,0,ncomp_cons,ngrow_state,ngrow_state);
560  temp_lev_new[Vars::xvel].ParallelCopy(vars_new[lev][Vars::xvel],0,0, 1,ngrow_vels,ngrow_vels);
561  temp_lev_new[Vars::yvel].ParallelCopy(vars_new[lev][Vars::yvel],0,0, 1,ngrow_vels,ngrow_vels);
562 
563  temp_lev_new[Vars::zvel].setVal(0.);
564  temp_lev_new[Vars::zvel].ParallelCopy(vars_new[lev][Vars::zvel],0,0, 1,
565  IntVect(ngrow_vels,ngrow_vels,0),IntVect(ngrow_vels,ngrow_vels,0));
566  }
567 
568  // Now swap the pointers since we needed both old and new in the FillPatch
569  std::swap(temp_base_state, base_state[lev]);
570 
571  // ********************************************************************************************
572  // Copy from new into old just in case
573  // ********************************************************************************************
574  MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
575  MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0, 1,ngrow_vels);
576  MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0, 1,ngrow_vels);
577  MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0, 1,IntVect(ngrow_vels,ngrow_vels,0));
578 
579  // ********************************************************************************************
580  // Now swap the pointers
581  // ********************************************************************************************
582  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
583  std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
584  std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
585  }
586 
587  t_new[lev] = time;
588  t_old[lev] = time - 1.e200;
589 
590  // ********************************************************************************************
591  // Build the data structures for calculating diffusive/turbulent terms
592  // ********************************************************************************************
593  update_diffusive_arrays(lev, ba, dm);
594 
595  //********************************************************************************************
596  // Microphysics
597  // *******************************************************************************************
598  int q_size = micro->Get_Qmoist_Size(lev);
599  qmoist[lev].resize(q_size);
600  micro->Define(lev, solverChoice);
601  if (solverChoice.moisture_type != MoistureType::None)
602  {
603  micro->Init(lev, vars_new[lev][Vars::cons],
604  grids[lev], Geom(lev), 0.0,
605  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
606  }
607  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
608  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
609  }
610 
611  //********************************************************************************************
612  // Radiation
613  // *******************************************************************************************
614  if (solverChoice.rad_type != RadiationType::None)
615  {
616  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
617  }
618 
619  // ********************************************************************************************
620  // Initialize the integrator class
621  // ********************************************************************************************
623 
624  // We need to re-define the FillPatcher if the grids have changed
625  if (lev > 0 && cf_width >= 0) {
626  bool ba_changed = (ba != ba_old);
627  bool dm_changed = (dm != dm_old);
628  if (ba_changed || dm_changed) {
630  }
631  }
632 
633  // ********************************************************************************************
634  // Update the SurfaceLayer arrays at this level
635  // ********************************************************************************************
636  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
637  int nlevs = finest_level+1;
638  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
639  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
640  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
641  mfv_old, Theta_prim[lev], Qv_prim[lev],
642  Qr_prim[lev], z_phys_nd[lev],
643  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
644  lsm_data[lev], lsm_flux[lev], sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
645  }
646 
647  // These calls are done in AmrCore::regrid if this is a regrid at lev > 0
648  // For a level 0 regrid we must explicitly do them here
649  if (lev == 0) {
650  // Define grids[lev] to be ba
651  SetBoxArray(lev, ba);
652 
653  // Define dmap[lev] to be dm
654  SetDistributionMap(lev, dm);
655  }
656 
657 #ifdef ERF_USE_PARTICLES
658  particleData.Redistribute();
659 #endif
660 }
void remake_zphys(int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:589

◆ restart()

void ERF::restart ( )
1630 {
1632 
1633  // We set this here so that we don't over-write the checkpoint file we just started from
1635 
1637  //
1638  // Coarsening before we split the grids ensures that each resulting
1639  // grid will have an even number of cells in each direction.
1640  //
1641  BoxArray new_ba(amrex::coarsen(Geom(0).Domain(),2));
1642  //
1643  // Now split up into list of grids within max_grid_size[0] limit.
1644  //
1645  new_ba.maxSize(max_grid_size[0]/2);
1646  //
1647  // Now refine these boxes back to level 0.
1648  //
1649  new_ba.refine(2);
1650 
1651  if (refine_grid_layout) {
1652  ChopGrids(0, new_ba, ParallelDescriptor::NProcs());
1653  }
1654 
1655  if (new_ba != grids[0]) {
1656  DistributionMapping new_dm(new_ba);
1657  RemakeLevel(0,t_new[0],new_ba,new_dm);
1658  }
1659  }
1660 
1661 #ifdef ERF_USE_PARTICLES
1662  // We call this here without knowing whether the particles have already been initialized or not
1663  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,t_new[0]);
1664 #endif
1665 }
void RemakeLevel(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:444
void ReadCheckpointFile()
Definition: ERF_Checkpoint.cpp:439

◆ sample_lines()

void ERF::sample_lines ( int  lev,
amrex::Real  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters
levCurrent level
timeCurrent time
cellIntVect containing the x,y-dimension indices to sample along z
mfMultiFab from which we sample the data
564 {
565  int ifile = 0;
566 
567  const int ncomp = mf.nComp(); // cell-centered state vars
568 
569  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
570  average_face_to_cellcenter(mf_vels, 0,
571  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
572 
573  //
574  // Sample the data at a line (in direction "dir") in space
575  // In this case we sample in the vertical direction so dir = 2
576  // The "k" value of "cell" is ignored
577  //
578  int dir = 2;
579  MultiFab my_line = get_line_data(mf, dir, cell);
580  MultiFab my_line_vels = get_line_data(mf_vels, dir, cell);
581  MultiFab my_line_tau11 = get_line_data(*Tau[lev][TauType::tau11], dir, cell);
582  MultiFab my_line_tau12 = get_line_data(*Tau[lev][TauType::tau12], dir, cell);
583  MultiFab my_line_tau13 = get_line_data(*Tau[lev][TauType::tau13], dir, cell);
584  MultiFab my_line_tau22 = get_line_data(*Tau[lev][TauType::tau22], dir, cell);
585  MultiFab my_line_tau23 = get_line_data(*Tau[lev][TauType::tau23], dir, cell);
586  MultiFab my_line_tau33 = get_line_data(*Tau[lev][TauType::tau33], dir, cell);
587 
588  for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
589  {
590  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
591 
592  std::ostream& sample_log = SampleLineLog(ifile);
593  if (sample_log.good()) {
594  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
595  const auto& my_line_arr = my_line[0].const_array();
596  const auto& my_line_vels_arr = my_line_vels[0].const_array();
597  const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
598  const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
599  const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
600  const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
601  const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
602  const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
603  const Box& my_box = my_line[0].box();
604  const int klo = my_box.smallEnd(2);
605  const int khi = my_box.bigEnd(2);
606  int i = cell[0];
607  int j = cell[1];
608  for (int n = 0; n < ncomp; n++) {
609  for (int k = klo; k <= khi; k++) {
610  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
611  }
612  }
613  for (int n = 0; n < AMREX_SPACEDIM; n++) {
614  for (int k = klo; k <= khi; k++) {
615  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
616  }
617  }
618  for (int k = klo; k <= khi; k++) {
619  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
620  }
621  for (int k = klo; k <= khi; k++) {
622  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
623  }
624  for (int k = klo; k <= khi; k++) {
625  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
626  }
627  for (int k = klo; k <= khi; k++) {
628  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
629  }
630  for (int k = klo; k <= khi; k++) {
631  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
632  }
633  for (int k = klo; k <= khi; k++) {
634  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
635  }
636  sample_log << std::endl;
637  } // if good
638  } // mfi
639 }
const int datwidth
Definition: ERF.H:965
AMREX_FORCE_INLINE std::ostream & SampleLineLog(int i)
Definition: ERF.H:1378
const int datprecision
Definition: ERF.H:966

◆ sample_points()

void ERF::sample_points ( int  lev,
amrex::Real  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling MultiFab data at a specified cell index.

Parameters
levLevel for the associated MultiFab data
timeCurrent time
cellIntVect containing the indexes for the cell where we want to sample
mfMultiFab from which we wish to sample data
528 {
529  int ifile = 0;
530 
531  //
532  // Sample the data at a single point in space
533  //
534  int ncomp = mf.nComp();
535  Vector<Real> my_point = get_cell_data(mf, cell);
536 
537  if (!my_point.empty()) {
538 
539  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
540 
541  std::ostream& sample_log = SamplePointLog(ifile);
542  if (sample_log.good()) {
543  sample_log << std::setw(datwidth) << time;
544  for (int i = 0; i < ncomp; ++i)
545  {
546  sample_log << std::setw(datwidth) << my_point[i];
547  }
548  sample_log << std::endl;
549  } // if good
550  } // only write from processor that holds the cell
551 }
AMREX_FORCE_INLINE std::ostream & SamplePointLog(int i)
Definition: ERF.H:1364

◆ SampleLine()

amrex::IntVect& ERF::SampleLine ( int  i)
inlineprivate
1405  {
1406  return sampleline[i];
1407  }

◆ SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int  i)
inlineprivate
1379  {
1380  return *samplelinelog[i];
1381  }

◆ SampleLineLogName()

std::string ERF::SampleLineLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith samplelinelog file.

1535 { return samplelinelogname[i]; }

◆ SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int  i)
inlineprivate
1392  {
1393  return samplepoint[i];
1394  }

◆ SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int  i)
inlineprivate
1365  {
1366  return *sampleptlog[i];
1367  }

◆ SamplePointLogName()

std::string ERF::SamplePointLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith sampleptlog file.

1532 { return sampleptlogname[i]; }

◆ setPlotVariables()

void ERF::setPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
24 {
25  ParmParse pp(pp_prefix);
26 
27  if (pp.contains(pp_plot_var_names.c_str()))
28  {
29  std::string nm;
30 
31  int nPltVars = pp.countval(pp_plot_var_names.c_str());
32 
33  for (int i = 0; i < nPltVars; i++)
34  {
35  pp.get(pp_plot_var_names.c_str(), nm, i);
36 
37  // Add the named variable to our list of plot variables
38  // if it is not already in the list
39  if (!containerHasElement(plot_var_names, nm)) {
40  plot_var_names.push_back(nm);
41  }
42  }
43  } else {
44  //
45  // The default is to add none of the variables to the list
46  //
47  plot_var_names.clear();
48  }
49 
50  // Get state variables in the same order as we define them,
51  // since they may be in any order in the input list
52  Vector<std::string> tmp_plot_names;
53 
54  for (int i = 0; i < cons_names.size(); ++i) {
55  if ( containerHasElement(plot_var_names, cons_names[i]) ) {
56  if (solverChoice.moisture_type == MoistureType::None) {
57  if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
58  cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
59  {
60  tmp_plot_names.push_back(cons_names[i]);
61  }
62  } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
63  if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
64  {
65  tmp_plot_names.push_back(cons_names[i]);
66  }
67  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
68  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
69  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
70  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
71  cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
72  {
73  tmp_plot_names.push_back(cons_names[i]);
74  }
75  } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
76  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow rhoQ1, rhoQ2, rhoQ4
77  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
78  {
79  tmp_plot_names.push_back(cons_names[i]);
80  }
81  } else
82  {
83  // For moisture_type SAM and Morrison we have all six variables
84  tmp_plot_names.push_back(cons_names[i]);
85  }
86  }
87  }
88 
89  // check for velocity since it's not in cons_names
90  // if we are asked for any velocity component, we will need them all
91  if (containerHasElement(plot_var_names, "x_velocity") ||
92  containerHasElement(plot_var_names, "y_velocity") ||
93  containerHasElement(plot_var_names, "z_velocity")) {
94  tmp_plot_names.push_back("x_velocity");
95  tmp_plot_names.push_back("y_velocity");
96  tmp_plot_names.push_back("z_velocity");
97  }
98 
99  //
100  // If the model we are running doesn't have the variable listed in the inputs file,
101  // just ignore it rather than aborting
102  //
103  for (int i = 0; i < derived_names.size(); ++i) {
104  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
105  bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing) ||
106  (derived_names[i] != "terrain_IB_mask") );
107  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
108  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
109  (derived_names[i] != "detJ") );
110  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
111  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
112  (derived_names[i] != "z_phys") );
113 #ifndef ERF_USE_WINDFARM
114  ok_to_add &= (derived_names[i] != "SMark0" && derived_names[i] != "SMark1");
115 #endif
116  if (ok_to_add)
117  {
118  if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
119  if (derived_names[i] != "qv" && derived_names[i] != "qc" && derived_names[i] != "qrain" &&
120  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
121  derived_names[i] != "qt" && derived_names[i] != "qn" && derived_names[i] != "qp" &&
122  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
123  {
124  tmp_plot_names.push_back(derived_names[i]);
125  }
126  } else if ( (solverChoice.moisture_type == MoistureType::Kessler ) ||
127  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
128  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow qv, qc, qrain
129  if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
130  derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
131  {
132  tmp_plot_names.push_back(derived_names[i]);
133  }
134  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
135  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
136  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
137  if (derived_names[i] != "qrain" &&
138  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
139  derived_names[i] != "qp" &&
140  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
141  {
142  tmp_plot_names.push_back(derived_names[i]);
143  }
144  } else
145  {
146  // For moisture_type SAM and Morrison we have all moist quantities
147  tmp_plot_names.push_back(derived_names[i]);
148  }
149  } // use_terrain?
150  } // hasElement
151  }
152 
153 #ifdef ERF_USE_WINDFARM
154  for (int i = 0; i < derived_names.size(); ++i) {
155  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
156  if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP) {
157  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0") {
158  tmp_plot_names.push_back(derived_names[i]);
159  }
160  }
161  if( solverChoice.windfarm_type == WindFarmType::SimpleAD or
162  solverChoice.windfarm_type == WindFarmType::GeneralAD ) {
163  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0" or derived_names[i] == "SMark1") {
164  tmp_plot_names.push_back(derived_names[i]);
165  }
166  }
167  }
168  }
169 #endif
170 
171 #ifdef ERF_USE_PARTICLES
172  const auto& particles_namelist( particleData.getNamesUnalloc() );
173  for (auto it = particles_namelist.cbegin(); it != particles_namelist.cend(); ++it) {
174  std::string tmp( (*it)+"_count" );
175  if (containerHasElement(plot_var_names, tmp) ) {
176  tmp_plot_names.push_back(tmp);
177  }
178  }
179 #endif
180  plot_var_names = tmp_plot_names;
181 }
const amrex::Vector< std::string > derived_names
Definition: ERF.H:1048
const amrex::Vector< std::string > cons_names
Definition: ERF.H:1041
Here is the call graph for this function:

◆ setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool  restarting)
private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
56 {
57  // If we are restarting then we haven't read the input_sounding file yet
58  // so we need to read it here
59  // TODO: should we store this information in the checkpoint file instead?
60  if (restarting) {
62  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
64  }
65  }
66 
67  const Real* z_inp_sound = input_sounding_data.z_inp_sound[0].dataPtr();
68  const Real* U_inp_sound = input_sounding_data.U_inp_sound[0].dataPtr();
69  const Real* V_inp_sound = input_sounding_data.V_inp_sound[0].dataPtr();
70  const Real* theta_inp_sound = input_sounding_data.theta_inp_sound[0].dataPtr();
71  const int inp_sound_size = input_sounding_data.size(0);
72 
73  int refine_fac{1};
74  for (int lev = 0; lev <= finest_level; lev++)
75  {
76  const int klo = geom[lev].Domain().smallEnd(2);
77  const int khi = geom[lev].Domain().bigEnd(2);
78  const int Nz = khi - klo + 1;
79 
80  Vector<Real> zcc(Nz);
81  Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
82  zlevels_stag[0].begin()+khi/refine_fac+2);
83  expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
84 #if 0
85  amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
86  for (int k = 0; k < zlevels_sub.size(); k++) { amrex::AllPrint() << zlevels_sub[k] << " "; }
87  amrex::AllPrint() << " --> ";
88  for (int k = 0; k < Nz; k++) { amrex::AllPrint() << zcc[k] << " "; }
89  amrex::AllPrint() << std::endl;
90 #endif
91 
92  for (int k = 0; k < Nz; k++)
93  {
94  h_rayleigh_ptrs[lev][Rayleigh::ubar][k] = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
95  h_rayleigh_ptrs[lev][Rayleigh::vbar][k] = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
96  h_rayleigh_ptrs[lev][Rayleigh::wbar][k] = Real(0.0);
97  h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
98  }
99 
100  // Copy from host version to device version
101  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
102  d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
103  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
104  d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
105  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
106  d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
107  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
108  d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
109 
110  refine_fac *= ref_ratio[lev][2];
111  }
112 }
AMREX_FORCE_INLINE void expand_and_interpolate_1d(amrex::Vector< amrex::Real > &znew, const amrex::Vector< amrex::Real > &zorig, int refine_fac, bool destag=false)
Definition: ERF_Interpolation_1D.H:85
amrex::Vector< amrex::Vector< amrex::Real > > theta_inp_sound
Definition: ERF_InputSoundingData.H:407
amrex::Vector< amrex::Vector< amrex::Real > > z_inp_sound
Definition: ERF_InputSoundingData.H:407
amrex::Vector< amrex::Vector< amrex::Real > > U_inp_sound
Definition: ERF_InputSoundingData.H:407
amrex::Vector< amrex::Vector< amrex::Real > > V_inp_sound
Definition: ERF_InputSoundingData.H:407
int size(int itime) const
Definition: ERF_InputSoundingData.H:382
Here is the call graph for this function:

◆ setRecordDataInfo()

void ERF::setRecordDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1435  {
1436  if (amrex::ParallelDescriptor::IOProcessor())
1437  {
1438  datalog[i] = std::make_unique<std::fstream>();
1439  datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1440  if (!datalog[i]->good()) {
1441  amrex::FileOpenFailed(filename);
1442  }
1443  }
1444  amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
1445  }

◆ setRecordDerDataInfo()

void ERF::setRecordDerDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1448  {
1449  if (amrex::ParallelDescriptor::IOProcessor())
1450  {
1451  der_datalog[i] = std::make_unique<std::fstream>();
1452  der_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1453  if (!der_datalog[i]->good()) {
1454  amrex::FileOpenFailed(filename);
1455  }
1456  }
1457  amrex::ParallelDescriptor::Barrier("ERF::setRecordDerDataInfo");
1458  }

◆ setRecordEnergyDataInfo()

void ERF::setRecordEnergyDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1461  {
1462  if (amrex::ParallelDescriptor::IOProcessor())
1463  {
1464  tot_e_datalog[i] = std::make_unique<std::fstream>();
1465  tot_e_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1466  if (!tot_e_datalog[i]->good()) {
1467  amrex::FileOpenFailed(filename);
1468  }
1469  }
1470  amrex::ParallelDescriptor::Barrier("ERF::setRecordEnergyDataInfo");
1471  }

◆ setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1491  {
1492  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1493  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1494  {
1495  const amrex::Box& bx = mfi.validbox();
1496  if (bx.contains(cell)) {
1497  samplelinelog[i] = std::make_unique<std::fstream>();
1498  samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1499  if (!samplelinelog[i]->good()) {
1500  amrex::FileOpenFailed(filename);
1501  }
1502  }
1503  }
1504  amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
1505  }

◆ setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1474  {
1475  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1476  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1477  {
1478  const amrex::Box& bx = mfi.validbox();
1479  if (bx.contains(cell)) {
1480  sampleptlog[i] = std::make_unique<std::fstream>();
1481  sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1482  if (!sampleptlog[i]->good()) {
1483  amrex::FileOpenFailed(filename);
1484  }
1485  }
1486  }
1487  amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
1488  }

◆ setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool  restarting)
private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
66 {
67  // If we are restarting then we haven't read the input_sponge file yet
68  // so we need to read it here
69  // TODO: should we store this information in the checkpoint file instead?
70  if (restarting) {
72  }
73 
74  const Real* z_inp_sponge = input_sponge_data.z_inp_sponge.dataPtr();
75  const Real* U_inp_sponge = input_sponge_data.U_inp_sponge.dataPtr();
76  const Real* V_inp_sponge = input_sponge_data.V_inp_sponge.dataPtr();
77  const int inp_sponge_size = input_sponge_data.size();
78 
79  for (int lev = 0; lev <= finest_level; lev++)
80  {
81  const int khi = geom[lev].Domain().bigEnd()[2];
82  Vector<Real> zcc(khi+1);
83 
84  if (z_phys_cc[lev]) {
85  // use_terrain=1
86  // calculate the damping strength based on the max height at each k
88  } else {
89  const auto *const prob_lo = geom[lev].ProbLo();
90  const auto *const dx = geom[lev].CellSize();
91  for (int k = 0; k <= khi; k++)
92  {
93  zcc[k] = prob_lo[2] + (k+0.5) * dx[2];
94  }
95  }
96 
97  for (int k = 0; k <= khi; k++)
98  {
99  h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
100  h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
101  }
102 
103  // Copy from host version to device version
104  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
105  d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
106  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
107  d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
108  }
109 }
AMREX_FORCE_INLINE void reduce_to_max_per_height(amrex::Vector< amrex::Real > &v, std::unique_ptr< amrex::MultiFab > &mf)
Definition: ERF_ParFunctions.H:8
amrex::Vector< amrex::Real > V_inp_sponge
Definition: ERF_InputSpongeData.H:114
amrex::Vector< amrex::Real > z_inp_sponge
Definition: ERF_InputSpongeData.H:114
amrex::Vector< amrex::Real > U_inp_sponge
Definition: ERF_InputSpongeData.H:114
int size() const
Definition: ERF_InputSpongeData.H:102
Here is the call graph for this function:

◆ solve_with_EB_mlmg()

void ERF::solve_with_EB_mlmg ( int  lev,
amrex::Vector< amrex::MultiFab > &  rhs,
amrex::Vector< amrex::MultiFab > &  p,
amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &  fluxes 
)

Solve the Poisson equation using EB_enabled MLMG Note that the level may or may not be level 0.

20 {
21  BL_PROFILE("ERF::solve_with_EB_mlmg()");
22 
23  auto const dom_lo = lbound(geom[lev].Domain());
24  auto const dom_hi = ubound(geom[lev].Domain());
25 
26  LPInfo info;
27  // Allow a hidden direction if the domain is one cell wide in any lateral direction
28  if (dom_lo.x == dom_hi.x) {
29  info.setHiddenDirection(0);
30  } else if (dom_lo.y == dom_hi.y) {
31  info.setHiddenDirection(1);
32  }
33 
34  // Make sure the solver only sees the levels over which we are solving
35  Vector<BoxArray> ba_tmp; ba_tmp.push_back(rhs[0].boxArray());
36  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(rhs[0].DistributionMap());
37  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
38 
39  auto bclo = get_projection_bc(Orientation::low);
40  auto bchi = get_projection_bc(Orientation::high);
41 
42  // amrex::Print() << "BCLO " << bclo[0] << " " << bclo[1] << " " << bclo[2] << std::endl;
43  // amrex::Print() << "BCHI " << bchi[0] << " " << bchi[1] << " " << bchi[2] << std::endl;
44 
47 
48  // ****************************************************************************
49  // Multigrid solve
50  // ****************************************************************************
51 
52  MLEBABecLap mleb (geom_tmp, ba_tmp, dm_tmp, info, {&EBFactory(lev)});
53 
54  mleb.setMaxOrder(2);
55  mleb.setDomainBC(bclo, bchi);
56  mleb.setLevelBC(0, nullptr);
57 
58  //
59  // This sets A = 0, B = 1 so that
60  // the operator A alpha - b del dot beta grad to b
61  // becomes - del dot beta grad
62  //
63  mleb.setScalars(0.0, 1.0);
64 
65  Array<MultiFab,AMREX_SPACEDIM> bcoef;
66  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
67  bcoef[idim].define(convert(ba_tmp[0],IntVect::TheDimensionVector(idim)),
68  dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
69  bcoef[idim].setVal(-1.0);
70  }
71  mleb.setBCoeffs(0, amrex::GetArrOfConstPtrs(bcoef));
72 
73  MLMG mlmg(mleb);
74 
75  int max_iter = 100;
76  mlmg.setMaxIter(max_iter);
77  mlmg.setVerbose(mg_verbose);
78  mlmg.setBottomVerbose(0);
79 
80  mlmg.solve(GetVecOfPtrs(phi), GetVecOfConstPtrs(rhs), reltol, abstol);
81 
82  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
83 
84  ImposeBCsOnPhi(lev,phi[0], geom[lev].Domain());
85 
86  //
87  // This arises because we solve MINUS del dot beta grad phi = div (rho u)
88  //
89  fluxes[0][0].mult(-1.);
90  fluxes[0][1].mult(-1.);
91  fluxes[0][2].mult(-1.);
92 }
void ImposeBCsOnPhi(int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
Definition: ERF_ImposeBCsOnPhi.cpp:12

◆ solve_with_gmres()

void ERF::solve_with_gmres ( int  lev,
const amrex::Box &  subdomain,
amrex::MultiFab &  rhs,
amrex::MultiFab &  p,
amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &  fluxes,
amrex::MultiFab &  ax_sub,
amrex::MultiFab &  ay_sub,
amrex::MultiFab &  znd_sub 
)

Solve the Poisson equation using FFT-preconditioned GMRES

15 {
16 #ifdef ERF_USE_FFT
17  BL_PROFILE("ERF::solve_with_gmres()");
18 
21 
22  auto const dom_lo = lbound(Geom(lev).Domain());
23  auto const dom_hi = ubound(Geom(lev).Domain());
24 
25  auto const sub_lo = lbound(subdomain);
26  auto const sub_hi = ubound(subdomain);
27 
28  auto dx = Geom(lev).CellSizeArray();
29 
30  Geometry my_geom;
31 
32  Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
33  if (Geom(lev).isPeriodic(0) && sub_lo.x == dom_lo.x && sub_hi.x == dom_hi.x) { is_per[0] = 1;}
34  if (Geom(lev).isPeriodic(1) && sub_lo.y == dom_lo.y && sub_hi.y == dom_hi.y) { is_per[1] = 1;}
35 
36  int coord_sys = 0;
37 
38  // If subdomain == domain then we pass Geom(lev) to the FFT solver
39  if (subdomain == Geom(lev).Domain()) {
40  my_geom.define(Geom(lev).Domain(), Geom(lev).ProbDomain(), coord_sys, is_per);
41  } else {
42  // else we create a new geometry based only on the subdomain
43  // The information in my_geom used by the FFT routines is:
44  // 1) my_geom.Domain()
45  // 2) my_geom.CellSize()
46  // 3) my_geom.isAllPeriodic() / my_geom.periodicity()
47  RealBox rb( sub_lo.x *dx[0], sub_lo.y *dx[1], sub_lo.z *dx[2],
48  (sub_hi.x+1)*dx[0], (sub_hi.y+1)*dx[1], (sub_hi.z+1)*dx[2]);
49  my_geom.define(subdomain, rb, coord_sys, is_per);
50  }
51 
52  amrex::GMRES<MultiFab, TerrainPoisson> gmsolver;
53 
54  TerrainPoisson tp(my_geom, rhs.boxArray(), rhs.DistributionMap(), domain_bc_type,
55  stretched_dz_d[lev], ax_sub, ay_sub, &znd_sub);
56 
57  gmsolver.define(tp);
58 
59  gmsolver.setVerbose(mg_verbose);
60 
61  gmsolver.setRestartLength(50);
62 
63  tp.usePrecond(true);
64 
65  gmsolver.solve(phi, rhs, reltol, abstol);
66 
67  tp.getFluxes(phi, fluxes);
68 #else
69  amrex::ignore_unused(lev, rhs, phi, fluxes, ax_sub, ay_sub, znd_sub);
70 #endif
71 
72  // ****************************************************************************
73  // Impose bc's on pprime
74  // ****************************************************************************
75  ImposeBCsOnPhi(lev, phi, subdomain);
76 }

◆ solve_with_mlmg()

void ERF::solve_with_mlmg ( int  lev,
amrex::Vector< amrex::MultiFab > &  rhs,
amrex::Vector< amrex::MultiFab > &  p,
amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &  fluxes 
)

Solve the Poisson equation using MLMG Note that the level may or may not be level 0.

41 {
42  BL_PROFILE("ERF::solve_with_mlmg()");
43 
44  auto const dom_lo = lbound(geom[lev].Domain());
45  auto const dom_hi = ubound(geom[lev].Domain());
46 
47  LPInfo info;
48  // Allow a hidden direction if the domain is one cell wide in any lateral direction
49  if (dom_lo.x == dom_hi.x) {
50  info.setHiddenDirection(0);
51  } else if (dom_lo.y == dom_hi.y) {
52  info.setHiddenDirection(1);
53  }
54 
55  // Make sure the solver only sees the levels over which we are solving
56  Vector<BoxArray> ba_tmp; ba_tmp.push_back(rhs[0].boxArray());
57  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(rhs[0].DistributionMap());
58  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
59 
60  auto bclo = get_projection_bc(Orientation::low);
61  auto bchi = get_projection_bc(Orientation::high);
62 
63  // amrex::Print() << "BCLO " << bclo[0] << " " << bclo[1] << " " << bclo[2] << std::endl;
64  // amrex::Print() << "BCHI " << bchi[0] << " " << bchi[1] << " " << bchi[2] << std::endl;
65 
68 
69  // ****************************************************************************
70  // Multigrid solve
71  // ****************************************************************************
72 
73  MLPoisson mlpoisson(geom_tmp, ba_tmp, dm_tmp, info);
74  mlpoisson.setDomainBC(bclo, bchi);
75  if (lev > 0) {
76  mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
77  }
78  mlpoisson.setLevelBC(0, nullptr);
79 
80  // Use low order for outflow at physical boundaries
81  mlpoisson.setMaxOrder(2);
82 
83  MLMG mlmg(mlpoisson);
84  int max_iter = 100;
85  mlmg.setMaxIter(max_iter);
86 
87  mlmg.setVerbose(mg_verbose);
88  mlmg.setBottomVerbose(0);
89 
90  mlmg.solve(GetVecOfPtrs(phi),
91  GetVecOfConstPtrs(rhs),
92  reltol, abstol);
93  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
94 
95  // ****************************************************************************
96  // Impose bc's on pprime
97  // ****************************************************************************
98  ImposeBCsOnPhi(lev, phi[0], geom[lev].Domain());
99 }

◆ sum_derived_quantities()

void ERF::sum_derived_quantities ( amrex::Real  time)
188 {
189  if (verbose <= 0 || NumDerDataLogs() <= 0) return;
190 
191  int lev = 0;
192 
193  AMREX_ALWAYS_ASSERT(lev == 0);
194 
195  // ************************************************************************
196  // WARNING: we are not filling ghost cells other than periodic outside the domain
197  // ************************************************************************
198 
199  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
200  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
201 
202  // Average all three components of velocity (on faces) to the cell center
203  average_face_to_cellcenter(mf_cc_vel,0,
204  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
205  &vars_new[lev][Vars::yvel],
206  &vars_new[lev][Vars::zvel]});
207  mf_cc_vel.FillBoundary(geom[lev].periodicity());
208 
209  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
210  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
211  }
212 
213  MultiFab r_wted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
214  MultiFab unwted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
215  MultiFab enstrophysq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
216  MultiFab theta_mf(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
217 
218 #ifdef _OPENMP
219 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
220 #endif
221  for (MFIter mfi(unwted_magvelsq, TilingIfNotGPU()); mfi.isValid(); ++mfi)
222  {
223  const Box& bx = mfi.tilebox();
224  auto& src_fab = mf_cc_vel[mfi];
225 
226  auto& dest1_fab = unwted_magvelsq[mfi];
227  derived::erf_dermagvelsq(bx, dest1_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
228 
229  auto& dest2_fab = enstrophysq[mfi];
230  derived::erf_derenstrophysq(bx, dest2_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
231  }
232 
233  // Copy the MF holding 1/2(u^2 + v^2 + w^2) into the MF that will hold 1/2 rho (u^2 + v^2 + w^2)d
234  MultiFab::Copy(r_wted_magvelsq, unwted_magvelsq, 0, 0, 1, 0);
235 
236  // Multiply the MF holding 1/2(u^2 + v^2 + w^2) by rho to get 1/2 rho (u^2 + v^2 + w^2)
237  MultiFab::Multiply(r_wted_magvelsq, vars_new[lev][Vars::cons], 0, 0, 1, 0);
238 
239  // Copy the MF holding (rho theta) into "theta_mf"
240  MultiFab::Copy(theta_mf, vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, 0);
241 
242  // Divide (rho theta) by rho to get theta in the MF "theta_mf"
243  MultiFab::Divide(theta_mf, vars_new[lev][Vars::cons], Rho_comp, 0, 1, 0);
244 
245  Real unwted_avg = volWgtSumMF(lev, unwted_magvelsq, 0, false);
246  Real r_wted_avg = volWgtSumMF(lev, r_wted_magvelsq, 0, false);
247  Real enstrsq_avg = volWgtSumMF(lev, enstrophysq, 0, false);
248  Real theta_avg = volWgtSumMF(lev, theta_mf, 0, false);
249 
250  // Get volume including terrain (consistent with volWgtSumMF routine)
251  MultiFab volume(grids[lev], dmap[lev], 1, 0);
252  auto const& dx = geom[lev].CellSizeArray();
253  Real cell_vol = dx[0]*dx[1]*dx[2];
254  volume.setVal(cell_vol);
255  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
256  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
257  }
258 #ifdef _OPENMP
259 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
260 #endif
261  for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
262  {
263  const Box& tbx = mfi.tilebox();
264  auto dst = volume.array(mfi);
265  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
266  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
267  ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
268  {
269  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
270  });
271  }
272  Real vol = volume.sum();
273 
274  unwted_avg /= vol;
275  r_wted_avg /= vol;
276  enstrsq_avg /= vol;
277  theta_avg /= vol;
278 
279  const int nfoo = 4;
280  Real foo[nfoo] = {unwted_avg,r_wted_avg,enstrsq_avg,theta_avg};
281 #ifdef AMREX_LAZY
282  Lazy::QueueReduction([=]() mutable {
283 #endif
284  ParallelDescriptor::ReduceRealSum(
285  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
286 
287  if (ParallelDescriptor::IOProcessor()) {
288  int i = 0;
289  unwted_avg = foo[i++];
290  r_wted_avg = foo[i++];
291  enstrsq_avg = foo[i++];
292  theta_avg = foo[i++];
293 
294  std::ostream& data_log_der = DerDataLog(0);
295 
296  if (time == 0.0) {
297  data_log_der << std::setw(datwidth) << " time";
298  data_log_der << std::setw(datwidth) << " ke_den";
299  data_log_der << std::setw(datwidth) << " velsq";
300  data_log_der << std::setw(datwidth) << " enstrophy";
301  data_log_der << std::setw(datwidth) << " int_energy";
302  data_log_der << std::endl;
303  }
304  data_log_der << std::setw(datwidth) << std::setprecision(timeprecision) << time;
305  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << unwted_avg;
306  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << r_wted_avg;
307  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << enstrsq_avg;
308  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << theta_avg;
309  data_log_der << std::endl;
310 
311  } // if IOProcessor
312 #ifdef AMREX_LAZY
313  }
314 #endif
315 }
AMREX_FORCE_INLINE std::ostream & DerDataLog(int i)
Definition: ERF.H:1342
AMREX_FORCE_INLINE int NumDerDataLogs() noexcept
Definition: ERF.H:1356
void erf_dermagvelsq(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:346
void erf_derenstrophysq(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:284
Here is the call graph for this function:

◆ sum_energy_quantities()

void ERF::sum_energy_quantities ( amrex::Real  time)
319 {
320  if ( (verbose <= 0) || (tot_e_datalog.size() < 1) ) { return; }
321 
322  int lev = 0;
323 
324  AMREX_ALWAYS_ASSERT(lev == 0);
325 
326  // ************************************************************************
327  // WARNING: we are not filling ghost cells other than periodic outside the domain
328  // ************************************************************************
329 
330  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
331  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
332 
333  // Average all three components of velocity (on faces) to the cell center
334  average_face_to_cellcenter(mf_cc_vel,0,
335  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
336  &vars_new[lev][Vars::yvel],
337  &vars_new[lev][Vars::zvel]});
338  mf_cc_vel.FillBoundary(geom[lev].periodicity());
339 
340  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
341  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
342  }
343 
344  MultiFab tot_mass (grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
345  MultiFab tot_energy(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
346 
347  auto const& dx = geom[lev].CellSizeArray();
348  bool is_moist = (solverChoice.moisture_type != MoistureType::None);
349 
350 #ifdef _OPENMP
351 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
352 #endif
353  for (MFIter mfi(tot_mass, TilingIfNotGPU()); mfi.isValid(); ++mfi)
354  {
355  const Box& bx = mfi.tilebox();
356 
357  const Array4<Real>& cc_vel_arr = mf_cc_vel.array(mfi);
358  const Array4<Real>& tot_mass_arr = tot_mass.array(mfi);
359  const Array4<Real>& tot_energy_arr = tot_energy.array(mfi);
360  const Array4<const Real>& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
361  const Array4<const Real>& z_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) :
362  Array4<const Real>{};
363  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
364  {
365  Real Qv = (is_moist) ? cons_arr(i,j,k,RhoQ1_comp) : 0.0;
366  Real Qc = (is_moist) ? cons_arr(i,j,k,RhoQ2_comp) : 0.0;
367  Real Qt = Qv + Qc;
368  Real Rhod = cons_arr(i,j,k,Rho_comp);
369  Real Rhot = Rhod * (1.0 + Qt);
370  Real Temp = getTgivenRandRTh(Rhod, cons_arr(i,j,k,RhoTheta_comp), Qv);
371  Real TKE = 0.5 * ( cc_vel_arr(i,j,k,0)*cc_vel_arr(i,j,k,0)
372  + cc_vel_arr(i,j,k,1)*cc_vel_arr(i,j,k,1)
373  + cc_vel_arr(i,j,k,2)*cc_vel_arr(i,j,k,2) );
374  Real zval = (z_arr) ? z_arr(i,j,k) : Real(k)*dx[2];
375 
376  Real Cv = Cp_d - R_d;
377  Real Cvv = Cp_v - R_v;
378  Real Cpv = Cp_v;
379 
380  tot_mass_arr(i,j,k) = Rhot;
381  tot_energy_arr(i,j,k) = Rhod * ( (Cv + Cvv*Qv + Cpv*Qc)*Temp - L_v*Qc
382  + (1.0 + Qt)*TKE + (1.0 + Qt)*CONST_GRAV*zval );
383 
384  });
385 
386  }
387 
388  Real tot_mass_avg = volWgtSumMF(lev, tot_mass , 0, false);
389  Real tot_energy_avg = volWgtSumMF(lev, tot_energy, 0, false);
390 
391  // Get volume including terrain (consistent with volWgtSumMF routine)
392  MultiFab volume(grids[lev], dmap[lev], 1, 0);
393  Real cell_vol = dx[0]*dx[1]*dx[2];
394  volume.setVal(cell_vol);
395  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
396  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
397  }
398 #ifdef _OPENMP
399 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
400 #endif
401  for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
402  {
403  const Box& tbx = mfi.tilebox();
404  auto dst = volume.array(mfi);
405  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
406  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
407  ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
408  {
409  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
410  });
411  }
412  Real vol = volume.sum();
413 
414  // Divide by the volume
415  tot_mass_avg /= vol;
416  tot_energy_avg /= vol;
417 
418  const int nfoo = 2;
419  Real foo[nfoo] = {tot_mass_avg,tot_energy_avg};
420 #ifdef AMREX_LAZY
421  Lazy::QueueReduction([=]() mutable {
422 #endif
423  ParallelDescriptor::ReduceRealSum(
424  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
425 
426  if (ParallelDescriptor::IOProcessor()) {
427  int i = 0;
428  tot_mass_avg = foo[i++];
429  tot_energy_avg = foo[i++];
430 
431  std::ostream& data_log_energy = *tot_e_datalog[0];
432 
433  if (time == 0.0) {
434  data_log_energy << std::setw(datwidth) << " time";
435  data_log_energy << std::setw(datwidth) << " tot_mass";
436  data_log_energy << std::setw(datwidth) << " tot_energy";
437  data_log_energy << std::endl;
438  }
439  data_log_energy << std::setw(datwidth) << std::setprecision(timeprecision) << time;
440  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_mass_avg;
441  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_energy_avg;
442  data_log_energy << std::endl;
443 
444  } // if IOProcessor
445 #ifdef AMREX_LAZY
446  }
447 #endif
448 }
constexpr amrex::Real R_v
Definition: ERF_Constants.H:11
constexpr amrex::Real Cp_d
Definition: ERF_Constants.H:12
constexpr amrex::Real CONST_GRAV
Definition: ERF_Constants.H:21
constexpr amrex::Real Cp_v
Definition: ERF_Constants.H:13
constexpr amrex::Real R_d
Definition: ERF_Constants.H:10
constexpr amrex::Real L_v
Definition: ERF_Constants.H:16
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getTgivenRandRTh(const amrex::Real rho, const amrex::Real rhotheta, const amrex::Real qv=0.0)
Definition: ERF_EOS.H:46
Here is the call graph for this function:

◆ sum_integrated_quantities()

void ERF::sum_integrated_quantities ( amrex::Real  time)

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters
timeCurrent time
16 {
17  BL_PROFILE("ERF::sum_integrated_quantities()");
18 
19  if (verbose <= 0)
20  return;
21 
22  // Single level sum
23  Real mass_sl;
24 
25  // Multilevel sums
26  Real mass_ml = 0.0;
27  Real rhth_ml = 0.0;
28  Real scal_ml = 0.0;
29  Real mois_ml = 0.0;
30 
31 #if 1
32  mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,false);
33  for (int lev = 0; lev <= finest_level; lev++) {
34  mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,true);
35  }
36 #else
37  for (int lev = 0; lev <= finest_level; lev++) {
38  MultiFab pert_dens(vars_new[lev][Vars::cons].boxArray(),
39  vars_new[lev][Vars::cons].DistributionMap(),
40  1,0);
41  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
42  for ( MFIter mfi(pert_dens,TilingIfNotGPU()); mfi.isValid(); ++mfi)
43  {
44  const Box& bx = mfi.tilebox();
45  const Array4<Real >& pert_dens_arr = pert_dens.array(mfi);
46  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
47  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
48  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
49  pert_dens_arr(i, j, k, 0) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
50  });
51  }
52  if (lev == 0) {
53  mass_sl = volWgtSumMF(0,pert_dens,0,false);
54  }
55  mass_ml += volWgtSumMF(lev,pert_dens,0,true);
56  } // lev
57 #endif
58 
59  Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,false);
60  Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,false);
61  Real mois_sl = 0.0;
62  if (solverChoice.moisture_type != MoistureType::None) {
63  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
64  for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
65  mois_sl += volWgtSumMF(0,vars_new[0][Vars::cons],RhoQ1_comp+qoff,false);
66  }
67  }
68 
69  for (int lev = 0; lev <= finest_level; lev++) {
70  rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,true);
71  scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,true);
72  if (solverChoice.moisture_type != MoistureType::None) {
73  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
74  for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
75  mois_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoQ1_comp+qoff,false);
76  }
77  }
78  }
79 
80  Gpu::HostVector<Real> h_avg_ustar; h_avg_ustar.resize(1);
81  Gpu::HostVector<Real> h_avg_tstar; h_avg_tstar.resize(1);
82  Gpu::HostVector<Real> h_avg_olen; h_avg_olen.resize(1);
83  if ((m_SurfaceLayer != nullptr) && (NumDataLogs() > 0)) {
84  Box domain = geom[0].Domain();
85  int zdir = 2;
86  h_avg_ustar = sumToLine(*m_SurfaceLayer->get_u_star(0),0,1,domain,zdir);
87  h_avg_tstar = sumToLine(*m_SurfaceLayer->get_t_star(0),0,1,domain,zdir);
88  h_avg_olen = sumToLine(*m_SurfaceLayer->get_olen(0) ,0,1,domain,zdir);
89 
90  // Divide by the total number of cells we are averaging over
91  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
92  h_avg_ustar[0] /= area_z;
93  h_avg_tstar[0] /= area_z;
94  h_avg_olen[0] /= area_z;
95 
96  } else {
97  h_avg_ustar[0] = 0.;
98  h_avg_tstar[0] = 0.;
99  h_avg_olen[0] = 0.;
100  }
101 
102  const int nfoo = 8;
103  Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mois_sl,mass_ml,rhth_ml,scal_ml,mois_ml};
104 #ifdef AMREX_LAZY
105  Lazy::QueueReduction([=]() mutable {
106 #endif
107  ParallelDescriptor::ReduceRealSum(
108  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
109 
110  if (ParallelDescriptor::IOProcessor()) {
111  int i = 0;
112  mass_sl = foo[i++];
113  rhth_sl = foo[i++];
114  scal_sl = foo[i++];
115  mois_sl = foo[i++];
116  mass_ml = foo[i++];
117  rhth_ml = foo[i++];
118  scal_ml = foo[i++];
119  mois_ml = foo[i++];
120 
121  Print() << '\n';
122  Print() << "TIME= " << std::setw(datwidth) << std::setprecision(timeprecision) << std::left << time << '\n';
123  if (finest_level == 0) {
124 #if 1
125  Print() << " MASS = " << mass_sl << '\n';
126 #else
127  Print() << " PERT MASS = " << mass_sl << '\n';
128 #endif
129  Print() << " RHO THETA = " << rhth_sl << '\n';
130  Print() << " RHO SCALAR = " << scal_sl << '\n';
131  Print() << " RHO QTOTAL = " << mois_sl << '\n';
132  } else {
133 #if 1
134  Print() << " MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
135 #else
136  Print() << " PERT MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
137 #endif
138  Print() << " RHO THETA SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
139  Print() << " RHO SCALAR SL/ML = " << scal_sl << " " << scal_ml << '\n';
140  Print() << " RHO QTOTAL SL/ML = " << mois_sl << " " << mois_ml << '\n';
141  }
142 
143  // The first data log only holds scalars
144  if (NumDataLogs() > 0)
145  {
146  int n_d = 0;
147  std::ostream& data_log1 = DataLog(n_d);
148  if (data_log1.good()) {
149  if (time == 0.0) {
150  data_log1 << std::setw(datwidth) << " time";
151  data_log1 << std::setw(datwidth) << " u_star";
152  data_log1 << std::setw(datwidth) << " t_star";
153  data_log1 << std::setw(datwidth) << " olen";
154  data_log1 << std::endl;
155  } // time = 0
156 
157  // Write the quantities at this time
158  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time;
159  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_ustar[0];
160  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_tstar[0];
161  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_olen[0];
162  data_log1 << std::endl;
163  } // if good
164  } // loop over i
165  } // if IOProcessor
166 #ifdef AMREX_LAZY
167  });
168 #endif
169 
170  // This is just an alias for convenience
171  int lev = 0;
172  if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
173  for (int i = 0; i < NumSamplePoints(); ++i)
174  {
175  sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
176  }
177  }
178  if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
179  for (int i = 0; i < NumSampleLines(); ++i)
180  {
181  sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
182  }
183  }
184 }
AMREX_FORCE_INLINE int NumSampleLineLogs() noexcept
Definition: ERF.H:1385
AMREX_FORCE_INLINE int NumSamplePointLogs() noexcept
Definition: ERF.H:1371
amrex::IntVect & SampleLine(int i)
Definition: ERF.H:1404
AMREX_FORCE_INLINE int NumSamplePoints() noexcept
Definition: ERF.H:1398
AMREX_FORCE_INLINE int NumSampleLines() noexcept
Definition: ERF.H:1411
amrex::IntVect & SamplePoint(int i)
Definition: ERF.H:1391
void sample_points(int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:527
AMREX_FORCE_INLINE std::ostream & DataLog(int i)
Definition: ERF.H:1335
AMREX_FORCE_INLINE int NumDataLogs() noexcept
Definition: ERF.H:1349
void sample_lines(int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:563

◆ timeStep()

void ERF::timeStep ( int  lev,
amrex::Real  time,
int  iteration 
)
private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters
[in]levlevel of refinement (coarsest level is 0)
[in]timestart time for time advance
[in]iterationtime step counter
18 {
19  //
20  // We need to FillPatch the coarse level before assessing whether to regrid
21  // We have not done the swap yet so we fill the "new" which will become the "old"
22  //
23  MultiFab& S_new = vars_new[lev][Vars::cons];
24  MultiFab& U_new = vars_new[lev][Vars::xvel];
25  MultiFab& V_new = vars_new[lev][Vars::yvel];
26  MultiFab& W_new = vars_new[lev][Vars::zvel];
27 
28 #ifdef ERF_USE_NETCDF
29  //
30  // Since we now only read in a subset of the time slices in wrfbdy we need to check
31  // whether it's time to read in more
32  //
33  if (solverChoice.use_real_bcs && (lev==0)) {
34  Real dT = bdy_time_interval;
35 
36  int n_time_old = static_cast<int>( (time ) / dT);
37  int n_time_new = static_cast<int>( (time+dt[lev]) / dT);
38 
39  int ntimes = bdy_data_xlo.size();
40  for (int itime = 0; itime < ntimes; itime++)
41  {
42  //if (bdy_data_xlo[itime].size() > 0) {
43  // amrex::Print() << "HAVE DATA AT TIME " << itime << std::endl;
44  //} else {
45  // amrex::Print() << " NO DATA AT TIME " << itime << std::endl;
46  //}
47 
48  bool clear_itime = (itime < n_time_old);
49 
50  if (clear_itime && bdy_data_xlo[itime].size() > 0) {
51  bdy_data_xlo[itime].clear();
52  //amrex::Print() << "CLEAR DATA AT TIME " << itime << std::endl;
53  }
54 
55  bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
56  //if (need_itime) amrex::Print() << "NEED DATA AT TIME " << itime << std::endl;
57 
58  if (bdy_data_xlo[itime].size() == 0 && need_itime) {
59  read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
60  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
61  real_width);
62 
63  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
64  convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
65  *mf_MUB, *mf_C1H, *mf_C2H,
67  geom[lev], use_moist);
68  }
69  } // itime
70  } // use_real_bcs && lev == 0
71 #endif
72 
73  //
74  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
75  //
76  if (lev == 0) {
77  FillPatchCrseLevel(lev, time, {&S_new, &U_new, &V_new, &W_new});
78  } else if (lev < finest_level) {
79  FillPatchFineLevel(lev, time, {&S_new, &U_new, &V_new, &W_new},
80  {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
81  base_state[lev], base_state[lev]);
82  }
83 
84  if (regrid_int > 0) // We may need to regrid
85  {
86  // help keep track of whether a level was already regridded
87  // from a coarser level call to regrid
88  static Vector<int> last_regrid_step(max_level+1, 0);
89 
90  // regrid changes level "lev+1" so we don't regrid on max_level
91  // also make sure we don't regrid fine levels again if
92  // it was taken care of during a coarser regrid
93  if (lev < max_level)
94  {
95  if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
96  {
97  // regrid could add newly refine levels (if finest_level < max_level)
98  // so we save the previous finest level index
99  int old_finest = finest_level;
100 
101  regrid(lev, time);
102 
103 #ifdef ERF_USE_PARTICLES
104  if (finest_level != old_finest) {
105  particleData.Redistribute();
106  }
107 #endif
108 
109  // mark that we have regridded this level already
110  for (int k = lev; k <= finest_level; ++k) {
111  last_regrid_step[k] = istep[k];
112  }
113 
114  // if there are newly created levels, set the time step
115  for (int k = old_finest+1; k <= finest_level; ++k) {
116  dt[k] = dt[k-1] / MaxRefRatio(k-1);
117  }
118  } // if
119  } // lev
120  }
121 
122  // Update what we call "old" and "new" time
123  t_old[lev] = t_new[lev];
124  t_new[lev] += dt[lev];
125 
126  if (Verbose()) {
127  amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
128  amrex::Print() << std::setprecision(timeprecision)
129  << "ADVANCE from elapsed time = " << t_old[lev] << " to " << t_new[lev]
130  << " with dt = " << dt[lev] << std::endl;
131  }
132 
133 #ifdef ERF_USE_WW3_COUPLING
134  amrex::Print() << " About to call send_to_ww3 from ERF_Timestep" << std::endl;
135  send_to_ww3(lev);
136  amrex::Print() << " About to call read_waves from ERF_Timestep" << std::endl;
137  read_waves(lev);
138  //send_to_ww3(lev);
139  //read_waves(lev);
140  //send_to_ww3(lev);
141 #endif
142 
143  // Advance a single level for a single time step
144  Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
145 
146  ++istep[lev];
147 
148  if (Verbose()) {
149  amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
150  amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
151  }
152 
153  if (lev < finest_level)
154  {
155  // recursive call for next-finer level
156  for (int i = 1; i <= nsubsteps[lev+1]; ++i)
157  {
158  Real strt_time_for_fine = time + (i-1)*dt[lev+1];
159  timeStep(lev+1, strt_time_for_fine, i);
160  }
161  }
162 
163  if (verbose && lev == 0 && solverChoice.moisture_type != MoistureType::None) {
164  amrex::Print() << "Cloud fraction " << time << " " << cloud_fraction(time) << std::endl;
165  }
166 }
amrex::Real cloud_fraction(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:451
void Advance(int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
Definition: ERF_Advance.cpp:23

◆ turbPert_amplitude()

void ERF::turbPert_amplitude ( const int  lev)
private
33 {
34  // Accessing data
35  auto& lev_new = vars_new[lev];
36 
37  // Creating local data
38  int ncons = lev_new[Vars::cons].nComp();
39  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
40 
41  // Defining BoxArray type
42  auto m_ixtype = cons_data.boxArray().ixType();
43 
44 #ifdef _OPENMP
45 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
46 #endif
47  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
48  const Box &bx = mfi.validbox();
49  const auto &cons_pert_arr = cons_data.array(mfi); // Address of perturbation array
50  const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
51 
52  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cons_pert_arr, pert_cell);
53  } // mfi
54 }
Here is the call graph for this function:

◆ turbPert_update()

void ERF::turbPert_update ( const int  lev,
const amrex::Real  dt 
)
private
13 {
14  // Accessing data
15  auto& lev_new = vars_new[lev];
16 
17  // Create aliases to state data to pass to calc_tpi_update
18  int ncons = lev_new[Vars::cons].nComp();
19  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
20  MultiFab xvel_data(lev_new[Vars::xvel], make_alias, 0, 1);
21  MultiFab yvel_data(lev_new[Vars::yvel], make_alias, 0, 1);
22 
23  // Computing perturbation update time
24  turbPert.calc_tpi_update(lev, local_dt, xvel_data, yvel_data, cons_data);
25 
26  Print() << "Successfully initialized turbulent perturbation update time and amplitude with type: "<< turbPert.pt_type <<"\n";
27 }
int pt_type
Definition: ERF_TurbPertStruct.H:631

◆ update_diffusive_arrays()

void ERF::update_diffusive_arrays ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
private
429 {
430  // ********************************************************************************************
431  // Diffusive terms
432  // ********************************************************************************************
433  bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None);
434  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
435  bool l_use_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
436  l_use_kturb );
437  bool l_need_SmnSmn = solverChoice.turbChoice[lev].use_keqn;
438  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
439  bool l_rotate = ( solverChoice.use_rotate_surface_flux );
440 
441  BoxArray ba12 = convert(ba, IntVect(1,1,0));
442  BoxArray ba13 = convert(ba, IntVect(1,0,1));
443  BoxArray ba23 = convert(ba, IntVect(0,1,1));
444 
445  Tau[lev].resize(9);
446 
447  if (l_use_diff) {
448  //
449  // NOTE: We require ghost cells in the vertical when allowing grids that don't
450  // cover the entire vertical extent of the domain at this level
451  //
452  for (int i = 0; i < 3; i++) {
453  Tau[lev][i] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) );
454  }
455  Tau[lev][TauType::tau12] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
456  Tau[lev][TauType::tau13] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
457  Tau[lev][TauType::tau23] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
458  if (l_use_terrain) {
459  Tau[lev][TauType::tau21] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
460  Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
461  Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
462  } else {
463  Tau[lev][TauType::tau21] = nullptr;
464  Tau[lev][TauType::tau31] = nullptr;
465  Tau[lev][TauType::tau32] = nullptr;
466  }
467  SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
468  SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
469  SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
470  SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) );
471  SFS_hfx1_lev[lev]->setVal(0.);
472  SFS_hfx2_lev[lev]->setVal(0.);
473  SFS_hfx3_lev[lev]->setVal(0.);
474  SFS_diss_lev[lev]->setVal(0.);
475  if (l_use_moist) {
476  SFS_q1fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
477  SFS_q2fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
478  SFS_q1fx3_lev[lev]->setVal(0.0);
479  SFS_q2fx3_lev[lev]->setVal(0.0);
480  if (l_rotate) {
481  SFS_q1fx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
482  SFS_q1fx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
483  SFS_q1fx1_lev[lev]->setVal(0.0);
484  SFS_q1fx2_lev[lev]->setVal(0.0);
485  } else {
486  SFS_q1fx1_lev[lev] = nullptr;
487  SFS_q1fx2_lev[lev] = nullptr;
488  }
489  } else {
490  SFS_q1fx1_lev[lev] = nullptr;
491  SFS_q1fx2_lev[lev] = nullptr;
492  SFS_q1fx3_lev[lev] = nullptr;
493  SFS_q2fx3_lev[lev] = nullptr;
494  }
495  } else {
496  for (int i = 0; i < 9; i++) {
497  Tau[lev][i] = nullptr;
498  }
499  SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
500  SFS_diss_lev[lev] = nullptr;
501  }
502 
503  if (l_use_kturb) {
504  eddyDiffs_lev[lev] = std::make_unique<MultiFab>(ba, dm, EddyDiff::NumDiffs, 2);
505  eddyDiffs_lev[lev]->setVal(0.0);
506  if(l_need_SmnSmn) {
507  SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
508  } else {
509  SmnSmn_lev[lev] = nullptr;
510  }
511  } else {
512  eddyDiffs_lev[lev] = nullptr;
513  SmnSmn_lev[lev] = nullptr;
514  }
515 }
@ NumDiffs
Definition: ERF_IndexDefines.H:181

◆ update_terrain_arrays()

void ERF::update_terrain_arrays ( int  lev)
624 {
625  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
626  SolverChoice::mesh_type == MeshType::VariableDz) {
627  make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
628  make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
629  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
630  }
631 }
void make_areas(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &ax, MultiFab &ay, MultiFab &az)
Definition: ERF_TerrainMetrics.cpp:558
void make_J(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &detJ_cc)
Definition: ERF_TerrainMetrics.cpp:520
Here is the call graph for this function:

◆ volWgtSumMF()

Real ERF::volWgtSumMF ( int  lev,
const amrex::MultiFab &  mf,
int  comp,
bool  finemask 
)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters
levCurrent level
mfMultiFab on which we do the volume weighted sum
compIndex of the component we want to sum
localBoolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)
finemaskIf a finer level is available, determines whether we mask fine data
655 {
656  BL_PROFILE("ERF::volWgtSumMF()");
657 
658  Real sum = 0.0;
659  MultiFab tmp(grids[lev], dmap[lev], 1, 0);
660  MultiFab::Copy(tmp, mf, comp, 0, 1, 0);
661 
662  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
663  // m is the map scale factor at cell centers
664 #ifdef _OPENMP
665 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
666 #endif
667  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
668  const Box& bx = mfi.tilebox();
669  const auto dst = tmp.array(mfi);
670  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
671  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
672  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
673  {
674  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
675  });
676  } // mfi
677 
678  if (lev < finest_level && finemask) {
679  const MultiFab& mask = build_fine_mask(lev+1);
680  MultiFab::Multiply(tmp, mask, 0, 0, 1, 0);
681  }
682 
683  // Get volume including terrain (consistent with volWgtSumMF routine)
684  MultiFab volume(grids[lev], dmap[lev], 1, 0);
685  auto const& dx = geom[lev].CellSizeArray();
686  Real cell_vol = dx[0]*dx[1]*dx[2];
687  volume.setVal(cell_vol);
688  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
689  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
690  }
691 
692  //
693  // Note that when we send in local = true, NO ParallelAllReduce::Sum
694  // is called inside the Dot product -- we will do that before we print
695  //
696  bool local = true;
697  sum = MultiFab::Dot(tmp, 0, volume, 0, 1, 0, local);
698 
699  return sum;
700 }
amrex::MultiFab & build_fine_mask(int lev)
Definition: ERF_WriteScalarProfiles.cpp:710

◆ WeatherDataInterpolation()

void ERF::WeatherDataInterpolation ( const amrex::Real  time)
512 {
513  static Real next_read_forecast_time = -1.0;
514 
515  if (next_read_forecast_time < 0.0) {
516  int next_multiple = static_cast<int>(time / 10800.0);
517  next_read_forecast_time = next_multiple * 10800.0;
518  }
519 
520  if (time >= next_read_forecast_time) {
521 
522  std::string folder = "WeatherData";
523 
524  // Check if folder exists and is a directory
525  if (!fs::exists(folder) || !fs::is_directory(folder)) {
526  throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
527  }
528 
529  std::vector<std::string> bin_files;
530  for (const auto& entry : fs::directory_iterator(folder)) {
531  if (entry.is_regular_file() && entry.path().extension() == ".bin") {
532  bin_files.push_back(entry.path().string());
533  }
534  }
535 
536  // 2. Sort lexicographically
537  std::sort(bin_files.begin(), bin_files.end());
538 
539 
540  for (const auto& entry : fs::directory_iterator(folder)) {
541  if (entry.is_regular_file() && entry.path().extension() == ".bin") {
542  bin_files.push_back(entry.path().string());
543  }
544  }
545 
546  // Check if no .bin files were found
547  if (bin_files.empty()) {
548  throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
549  }
550 
551  std::string filename1, filename2;
552  Vector<MultiFab> weather_forecast_data_1, weather_forecast_data_2;
553  amrex::Geometry geom_weather;
554  BoxArray nba;
555  DistributionMapping dm;
556 
557  int idx1 = static_cast<int>(time / 10800.0);
558  int idx2 = static_cast<int>(time / 10800.0)+1;
559 
560  if (idx2 >= static_cast<int>(bin_files.size())) {
561  throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
562  }
563 
564  filename1 = bin_files[idx1];
565  filename2 = bin_files[idx2];
566 
567  //Read in weather_forecast_1
569  geom_weather,
570  nba,
571  dm);
572 
573  FillWeatherDataMultiFab(filename1,
574  geom_weather,
575  nba,
576  dm,
577  weather_forecast_data_1);
578 
579  FillWeatherDataMultiFab(filename2,
580  geom_weather,
581  nba,
582  dm,
583  weather_forecast_data_2);
584 
585  //Interpolate in time to get the weather_forecast_interp
586  int ncomp = weather_forecast_data_1[0].nComp();
587  MultiFab weather_forecast_interp(nba, dm, ncomp, 0);
588  Real alpha1 = 1.0 - (time - next_read_forecast_time)/10800.0;
589  Real alpha2 = 1.0 - alpha1;
590 
591  MultiFab::LinComb(weather_forecast_interp,
592  alpha1, weather_forecast_data_1[0], 0,
593  alpha2, weather_forecast_data_2[0], 0,
594  0, ncomp, 0);
595 
596  //Interpolate in space to get the erf_forecast_interp
597  InterpWeatherDataOntoMesh(geom_weather, weather_forecast_data_1[0]);
598  next_read_forecast_time += 10800.0;
599  }
600 }
void CreateWeatherDataGeomBoxArrayDistMap(const std::string &filename, amrex::Geometry &geom_weather, amrex::BoxArray &nba, amrex::DistributionMapping &dm)
Definition: ERF_WeatherDataInterpolation.cpp:155
void FillWeatherDataMultiFab(const std::string &filename, const amrex::Geometry &geom_weather, const amrex::BoxArray &nba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &weather_forecast_data)
Definition: ERF_WeatherDataInterpolation.cpp:451
void InterpWeatherDataOntoMesh(const amrex::Geometry &geom_weather, amrex::MultiFab &weather_forecast_interp)
Definition: ERF_WeatherDataInterpolation.cpp:249

◆ Write2DPlotFile()

void ERF::Write2DPlotFile ( int  which,
PlotFileType  plotfile_type,
amrex::Vector< std::string >  plot_var_names 
)
1860 {
1861  const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
1862  const int ncomp_mf = varnames.size();
1863 
1864  if (ncomp_mf == 0) return;
1865 
1866  // Vector of MultiFabs for cell-centered data
1867  Vector<MultiFab> mf(finest_level+1);
1868  for (int lev = 0; lev <= finest_level; ++lev) {
1869  mf[lev].define(ba2d[lev], dmap[lev], ncomp_mf, 0);
1870  }
1871 
1872 
1873  // **********************************************************************************************
1874  // (Effectively) 2D arrays
1875  // **********************************************************************************************
1876  for (int lev = 0; lev <= finest_level; ++lev)
1877  {
1878  int mf_comp = 0;
1879 
1880  // Set all components to zero in case they aren't defined below
1881  mf[lev].setVal(0.0);
1882 
1883  if (containerHasElement(plot_var_names, "mapfac")) {
1884 #ifdef _OPENMP
1885 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1886 #endif
1887  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1888  {
1889  const Box& bx = mfi.tilebox();
1890  const Array4<Real>& derdat = mf[lev].array(mfi);
1891  const Array4<Real>& mf_m = mapfac[lev][MapFacType::m_x]->array(mfi);
1892  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1893  derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
1894  });
1895  }
1896  mf_comp++;
1897  }
1898 
1899  if (containerHasElement(plot_var_names, "lat_m")) {
1900  if (lat_m[lev]) {
1901 #ifdef _OPENMP
1902 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1903 #endif
1904  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1905  {
1906  const Box& bx = mfi.tilebox();
1907  const Array4<Real>& derdat = mf[lev].array(mfi);
1908  const Array4<Real>& data = lat_m[lev]->array(mfi);
1909  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1910  derdat(i, j, k, mf_comp) = data(i,j,0);
1911  });
1912  }
1913  }
1914  mf_comp++;
1915  } // lat_m
1916 
1917  if (containerHasElement(plot_var_names, "lon_m")) {
1918  if (lon_m[lev]) {
1919 #ifdef _OPENMP
1920 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1921 #endif
1922  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1923  {
1924  const Box& bx = mfi.tilebox();
1925  const Array4<Real>& derdat = mf[lev].array(mfi);
1926  const Array4<Real>& data = lon_m[lev]->array(mfi);
1927  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1928  derdat(i, j, k, mf_comp) = data(i,j,0);
1929  });
1930  }
1931  } else {
1932  mf[lev].setVal(0.0,mf_comp,1,0);
1933  }
1934 
1935  mf_comp++;
1936 
1937  } // lon_m
1938  }
1939 
1940  // mf[0].setVal(1.0);
1941 
1942  std::string plotfilename;
1943  if (which == 1) {
1944  plotfilename = Concatenate(plot2d_file_1, istep[0], 5);
1945  } else if (which == 2) {
1946  plotfilename = Concatenate(plot2d_file_2, istep[0], 5);
1947  }
1948 
1949  Vector<Geometry> my_geom(finest_level+1);
1950 
1951  Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
1952  if (geom[0].isPeriodic(0)) { is_per[0] = 1;}
1953  if (geom[0].isPeriodic(1)) { is_per[1] = 1;}
1954 
1955  int coord_sys = 0;
1956 
1957  for (int lev = 0; lev <= finest_level; lev++)
1958  {
1959  Box slab = makeSlab(geom[lev].Domain(),2,0);
1960  auto const slab_lo = lbound(slab);
1961  auto const slab_hi = ubound(slab);
1962 
1963  // Create a new geometry based only on the 2D slab
1964  // We need
1965  // 1) my_geom.Domain()
1966  // 2) my_geom.CellSize()
1967  // 3) my_geom.periodicity()
1968  const auto dx = geom[lev].CellSize();
1969  RealBox rb( slab_lo.x *dx[0], slab_lo.y *dx[1], slab_lo.z *dx[2],
1970  (slab_hi.x+1)*dx[0], (slab_hi.y+1)*dx[1], (slab_hi.z+1)*dx[2]);
1971  my_geom[lev].define(slab, rb, coord_sys, is_per);
1972  }
1973 
1974  if (plotfile_type == PlotFileType::Amrex)
1975  {
1976  Print() << "Writing 2D native plotfile " << plotfilename << "\n";
1977  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1978  GetVecOfConstPtrs(mf),
1979  varnames, my_geom, t_new[0], istep, refRatio());
1980  writeJobInfo(plotfilename);
1981 
1982 #ifdef ERF_USE_NETCDF
1983  } else if (plotfile_type == PlotFileType::Netcdf) {
1984  int lev = 0;
1985  int l_which = 0;
1986  const Real* p_lo = my_geom[lev].ProbLo();
1987  const Real* p_hi = my_geom[lev].ProbHi();
1988  const auto dx = my_geom[lev].CellSize();
1989  writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1990  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],dx[2]}, {dx[0],dx[1],dx[2]},
1991  my_geom[lev].Domain(), t_new[0], start_bdy_time);
1992 #endif
1993  } else {
1994  // Here we assume the plotfile_type is PlotFileType::None
1995  Print() << "Writing no 2D plotfile since plotfile_type is none" << std::endl;
1996  }
1997 }
void writeNCPlotFile(int lev, int which_subdomain, const std::string &dir, const Vector< const MultiFab * > &plotMF, const Vector< std::string > &plot_var_names, const Vector< int > &, Array< Real, AMREX_SPACEDIM > prob_lo, Array< Real, AMREX_SPACEDIM > prob_hi, Array< Real, AMREX_SPACEDIM > dx_in, const Box &subdomain, const Real time, const Real start_bdy_time)
Definition: ERF_NCPlotFile.cpp:14
static amrex::Vector< std::string > PlotFileVarNames(amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:251
void writeJobInfo(const std::string &dir) const
Definition: ERF_WriteJobInfo.cpp:9
Here is the call graph for this function:

◆ Write3DPlotFile()

void ERF::Write3DPlotFile ( int  which,
PlotFileType  plotfile_type,
amrex::Vector< std::string >  plot_var_names 
)
264 {
265  const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
266  const int ncomp_mf = varnames.size();
267 
268  int ncomp_cons = vars_new[0][Vars::cons].nComp();
269 
270  if (ncomp_mf == 0) return;
271 
272  // We Fillpatch here because some of the derived quantities require derivatives
273  // which require ghost cells to be filled. We do not need to call FillPatcher
274  // because we don't need to set interior fine points.
275  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
276 
277  // Level 0 FilLPatch
279  &vars_new[0][Vars::yvel], &vars_new[0][Vars::zvel]});
280 
281  for (int lev = 1; lev <= finest_level; ++lev) {
282  bool fillset = false;
283  FillPatchFineLevel(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
284  &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
285  {&vars_new[lev][Vars::cons], &rU_new[lev], &rV_new[lev], &rW_new[lev]},
286  base_state[lev], base_state[lev], fillset);
287  }
288 
289  // Get qmoist pointers if using moisture
290  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
291  for (int lev = 0; lev <= finest_level; ++lev) {
292  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
293  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
294  }
295  }
296 
297  // Vector of MultiFabs for cell-centered data
298  Vector<MultiFab> mf(finest_level+1);
299  for (int lev = 0; lev <= finest_level; ++lev) {
300  mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
301  }
302 
303  // Vector of MultiFabs for nodal data
304  Vector<MultiFab> mf_nd(finest_level+1);
305  if ( SolverChoice::mesh_type != MeshType::ConstantDz) {
306  for (int lev = 0; lev <= finest_level; ++lev) {
307  BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
308  mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
309  mf_nd[lev].setVal(0.);
310  }
311  }
312 
313  // Vector of MultiFabs for face-centered velocity
314  Vector<MultiFab> mf_u(finest_level+1);
315  Vector<MultiFab> mf_v(finest_level+1);
316  Vector<MultiFab> mf_w(finest_level+1);
317  if (m_plot_face_vels) {
318  for (int lev = 0; lev <= finest_level; ++lev) {
319  BoxArray grid_stag_u(grids[lev]); grid_stag_u.surroundingNodes(0);
320  BoxArray grid_stag_v(grids[lev]); grid_stag_v.surroundingNodes(1);
321  BoxArray grid_stag_w(grids[lev]); grid_stag_w.surroundingNodes(2);
322  mf_u[lev].define(grid_stag_u, dmap[lev], 1, 0);
323  mf_v[lev].define(grid_stag_v, dmap[lev], 1, 0);
324  mf_w[lev].define(grid_stag_w, dmap[lev], 1, 0);
325  MultiFab::Copy(mf_u[lev],vars_new[lev][Vars::xvel],0,0,1,0);
326  MultiFab::Copy(mf_v[lev],vars_new[lev][Vars::yvel],0,0,1,0);
327  MultiFab::Copy(mf_w[lev],vars_new[lev][Vars::zvel],0,0,1,0);
328  }
329  }
330 
331  // Array of MultiFabs for cell-centered velocity
332  Vector<MultiFab> mf_cc_vel(finest_level+1);
333 
334  if (containerHasElement(plot_var_names, "x_velocity" ) ||
335  containerHasElement(plot_var_names, "y_velocity" ) ||
336  containerHasElement(plot_var_names, "z_velocity" ) ||
337  containerHasElement(plot_var_names, "magvel" ) ||
338  containerHasElement(plot_var_names, "vorticity_x") ||
339  containerHasElement(plot_var_names, "vorticity_y") ||
340  containerHasElement(plot_var_names, "vorticity_z") ) {
341 
342  for (int lev = 0; lev <= finest_level; ++lev) {
343  mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
344  mf_cc_vel[lev].setVal(-1.e20);
345  average_face_to_cellcenter(mf_cc_vel[lev],0,
346  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
347  &vars_new[lev][Vars::yvel],
348  &vars_new[lev][Vars::zvel]});
349  } // lev
350  } // if (vel or vort)
351 
352  // We need ghost cells if computing vorticity
353  if ( containerHasElement(plot_var_names, "vorticity_x")||
354  containerHasElement(plot_var_names, "vorticity_y") ||
355  containerHasElement(plot_var_names, "vorticity_z") )
356  {
357  amrex::Interpolater* mapper = &cell_cons_interp;
358  for (int lev = 1; lev <= finest_level; ++lev)
359  {
360  Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
361  Vector<Real> ftime = {t_new[lev], t_new[lev]};
362  Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
363  Vector<Real> ctime = {t_new[lev], t_new[lev]};
364 
365  FillBdyCCVels(mf_cc_vel,lev-1);
366 
367  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
368  FillPatchTwoLevels(mf_cc_vel[lev], mf_cc_vel[lev].nGrowVect(), IntVect(0,0,0),
369  t_new[lev], cmf, ctime, fmf, ftime,
370  0, 0, mf_cc_vel[lev].nComp(), geom[lev-1], geom[lev],
371  refRatio(lev-1), mapper, domain_bcs_type,
373  } // lev
374  FillBdyCCVels(mf_cc_vel);
375  } // if (vort)
376 
377 
378  for (int lev = 0; lev <= finest_level; ++lev)
379  {
380  int mf_comp = 0;
381 
382  BoxArray ba(vars_new[lev][Vars::cons].boxArray());
383  DistributionMapping dm = vars_new[lev][Vars::cons].DistributionMap();
384 
385  // First, copy any of the conserved state variables into the output plotfile
386  for (int i = 0; i < cons_names.size(); ++i) {
387  if (containerHasElement(plot_var_names, cons_names[i])) {
388  MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,0);
389  mf_comp++;
390  }
391  }
392 
393  // Next, check for velocities
394  if (containerHasElement(plot_var_names, "x_velocity")) {
395  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 0, mf_comp, 1, 0);
396  mf_comp += 1;
397  }
398  if (containerHasElement(plot_var_names, "y_velocity")) {
399  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 1, mf_comp, 1, 0);
400  mf_comp += 1;
401  }
402  if (containerHasElement(plot_var_names, "z_velocity")) {
403  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 2, mf_comp, 1, 0);
404  mf_comp += 1;
405  }
406 
407  // Create multifabs for HSE and pressure fields used to derive other quantities
408  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
409  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
410  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
411 
412  MultiFab pressure;
413 
414  if (solverChoice.anelastic[lev] == 0) {
415  if (containerHasElement(plot_var_names, "pressure") ||
416  containerHasElement(plot_var_names, "pert_pres") ||
417  containerHasElement(plot_var_names, "dpdx") ||
418  containerHasElement(plot_var_names, "dpdy") ||
419  containerHasElement(plot_var_names, "dpdz") ||
420  containerHasElement(plot_var_names, "eq_pot_temp") ||
421  containerHasElement(plot_var_names, "qsat"))
422  {
423  int ng = (containerHasElement(plot_var_names, "dpdx") || containerHasElement(plot_var_names, "dpdy") ||
424  containerHasElement(plot_var_names, "dpdz")) ? 1 : 0;
425 
426  // Allocate space for pressure
427  pressure.define(ba,dm,1,ng);
428 
429  if (ng > 0) {
430  // Default to p_hse as a way of filling ghost cells at domain boundaries
431  MultiFab::Copy(pressure,p_hse,0,0,1,1);
432  }
433 #ifdef _OPENMP
434 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
435 #endif
436  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
437  {
438  const Box& gbx = mfi.growntilebox(IntVect(ng,ng,0));
439 
440  const Array4<Real >& p_arr = pressure.array(mfi);
441  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
442  const int ncomp = vars_new[lev][Vars::cons].nComp();
443 
444  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
445  {
446  Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
447  const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
448  p_arr(i, j, k) = getPgivenRTh(rhotheta,qv_for_p);
449  });
450  } // mfi
451  pressure.FillBoundary(geom[lev].periodicity());
452  } // compute compressible pressure
453  } // not anelastic
454  else {
455  if (containerHasElement(plot_var_names, "dpdx") ||
456  containerHasElement(plot_var_names, "dpdy") ||
457  containerHasElement(plot_var_names, "dpdz") ||
458  containerHasElement(plot_var_names, "eq_pot_temp") ||
459  containerHasElement(plot_var_names, "qsat"))
460  {
461  // Copy p_hse into pressure if using anelastic
462  pressure.define(ba,dm,1,0);
463  MultiFab::Copy(pressure,p_hse,0,0,1,0);
464  }
465  }
466 
467  // Finally, check for any derived quantities and compute them, inserting
468  // them into our output multifab
469  auto calculate_derived = [&](const std::string& der_name,
470  MultiFab& src_mf,
471  decltype(derived::erf_dernull)& der_function)
472  {
473  if (containerHasElement(plot_var_names, der_name)) {
474  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
475 #ifdef _OPENMP
476 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
477 #endif
478  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
479  {
480  const Box& bx = mfi.tilebox();
481  auto& dfab = dmf[mfi];
482  auto& sfab = src_mf[mfi];
483  der_function(bx, dfab, 0, 1, sfab, Geom(lev), t_new[0], nullptr, lev);
484  }
485 
486  mf_comp++;
487  }
488  };
489 
490  // *****************************************************************************************
491  // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
492  // defined in ERF.H
493  // *****************************************************************************************
494 
495  calculate_derived("soundspeed", vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
496  if (use_moisture) {
497  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dermoisttemp);
498  } else {
499  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dertemp);
500  }
501  calculate_derived("theta", vars_new[lev][Vars::cons], derived::erf_dertheta);
502  calculate_derived("KE", vars_new[lev][Vars::cons], derived::erf_derKE);
503  calculate_derived("scalar", vars_new[lev][Vars::cons], derived::erf_derscalar);
504  calculate_derived("vorticity_x", mf_cc_vel[lev] , derived::erf_dervortx);
505  calculate_derived("vorticity_y", mf_cc_vel[lev] , derived::erf_dervorty);
506  calculate_derived("vorticity_z", mf_cc_vel[lev] , derived::erf_dervortz);
507  calculate_derived("magvel" , mf_cc_vel[lev] , derived::erf_dermagvel);
508 
509  if (containerHasElement(plot_var_names, "divU"))
510  {
511  // TODO TODO TODO -- we need to convert w to omega here!!
512  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
513  Array<MultiFab const*, AMREX_SPACEDIM> u;
514  u[0] = &(vars_new[lev][Vars::xvel]);
515  u[1] = &(vars_new[lev][Vars::yvel]);
516  u[2] = &(vars_new[lev][Vars::zvel]);
517  compute_divergence (lev, dmf, u, geom[lev]);
518  mf_comp += 1;
519  }
520 
521  if (containerHasElement(plot_var_names, "pres_hse"))
522  {
523  MultiFab::Copy(mf[lev],p_hse,0,mf_comp,1,0);
524  mf_comp += 1;
525  }
526  if (containerHasElement(plot_var_names, "dens_hse"))
527  {
528  MultiFab::Copy(mf[lev],r_hse,0,mf_comp,1,0);
529  mf_comp += 1;
530  }
531  if (containerHasElement(plot_var_names, "theta_hse"))
532  {
533  MultiFab::Copy(mf[lev],th_hse,0,mf_comp,1,0);
534  mf_comp += 1;
535  }
536 
537  if (containerHasElement(plot_var_names, "pressure"))
538  {
539  if (solverChoice.anelastic[lev] == 1) {
540  MultiFab::Copy(mf[lev], p_hse, 0, mf_comp, 1, 0);
541  } else {
542  MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
543  }
544 
545  mf_comp += 1;
546  }
547 
548  if (containerHasElement(plot_var_names, "pert_pres"))
549  {
550  if (solverChoice.anelastic[lev] == 1) {
551  MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0);
552  } else {
553  MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
554  MultiFab::Subtract(mf[lev],p_hse,0,mf_comp,1,IntVect{0});
555  }
556  mf_comp += 1;
557  }
558 
559  if (containerHasElement(plot_var_names, "pert_dens"))
560  {
561 #ifdef _OPENMP
562 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
563 #endif
564  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
565  {
566  const Box& bx = mfi.tilebox();
567  const Array4<Real>& derdat = mf[lev].array(mfi);
568  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
569  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
570  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
571  derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
572  });
573  }
574  mf_comp ++;
575  }
576 
577  if (containerHasElement(plot_var_names, "eq_pot_temp"))
578  {
579 #ifdef _OPENMP
580 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
581 #endif
582  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
583  {
584  const Box& bx = mfi.tilebox();
585  const Array4<Real>& derdat = mf[lev].array(mfi);
586  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
587  const Array4<Real const>& p_arr = pressure.const_array(mfi);
588  const int ncomp = vars_new[lev][Vars::cons].nComp();
589  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
590  Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
591  Real qc = (use_moisture && (ncomp > RhoQ2_comp)) ? S_arr(i,j,k,RhoQ2_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
592  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
593  Real fac = Cp_d + Cp_l*(qv + qc);
594  Real pv = erf_esatw(T)*100.0;
595 
596  derdat(i, j, k, mf_comp) = T*std::pow((p_arr(i,j,k) - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
597  });
598  }
599  mf_comp ++;
600  }
601 
602 #ifdef ERF_USE_WINDFARM
603  if ( containerHasElement(plot_var_names, "num_turb") and
604  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
605  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
606  {
607  MultiFab::Copy(mf[lev],Nturb[lev],0,mf_comp,1,0);
608  mf_comp ++;
609  }
610 
611  if ( containerHasElement(plot_var_names, "SMark0") and
612  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
613  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
614  {
615  MultiFab::Copy(mf[lev],SMark[lev],0,mf_comp,1,0);
616  mf_comp ++;
617  }
618 
619  if (containerHasElement(plot_var_names, "SMark1") and
620  (solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD))
621  {
622  MultiFab::Copy(mf[lev],SMark[lev],1,mf_comp,1,0);
623  mf_comp ++;
624  }
625 #endif
626 
627  // **********************************************************************************************
628  // Allocate space if we are computing any pressure gradients
629  // **********************************************************************************************
630 
631  Vector<MultiFab> gradp_temp; gradp_temp.resize(AMREX_SPACEDIM);
632  if (containerHasElement(plot_var_names, "dpdx") ||
633  containerHasElement(plot_var_names, "dpdy") ||
634  containerHasElement(plot_var_names, "dpdz") ||
635  containerHasElement(plot_var_names, "pres_hse_x") ||
636  containerHasElement(plot_var_names, "pres_hse_y"))
637  {
638  gradp_temp[GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp_temp[GpVars::gpx].setVal(0.);
639  gradp_temp[GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp_temp[GpVars::gpy].setVal(0.);
640  gradp_temp[GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp_temp[GpVars::gpz].setVal(0.);
641  }
642 
643  // **********************************************************************************************
644  // These are based on computing gradient of full pressure
645  // **********************************************************************************************
646 
647  if (solverChoice.anelastic[lev] == 0) {
648  if ( (containerHasElement(plot_var_names, "dpdx")) ||
649  (containerHasElement(plot_var_names, "dpdy")) ||
650  (containerHasElement(plot_var_names, "dpdz")) ) {
651  compute_gradp(pressure, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), get_eb(lev), gradp_temp, solverChoice);
652  }
653  }
654 
655  if (containerHasElement(plot_var_names, "dpdx"))
656  {
657  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
658  {
659  const Box& bx = mfi.tilebox();
660  const Array4<Real >& derdat = mf[lev].array(mfi);
661  const Array4<Real const>& gpx_arr = (solverChoice.anelastic[lev] == 1) ?
662  gradp[lev][GpVars::gpx].array(mfi) : gradp_temp[GpVars::gpx].array(mfi);
663  const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
664  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
665  derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
666  });
667  }
668  mf_comp ++;
669  } // dpdx
670  if (containerHasElement(plot_var_names, "dpdy"))
671  {
672  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
673  {
674  const Box& bx = mfi.tilebox();
675  const Array4<Real >& derdat = mf[lev].array(mfi);
676  const Array4<Real const>& gpy_arr = (solverChoice.anelastic[lev] == 1) ?
677  gradp[lev][GpVars::gpy].array(mfi) : gradp_temp[GpVars::gpy].array(mfi);
678  const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
679  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
680  derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
681  });
682  }
683  mf_comp ++;
684  } // dpdy
685  if (containerHasElement(plot_var_names, "dpdz"))
686  {
687  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
688  {
689  const Box& bx = mfi.tilebox();
690  const Array4<Real >& derdat = mf[lev].array(mfi);
691  const Array4<Real const>& gpz_arr = (solverChoice.anelastic[lev] == 1) ?
692  gradp[lev][GpVars::gpz].array(mfi) : gradp_temp[GpVars::gpz].array(mfi);
693  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
694  derdat(i ,j ,k, mf_comp) = 0.5 * (gpz_arr(i,j,k+1) + gpz_arr(i,j,k));
695  });
696  }
697  mf_comp ++;
698  } // dpdz
699 
700  // **********************************************************************************************
701  // These are based on computing gradient of basestate pressure
702  // **********************************************************************************************
703 
704  if ( (containerHasElement(plot_var_names, "pres_hse_x")) ||
705  (containerHasElement(plot_var_names, "pres_hse_y")) ) {
706  compute_gradp(p_hse, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), get_eb(lev), gradp_temp, solverChoice);
707  }
708 
709  if (containerHasElement(plot_var_names, "pres_hse_x"))
710  {
711  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
712  {
713  const Box& bx = mfi.tilebox();
714  const Array4<Real >& derdat = mf[lev].array(mfi);
715  const Array4<Real const>& gpx_arr = gradp_temp[0].array(mfi);
716  const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
717  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
718  derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
719  });
720  }
721  mf_comp += 1;
722  } // pres_hse_x
723 
724  if (containerHasElement(plot_var_names, "pres_hse_y"))
725  {
726  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
727  {
728  const Box& bx = mfi.tilebox();
729  const Array4<Real >& derdat = mf[lev].array(mfi);
730  const Array4<Real const>& gpy_arr = gradp_temp[1].array(mfi);
731  const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
732  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
733  derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
734  });
735  }
736  mf_comp += 1;
737  } // pres_hse_y
738 
739  // **********************************************************************************************
740  // Metric terms
741  // **********************************************************************************************
742 
743  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
744  if (containerHasElement(plot_var_names, "z_phys"))
745  {
746  MultiFab::Copy(mf[lev],*z_phys_cc[lev],0,mf_comp,1,0);
747  mf_comp ++;
748  }
749 
750  if (containerHasElement(plot_var_names, "detJ"))
751  {
752  MultiFab::Copy(mf[lev],*detJ_cc[lev],0,mf_comp,1,0);
753  mf_comp ++;
754  }
755  } // use_terrain
756 
757  if (containerHasElement(plot_var_names, "mapfac")) {
758  amrex::Print() << "You are plotting a 3D version of mapfac; we suggest using the 2D plotfile instead" << std::endl;
759 #ifdef _OPENMP
760 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
761 #endif
762  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
763  {
764  const Box& bx = mfi.tilebox();
765  const Array4<Real>& derdat = mf[lev].array(mfi);
766  const Array4<Real>& mf_m = mapfac[lev][MapFacType::m_x]->array(mfi);
767  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
768  derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
769  });
770  }
771  mf_comp ++;
772  }
773 
774  if (containerHasElement(plot_var_names, "lat_m")) {
775  amrex::Print() << "You are plotting a 3D version of lat_m; we suggest using the 2D plotfile instead" << std::endl;
776  if (lat_m[lev]) {
777 #ifdef _OPENMP
778 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
779 #endif
780  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
781  {
782  const Box& bx = mfi.tilebox();
783  const Array4<Real>& derdat = mf[lev].array(mfi);
784  const Array4<Real>& data = lat_m[lev]->array(mfi);
785  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
786  derdat(i, j, k, mf_comp) = data(i,j,0);
787  });
788  }
789  } else {
790  mf[lev].setVal(0.0,mf_comp,1,0);
791  }
792  mf_comp++;
793  } // lat_m
794 
795  if (containerHasElement(plot_var_names, "lon_m")) {
796  amrex::Print() << "You are plotting a 3D version of lon_m; we suggest using the 2D plotfile instead" << std::endl;
797  if (lon_m[lev]) {
798 #ifdef _OPENMP
799 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
800 #endif
801  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
802  {
803  const Box& bx = mfi.tilebox();
804  const Array4<Real>& derdat = mf[lev].array(mfi);
805  const Array4<Real>& data = lon_m[lev]->array(mfi);
806  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
807  derdat(i, j, k, mf_comp) = data(i,j,0);
808  });
809  }
810  } else {
811  mf[lev].setVal(0.0,mf_comp,1,0);
812  }
813  mf_comp++;
814  } // lon_m
815 
817  if (containerHasElement(plot_var_names, "u_t_avg")) {
818 #ifdef _OPENMP
819 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
820 #endif
821  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
822  {
823  const Box& bx = mfi.tilebox();
824  const Array4<Real>& derdat = mf[lev].array(mfi);
825  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
826  const Real norm = t_avg_cnt[lev];
827  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
828  {
829  derdat(i ,j ,k, mf_comp) = data(i,j,k,0) / norm;
830  });
831  }
832  mf_comp ++;
833  }
834 
835  if (containerHasElement(plot_var_names, "v_t_avg")) {
836 #ifdef _OPENMP
837 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
838 #endif
839  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
840  {
841  const Box& bx = mfi.tilebox();
842  const Array4<Real>& derdat = mf[lev].array(mfi);
843  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
844  const Real norm = t_avg_cnt[lev];
845  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
846  {
847  derdat(i ,j ,k, mf_comp) = data(i,j,k,1) / norm;
848  });
849  }
850  mf_comp ++;
851  }
852 
853  if (containerHasElement(plot_var_names, "w_t_avg")) {
854 #ifdef _OPENMP
855 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
856 #endif
857  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
858  {
859  const Box& bx = mfi.tilebox();
860  const Array4<Real>& derdat = mf[lev].array(mfi);
861  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
862  const Real norm = t_avg_cnt[lev];
863  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
864  {
865  derdat(i ,j ,k, mf_comp) = data(i,j,k,2) / norm;
866  });
867  }
868  mf_comp ++;
869  }
870 
871  if (containerHasElement(plot_var_names, "umag_t_avg")) {
872 #ifdef _OPENMP
873 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
874 #endif
875  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
876  {
877  const Box& bx = mfi.tilebox();
878  const Array4<Real>& derdat = mf[lev].array(mfi);
879  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
880  const Real norm = t_avg_cnt[lev];
881  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
882  {
883  derdat(i ,j ,k, mf_comp) = data(i,j,k,3) / norm;
884  });
885  }
886  mf_comp ++;
887  }
888  }
889 
890  if (containerHasElement(plot_var_names, "nut")) {
891  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
892  MultiFab cmf(vars_new[lev][Vars::cons], make_alias, 0, 1); // to provide rho only
893 #ifdef _OPENMP
894 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
895 #endif
896  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
897  {
898  const Box& bx = mfi.tilebox();
899  auto prim = dmf[mfi].array();
900  auto const cons = cmf[mfi].const_array();
901  auto const diff = (*eddyDiffs_lev[lev])[mfi].const_array();
902  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
903  {
904  const Real rho = cons(i, j, k, Rho_comp);
905  const Real Kmv = diff(i, j, k, EddyDiff::Mom_v);
906  prim(i,j,k) = Kmv / rho;
907  });
908  }
909 
910  mf_comp++;
911  }
912 
913  if (containerHasElement(plot_var_names, "Kmv")) {
914  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_v,mf_comp,1,0);
915  mf_comp ++;
916  }
917  if (containerHasElement(plot_var_names, "Kmh")) {
918  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
919  mf_comp ++;
920  }
921  if (containerHasElement(plot_var_names, "Khv")) {
922  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_v,mf_comp,1,0);
923  mf_comp ++;
924  }
925  if (containerHasElement(plot_var_names, "Khh")) {
926  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
927  mf_comp ++;
928  }
929  if (containerHasElement(plot_var_names, "Lturb")) {
930  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Turb_lengthscale,mf_comp,1,0);
931  mf_comp ++;
932  }
933  if (containerHasElement(plot_var_names, "walldist")) {
934  MultiFab::Copy(mf[lev],*walldist[lev],0,mf_comp,1,0);
935  mf_comp ++;
936  }
937  if (containerHasElement(plot_var_names, "diss")) {
938  MultiFab::Copy(mf[lev],*SFS_diss_lev[lev],0,mf_comp,1,0);
939  mf_comp ++;
940  }
941 
942  // TODO: The size of the q variables can vary with different
943  // moisture models. Therefore, certain components may
944  // reside at different indices. For example, Kessler is
945  // warm but precipitating. This puts qp at index 3.
946  // However, SAM is cold and precipitating so qp is index 4.
947  // Need to built an external enum struct or a better pathway.
948 
949  // NOTE: Protect against accessing non-existent data
950  if (use_moisture) {
951  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
952 
953  // Moist density
954  if(containerHasElement(plot_var_names, "moist_density"))
955  {
956  int n_start = RhoQ1_comp; // qv
957  int n_end = RhoQ2_comp; // qc
958  if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
959  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
960  for (int n_comp(n_start); n_comp <= n_end; ++n_comp) {
961  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
962  }
963  mf_comp += 1;
964  }
965 
966  if(containerHasElement(plot_var_names, "qv") && (n_qstate_moist >= 1))
967  {
968  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ1_comp, mf_comp, 1, 0);
969  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
970  mf_comp += 1;
971  }
972 
973  if(containerHasElement(plot_var_names, "qc") && (n_qstate_moist >= 2))
974  {
975  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ2_comp, mf_comp, 1, 0);
976  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
977  mf_comp += 1;
978  }
979 
980  if(containerHasElement(plot_var_names, "qi") && (n_qstate_moist >= 4))
981  {
982  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ3_comp, mf_comp, 1, 0);
983  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
984  mf_comp += 1;
985  }
986 
987  if(containerHasElement(plot_var_names, "qrain") && (n_qstate_moist >= 3))
988  {
989  int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
990  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start , mf_comp, 1, 0);
991  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
992  mf_comp += 1;
993  }
994 
995  if(containerHasElement(plot_var_names, "qsnow") && (n_qstate_moist >= 5))
996  {
997  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ5_comp, mf_comp, 1, 0);
998  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
999  mf_comp += 1;
1000  }
1001 
1002  if(containerHasElement(plot_var_names, "qgraup") && (n_qstate_moist >= 6))
1003  {
1004  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ6_comp, mf_comp, 1, 0);
1005  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1006  mf_comp += 1;
1007  }
1008 
1009  // Precipitating + non-precipitating components
1010  //--------------------------------------------------------------------------
1011  if(containerHasElement(plot_var_names, "qt"))
1012  {
1013  int n_start = RhoQ1_comp; // qv
1014  int n_end = n_start + n_qstate_moist;
1015  MultiFab::Copy(mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1016  for (int n_comp(n_start+1); n_comp < n_end; ++n_comp) {
1017  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1018  }
1019  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1020  mf_comp += 1;
1021  }
1022 
1023  // Non-precipitating components
1024  //--------------------------------------------------------------------------
1025  if (containerHasElement(plot_var_names, "qn"))
1026  {
1027  int n_start = RhoQ1_comp; // qv
1028  int n_end = RhoQ2_comp; // qc
1029  if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
1030  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1031  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1032  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1033  }
1034  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1035  mf_comp += 1;
1036  }
1037 
1038  // Precipitating components
1039  //--------------------------------------------------------------------------
1040  if(containerHasElement(plot_var_names, "qp") && (n_qstate_moist >= 3))
1041  {
1042  int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
1043  int n_end = ncomp_cons - 1;
1044  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1045  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1046  MultiFab::Add( mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1047  }
1048  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1049  mf_comp += 1;
1050  }
1051 
1052  if (containerHasElement(plot_var_names, "qsat"))
1053  {
1054 #ifdef _OPENMP
1055 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1056 #endif
1057  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1058  {
1059  const Box& bx = mfi.tilebox();
1060  const Array4<Real>& derdat = mf[lev].array(mfi);
1061  const Array4<Real const>& p_arr = pressure.array(mfi);
1062  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1063  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1064  {
1065  Real qv = S_arr(i,j,k,RhoQ1_comp) / S_arr(i,j,k,Rho_comp);
1066  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
1067  Real p = p_arr(i,j,k) * Real(0.01);
1068  erf_qsatw(T, p, derdat(i,j,k,mf_comp));
1069  });
1070  }
1071  mf_comp ++;
1072  }
1073 
1074  if ( (solverChoice.moisture_type == MoistureType::Kessler) ||
1075  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
1076  (solverChoice.moisture_type == MoistureType::SAM_NoIce) )
1077  {
1078  int offset = (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ? 5 : 0;
1079  if (containerHasElement(plot_var_names, "rain_accum"))
1080  {
1081  MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
1082  mf_comp += 1;
1083  }
1084  }
1085  else if ( (solverChoice.moisture_type == MoistureType::SAM) ||
1086  (solverChoice.moisture_type == MoistureType::Morrison) )
1087  {
1088  int offset = (solverChoice.moisture_type == MoistureType::Morrison) ? 5 : 0;
1089  if (containerHasElement(plot_var_names, "rain_accum"))
1090  {
1091  MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
1092  mf_comp += 1;
1093  }
1094  if (containerHasElement(plot_var_names, "snow_accum"))
1095  {
1096  MultiFab::Copy(mf[lev],*(qmoist[lev][offset+1]),0,mf_comp,1,0);
1097  mf_comp += 1;
1098  }
1099  if (containerHasElement(plot_var_names, "graup_accum"))
1100  {
1101  MultiFab::Copy(mf[lev],*(qmoist[lev][offset+2]),0,mf_comp,1,0);
1102  mf_comp += 1;
1103  }
1104  }
1105 
1106  if (containerHasElement(plot_var_names, "reflectivity")) {
1107  if (solverChoice.moisture_type == MoistureType::Morrison) {
1108 
1109  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1110  const Box& bx = mfi.tilebox();
1111  const Array4<Real>& derdat = mf[lev].array(mfi);
1112  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1113  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1114 
1115  Real rho = S_arr(i,j,k,Rho_comp);
1116  Real qv = std::max(0.0,S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp));
1117  Real qpr = std::max(0.0,S_arr(i,j,k,RhoQ4_comp)/S_arr(i,j,k,Rho_comp));
1118  Real qps = std::max(0.0,S_arr(i,j,k,RhoQ5_comp)/S_arr(i,j,k,Rho_comp));
1119  Real qpg = std::max(0.0,S_arr(i,j,k,RhoQ6_comp)/S_arr(i,j,k,Rho_comp));
1120 
1121  Real temp = getTgivenRandRTh(S_arr(i,j,k,Rho_comp),
1122  S_arr(i,j,k,RhoTheta_comp),
1123  qv);
1124  derdat(i, j, k, mf_comp) = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
1125  1, 1, 1, 1) ;
1126  });
1127  }
1128  mf_comp ++;
1129  }
1130  }
1131 
1132  if (solverChoice.moisture_type == MoistureType::Morrison) {
1133  if (containerHasElement(plot_var_names, "max_reflectivity")) {
1134  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1135  const Box& bx = mfi.tilebox();
1136 
1137  const Array4<Real>& derdat = mf[lev].array(mfi);
1138  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1139 
1140  // collapse to i,j box (ignore vertical for now)
1141  Box b2d = bx;
1142  b2d.setSmall(2,0);
1143  b2d.setBig(2,0);
1144 
1145  ParallelFor(b2d, [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept {
1146 
1147  Real max_dbz = -1.0e30;
1148 
1149  // find max reflectivity over k
1150  for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
1151  Real rho = S_arr(i,j,k,Rho_comp);
1152  Real qv = std::max(0.0, S_arr(i,j,k,RhoQ1_comp)/rho);
1153  Real qpr = std::max(0.0, S_arr(i,j,k,RhoQ4_comp)/rho);
1154  Real qps = std::max(0.0, S_arr(i,j,k,RhoQ5_comp)/rho);
1155  Real qpg = std::max(0.0, S_arr(i,j,k,RhoQ6_comp)/rho);
1156 
1157  Real temp = getTgivenRandRTh(rho, S_arr(i,j,k,RhoTheta_comp), qv);
1158 
1159  Real dbz = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
1160  1, 1, 1, 1);
1161  max_dbz = amrex::max(max_dbz, dbz);
1162  }
1163 
1164  // store max_dbz into *all* levels for this (i,j)
1165  for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
1166  derdat(i, j, k, mf_comp) = max_dbz;
1167  }
1168  });
1169  }
1170  mf_comp++;
1171  }
1172  }
1173  } // use_moisture
1174 
1175  if (containerHasElement(plot_var_names, "terrain_IB_mask"))
1176  {
1177  MultiFab* terrain_blank = terrain_blanking[lev].get();
1178  MultiFab::Copy(mf[lev],*terrain_blank,0,mf_comp,1,0);
1179  mf_comp ++;
1180  }
1181 
1182  if (containerHasElement(plot_var_names, "volfrac")) {
1183  if ( solverChoice.terrain_type == TerrainType::EB ||
1184  solverChoice.terrain_type == TerrainType::ImmersedForcing)
1185  {
1186  MultiFab::Copy(mf[lev], EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
1187  } else {
1188  mf[lev].setVal(1.0, mf_comp, 1, 0);
1189  }
1190  mf_comp += 1;
1191  }
1192 
1193 #ifdef ERF_COMPUTE_ERROR
1194  // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
1195  if (containerHasElement(plot_var_names, "xvel_err") ||
1196  containerHasElement(plot_var_names, "yvel_err") ||
1197  containerHasElement(plot_var_names, "zvel_err"))
1198  {
1199  //
1200  // Moving terrain ANALYTICAL
1201  //
1202  Real H = geom[lev].ProbHi()[2];
1203  Real Ampl = 0.16;
1204  Real wavelength = 100.;
1205  Real kp = 2. * PI / wavelength;
1206  Real g = CONST_GRAV;
1207  Real omega = std::sqrt(g * kp);
1208  Real omega_t = omega * t_new[lev];
1209 
1210  const auto dx = geom[lev].CellSizeArray();
1211 
1212 #ifdef _OPENMP
1213 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1214 #endif
1215  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1216  {
1217  const Box& bx = mfi.validbox();
1218  Box xbx(bx); xbx.surroundingNodes(0);
1219  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1220  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1221 
1222  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1223 
1224  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1225  {
1226  Real x = i * dx[0];
1227  Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1228 
1229  Real z_base = Ampl * std::sin(kp * x - omega_t);
1230  z -= z_base;
1231 
1232  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1233 
1234  xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
1235  });
1236 
1237  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1238  {
1239  Real x = (i + 0.5) * dx[0];
1240  Real z = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1241 
1242  Real z_base = Ampl * std::sin(kp * x - omega_t);
1243  z -= z_base;
1244 
1245  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1246 
1247  zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
1248  });
1249  }
1250 
1251  MultiFab temp_mf(mf[lev].boxArray(), mf[lev].DistributionMap(), AMREX_SPACEDIM, 0);
1252  average_face_to_cellcenter(temp_mf,0,
1253  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
1254 
1255  if (containerHasElement(plot_var_names, "xvel_err")) {
1256  MultiFab::Copy(mf[lev],temp_mf,0,mf_comp,1,0);
1257  mf_comp += 1;
1258  }
1259  if (containerHasElement(plot_var_names, "yvel_err")) {
1260  MultiFab::Copy(mf[lev],temp_mf,1,mf_comp,1,0);
1261  mf_comp += 1;
1262  }
1263  if (containerHasElement(plot_var_names, "zvel_err")) {
1264  MultiFab::Copy(mf[lev],temp_mf,2,mf_comp,1,0);
1265  mf_comp += 1;
1266  }
1267 
1268  // Now restore the velocities to what they were
1269 #ifdef _OPENMP
1270 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1271 #endif
1272  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1273  {
1274  const Box& bx = mfi.validbox();
1275  Box xbx(bx); xbx.surroundingNodes(0);
1276 
1277  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1278  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1279 
1280  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1281 
1282  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1283  {
1284  Real x = i * dx[0];
1285  Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1286  Real z_base = Ampl * std::sin(kp * x - omega_t);
1287 
1288  z -= z_base;
1289 
1290  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1291  xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
1292  });
1293  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1294  {
1295  Real x = (i + 0.5) * dx[0];
1296  Real z = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1297  Real z_base = Ampl * std::sin(kp * x - omega_t);
1298 
1299  z -= z_base;
1300  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1301 
1302  zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
1303  });
1304  }
1305  } // end xvel_err, yvel_err, zvel_err
1306 
1307  if (containerHasElement(plot_var_names, "pp_err"))
1308  {
1309  // Moving terrain ANALYTICAL
1310 #ifdef _OPENMP
1311 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1312 #endif
1313  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1314  {
1315  const Box& bx = mfi.tilebox();
1316  const Array4<Real>& derdat = mf[lev].array(mfi);
1317  const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
1318  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1319 
1320  const auto dx = geom[lev].CellSizeArray();
1321  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1322  const Array4<Real const>& p_arr = pressure.const_array(mfi);
1323  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
1324 
1325  Real H = geom[lev].ProbHi()[2];
1326  Real Ampl = 0.16;
1327  Real wavelength = 100.;
1328  Real kp = 2. * PI / wavelength;
1329  Real g = CONST_GRAV;
1330  Real omega = std::sqrt(g * kp);
1331  Real omega_t = omega * t_new[lev];
1332 
1333  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1334  {
1335  derdat(i, j, k, mf_comp) = p_arr(i,j,k) - p0_arr(i,j,k);
1336 
1337  Real rho_hse = r0_arr(i,j,k);
1338 
1339  Real x = (i + 0.5) * dx[0];
1340  Real z = 0.125 * ( z_nd(i,j,k ) + z_nd(i+1,j,k ) + z_nd(i,j+1,k ) + z_nd(i+1,j+1,k )
1341  +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
1342  Real z_base = Ampl * std::sin(kp * x - omega_t);
1343 
1344  z -= z_base;
1345  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1346  Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
1347  std::sin(kp * x - omega_t) * r0_arr(i,j,k);
1348 
1349  derdat(i,j,k,mf_comp) -= pprime_exact;
1350  });
1351  }
1352  mf_comp += 1;
1353  }
1354 #endif
1355 
1356  if (solverChoice.rad_type != RadiationType::None) {
1357  if (containerHasElement(plot_var_names, "qsrc_sw")) {
1358  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 0, mf_comp, 1, 0);
1359  mf_comp += 1;
1360  }
1361  if (containerHasElement(plot_var_names, "qsrc_lw")) {
1362  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 1, mf_comp, 1, 0);
1363  mf_comp += 1;
1364  }
1365  }
1366 
1367  // *****************************************************************************************
1368  // End of derived variables corresponding to "derived_names" in ERF.H
1369  //
1370  // Particles and microphysics can provide additional outputs, which are handled below.
1371  // *****************************************************************************************
1372 
1373 #ifdef ERF_USE_PARTICLES
1374  const auto& particles_namelist( particleData.getNames() );
1375 
1376  if (containerHasElement(plot_var_names, "tracer_particles_count")) {
1377  if (particles_namelist.size() == 0) {
1378  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
1379  temp_dat.setVal(0);
1380  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1381  mf_comp += 1;
1382  } else {
1383  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++) {
1384  if (containerHasElement(plot_var_names, std::string(particles_namelist[i]+"_count"))) {
1385  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
1386  temp_dat.setVal(0);
1387  if (particleData.HasSpecies(particles_namelist[i])) {
1388  particleData[particles_namelist[i]]->Increment(temp_dat, lev);
1389  }
1390  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1391  mf_comp += 1;
1392  }
1393  }
1394  }
1395  }
1396 
1397  Vector<std::string> particle_mesh_plot_names(0);
1398  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
1399 
1400  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
1401  std::string plot_var_name(particle_mesh_plot_names[i]);
1402  if (containerHasElement(plot_var_names, plot_var_name) ) {
1403  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
1404  temp_dat.setVal(0);
1405  particleData.GetMeshPlotVar(plot_var_name, temp_dat, lev);
1406  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1407  mf_comp += 1;
1408  }
1409  }
1410 #endif
1411 
1412  {
1413  Vector<std::string> microphysics_plot_names;
1414  micro->GetPlotVarNames(microphysics_plot_names);
1415  for (auto& plot_name : microphysics_plot_names) {
1416  if (containerHasElement(plot_var_names, plot_name)) {
1417  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
1418  temp_dat.setVal(0);
1419  micro->GetPlotVar(plot_name, temp_dat, lev);
1420  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1421  mf_comp += 1;
1422  }
1423  }
1424  }
1425 
1426 
1427  }
1428 
1429  if (solverChoice.terrain_type == TerrainType::EB)
1430  {
1431  for (int lev = 0; lev <= finest_level; ++lev) {
1432  EB_set_covered(mf[lev], 0.0);
1433  }
1434  }
1435 
1436  // Fill terrain distortion MF (nu_nd)
1437  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1438  for (int lev(0); lev <= finest_level; ++lev) {
1439  MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
1440  Real dz = Geom()[lev].CellSizeArray()[2];
1441  for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1442  const Box& bx = mfi.tilebox();
1443  Array4<Real> mf_arr = mf_nd[lev].array(mfi);
1444  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1445  {
1446  mf_arr(i,j,k,2) -= k * dz;
1447  });
1448  }
1449  }
1450  }
1451 
1452  std::string plotfilename;
1453  std::string plotfilenameU;
1454  std::string plotfilenameV;
1455  std::string plotfilenameW;
1456  if (which == 1) {
1457  plotfilename = Concatenate(plot3d_file_1, istep[0], 5);
1458  plotfilenameU = Concatenate(plot3d_file_1+"U", istep[0], 5);
1459  plotfilenameV = Concatenate(plot3d_file_1+"V", istep[0], 5);
1460  plotfilenameW = Concatenate(plot3d_file_1+"W", istep[0], 5);
1461  } else if (which == 2) {
1462  plotfilename = Concatenate(plot3d_file_2, istep[0], 5);
1463  plotfilenameU = Concatenate(plot3d_file_2+"U", istep[0], 5);
1464  plotfilenameV = Concatenate(plot3d_file_2+"V", istep[0], 5);
1465  plotfilenameW = Concatenate(plot3d_file_2+"W", istep[0], 5);
1466  }
1467 
1468  // LSM writes it's own data
1469  if (which==1 && plot_lsm) {
1470  lsm.Plot_Lsm_Data(t_new[0], istep, refRatio());
1471  }
1472 
1473 #ifdef ERF_USE_RRTMGP
1474  /*
1475  // write additional RRTMGP data
1476  // TODO: currently single level only
1477  if (which==1 && plot_rad) {
1478  rad[0]->writePlotfile(plot_file_1, t_new[0], istep[0]);
1479  }
1480  */
1481 #endif
1482 
1483  // Single level
1484  if (finest_level == 0)
1485  {
1486  if (plotfile_type == PlotFileType::Amrex)
1487  {
1488  Print() << "Writing native 3D plotfile " << plotfilename << "\n";
1489  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1490  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1491  GetVecOfConstPtrs(mf),
1492  GetVecOfConstPtrs(mf_nd),
1493  varnames,
1494  Geom(), t_new[0], istep, refRatio());
1495  } else {
1496  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1497  GetVecOfConstPtrs(mf),
1498  varnames,
1499  Geom(), t_new[0], istep, refRatio());
1500  }
1501  writeJobInfo(plotfilename);
1502 
1503  if (m_plot_face_vels) {
1504  Print() << "Writing face velocities" << std::endl;
1505  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1506  GetVecOfConstPtrs(mf_u),
1507  {"x_velocity_stag"},
1508  Geom(), t_new[0], istep, refRatio());
1509  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1510  GetVecOfConstPtrs(mf_v),
1511  {"y_velocity_stag"},
1512  Geom(), t_new[0], istep, refRatio());
1513  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1514  GetVecOfConstPtrs(mf_w),
1515  {"z_velocity_stag"},
1516  Geom(), t_new[0], istep, refRatio());
1517  }
1518 
1519 #ifdef ERF_USE_PARTICLES
1520  particleData.writePlotFile(plotfilename);
1521 #endif
1522 #ifdef ERF_USE_NETCDF
1523  } else if (plotfile_type == PlotFileType::Netcdf) {
1524  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
1525  int lev = 0;
1526  int l_which = 0;
1527  const Real* p_lo = geom[lev].ProbLo();
1528  const Real* p_hi = geom[lev].ProbHi();
1529  const auto dx = geom[lev].CellSize();
1530  writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1531  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
1532  geom[lev].Domain(), t_new[0], start_bdy_time);
1533 #endif
1534  } else {
1535  // Here we assume the plotfile_type is PlotFileType::None
1536  Print() << "Writing no 3D plotfile since plotfile_type is none" << std::endl;
1537  }
1538 
1539  } else { // Multilevel
1540 
1541  if (plotfile_type == PlotFileType::Amrex) {
1542 
1543  int lev0 = 0;
1544  int desired_ratio = std::max(std::max(ref_ratio[lev0][0],ref_ratio[lev0][1]),ref_ratio[lev0][2]);
1545  bool any_ratio_one = ( ( (ref_ratio[lev0][0] == 1) || (ref_ratio[lev0][1] == 1) ) ||
1546  (ref_ratio[lev0][2] == 1) );
1547  for (int lev = 1; lev < finest_level; lev++) {
1548  any_ratio_one = any_ratio_one ||
1549  ( ( (ref_ratio[lev][0] == 1) || (ref_ratio[lev][1] == 1) ) ||
1550  (ref_ratio[lev][2] == 1) );
1551  }
1552 
1553  if (any_ratio_one && m_expand_plotvars_to_unif_rr)
1554  {
1555  Vector<IntVect> r2(finest_level);
1556  Vector<Geometry> g2(finest_level+1);
1557  Vector<MultiFab> mf2(finest_level+1);
1558 
1559  mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
1560 
1561  // Copy level 0 as is
1562  MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
1563 
1564  // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
1565  Array<int,AMREX_SPACEDIM> periodicity =
1566  {Geom()[lev0].isPeriodic(0),Geom()[lev0].isPeriodic(1),Geom()[lev0].isPeriodic(2)};
1567  g2[lev0].define(Geom()[lev0].Domain(),&(Geom()[lev0].ProbDomain()),0,periodicity.data());
1568 
1569  r2[0] = IntVect(desired_ratio/ref_ratio[lev0][0],
1570  desired_ratio/ref_ratio[lev0][1],
1571  desired_ratio/ref_ratio[lev0][2]);
1572 
1573  for (int lev = 1; lev <= finest_level; ++lev) {
1574  if (lev > 1) {
1575  r2[lev-1][0] = r2[lev-2][0] * desired_ratio / ref_ratio[lev-1][0];
1576  r2[lev-1][1] = r2[lev-2][1] * desired_ratio / ref_ratio[lev-1][1];
1577  r2[lev-1][2] = r2[lev-2][2] * desired_ratio / ref_ratio[lev-1][2];
1578  }
1579 
1580  mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
1581 
1582  // Set the new problem domain
1583  Box d2(Geom()[lev].Domain());
1584  d2.refine(r2[lev-1]);
1585 
1586  g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
1587  }
1588 
1589  //
1590  // We need to make a temporary that is the size of ncomp_mf
1591  // in order to not get an out of bounds error
1592  // even though the values will not be used
1593  //
1594  Vector<BCRec> temp_domain_bcs_type;
1595  temp_domain_bcs_type.resize(ncomp_mf);
1596 
1597  //
1598  // Do piecewise constant interpolation of mf into mf2
1599  //
1600  for (int lev = 1; lev <= finest_level; ++lev) {
1601  Interpolater* mapper_c = &pc_interp;
1602  InterpFromCoarseLevel(mf2[lev], t_new[lev], mf[lev],
1603  0, 0, ncomp_mf,
1604  geom[lev], g2[lev],
1606  r2[lev-1], mapper_c, temp_domain_bcs_type, 0);
1607  }
1608 
1609  // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
1610  Vector<IntVect> rr(finest_level);
1611  for (int lev = 0; lev < finest_level; ++lev) {
1612  rr[lev] = IntVect(desired_ratio);
1613  }
1614 
1615  Print() << "Writing 3D plotfile " << plotfilename << "\n";
1616  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1617  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1618  GetVecOfConstPtrs(mf2),
1619  GetVecOfConstPtrs(mf_nd),
1620  varnames,
1621  g2, t_new[0], istep, rr);
1622  } else {
1623  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1624  GetVecOfConstPtrs(mf2), varnames,
1625  g2, t_new[0], istep, rr);
1626  }
1627 
1628  } else {
1629  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1630  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1631  GetVecOfConstPtrs(mf),
1632  GetVecOfConstPtrs(mf_nd),
1633  varnames,
1634  geom, t_new[0], istep, ref_ratio);
1635  } else {
1636  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1637  GetVecOfConstPtrs(mf), varnames,
1638  geom, t_new[0], istep, ref_ratio);
1639  }
1640  if (m_plot_face_vels) {
1641  Print() << "Writing face velocities" << std::endl;
1642  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1643  GetVecOfConstPtrs(mf_u),
1644  {"x_velocity_stag"},
1645  geom, t_new[0], istep, ref_ratio);
1646  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1647  GetVecOfConstPtrs(mf_v),
1648  {"y_velocity_stag"},
1649  geom, t_new[0], istep, ref_ratio);
1650  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1651  GetVecOfConstPtrs(mf_w),
1652  {"z_velocity_stag"},
1653  geom, t_new[0], istep, ref_ratio);
1654  }
1655  } // ref_ratio test
1656 
1657  writeJobInfo(plotfilename);
1658 
1659 #ifdef ERF_USE_PARTICLES
1660  particleData.writePlotFile(plotfilename);
1661 #endif
1662 
1663 #ifdef ERF_USE_NETCDF
1664  } else if (plotfile_type == PlotFileType::Netcdf) {
1665  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
1666  for (int lev = 0; lev <= finest_level; ++lev) {
1667  for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
1668  Box bounding_region = (lev == 0) ? geom[lev].Domain() : boxes_at_level[lev][which_box];
1669  const Real* p_lo = geom[lev].ProbLo();
1670  const Real* p_hi = geom[lev].ProbHi();
1671  const auto dx = geom[lev].CellSizeArray();
1672  writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1673  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
1674  bounding_region, t_new[0], start_bdy_time);
1675  }
1676  }
1677 #endif
1678  }
1679  } // end multi-level
1680 }
constexpr amrex::Real PI
Definition: ERF_Constants.H:6
constexpr amrex::Real Cp_l
Definition: ERF_Constants.H:14
#define RhoQ4_comp
Definition: ERF_IndexDefines.H:45
void compute_gradp(const MultiFab &p, const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc, const eb_ &ebfact, Vector< MultiFab > &gradp, const SolverChoice &solverChoice)
Definition: ERF_MakeGradP.cpp:74
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real erf_esatw(amrex::Real t)
Definition: ERF_MicrophysicsUtils.H:65
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void erf_qsatw(amrex::Real t, amrex::Real p, amrex::Real &qsatw)
Definition: ERF_MicrophysicsUtils.H:163
PhysBCFunctNoOp null_bc_for_fill
Definition: ERF_Plotfile.cpp:7
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real compute_max_reflectivity_dbz(amrex::Real rho_air, amrex::Real tmk, amrex::Real qra, amrex::Real qsn, amrex::Real qgr, int in0r, int in0s, int in0g, int iliqskin)
Definition: ERF_StormDiagnostics.H:13
void WriteMultiLevelPlotfileWithTerrain(const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
Definition: ERF_Plotfile.cpp:1683
void Plot_Lsm_Data(amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &ref_ratio)
Definition: ERF_LandSurface.H:101
@ Turb_lengthscale
Definition: ERF_IndexDefines.H:180
@ Mom_h
Definition: ERF_IndexDefines.H:170
@ Theta_h
Definition: ERF_IndexDefines.H:171
@ qpg
Definition: ERF_Morrison.H:41
@ qps
Definition: ERF_Morrison.H:40
@ qpr
Definition: ERF_Morrison.H:39
void erf_dervortx(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:200
void erf_dervorty(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:228
void erf_dermagvel(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:319
void erf_dernull(const Box &, FArrayBox &, int, int, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:39
void erf_dertemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:91
void erf_derKE(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:186
void erf_dermoisttemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:113
void erf_dersoundspeed(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:58
real(c_double), parameter g
Definition: ERF_module_model_constants.F90:19
Here is the call graph for this function:

◆ write_1D_profiles()

void ERF::write_1D_profiles ( amrex::Real  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters
timeCurrent time
18 {
19  BL_PROFILE("ERF::write_1D_profiles()");
20 
21  if (NumDataLogs() > 1)
22  {
23  // Define the 1d arrays we will need
24  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
25  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
26  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
27  Gpu::HostVector<Real> h_avg_wthv;
28  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
29  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
30  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
31  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
32  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
33  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
34 
35  if (NumDataLogs() > 1) {
37  h_avg_u, h_avg_v, h_avg_w,
38  h_avg_rho, h_avg_th, h_avg_ksgs,
39  h_avg_Kmv, h_avg_Khv,
40  h_avg_qv, h_avg_qc, h_avg_qr,
41  h_avg_wqv, h_avg_wqc, h_avg_wqr,
42  h_avg_qi, h_avg_qs, h_avg_qg,
43  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
44  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
45  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
46  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
47  h_avg_wthv);
48  }
49 
50  if (NumDataLogs() > 3 && time > 0.) {
51  derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
52  h_avg_tau22, h_avg_tau23, h_avg_tau33,
53  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
54  h_avg_sgsdiss);
55  }
56 
57  int hu_size = h_avg_u.size();
58 
59  auto const& dx = geom[0].CellSizeArray();
60  if (ParallelDescriptor::IOProcessor()) {
61  if (NumDataLogs() > 1) {
62  std::ostream& data_log1 = DataLog(1);
63  if (data_log1.good()) {
64  // Write the quantities at this time
65  for (int k = 0; k < hu_size; k++) {
66  Real z;
67  if (zlevels_stag[0].size() > 1) {
68  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
69  } else {
70  z = (k + 0.5)* dx[2];
71  }
72  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
73  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
74  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
75  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
76  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
77  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
78  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
79  << std::endl;
80  } // loop over z
81  } // if good
82  } // NumDataLogs
83 
84  if (NumDataLogs() > 2) {
85  std::ostream& data_log2 = DataLog(2);
86  if (data_log2.good()) {
87  // Write the perturbational quantities at this time
88  for (int k = 0; k < hu_size; k++) {
89  Real z;
90  if (zlevels_stag[0].size() > 1) {
91  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
92  } else {
93  z = (k + 0.5)* dx[2];
94  }
95  Real thv = h_avg_th[k] * (1 + 0.61*h_avg_qv[k] - h_avg_qc[k] - h_avg_qr[k]);
96  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
97  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
98  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " "
99  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " "
100  << h_avg_uw[k] - h_avg_u[k]*h_avg_w[k] << " "
101  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " "
102  << h_avg_vw[k] - h_avg_v[k]*h_avg_w[k] << " "
103  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " "
104  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " "
105  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " "
106  << h_avg_wth[k] - h_avg_w[k]*h_avg_th[k] << " "
107  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
108  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
109  // - <u_i u_i> * <u_j>
110  // - 2*<u_i> * <u_i u_j>
111  // + 2*<u_i>*<u_i> * <u_j>
112  << h_avg_uiuiu[k]
113  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
114  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
115  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
116  << " " // (u'_i u'_i)u'
117  << h_avg_uiuiv[k]
118  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
119  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
120  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
121  << " " // (u'_i u'_i)v'
122  << h_avg_uiuiw[k]
123  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
124  - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
125  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
126  << " " // (u'_i u'_i)w'
127  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " "
128  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " "
129  << h_avg_pw[k] - h_avg_p[k]*h_avg_w[k] << " "
130  << h_avg_wqv[k] - h_avg_qv[k]*h_avg_w[k] << " "
131  << h_avg_wqc[k] - h_avg_qc[k]*h_avg_w[k] << " "
132  << h_avg_wqr[k] - h_avg_qr[k]*h_avg_w[k] << " "
133  << h_avg_wthv[k] - h_avg_w[k]*thv
134  << std::endl;
135  } // loop over z
136  } // if good
137  } // NumDataLogs
138 
139  if (NumDataLogs() > 3 && time > 0.) {
140  std::ostream& data_log3 = DataLog(3);
141  if (data_log3.good()) {
142  // Write the average stresses
143  for (int k = 0; k < hu_size; k++) {
144  Real z;
145  if (zlevels_stag[0].size() > 1) {
146  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
147  } else {
148  z = (k + 0.5)* dx[2];
149  }
150  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
151  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
152  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
153  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
154  << h_avg_sgshfx[k] << " "
155  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
156  << h_avg_sgsdiss[k]
157  << std::endl;
158  } // loop over z
159  } // if good
160  } // if (NumDataLogs() > 3)
161  } // if IOProcessor
162  } // if (NumDataLogs() > 1)
163 }
void derive_diag_profiles(amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles.cpp:190
void derive_stress_profiles(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles.cpp:475

◆ write_1D_profiles_stag()

void ERF::write_1D_profiles_stag ( amrex::Real  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Quantities are output at their native grid locations. Therefore, w and associated flux quantities <(•)'w'>, tau13, and tau23 (where '•' includes u, v, p, theta, ...) will be output at staggered heights (i.e., coincident with z faces) rather than cell-center heights to avoid performing additional averaging. Unstaggered (i.e., cell-centered) quantities are output alongside staggered quantities at the lower cell faces in the log file; these quantities will have a zero value at the big end, corresponding to k=Nz+1.

The structure of file should follow ERF_Write1DProfiles.cpp

Parameters
timeCurrent time
26 {
27  BL_PROFILE("ERF::write_1D_profiles()");
28 
29  if (NumDataLogs() > 1)
30  {
31  // Define the 1d arrays we will need
32  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
33  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
34  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
35  Gpu::HostVector<Real> h_avg_wthv;
36  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
37  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
38  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
39  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
40  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
41  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
42 
43  if (NumDataLogs() > 1) {
45  h_avg_u, h_avg_v, h_avg_w,
46  h_avg_rho, h_avg_th, h_avg_ksgs,
47  h_avg_Kmv, h_avg_Khv,
48  h_avg_qv, h_avg_qc, h_avg_qr,
49  h_avg_wqv, h_avg_wqc, h_avg_wqr,
50  h_avg_qi, h_avg_qs, h_avg_qg,
51  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
52  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
53  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
54  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
55  h_avg_wthv);
56  }
57 
58  if (NumDataLogs() > 3 && time > 0.) {
59  derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
60  h_avg_tau22, h_avg_tau23, h_avg_tau33,
61  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
62  h_avg_sgsdiss);
63  }
64 
65  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
66 
67  auto const& dx = geom[0].CellSizeArray();
68  if (ParallelDescriptor::IOProcessor()) {
69  if (NumDataLogs() > 1) {
70  std::ostream& data_log1 = DataLog(1);
71  if (data_log1.good()) {
72  // Write the quantities at this time
73  for (int k = 0; k < unstag_size; k++) {
74  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
75  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
76  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
77  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
78  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
79  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
80  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
81  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
82  << std::endl;
83  } // loop over z
84  // Write top face values
85  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
86  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
87  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
88  << 0 << " " << 0 << " " << h_avg_w[unstag_size] << " "
89  << 0 << " " << 0 << " " << 0 << " " // rho, theta, ksgs
90  << 0 << " " << 0 << " " // Kmv, Khv
91  << 0 << " " << 0 << " " << 0 << " " // qv, qc, qr
92  << 0 << " " << 0 << " " << 0 // qi, qs, qg
93  << std::endl;
94  } // if good
95  } // NumDataLogs
96 
97  if (NumDataLogs() > 2) {
98  std::ostream& data_log2 = DataLog(2);
99  if (data_log2.good()) {
100  // Write the perturbational quantities at this time
101  // For surface values (k=0), assume w = uw = vw = ww = 0
102  Real w_cc = h_avg_w[1] / 2; // w at first cell center
103  Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
104  Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
105  Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
106  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
107  << std::setw(datwidth) << std::setprecision(datprecision) << 0 << " "
108  << h_avg_uu[0] - h_avg_u[0]*h_avg_u[0] << " " // u'u'
109  << h_avg_uv[0] - h_avg_u[0]*h_avg_v[0] << " " // u'v'
110  << 0 << " " // u'w'
111  << h_avg_vv[0] - h_avg_v[0]*h_avg_v[0] << " " // v'v'
112  << 0 << " " // v'w'
113  << 0 << " " // w'w'
114  << h_avg_uth[0] - h_avg_u[0]*h_avg_th[0] << " " // u'th'
115  << h_avg_vth[0] - h_avg_v[0]*h_avg_th[0] << " " // v'th'
116  << 0 << " " // w'th'
117  << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
118  << h_avg_uiuiu[0]
119  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
120  - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
121  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
122  << " " // (u'_i u'_i)u'
123  << h_avg_uiuiv[0]
124  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
125  - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
126  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
127  << " " // (u'_i u'_i)v'
128  << 0 << " " // (u'_i u'_i)w'
129  << h_avg_pu[0] - h_avg_p[0]*h_avg_u[0] << " " // p'u'
130  << h_avg_pv[0] - h_avg_p[0]*h_avg_v[0] << " " // p'v'
131  << 0 << " " // p'w'
132  << 0 << " " // qv'w'
133  << 0 << " " // qc'w'
134  << 0 << " " // qr'w'
135  << 0 // thv'w'
136  << std::endl;
137 
138  // For internal values, interpolate scalar quantities to faces
139  for (int k = 1; k < unstag_size; k++) {
140  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
141  Real uface = 0.5*(h_avg_u[k] + h_avg_u[k-1]);
142  Real vface = 0.5*(h_avg_v[k] + h_avg_v[k-1]);
143  Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]);
144  Real pface = 0.5*(h_avg_p[k] + h_avg_p[k-1]);
145  Real qvface = 0.5*(h_avg_qv[k] + h_avg_qv[k-1]);
146  Real qcface = 0.5*(h_avg_qc[k] + h_avg_qc[k-1]);
147  Real qrface = 0.5*(h_avg_qr[k] + h_avg_qr[k-1]);
148  Real uuface = 0.5*(h_avg_uu[k] + h_avg_uu[k-1]);
149  Real vvface = 0.5*(h_avg_vv[k] + h_avg_vv[k-1]);
150  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
151  w_cc = 0.5*(h_avg_w[k-1] + h_avg_w[k]);
152  uw_cc = 0.5*(h_avg_uw[k-1] + h_avg_uw[k]);
153  vw_cc = 0.5*(h_avg_vw[k-1] + h_avg_vw[k]);
154  ww_cc = 0.5*(h_avg_ww[k-1] + h_avg_ww[k]);
155  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
156  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
157  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " " // u'u'
158  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " " // u'v'
159  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
160  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " " // v'v'
161  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
162  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
163  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " " // u'th'
164  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " " // v'th'
165  << h_avg_wth[k] - h_avg_w[k]*thface << " " // w'th'
166  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
167  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
168  // - <u_i u_i> * <u_j>
169  // - 2*<u_i> * <u_i u_j>
170  // + 2*<u_i>*<u_i> * <u_j>
171  << h_avg_uiuiu[k]
172  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
173  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
174  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
175  << " " // cell-centered (u'_i u'_i)u'
176  << h_avg_uiuiv[k]
177  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
178  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
179  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
180  << " " // cell-centered (u'_i u'_i)v'
181  << h_avg_uiuiw[k]
182  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
183  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
184  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
185  << " " // face-centered (u'_i u'_i)w'
186  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " " // cell-centered p'u'
187  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " " // cell-centered p'v'
188  << h_avg_pw[k] - pface*h_avg_w[k] << " " // face-centered p'w'
189  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
190  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
191  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
192  << h_avg_wthv[k] - thvface*h_avg_w[k]
193  << std::endl;
194  } // loop over z
195 
196  // Write top face values, extrapolating scalar quantities
197  const int k = unstag_size;
198  Real uface = 1.5*h_avg_u[k-1] - 0.5*h_avg_u[k-2];
199  Real vface = 1.5*h_avg_v[k-1] - 0.5*h_avg_v[k-2];
200  Real thface = 1.5*h_avg_th[k-1] - 0.5*h_avg_th[k-2];
201  Real pface = 1.5*h_avg_p[k-1] - 0.5*h_avg_p[k-2];
202  Real qvface = 1.5*h_avg_qv[k-1] - 0.5*h_avg_qv[k-2];
203  Real qcface = 1.5*h_avg_qc[k-1] - 0.5*h_avg_qc[k-2];
204  Real qrface = 1.5*h_avg_qr[k-1] - 0.5*h_avg_qr[k-2];
205  Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2];
206  Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2];
207  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
208  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
209  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
210  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
211  << 0 << " " // u'u'
212  << 0 << " " // u'v'
213  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
214  << 0 << " " // v'v'
215  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
216  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
217  << 0 << " " // u'th'
218  << 0 << " " // v'th'
219  << h_avg_wth[k] - thface*h_avg_w[k] << " " // w'th'
220  << 0 << " " // th'th'
221  << 0 << " " // (u'_i u'_i)u'
222  << 0 << " " // (u'_i u'_i)v'
223  << h_avg_uiuiw[k]
224  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
225  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
226  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
227  << " " // (u'_i u'_i)w'
228  << 0 << " " // pu'
229  << 0 << " " // pv'
230  << h_avg_pw[k] - pface*h_avg_w[k] << " " // pw'
231  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
232  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
233  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
234  << h_avg_wthv[k] - thvface*h_avg_w[k]
235  << std::endl;
236  } // if good
237  } // NumDataLogs
238 
239  if (NumDataLogs() > 3 && time > 0.) {
240  std::ostream& data_log3 = DataLog(3);
241  if (data_log3.good()) {
242  // Write the average stresses
243  for (int k = 0; k < unstag_size; k++) {
244  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
245  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
246  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
247  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
248  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
249  << h_avg_sgshfx[k] << " "
250  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
251  << h_avg_sgsdiss[k]
252  << std::endl;
253  } // loop over z
254  // Write top face values
255  Real NANval = 0.0;
256  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
257  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
258  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
259  << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
260  << NANval << " " << h_avg_tau23[unstag_size] << " " << NANval << " "
261  << h_avg_sgshfx[unstag_size] << " "
262  << h_avg_sgsq1fx[unstag_size] << " " << h_avg_sgsq2fx[unstag_size] << " "
263  << NANval
264  << std::endl;
265  } // if good
266  } // if (NumDataLogs() > 3)
267  } // if IOProcessor
268  } // if (NumDataLogs() > 1)
269 }
void derive_diag_profiles_stag(amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles_stag.cpp:296
void derive_stress_profiles_stag(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles_stag.cpp:605

◆ writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream &  os)
static
138 {
139  std::string PrettyLine = std::string(78, '=') + "\n";
140  std::string OtherLine = std::string(78, '-') + "\n";
141  std::string SkipSpace = std::string(8, ' ');
142 
143  // build information
144  os << PrettyLine;
145  os << " ERF Build Information\n";
146  os << PrettyLine;
147 
148  os << "build date: " << buildInfoGetBuildDate() << "\n";
149  os << "build machine: " << buildInfoGetBuildMachine() << "\n";
150  os << "build dir: " << buildInfoGetBuildDir() << "\n";
151  os << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
152 
153  os << "\n";
154 
155  os << "COMP: " << buildInfoGetComp() << "\n";
156  os << "COMP version: " << buildInfoGetCompVersion() << "\n";
157 
158  os << "C++ compiler: " << buildInfoGetCXXName() << "\n";
159  os << "C++ flags: " << buildInfoGetCXXFlags() << "\n";
160 
161  os << "\n";
162 
163  os << "Link flags: " << buildInfoGetLinkFlags() << "\n";
164  os << "Libraries: " << buildInfoGetLibraries() << "\n";
165 
166  os << "\n";
167 
168  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
169  os << buildInfoGetModuleName(n) << ": "
170  << buildInfoGetModuleVal(n) << "\n";
171  }
172 
173  os << "\n";
174  const char* githash1 = buildInfoGetGitHash(1);
175  const char* githash2 = buildInfoGetGitHash(2);
176  if (strlen(githash1) > 0) {
177  os << "ERF git hash: " << githash1 << "\n";
178  }
179  if (strlen(githash2) > 0) {
180  os << "AMReX git hash: " << githash2 << "\n";
181  }
182 
183  const char* buildgithash = buildInfoGetBuildGitHash();
184  const char* buildgitname = buildInfoGetBuildGitName();
185  if (strlen(buildgithash) > 0) {
186  os << buildgitname << " git hash: " << buildgithash << "\n";
187  }
188 
189  os << "\n";
190  os << " ERF Compile time variables: \n";
191 
192  os << "\n";
193  os << " ERF Defines: \n";
194 #ifdef _OPENMP
195  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
196  << std::endl;
197 #else
198  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
199  << std::endl;
200 #endif
201 
202 #ifdef MPI_VERSION
203  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
204  << MPI_VERSION << std::endl;
205 #else
206  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
207  << "UNDEFINED" << std::endl;
208 #endif
209 
210 #ifdef MPI_SUBVERSION
211  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
212  << MPI_SUBVERSION << std::endl;
213 #else
214  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
215  << "UNDEFINED" << std::endl;
216 #endif
217 
218  os << "\n\n";
219 }

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ WriteCheckpointFile()

void ERF::WriteCheckpointFile ( ) const

ERF function for writing a checkpoint file.

27 {
28  // chk00010 write a checkpoint file with this root directory
29  // chk00010/Header this contains information you need to save (e.g., finest_level, t_new, etc.) and also
30  // the BoxArrays at each level
31  // chk00010/Level_0/
32  // chk00010/Level_1/
33  // etc. these subdirectories will hold the MultiFab data at each level of refinement
34 
35  // checkpoint file name, e.g., chk00010
36  const std::string& checkpointname = Concatenate(check_file,istep[0],5);
37 
38  Print() << "Writing native checkpoint " << checkpointname << "\n";
39 
40  const int nlevels = finest_level+1;
41 
42  // ---- prebuild a hierarchy of directories
43  // ---- dirName is built first. if dirName exists, it is renamed. then build
44  // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
45  // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
46  // ---- after all directories are built
47  // ---- ParallelDescriptor::IOProcessor() creates the directories
48  PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
49 
50  int ncomp_cons = vars_new[0][Vars::cons].nComp();
51 
52  // write Header file
53  if (ParallelDescriptor::IOProcessor()) {
54 
55  std::string HeaderFileName(checkpointname + "/Header");
56  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
57  std::ofstream HeaderFile;
58  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
59  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
60  std::ofstream::trunc |
61  std::ofstream::binary);
62  if(! HeaderFile.good()) {
63  FileOpenFailed(HeaderFileName);
64  }
65 
66  HeaderFile.precision(17);
67 
68  // write out title line
69  HeaderFile << "Checkpoint file for ERF\n";
70 
71  // write out finest_level
72  HeaderFile << finest_level << "\n";
73 
74  // write the number of components
75  // for each variable we store
76 
77  // conservative, cell-centered vars
78  HeaderFile << ncomp_cons << "\n";
79 
80  // x-velocity on faces
81  HeaderFile << 1 << "\n";
82 
83  // y-velocity on faces
84  HeaderFile << 1 << "\n";
85 
86  // z-velocity on faces
87  HeaderFile << 1 << "\n";
88 
89  // write out array of istep
90  for (int i = 0; i < istep.size(); ++i) {
91  HeaderFile << istep[i] << " ";
92  }
93  HeaderFile << "\n";
94 
95  // write out array of dt
96  for (int i = 0; i < dt.size(); ++i) {
97  HeaderFile << dt[i] << " ";
98  }
99  HeaderFile << "\n";
100 
101  // write out array of t_new
102  for (int i = 0; i < t_new.size(); ++i) {
103  HeaderFile << t_new[i] << " ";
104  }
105  HeaderFile << "\n";
106 
107  // write the BoxArray at each level
108  for (int lev = 0; lev <= finest_level; ++lev) {
109  boxArray(lev).writeOn(HeaderFile);
110  HeaderFile << '\n';
111  }
112 
113  // Write separate file that tells how many components we have of the base state
114  std::string BaseStateFileName(checkpointname + "/num_base_state_comps");
115  std::ofstream BaseStateFile;
116  BaseStateFile.open(BaseStateFileName.c_str(), std::ofstream::out |
117  std::ofstream::trunc |
118  std::ofstream::binary);
119  if(! BaseStateFile.good()) {
120  FileOpenFailed(BaseStateFileName);
121  } else {
122  // write out number of components in base state
123  BaseStateFile << BaseState::num_comps << "\n";
124  BaseStateFile << base_state[0].nGrowVect() << "\n";
125  }
126  }
127 
128  // write the MultiFab data to, e.g., chk00010/Level_0/
129  // Here we make copies of the MultiFab with no ghost cells
130  for (int lev = 0; lev <= finest_level; ++lev)
131  {
132  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
133  MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
134  VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
135 
136  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
137  MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
138  VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
139 
140  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
141  MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
142  VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
143 
144  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
145  MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
146  VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
147 
148  if (solverChoice.anelastic[lev] == 1) {
149  MultiFab ppinc(grids[lev],dmap[lev],1,0);
150  MultiFab::Copy(ppinc,pp_inc[lev],0,0,1,0);
151  VisMF::Write(ppinc, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PP_Inc"));
152 
153  MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
154  MultiFab::Copy(gpx,gradp[lev][GpVars::gpx],0,0,1,0);
155  VisMF::Write(gpx, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpx"));
156 
157  MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
158  MultiFab::Copy(gpy,gradp[lev][GpVars::gpy],0,0,1,0);
159  VisMF::Write(gpy, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpy"));
160 
161  MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
162  MultiFab::Copy(gpz,gradp[lev][GpVars::gpz],0,0,1,0);
163  VisMF::Write(gpz, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpz"));
164  }
165 
166  // Note that we write the ghost cells of the base state (unlike above)
167  IntVect ng_base = base_state[lev].nGrowVect();
168  int ncomp_base = base_state[lev].nComp();
169  MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
170  MultiFab::Copy(base,base_state[lev],0,0,ncomp_base,ng_base);
171  VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
172 
173  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
174  // Note that we also write the ghost cells of z_phys_nd
175  IntVect ng = z_phys_nd[lev]->nGrowVect();
176  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
177  MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
178  VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
179  }
180 
181  // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
182  // Write the moisture model restart variables
183  std::vector<int> qmoist_indices;
184  std::vector<std::string> qmoist_names;
185  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
186  int qmoist_nvar = qmoist_indices.size();
187  for (int var = 0; var < qmoist_nvar; var++) {
188  const int ncomp = 1;
189  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
190  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
191  MultiFab::Copy(moist_vars,*(qmoist[lev][qmoist_indices[var]]),0,0,ncomp,ng_moist);
192  VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", qmoist_names[var]));
193  }
194 
195 #if defined(ERF_USE_WINDFARM)
196  if(solverChoice.windfarm_type == WindFarmType::Fitch or
197  solverChoice.windfarm_type == WindFarmType::EWP or
198  solverChoice.windfarm_type == WindFarmType::SimpleAD){
199  IntVect ng_turb = Nturb[lev].nGrowVect();
200  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng_turb);
201  MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng_turb);
202  VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
203  }
204 #endif
205 
206  if (solverChoice.lsm_type != LandSurfaceType::None) {
207  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
208  BoxArray ba = lsm_data[lev][mvar]->boxArray();
209  DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
210  IntVect ng = lsm_data[lev][mvar]->nGrowVect();
211  int nvar = lsm_data[lev][mvar]->nComp();
212  MultiFab lsm_vars(ba,dm,nvar,ng);
213  MultiFab::Copy(lsm_vars,*(lsm_data[lev][mvar]),0,0,nvar,ng);
214  VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmVars"));
215  }
216  }
217 
218  IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
219  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
220  MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_x],0,0,1,ng);
221  VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_mx"));
222 
223 #if 0
225  MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_y],0,0,1,ng);
226  VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_my"));
227  }
228 #endif
229 
230  ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
231  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
232  MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_x],0,0,1,ng);
233  VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_ux"));
234 
235 #if 0
237  MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_y],0,0,1,ng);
238  VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_uy"));
239  }
240 #endif
241 
242  ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
243  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
244  MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_x],0,0,1,ng);
245  VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vx"));
246 
247 #if 0
249  MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_y],0,0,1,ng);
250  VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vy"));
251  }
252 #endif
253 
254  if (m_SurfaceLayer) {
255  amrex::Print() << "Writing SurfaceLayer variables at level " << lev << std::endl;
256  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
257  MultiFab m_var(ba2d[lev],dmap[lev],1,ng);
258  MultiFab* src = nullptr;
259 
260  // U*
261  src = m_SurfaceLayer->get_u_star(lev);
262  MultiFab::Copy(m_var,*src,0,0,1,ng);
263  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Ustar"));
264 
265  // W*
266  src = m_SurfaceLayer->get_w_star(lev);
267  MultiFab::Copy(m_var,*src,0,0,1,ng);
268  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Wstar"));
269 
270  // T*
271  src = m_SurfaceLayer->get_t_star(lev);
272  MultiFab::Copy(m_var,*src,0,0,1,ng);
273  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Tstar"));
274 
275  // Q*
276  src = m_SurfaceLayer->get_q_star(lev);
277  MultiFab::Copy(m_var,*src,0,0,1,ng);
278  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qstar"));
279 
280  // Olen
281  src = m_SurfaceLayer->get_olen(lev);
282  MultiFab::Copy(m_var,*src,0,0,1,ng);
283  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Olen"));
284 
285  // Qsurf
286  src = m_SurfaceLayer->get_q_surf(lev);
287  MultiFab::Copy(m_var,*src,0,0,1,ng);
288  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qsurf"));
289 
290  // PBLH
291  src = m_SurfaceLayer->get_pblh(lev);
292  MultiFab::Copy(m_var,*src,0,0,1,ng);
293  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PBLH"));
294 
295  // Z0
296  src = m_SurfaceLayer->get_z0(lev);
297  MultiFab::Copy(m_var,*src,0,0,1,ng);
298  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z0"));
299  }
300 
301  if (sst_lev[lev][0]) {
302  int ntimes = 1;
303  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
304  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
305  for (int nt(0); nt<ntimes; ++nt) {
306  MultiFab::Copy(sst_at_t,*sst_lev[lev][nt],0,0,1,ng);
307  VisMF::Write(sst_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
308  "SST_" + std::to_string(nt)));
309  }
310  }
311 
312  if (tsk_lev[lev][0]) {
313  int ntimes = 1;
314  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
315  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
316  for (int nt(0); nt<ntimes; ++nt) {
317  MultiFab::Copy(tsk_at_t,*tsk_lev[lev][nt],0,0,1,ng);
318  VisMF::Write(tsk_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
319  "TSK_" + std::to_string(nt)));
320  }
321  }
322 
323  {
324  int ntimes = 1;
325  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
326  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
327  for (int nt(0); nt<ntimes; ++nt) {
328  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
329  const Box& bx = mfi.growntilebox();
330  Array4<int> const& src_arr = lmask_lev[lev][nt]->array(mfi);
331  Array4<Real> const& dst_arr = lmask_at_t.array(mfi);
332  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
333  {
334  dst_arr(i,j,k) = Real(src_arr(i,j,k));
335  });
336  }
337  VisMF::Write(lmask_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
338  "LMASK_" + std::to_string(nt)));
339  }
340  }
341 
342  IntVect ngv = ng; ngv[2] = 0;
343 
344  // Write lat/lon if it exists
345  if (lat_m[lev] && lon_m[lev] && solverChoice.has_lat_lon) {
346  amrex::Print() << "Writing Lat/Lon variables at level " << lev << std::endl;
347  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
348  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
349  MultiFab::Copy(lat,*lat_m[lev],0,0,1,ngv);
350  MultiFab::Copy(lon,*lon_m[lev],0,0,1,ngv);
351  VisMF::Write(lat, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LAT"));
352  VisMF::Write(lon, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LON"));
353  }
354 
355 
356 #ifdef ERF_USE_NETCDF
357  // Write sinPhi and cosPhi if it exists
358  if (cosPhi_m[lev] && sinPhi_m[lev] && solverChoice.variable_coriolis) {
359  amrex::Print() << "Writing Coriolis factors at level " << lev << std::endl;
360  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
361  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
362  MultiFab::Copy(sphi,*sinPhi_m[lev],0,0,1,ngv);
363  MultiFab::Copy(cphi,*cosPhi_m[lev],0,0,1,ngv);
364  VisMF::Write(sphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SinPhi"));
365  VisMF::Write(cphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "CosPhi"));
366  }
367 
368  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
369  amrex::Print() << "Writing C1H/C2H/MUB variables at level " << lev << std::endl;
370  MultiFab tmp1d(ba1d[0],dmap[0],1,0);
371 
372  MultiFab::Copy(tmp1d,*mf_C1H,0,0,1,0);
373  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C1H"));
374 
375  MultiFab::Copy(tmp1d,*mf_C2H,0,0,1,0);
376  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C2H"));
377 
378  MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
379 
380  MultiFab::Copy(tmp2d,*mf_MUB,0,0,1,mf_MUB->nGrowVect());
381  VisMF::Write(tmp2d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MUB"));
382  }
383 #endif
384 
385  } // for lev
386 
387 #ifdef ERF_USE_PARTICLES
388  particleData.Checkpoint(checkpointname);
389 #endif
390 
391 #if 0
392 #ifdef ERF_USE_NETCDF
393  // Write bdy_data files
394  if ( ParallelDescriptor::IOProcessor() &&
395  ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
397  {
398  // Vector dimensions
399  int num_time = bdy_data_xlo.size();
400  int num_var = bdy_data_xlo[0].size();
401 
402  // Open header file and write to it
403  std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
404  bdy_h_file << std::setprecision(1) << std::fixed;
405  bdy_h_file << num_time << "\n";
406  bdy_h_file << num_var << "\n";
407  bdy_h_file << start_bdy_time << "\n";
408  bdy_h_file << bdy_time_interval << "\n";
409  bdy_h_file << real_width << "\n";
410  for (int ivar(0); ivar<num_var; ++ivar) {
411  bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
412  bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
413  bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
414  bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
415  }
416 
417  // Open data file and write to it
418  std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
419  for (int itime(0); itime<num_time; ++itime) {
420  if (bdy_data_xlo[itime].size() > 0) {
421  for (int ivar(0); ivar<num_var; ++ivar) {
422  bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
423  bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
424  bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
425  bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
426  }
427  }
428  }
429  }
430 #endif
431 #endif
432 
433 }

◆ WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain ( std::ostream &  HeaderFile,
int  nlevels,
const amrex::Vector< amrex::BoxArray > &  bArray,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
amrex::Real  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName,
const std::string &  levelPrefix,
const std::string &  mfPrefix 
) const
1781 {
1782  AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
1783  AMREX_ALWAYS_ASSERT(nlevels <= my_ref_ratio.size()+1);
1784  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
1785 
1786  HeaderFile.precision(17);
1787 
1788  // ---- this is the generic plot file type name
1789  HeaderFile << versionName << '\n';
1790 
1791  HeaderFile << varnames.size() << '\n';
1792 
1793  for (int ivar = 0; ivar < varnames.size(); ++ivar) {
1794  HeaderFile << varnames[ivar] << "\n";
1795  }
1796  HeaderFile << AMREX_SPACEDIM << '\n';
1797  HeaderFile << my_time << '\n';
1798  HeaderFile << finest_level << '\n';
1799  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1800  HeaderFile << my_geom[0].ProbLo(i) << ' ';
1801  }
1802  HeaderFile << '\n';
1803  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1804  HeaderFile << my_geom[0].ProbHi(i) << ' ';
1805  }
1806  HeaderFile << '\n';
1807  for (int i = 0; i < finest_level; ++i) {
1808  HeaderFile << my_ref_ratio[i][0] << ' ';
1809  }
1810  HeaderFile << '\n';
1811  for (int i = 0; i <= finest_level; ++i) {
1812  HeaderFile << my_geom[i].Domain() << ' ';
1813  }
1814  HeaderFile << '\n';
1815  for (int i = 0; i <= finest_level; ++i) {
1816  HeaderFile << level_steps[i] << ' ';
1817  }
1818  HeaderFile << '\n';
1819  for (int i = 0; i <= finest_level; ++i) {
1820  for (int k = 0; k < AMREX_SPACEDIM; ++k) {
1821  HeaderFile << my_geom[i].CellSize()[k] << ' ';
1822  }
1823  HeaderFile << '\n';
1824  }
1825  HeaderFile << (int) my_geom[0].Coord() << '\n';
1826  HeaderFile << "0\n";
1827 
1828  for (int level = 0; level <= finest_level; ++level) {
1829  HeaderFile << level << ' ' << bArray[level].size() << ' ' << my_time << '\n';
1830  HeaderFile << level_steps[level] << '\n';
1831 
1832  const IntVect& domain_lo = my_geom[level].Domain().smallEnd();
1833  for (int i = 0; i < bArray[level].size(); ++i)
1834  {
1835  // Need to shift because the RealBox ctor we call takes the
1836  // physical location of index (0,0,0). This does not affect
1837  // the usual cases where the domain index starts with 0.
1838  const Box& b = shift(bArray[level][i], -domain_lo);
1839  RealBox loc = RealBox(b, my_geom[level].CellSize(), my_geom[level].ProbLo());
1840  for (int n = 0; n < AMREX_SPACEDIM; ++n) {
1841  HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
1842  }
1843  }
1844 
1845  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
1846  }
1847  HeaderFile << "1" << "\n";
1848  HeaderFile << "3" << "\n";
1849  HeaderFile << "amrexvec_nu_x" << "\n";
1850  HeaderFile << "amrexvec_nu_y" << "\n";
1851  HeaderFile << "amrexvec_nu_z" << "\n";
1852  std::string mf_nodal_prefix = "Nu_nd";
1853  for (int level = 0; level <= finest_level; ++level) {
1854  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
1855  }
1856 }
Coord
Definition: ERF_DataStruct.H:85

◆ writeJobInfo()

void ERF::writeJobInfo ( const std::string &  dir) const
10 {
11  // job_info file with details about the run
12  std::ofstream jobInfoFile;
13  std::string FullPathJobInfoFile = dir;
14  FullPathJobInfoFile += "/job_info";
15  jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out);
16 
17  std::string PrettyLine = "==================================================="
18  "============================\n";
19  std::string OtherLine = "----------------------------------------------------"
20  "----------------------------\n";
21  std::string SkipSpace = " ";
22 
23  // job information
24  jobInfoFile << PrettyLine;
25  jobInfoFile << " ERF Job Information\n";
26  jobInfoFile << PrettyLine;
27 
28  jobInfoFile << "inputs file: " << inputs_name << "\n\n";
29 
30  jobInfoFile << "number of MPI processes: "
31  << ParallelDescriptor::NProcs() << "\n";
32 #ifdef _OPENMP
33  jobInfoFile << "number of threads: " << omp_get_max_threads() << "\n";
34 #endif
35 
36  jobInfoFile << "\n";
37  jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
38  << getCPUTime() / 3600.0;
39 
40  jobInfoFile << "\n\n";
41 
42  // plotfile information
43  jobInfoFile << PrettyLine;
44  jobInfoFile << " Plotfile Information\n";
45  jobInfoFile << PrettyLine;
46 
47  time_t now = time(nullptr);
48 
49  // Convert now to tm struct for local timezone
50  tm* localtm = localtime(&now);
51  jobInfoFile << "output data / time: " << asctime(localtm);
52 
53  std::string currentDir = FileSystem::CurrentPath();
54  jobInfoFile << "output dir: " << currentDir << "\n";
55 
56  jobInfoFile << "\n\n";
57 
58  // build information
59  jobInfoFile << PrettyLine;
60  jobInfoFile << " Build Information\n";
61  jobInfoFile << PrettyLine;
62 
63  jobInfoFile << "build date: " << buildInfoGetBuildDate() << "\n";
64  jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
65  jobInfoFile << "build dir: " << buildInfoGetBuildDir() << "\n";
66  jobInfoFile << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
67 
68  jobInfoFile << "\n";
69 
70  jobInfoFile << "COMP: " << buildInfoGetComp() << "\n";
71  jobInfoFile << "COMP version: " << buildInfoGetCompVersion() << "\n";
72 
73  jobInfoFile << "\n";
74 
75  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
76  jobInfoFile << buildInfoGetModuleName(n) << ": "
77  << buildInfoGetModuleVal(n) << "\n";
78  }
79 
80  jobInfoFile << "\n";
81 
82  const char* githash1 = buildInfoGetGitHash(1);
83  const char* githash2 = buildInfoGetGitHash(2);
84  if (strlen(githash1) > 0) {
85  jobInfoFile << "ERF git hash: " << githash1 << "\n";
86  }
87  if (strlen(githash2) > 0) {
88  jobInfoFile << "AMReX git hash: " << githash2 << "\n";
89  }
90 
91  const char* buildgithash = buildInfoGetBuildGitHash();
92  const char* buildgitname = buildInfoGetBuildGitName();
93  if (strlen(buildgithash) > 0) {
94  jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
95  }
96 
97  jobInfoFile << "\n\n";
98 
99  // grid information
100  jobInfoFile << PrettyLine;
101  jobInfoFile << " Grid Information\n";
102  jobInfoFile << PrettyLine;
103 
104  int f_lev = finest_level;
105 
106  for (int i = 0; i <= f_lev; i++) {
107  jobInfoFile << " level: " << i << "\n";
108  jobInfoFile << " number of boxes = " << grids[i].size() << "\n";
109  jobInfoFile << " maximum zones = ";
110  for (int n = 0; n < AMREX_SPACEDIM; n++) {
111  jobInfoFile << geom[i].Domain().length(n) << " ";
112  }
113  jobInfoFile << "\n\n";
114  }
115 
116  jobInfoFile << " Boundary conditions\n";
117 
118  jobInfoFile << " -x: " << domain_bc_type[0] << "\n";
119  jobInfoFile << " +x: " << domain_bc_type[3] << "\n";
120  jobInfoFile << " -y: " << domain_bc_type[1] << "\n";
121  jobInfoFile << " +y: " << domain_bc_type[4] << "\n";
122  jobInfoFile << " -z: " << domain_bc_type[2] << "\n";
123  jobInfoFile << " +z: " << domain_bc_type[5] << "\n";
124 
125  jobInfoFile << "\n\n";
126 
127  // runtime parameters
128  jobInfoFile << PrettyLine;
129  jobInfoFile << " Inputs File Parameters\n";
130  jobInfoFile << PrettyLine;
131 
132  ParmParse::dumpTable(jobInfoFile, true);
133  jobInfoFile.close();
134 }
std::string inputs_name
Definition: main.cpp:14
static amrex::Real getCPUTime()
Definition: ERF.H:1420
Here is the call graph for this function:

◆ WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain ( const std::string &  plotfilename,
int  nlevels,
const amrex::Vector< const amrex::MultiFab * > &  mf,
const amrex::Vector< const amrex::MultiFab * > &  mf_nd,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
amrex::Real  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName = "HyperCLaw-V1.1",
const std::string &  levelPrefix = "Level_",
const std::string &  mfPrefix = "Cell",
const amrex::Vector< std::string > &  extra_dirs = amrex::Vector<std::string>() 
) const
1695 {
1696  BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
1697 
1698  AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
1699  AMREX_ALWAYS_ASSERT(nlevels <= rr.size()+1);
1700  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
1701  AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
1702 
1703  bool callBarrier(false);
1704  PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
1705  if (!extra_dirs.empty()) {
1706  for (const auto& d : extra_dirs) {
1707  const std::string ed = plotfilename+"/"+d;
1708  PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
1709  }
1710  }
1711  ParallelDescriptor::Barrier();
1712 
1713  if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
1714  Vector<BoxArray> boxArrays(nlevels);
1715  for(int level(0); level < boxArrays.size(); ++level) {
1716  boxArrays[level] = mf[level]->boxArray();
1717  }
1718 
1719  auto f = [=]() {
1720  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
1721  std::string HeaderFileName(plotfilename + "/Header");
1722  std::ofstream HeaderFile;
1723  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
1724  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
1725  std::ofstream::trunc |
1726  std::ofstream::binary);
1727  if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
1728  WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
1729  my_geom, time, level_steps, rr, versionName,
1730  levelPrefix, mfPrefix);
1731  };
1732 
1733  if (AsyncOut::UseAsyncOut()) {
1734  AsyncOut::Submit(std::move(f));
1735  } else {
1736  f();
1737  }
1738  }
1739 
1740  std::string mf_nodal_prefix = "Nu_nd";
1741  for (int level = 0; level <= finest_level; ++level)
1742  {
1743  if (AsyncOut::UseAsyncOut()) {
1744  VisMF::AsyncWrite(*mf[level],
1745  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
1746  true);
1747  VisMF::AsyncWrite(*mf_nd[level],
1748  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
1749  true);
1750  } else {
1751  const MultiFab* data;
1752  std::unique_ptr<MultiFab> mf_tmp;
1753  if (mf[level]->nGrowVect() != 0) {
1754  mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
1755  mf[level]->DistributionMap(),
1756  mf[level]->nComp(), 0, MFInfo(),
1757  mf[level]->Factory());
1758  MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
1759  data = mf_tmp.get();
1760  } else {
1761  data = mf[level];
1762  }
1763  VisMF::Write(*data , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
1764  VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
1765  }
1766  }
1767 }
void WriteGenericPlotfileHeaderWithTerrain(std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
Definition: ERF_Plotfile.cpp:1770

◆ WriteMyEBSurface()

void ERF::WriteMyEBSurface ( )
6 {
7  using namespace amrex;
8 
9  amrex::Print() << "Writing the geometry to a vtp file.\n" << std::endl;
10 
11  // Only write at the finest level!
12  int lev = finest_level;
13 
14  BoxArray & ba = grids[lev];
15  DistributionMapping & dm = dmap[lev];
16 
17  const EBFArrayBoxFactory* ebfact = &EBFactory(lev);
18 
19  WriteEBSurface(ba,dm,Geom(lev),ebfact);
20 }
Definition: ERF_ConsoleIO.cpp:12
Here is the call graph for this function:

◆ writeNow()

bool ERF::writeNow ( const amrex::Real  cur_time,
const amrex::Real  dt,
const int  nstep,
const int  plot_int,
const amrex::Real  plot_per 
)
2449 {
2450  bool write_now = false;
2451 
2452  if ( plot_int > 0 && (nstep % plot_int == 0) ) {
2453  write_now = true;
2454 
2455  } else if (plot_per > 0.0) {
2456 
2457  // Check to see if we've crossed a plot_per interval by comparing
2458  // the number of intervals that have elapsed for both the current
2459  // time and the time at the beginning of this timestep.
2460 
2461  const Real eps = std::numeric_limits<Real>::epsilon() * Real(10.0) * std::abs(cur_time);
2462 
2463  int num_per_old = static_cast<int>(std::floor((cur_time-eps-dt_lev) / plot_per));
2464  int num_per_new = static_cast<int>(std::floor((cur_time-eps ) / plot_per));
2465 
2466  // Before using these, however, we must test for the case where we're
2467  // within machine epsilon of the next interval. In that case, increment
2468  // the counter, because we have indeed reached the next plot_per interval
2469  // at this point.
2470 
2471  const Real next_plot_time = (num_per_old + 1) * plot_per;
2472 
2473  if ((num_per_new == num_per_old) && std::abs(cur_time - next_plot_time) <= eps)
2474  {
2475  num_per_new += 1;
2476  }
2477 
2478  // Similarly, we have to account for the case where the old time is within
2479  // machine epsilon of the beginning of this interval, so that we don't double
2480  // count that time threshold -- we already plotted at that time on the last timestep.
2481 
2482  if ((num_per_new != num_per_old) && std::abs((cur_time - dt_lev) - next_plot_time) <= eps)
2483  num_per_old += 1;
2484 
2485  if (num_per_old != num_per_new)
2486  write_now = true;
2487  }
2488  return write_now;
2489 }
real(c_double), parameter epsilon
Definition: ERF_module_model_constants.F90:12

◆ WriteSubvolume()

void ERF::WriteSubvolume ( )
10 {
11  ParmParse pp("erf.subvol");
12 
13  Vector<Real> origin;
14  Vector< int> ncell;
15  Vector<Real> delta;
16 
17  // **************************************************************
18  // Read in the origin, number of cells in each dir, and resolution
19  // **************************************************************
20  pp.getarr("origin",origin,0,AMREX_SPACEDIM);
21  pp.getarr("nxnynz", ncell,0,AMREX_SPACEDIM);
22  pp.getarr("dxdydz", delta,0,AMREX_SPACEDIM);
23 
24  int lev_for_sub = 0;
25 
26  bool found = false;
27  for (int i = 0; i <= finest_level; i++) {
28  if (!found) {
29  if (almostEqual(delta[0],geom[i].CellSize(0)) &&
30  almostEqual(delta[1],geom[i].CellSize(1)) &&
31  almostEqual(delta[2],geom[i].CellSize(2)) ) {
32 
33  // amrex::Print() << "XDIR " << delta[0] << " " << geom[i].CellSize(0) << std::endl;
34  // amrex::Print() << "YDIR " << delta[1] << " " << geom[i].CellSize(1) << std::endl;
35  // amrex::Print() << "ZDIR " << delta[2] << " " << geom[i].CellSize(2) << std::endl;
36  amrex::Print() << "Resolution specified matches that of level " << i << std::endl;
37  found = true;
38  lev_for_sub = i;
39  }
40  }
41  }
42 
43  if (!found) {
44  amrex::Abort("Resolution specified for subvol does not match the resolution of any of the levels.");
45  }
46 
47  // **************************************************************
48  // Now that we know which level we're at, we can figure out which (i,j,k) the origin corresponds to
49  // Note we use 1.0001 as a fudge factor since the division of two reals --> integer will do a floor
50  // **************************************************************
51  int i0 = static_cast<int>((origin[0] - geom[lev_for_sub].ProbLo(0)) * 1.0001 / delta[0]);
52  int j0 = static_cast<int>((origin[1] - geom[lev_for_sub].ProbLo(1)) * 1.0001 / delta[1]);
53  int k0 = static_cast<int>((origin[2] - geom[lev_for_sub].ProbLo(2)) * 1.0001 / delta[2]);
54 
55  found = false;
56  if (almostEqual(geom[lev_for_sub].ProbLo(0)+i0*delta[0],origin[0]) &&
57  almostEqual(geom[lev_for_sub].ProbLo(1)+j0*delta[1],origin[1]) &&
58  almostEqual(geom[lev_for_sub].ProbLo(2)+k0*delta[2],origin[2]) )
59  {
60  amrex::Print() << "Specified origin is the lower left corner of cell " << IntVect(i0,j0,k0) << std::endl;
61  found = true;
62  }
63 
64  if (!found) {
65  amrex::Abort("Origin specified does not correspond to a node at this level.");
66  }
67 
68  Box domain(geom[lev_for_sub].Domain());
69 
70  Box bx(IntVect(i0,j0,k0),IntVect(i0+ncell[0]-1,j0+ncell[1]-1,k0+ncell[2]-1));
71  amrex::Print() << "Box requested is " << bx << std::endl;
72 
73  if (!domain.contains(bx))
74  {
75  amrex::Abort("Box requested is larger than the existing domain");
76  }
77 
78  Vector<int> cs(3);
79  int count = pp.countval("chunk_size");
80  if (count > 0) {
81  pp.queryarr("chunk_size",cs,0,AMREX_SPACEDIM);
82  } else {
83  cs[0] = max_grid_size[0][0];
84  cs[1] = max_grid_size[0][1];
85  cs[2] = max_grid_size[0][2];
86  }
87  IntVect chunk_size(cs[0],cs[1],cs[2]);
88 
89  BoxArray ba(bx);
90  ba.maxSize(chunk_size);
91 
92  amrex::Print() << "BoxArray is " << ba << std::endl;
93 
94  int ncomp_mf = AMREX_SPACEDIM;
95 
96  DistributionMapping dm(ba);
97 
98  MultiFab mf(ba, dm, ncomp_mf, 0);
99 
100  MultiFab mf_cc_vel(grids[lev_for_sub], dmap[lev_for_sub], ncomp_mf, 0);
101  average_face_to_cellcenter(mf_cc_vel,0,
102  Array<const MultiFab*,3>{&vars_new[lev_for_sub][Vars::xvel],
103  &vars_new[lev_for_sub][Vars::yvel],
104  &vars_new[lev_for_sub][Vars::zvel]});
105 
106  mf.ParallelCopy(mf_cc_vel,0,0,AMREX_SPACEDIM,0,0);
107 
108  std::string subvol_filename = Concatenate(subvol_file, istep[0], 5);
109 
110  Vector<std::string> varnames;
111  varnames.push_back("x_velocity");
112  varnames.push_back("y_velocity");
113  varnames.push_back("z_velocity");
114 
115  Real time = t_new[lev_for_sub];
116 
117  amrex::Print() <<"Writing subvolume into " << subvol_filename << std::endl;
118  WriteSingleLevelPlotfile(subvol_filename,mf,varnames,geom[lev_for_sub],time,istep[0]);
119 }
real(c_double), private cs
Definition: ERF_module_mp_morr_two_moment.F90:202
Here is the call graph for this function:

◆ WriteVTKPolyline()

void ERF::WriteVTKPolyline ( const std::string &  filename,
amrex::Vector< std::array< amrex::Real, 2 >> &  points_xy 
)
595 {
596  std::ofstream vtkfile(filename);
597  if (!vtkfile.is_open()) {
598  std::cerr << "Error: Cannot open file " << filename << std::endl;
599  return;
600  }
601 
602  int num_points = points_xy.size();
603  if (num_points == 0) {
604  vtkfile << "# vtk DataFile Version 3.0\n";
605  vtkfile << "Hurricane Track\n";
606  vtkfile << "ASCII\n";
607  vtkfile << "DATASET POLYDATA\n";
608  vtkfile << "POINTS " << num_points << " float\n";
609  vtkfile.close();
610  return;
611  }
612  if (num_points < 2) {
613  points_xy.push_back(points_xy[0]);
614  }
615  num_points = points_xy.size();
616 
617  vtkfile << "# vtk DataFile Version 3.0\n";
618  vtkfile << "Hurricane Track\n";
619  vtkfile << "ASCII\n";
620  vtkfile << "DATASET POLYDATA\n";
621 
622  // Write points (Z=0 assumed)
623  vtkfile << "POINTS " << num_points << " float\n";
624  for (const auto& pt : points_xy) {
625  vtkfile << pt[0] << " " << pt[1] << " 10000.0\n";
626  }
627 
628  // Write polyline connectivity
629  vtkfile << "LINES 1 " << num_points + 1 << "\n";
630  vtkfile << num_points << " ";
631  for (int i = 0; i < num_points; ++i) {
632  vtkfile << i << " ";
633  }
634  vtkfile << "\n";
635 
636  vtkfile.close();
637 }

Member Data Documentation

◆ advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg
private

Referenced by getAdvFluxReg().

◆ ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax
private

◆ ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src
private

◆ ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay
private

◆ ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src
private

◆ az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az
private

◆ az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src
private

◆ ba1d

amrex::Vector<amrex::BoxArray> ERF::ba1d
private

◆ ba2d

amrex::Vector<amrex::BoxArray> ERF::ba2d
private

◆ base_state

amrex::Vector<amrex::MultiFab> ERF::base_state
private

◆ base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new
private

◆ bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1
staticprivate

◆ bndry_output_planes_per

Real ERF::bndry_output_planes_per = -1.0
staticprivate

◆ bndry_output_planes_start_time

Real ERF::bndry_output_planes_start_time = 0.0
staticprivate

◆ boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level
private

◆ cf_set_width

int ERF::cf_set_width {0}
private

◆ cf_width

int ERF::cf_width {0}
private

◆ cfl

Real ERF::cfl = 0.8
staticprivate

◆ change_max

Real ERF::change_max = 1.1
staticprivate

◆ check_file

std::string ERF::check_file {"chk"}
private

◆ column_file_name

std::string ERF::column_file_name = "column_data.nc"
staticprivate

◆ column_interval

int ERF::column_interval = -1
staticprivate

◆ column_loc_x

Real ERF::column_loc_x = 0.0
staticprivate

◆ column_loc_y

Real ERF::column_loc_y = 0.0
staticprivate

◆ column_per

Real ERF::column_per = -1.0
staticprivate

◆ cons_names

const amrex::Vector<std::string> ERF::cons_names
private
Initial value:
{"density", "rhotheta", "rhoKE", "rhoadv_0",
"rhoQ1", "rhoQ2", "rhoQ3",
"rhoQ4", "rhoQ5", "rhoQ6"}

◆ cosPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::cosPhi_m
private

◆ d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density
private

◆ d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure
private

◆ d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc
private

◆ d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv
private

◆ d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature
private

◆ d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs
private

◆ d_rhoqt_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhoqt_src
private

◆ d_rhotheta_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhotheta_src
private

◆ d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs
private

◆ d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos
private

◆ d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos
private

◆ d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid
private

◆ data_sampler

std::unique_ptr<SampleData> ERF::data_sampler = nullptr
private

◆ datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog
private

◆ datalogname

amrex::Vector<std::string> ERF::datalogname
private

Referenced by DataLogName().

◆ datetime_format

const std::string ERF::datetime_format = "%Y-%m-%d %H:%M:%S"
private

◆ datprecision

const int ERF::datprecision = 6
private

◆ datwidth

const int ERF::datwidth = 14
private

◆ der_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::der_datalog
private

◆ der_datalogname

amrex::Vector<std::string> ERF::der_datalogname
private

Referenced by DerDataLogName().

◆ derived_names

const amrex::Vector<std::string> ERF::derived_names
private

◆ destag_profiles

bool ERF::destag_profiles = true
private

◆ detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc
private

◆ detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new
private

◆ detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src
private

◆ domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type
private

◆ domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type
private

◆ domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d
private

◆ dt

amrex::Vector<amrex::Real> ERF::dt
private

◆ dt_max

Real ERF::dt_max = 1.0e9
staticprivate

◆ dt_max_initial

Real ERF::dt_max_initial = 2.0e100
staticprivate

◆ dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio
private

◆ dz_min

amrex::Vector<amrex::Real> ERF::dz_min
private

◆ eb

amrex::Vector<std::unique_ptr<eb_> > ERF::eb
private

Referenced by EBFactory(), and get_eb().

◆ eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev
private

◆ fine_mask

amrex::MultiFab ERF::fine_mask
private

◆ finished_wave

bool ERF::finished_wave = false
private

◆ fixed_dt

amrex::Vector<amrex::Real> ERF::fixed_dt
private

◆ fixed_fast_dt

amrex::Vector<amrex::Real> ERF::fixed_fast_dt
private

◆ fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0
staticprivate

◆ FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c
private

◆ FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u
private

◆ FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v
private

◆ FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w
private

◆ gradp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::gradp
private

◆ h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density
private

◆ h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure
private

◆ h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc
private

◆ h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv
private

◆ h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature
private

◆ h_rayleigh_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs
private

◆ h_rhoqt_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhoqt_src
private

◆ h_rhotheta_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhotheta_src
private

◆ h_sponge_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs
private

◆ h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos
private

◆ h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos
private

◆ h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid
private

◆ hurricane_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_track_xy

◆ Hwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave
private

◆ Hwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave_onegrid
private

◆ init_shrink

Real ERF::init_shrink = 1.0
staticprivate

◆ initial_state

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::initial_state

◆ input_bndry_planes

int ERF::input_bndry_planes = 0
staticprivate

◆ input_sounding_data

InputSoundingData ERF::input_sounding_data
private

◆ input_sponge_data

InputSpongeData ERF::input_sponge_data
private

◆ interpolation_type

StateInterpType ERF::interpolation_type
staticprivate

◆ istep

amrex::Vector<int> ERF::istep
private

◆ last_check_file_step

int ERF::last_check_file_step
private

◆ last_plot2d_file_step_1

int ERF::last_plot2d_file_step_1
private

◆ last_plot2d_file_step_2

int ERF::last_plot2d_file_step_2
private

◆ last_plot3d_file_step_1

int ERF::last_plot3d_file_step_1
private

◆ last_plot3d_file_step_2

int ERF::last_plot3d_file_step_2
private

◆ last_subvol

int ERF::last_subvol
private

◆ lat_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lat_m
private

◆ lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev
private

◆ lon_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lon_m
private

◆ lsm

LandSurface ERF::lsm
private

◆ lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data
private

◆ lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux
private

◆ Lwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave
private

◆ Lwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave_onegrid
private

◆ m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_extdir_vals
private

◆ m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_neumann_vals
private

◆ m_check_int

int ERF::m_check_int = -1
private

◆ m_check_per

amrex::Real ERF::m_check_per = -1.0
private

◆ m_expand_plotvars_to_unif_rr

bool ERF::m_expand_plotvars_to_unif_rr = false
private

◆ m_forest_drag

amrex::Vector<std::unique_ptr<ForestDrag> > ERF::m_forest_drag
private

◆ m_plot2d_int_1

int ERF::m_plot2d_int_1 = -1
private

◆ m_plot2d_int_2

int ERF::m_plot2d_int_2 = -1
private

◆ m_plot2d_per_1

amrex::Real ERF::m_plot2d_per_1 = -1.0
private

◆ m_plot2d_per_2

amrex::Real ERF::m_plot2d_per_2 = -1.0
private

◆ m_plot3d_int_1

int ERF::m_plot3d_int_1 = -1
private

◆ m_plot3d_int_2

int ERF::m_plot3d_int_2 = -1
private

◆ m_plot3d_per_1

amrex::Real ERF::m_plot3d_per_1 = -1.0
private

◆ m_plot3d_per_2

amrex::Real ERF::m_plot3d_per_2 = -1.0
private

◆ m_plot_face_vels

bool ERF::m_plot_face_vels = false
private

◆ m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr
private

◆ m_subvol_int

int ERF::m_subvol_int = -1
private

◆ m_subvol_per

amrex::Real ERF::m_subvol_per = -1.0
private

◆ m_SurfaceLayer

std::unique_ptr<SurfaceLayer> ERF::m_SurfaceLayer = nullptr
private

◆ m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr
private

◆ mapfac

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::mapfac
private

◆ max_step

int ERF::max_step = std::numeric_limits<int>::max()
private

◆ metgrid_basic_linear

bool ERF::metgrid_basic_linear {false}
private

◆ metgrid_debug_dry

bool ERF::metgrid_debug_dry {false}
private

◆ metgrid_debug_isothermal

bool ERF::metgrid_debug_isothermal {false}
private

◆ metgrid_debug_msf

bool ERF::metgrid_debug_msf {false}
private

◆ metgrid_debug_psfc

bool ERF::metgrid_debug_psfc {false}
private

◆ metgrid_debug_quiescent

bool ERF::metgrid_debug_quiescent {false}
private

◆ metgrid_force_sfc_k

int ERF::metgrid_force_sfc_k {6}
private

◆ metgrid_interp_theta

bool ERF::metgrid_interp_theta {false}
private

◆ metgrid_order

int ERF::metgrid_order {2}
private

◆ metgrid_proximity

amrex::Real ERF::metgrid_proximity {500.0}
private

◆ metgrid_retain_sfc

bool ERF::metgrid_retain_sfc {false}
private

◆ metgrid_use_below_sfc

bool ERF::metgrid_use_below_sfc {true}
private

◆ metgrid_use_sfc

bool ERF::metgrid_use_sfc {true}
private

◆ mf_C1H

std::unique_ptr<amrex::MultiFab> ERF::mf_C1H
private

◆ mf_C2H

std::unique_ptr<amrex::MultiFab> ERF::mf_C2H
private

◆ mf_MUB

std::unique_ptr<amrex::MultiFab> ERF::mf_MUB
private

◆ mf_PSFC

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_PSFC
private

◆ mg_verbose

int ERF::mg_verbose = 0
staticprivate

◆ micro

std::unique_ptr<Microphysics> ERF::micro
private

◆ mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem
private

◆ nc_bdy_file

std::string ERF::nc_bdy_file
staticprivate

◆ nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}
staticprivate

◆ nc_low_file

std::string ERF::nc_low_file
staticprivate

◆ ng_dens_hse

int ERF::ng_dens_hse
staticprivate

◆ ng_pres_hse

int ERF::ng_pres_hse
staticprivate

◆ nsubsteps

amrex::Vector<int> ERF::nsubsteps
private

◆ num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level
private

◆ num_files_at_level

amrex::Vector<int> ERF::num_files_at_level
private

◆ output_1d_column

int ERF::output_1d_column = 0
staticprivate

◆ output_bndry_planes

int ERF::output_bndry_planes = 0
staticprivate

◆ pert_interval

int ERF::pert_interval = -1
staticprivate

◆ phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type
private

◆ physbcs_base

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_base> > ERF::physbcs_base
private

◆ physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons
private

◆ physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u
private

◆ physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v
private

◆ physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w
private

◆ plot2d_file_1

std::string ERF::plot2d_file_1 {"plt2d_1_"}
private

◆ plot2d_file_2

std::string ERF::plot2d_file_2 {"plt2d_2_"}
private

◆ plot2d_var_names_1

amrex::Vector<std::string> ERF::plot2d_var_names_1
private

◆ plot2d_var_names_2

amrex::Vector<std::string> ERF::plot2d_var_names_2
private

◆ plot3d_file_1

std::string ERF::plot3d_file_1 {"plt_1_"}
private

◆ plot3d_file_2

std::string ERF::plot3d_file_2 {"plt_2_"}
private

◆ plot3d_var_names_1

amrex::Vector<std::string> ERF::plot3d_var_names_1
private

◆ plot3d_var_names_2

amrex::Vector<std::string> ERF::plot3d_var_names_2
private

◆ plot_file_on_restart

int ERF::plot_file_on_restart = 1
private

◆ plot_lsm

bool ERF::plot_lsm = false
private

◆ plot_rad

bool ERF::plot_rad = false
private

◆ plotfile2d_type_1

PlotFileType ERF::plotfile2d_type_1 = PlotFileType::None
staticprivate

◆ plotfile2d_type_2

PlotFileType ERF::plotfile2d_type_2 = PlotFileType::None
staticprivate

◆ plotfile3d_type_1

PlotFileType ERF::plotfile3d_type_1 = PlotFileType::None
staticprivate

◆ plotfile3d_type_2

PlotFileType ERF::plotfile3d_type_2 = PlotFileType::None
staticprivate

◆ pp_inc

amrex::Vector<amrex::MultiFab> ERF::pp_inc
private

◆ pp_prefix

std::string ERF::pp_prefix {"erf"}

◆ previousCPUTimeUsed

Real ERF::previousCPUTimeUsed = 0.0
staticprivate

Referenced by getCPUTime().

◆ prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr
private

◆ profile_int

int ERF::profile_int = -1
private

◆ qheating_rates

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::qheating_rates
private

◆ qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist
private

◆ Qr_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qr_prim
private

◆ Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim
private

◆ rad

amrex::Vector<std::unique_ptr<IRadiation> > ERF::rad
private

◆ rad_datalog_int

int ERF::rad_datalog_int = -1
private

◆ real_extrap_w

bool ERF::real_extrap_w {true}
private

◆ real_set_width

int ERF::real_set_width {0}
private

◆ real_width

int ERF::real_width {0}
private

◆ ref_tags

Vector< AMRErrorTag > ERF::ref_tags
staticprivate

◆ regrid_int

int ERF::regrid_int = -1
private

◆ regrid_level_0_on_restart

bool ERF::regrid_level_0_on_restart = false
private

◆ restart_chkfile

std::string ERF::restart_chkfile = ""
private

◆ rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new
private

◆ rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old
private

◆ rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new
private

◆ rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old
private

◆ rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new
private

◆ rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old
private

◆ sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline
private

Referenced by NumSampleLines(), and SampleLine().

◆ samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog
private

◆ samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname
private

Referenced by SampleLineLogName().

◆ samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint
private

Referenced by NumSamplePoints(), and SamplePoint().

◆ sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog
private

◆ sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname
private

Referenced by SamplePointLogName().

◆ sampler_interval

int ERF::sampler_interval = -1
private

◆ sampler_per

amrex::Real ERF::sampler_per = -1.0
private

◆ SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev
private

◆ SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev
private

◆ SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev
private

◆ SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev
private

◆ SFS_q1fx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx1_lev
private

◆ SFS_q1fx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx2_lev
private

◆ SFS_q1fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx3_lev
private

◆ SFS_q2fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q2fx3_lev
private

◆ sinPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sinPhi_m
private

◆ SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev
private

◆ solar_zenith

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::solar_zenith
private

◆ solverChoice

SolverChoice ERF::solverChoice
staticprivate

◆ sponge_type

std::string ERF::sponge_type
staticprivate

◆ sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev
private

◆ start_time

Real ERF::start_time = 0.0
staticprivate

◆ startCPUTime

Real ERF::startCPUTime = 0.0
staticprivate

Referenced by getCPUTime().

◆ stop_time

Real ERF::stop_time = std::numeric_limits<amrex::Real>::max()
staticprivate

◆ stretched_dz_d

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::stretched_dz_d
private

◆ stretched_dz_h

amrex::Vector<amrex::Vector<amrex::Real> > ERF::stretched_dz_h
private

◆ sub_cfl

Real ERF::sub_cfl = 1.0
staticprivate

◆ subdomains

amrex::Vector<amrex::Vector<amrex::BoxArray> > ERF::subdomains
private

◆ subvol_file

std::string ERF::subvol_file {"subvol"}
private

◆ sum_interval

int ERF::sum_interval = -1
staticprivate

◆ sum_per

Real ERF::sum_per = -1.0
staticprivate

◆ sw_lw_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sw_lw_fluxes
private

◆ t_avg_cnt

amrex::Vector<amrex::Real> ERF::t_avg_cnt
private

◆ t_new

amrex::Vector<amrex::Real> ERF::t_new
private

◆ t_old

amrex::Vector<amrex::Real> ERF::t_old
private

◆ Tau

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau
private

◆ terrain_blanking

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking
private

◆ th_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::th_bc_data
private

◆ Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim
private

◆ thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce
private

◆ thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce
private

◆ thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce
private

◆ timeprecision

const int ERF::timeprecision = 13
private

◆ tot_e_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::tot_e_datalog
private

Referenced by setRecordEnergyDataInfo().

◆ tot_e_datalogname

amrex::Vector<std::string> ERF::tot_e_datalogname
private

◆ tsk_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::tsk_lev
private

◆ turbPert

TurbulentPerturbation ERF::turbPert
private

◆ use_datetime

bool ERF::use_datetime = false
private

◆ use_fft

bool ERF::use_fft = false
staticprivate

◆ vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new
private

◆ vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old
private

◆ vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg
private

◆ verbose

int ERF::verbose = 0
staticprivate

◆ walldist

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::walldist
private

◆ xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask
private

◆ xvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::xvel_bc_data
private

◆ yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask
private

◆ yvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::yvel_bc_data
private

◆ z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc
private

◆ z_phys_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc_src
private

◆ z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd
private

◆ z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new
private

◆ z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src
private

◆ z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk
private

◆ zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask
private

◆ zlevels_stag

amrex::Vector<amrex::Vector<amrex::Real> > ERF::zlevels_stag
private

◆ zmom_crse_rhs

amrex::Vector<amrex::MultiFab> ERF::zmom_crse_rhs
private

◆ zvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::zvel_bc_data
private

The documentation for this class was generated from the following files: