ERF
Energy Research and Forecasting: An Atmospheric Modeling Code
ERF Class Reference

#include <ERF.H>

Inheritance diagram for ERF:
Collaboration diagram for ERF:

Public Member Functions

 ERF ()
 
 ~ERF () override
 
void ERF_shared ()
 
 ERF (ERF &&) noexcept=delete
 
ERFoperator= (ERF &&other) noexcept=delete
 
 ERF (const ERF &other)=delete
 
ERFoperator= (const ERF &other)=delete
 
void Evolve ()
 
void ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override
 
void HurricaneTracker (int lev, const amrex::MultiFab &U_new, const amrex::MultiFab &V_new, const amrex::MultiFab &W_new, const amrex::Real velmag_threshold, const bool is_track_io, amrex::TagBoxArray *tags=nullptr)
 
std::string MakeVTKFilename (int nstep)
 
void WriteVTKPolyline (const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
 
void InitData ()
 
void InitData_pre ()
 
void InitData_post ()
 
void WriteMyEBSurface ()
 
void compute_divergence (int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
 
void project_velocity (int lev, amrex::Real dt)
 
void project_momenta (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
 
void project_velocity_tb (int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
 
void poisson_wall_dist (int lev)
 
void make_subdomains (const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
 
void solve_with_EB_mlmg (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
 
void solve_with_gmres (int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &znd_sub)
 
void solve_with_mlmg (int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
 
void ImposeBCsOnPhi (int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
 
amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > get_projection_bc (amrex::Orientation::Side side) const noexcept
 
bool projection_has_dirichlet (amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > bcs) const
 
void init_only (int lev, amrex::Real time)
 
void restart ()
 
bool writeNow (const amrex::Real cur_time, const int nstep, const int plot_int, const amrex::Real plot_per, const amrex::Real dt_0, amrex::Real &last_file_time)
 
void post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev)
 
void sum_integrated_quantities (amrex::Real time)
 
void sum_derived_quantities (amrex::Real time)
 
void sum_energy_quantities (amrex::Real time)
 
void write_1D_profiles (amrex::Real time)
 
void write_1D_profiles_stag (amrex::Real time)
 
amrex::Real cloud_fraction (amrex::Real time)
 
void FillBdyCCVels (amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)
 
void sample_points (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void sample_lines (int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
 
void derive_diag_profiles (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_diag_profiles_stag (amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
 
void derive_stress_profiles (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
void derive_stress_profiles_stag (amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
 
amrex::Real volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, bool finemask)
 
void MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
void ClearLevel (int lev) override
 
void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
 
amrex::Real estTimeStep (int lev, long &dt_fast_ratio) const
 
void advance_dycore (int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
 
void advance_microphysics (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
 
void advance_lsm (int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)
 
void advance_radiation (int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance)
 
amrex::MultiFab & build_fine_mask (int lev)
 
void MakeHorizontalAverages ()
 
void MakeDiagnosticAverage (amrex::Vector< amrex::Real > &h_havg, amrex::MultiFab &S, int n)
 
void derive_upwp (amrex::Vector< amrex::Real > &h_havg)
 
void Write3DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
 
void Write2DPlotFile (int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
 
void WriteSubvolume ()
 
void WriteMultiLevelPlotfileWithTerrain (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
 
void WriteGenericPlotfileHeaderWithTerrain (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
 
void erf_enforce_hse (int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
 
void init_from_input_sounding (int lev)
 
void input_sponge (int lev)
 
void init_from_hse (int lev)
 
void init_thin_body (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void CreateWeatherDataGeomBoxArrayDistMap (const std::string &filename, amrex::Geometry &geom_weather, amrex::BoxArray &nba, amrex::DistributionMapping &dm)
 
void FillWeatherDataMultiFab (const std::string &filename, const amrex::Geometry &geom_weather, const amrex::BoxArray &nba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &weather_forecast_data)
 
void InterpWeatherDataOntoMesh (const amrex::Geometry &geom_weather, amrex::MultiFab &weather_forecast_interp, amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
 
void CreateForecastStateMultiFabs (amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
 
void WeatherDataInterpolation (const amrex::Real time)
 
void fill_from_bndryregs (const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
 
void MakeEBGeometry ()
 
void make_eb_box ()
 
void make_eb_regular ()
 
void AverageDownTo (int crse_lev, int scomp, int ncomp)
 
void WriteCheckpointFile () const
 
void ReadCheckpointFile ()
 
void ReadCheckpointFileSurfaceLayer ()
 
void init_zphys (int lev, amrex::Real time)
 
void remake_zphys (int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
 
void update_terrain_arrays (int lev)
 
void writeJobInfo (const std::string &dir) const
 

Static Public Member Functions

static bool is_it_time_for_action (int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
 
static void writeBuildInfo (std::ostream &os)
 
static void print_banner (MPI_Comm, std::ostream &)
 
static void print_usage (MPI_Comm, std::ostream &)
 
static void print_error (MPI_Comm, const std::string &msg)
 
static void print_summary (std::ostream &)
 
static void print_tpls (std::ostream &)
 

Public Attributes

amrex::Vector< std::array< amrex::Real, 2 > > hurricane_track_xy
 
amrex::Vector< amrex::MultiFab > weather_forecast_data_1
 
amrex::Vector< amrex::MultiFab > weather_forecast_data_2
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_1
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_2
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_interp
 
std::string pp_prefix {"erf"}
 

Private Member Functions

void ReadParameters ()
 
void ParameterSanityChecks ()
 
void AverageDown ()
 
void update_diffusive_arrays (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
 
void Construct_ERFFillPatchers (int lev)
 
void Define_ERFFillPatchers (int lev)
 
void init1DArrays ()
 
void init_bcs ()
 
void init_custom (int lev)
 
void init_uniform (int lev)
 
void init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
 
void turbPert_update (const int lev, const amrex::Real dt)
 
void turbPert_amplitude (const int lev)
 
void initialize_integrator (int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
 
void make_physbcs (int lev)
 
void initializeMicrophysics (const int &)
 
void FillPatchCrseLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
 
void FillPatchFineLevel (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
 
void FillIntermediatePatch (int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
 
void FillCoarsePatch (int lev, amrex::Real time)
 
void timeStep (int lev, amrex::Real time, int iteration)
 
void Advance (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
 
void initHSE ()
 Initialize HSE. More...
 
void initHSE (int lev)
 
void initRayleigh ()
 Initialize Rayleigh damping profiles. More...
 
void initSponge ()
 Initialize sponge profiles. More...
 
void setRayleighRefFromSounding (bool restarting)
 Set Rayleigh mean profiles from input sounding. More...
 
void setSpongeRefFromSounding (bool restarting)
 Set sponge mean profiles from input sounding. More...
 
void ComputeDt (int step=-1)
 
std::string PlotFileName (int lev) const
 
void setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void appendPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
 
void init_Dirichlet_bc_data (const std::string input_file)
 
void InitializeFromFile ()
 
void InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)
 
void post_update (amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
 
void fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, amrex::Real time, const amrex::Geometry &geom)
 
void init_geo_wind_profile (const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
 
void refinement_criteria_setup ()
 
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg (int lev)
 
AMREX_FORCE_INLINE std::ostream & DataLog (int i)
 
AMREX_FORCE_INLINE std::ostream & DerDataLog (int i)
 
AMREX_FORCE_INLINE int NumDataLogs () noexcept
 
AMREX_FORCE_INLINE int NumDerDataLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SamplePointLog (int i)
 
AMREX_FORCE_INLINE int NumSamplePointLogs () noexcept
 
AMREX_FORCE_INLINE std::ostream & SampleLineLog (int i)
 
AMREX_FORCE_INLINE int NumSampleLineLogs () noexcept
 
amrex::IntVect & SamplePoint (int i)
 
AMREX_FORCE_INLINE int NumSamplePoints () noexcept
 
amrex::IntVect & SampleLine (int i)
 
AMREX_FORCE_INLINE int NumSampleLines () noexcept
 
void setRecordDataInfo (int i, const std::string &filename)
 
void setRecordDerDataInfo (int i, const std::string &filename)
 
void setRecordEnergyDataInfo (int i, const std::string &filename)
 
void setRecordSamplePointInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
void setRecordSampleLineInfo (int i, int lev, amrex::IntVect &cell, const std::string &filename)
 
std::string DataLogName (int i) const noexcept
 The filename of the ith datalog file. More...
 
std::string DerDataLogName (int i) const noexcept
 
std::string SamplePointLogName (int i) const noexcept
 The filename of the ith sampleptlog file. More...
 
std::string SampleLineLogName (int i) const noexcept
 The filename of the ith samplelinelog file. More...
 
eb_ const & get_eb (int lev) const noexcept
 
amrex::EBFArrayBoxFactory const & EBFactory (int lev) const noexcept
 

Static Private Member Functions

static amrex::Vector< std::string > PlotFileVarNames (amrex::Vector< std::string > plot_var_names)
 
static void GotoNextLine (std::istream &is)
 
static AMREX_FORCE_INLINE int ComputeGhostCells (const SolverChoice &sc)
 
static amrex::Real getCPUTime ()
 
static int nghost_eb_basic ()
 
static int nghost_eb_volume ()
 
static int nghost_eb_full ()
 

Private Attributes

amrex::Vector< std::unique_ptr< amrex::MultiFab > > lat_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lon_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
 
InputSoundingData input_sounding_data
 
InputSpongeData input_sponge_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
 
std::unique_ptr< ProblemBaseprob = nullptr
 
amrex::Vector< int > num_boxes_at_level
 
amrex::Vector< int > num_files_at_level
 
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
 
amrex::Vector< int > istep
 
amrex::Vector< int > nsubsteps
 
amrex::Vector< amrex::Realt_new
 
amrex::Vector< amrex::Realt_old
 
amrex::Vector< amrex::Realdt
 
amrex::Vector< long > dt_mri_ratio
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
 
amrex::Vector< amrex::Vector< amrex::MultiFab > > gradp
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
 
amrex::Vector< amrex::Realt_avg_cnt
 
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
 
amrex::Vector< amrex::MultiFab > pp_inc
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
 
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
 
amrex::Vector< amrex::MultiFab > rU_old
 
amrex::Vector< amrex::MultiFab > rU_new
 
amrex::Vector< amrex::MultiFab > rV_old
 
amrex::Vector< amrex::MultiFab > rV_new
 
amrex::Vector< amrex::MultiFab > rW_old
 
amrex::Vector< amrex::MultiFab > rW_new
 
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
 
std::unique_ptr< Microphysicsmicro
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
 
LandSurface lsm
 
amrex::Vector< std::string > lsm_data_name
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
 
amrex::Vector< std::string > lsm_flux_name
 
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
 
amrex::Vector< std::unique_ptr< IRadiation > > rad
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > qheating_rates
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sw_lw_fluxes
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > solar_zenith
 
bool plot_rad = false
 
int rad_datalog_int = -1
 
int cf_width {0}
 
int cf_set_width {0}
 
amrex::Vector< ERFFillPatcherFPr_c
 
amrex::Vector< ERFFillPatcherFPr_u
 
amrex::Vector< ERFFillPatcherFPr_v
 
amrex::Vector< ERFFillPatcherFPr_w
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
 
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
 
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > mapfac
 
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
 
amrex::Vector< amrex::MultiFab > base_state
 
amrex::Vector< amrex::MultiFab > base_state_new
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
 
bool finished_wave = false
 
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
 
amrex::Vector< amrex::BCRec > domain_bcs_type
 
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
 
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_maxm_bc_extdir_vals
 
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_maxm_bc_neumann_vals
 
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
 
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
 
const int datwidth = 14
 
const int datprecision = 6
 
const int timeprecision = 13
 
int max_step = -1
 
bool use_datetime = false
 
const std::string datetime_format = "%Y-%m-%d %H:%M:%S"
 
std::string restart_chkfile = ""
 
amrex::Vector< amrex::Realfixed_dt
 
amrex::Vector< amrex::Realfixed_fast_dt
 
int regrid_int = -1
 
bool regrid_level_0_on_restart = false
 
std::string plot3d_file_1 {"plt_1_"}
 
std::string plot3d_file_2 {"plt_2_"}
 
std::string plot2d_file_1 {"plt2d_1_"}
 
std::string plot2d_file_2 {"plt2d_2_"}
 
std::string subvol_file {"subvol"}
 
bool m_expand_plotvars_to_unif_rr = false
 
int m_plot3d_int_1 = -1
 
int m_plot3d_int_2 = -1
 
int m_plot2d_int_1 = -1
 
int m_plot2d_int_2 = -1
 
int m_subvol_int = -1
 
amrex::Real m_plot3d_per_1 = -1.0
 
amrex::Real m_plot3d_per_2 = -1.0
 
amrex::Real m_plot2d_per_1 = -1.0
 
amrex::Real m_plot2d_per_2 = -1.0
 
amrex::Real m_subvol_per = -1.0
 
bool m_plot_face_vels = false
 
bool plot_lsm = false
 
int profile_int = -1
 
bool destag_profiles = true
 
std::string check_file {"chk"}
 
int m_check_int = -1
 
amrex::Real m_check_per = -1.0
 
amrex::Vector< std::string > plot3d_var_names_1
 
amrex::Vector< std::string > plot3d_var_names_2
 
amrex::Vector< std::string > plot2d_var_names_1
 
amrex::Vector< std::string > plot2d_var_names_2
 
const amrex::Vector< std::string > cons_names
 
const amrex::Vector< std::string > derived_names
 
TurbulentPerturbation turbPert
 
int real_width {0}
 
int real_set_width {0}
 
bool real_extrap_w {true}
 
bool metgrid_debug_quiescent {false}
 
bool metgrid_debug_isothermal {false}
 
bool metgrid_debug_dry {false}
 
bool metgrid_debug_psfc {false}
 
bool metgrid_debug_msf {false}
 
bool metgrid_interp_theta {false}
 
bool metgrid_basic_linear {false}
 
bool metgrid_use_below_sfc {true}
 
bool metgrid_use_sfc {true}
 
bool metgrid_retain_sfc {false}
 
amrex::Real metgrid_proximity {500.0}
 
int metgrid_order {2}
 
int metgrid_force_sfc_k {6}
 
amrex::Vector< amrex::BoxArray > ba1d
 
amrex::Vector< amrex::BoxArray > ba2d
 
std::unique_ptr< amrex::MultiFab > mf_C1H
 
std::unique_ptr< amrex::MultiFab > mf_C2H
 
std::unique_ptr< amrex::MultiFab > mf_MUB
 
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_PSFC
 
amrex::Vector< amrex::Vector< amrex::Real > > h_rhotheta_src
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhotheta_src
 
amrex::Vector< amrex::Vector< amrex::Real > > h_rhoqt_src
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhoqt_src
 
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
 
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
 
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
 
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
 
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
 
amrex::Vector< amrex::Realh_havg_density
 
amrex::Vector< amrex::Realh_havg_temperature
 
amrex::Vector< amrex::Realh_havg_pressure
 
amrex::Vector< amrex::Realh_havg_qv
 
amrex::Vector< amrex::Realh_havg_qc
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_density
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_temperature
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_pressure
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_qv
 
amrex::Gpu::DeviceVector< amrex::Reald_havg_qc
 
std::unique_ptr< WriteBndryPlanesm_w2d = nullptr
 
std::unique_ptr< ReadBndryPlanesm_r2d = nullptr
 
std::unique_ptr< SurfaceLayerm_SurfaceLayer = nullptr
 
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
 
amrex::Vector< amrex::Vector< amrex::BoxArray > > subdomains
 
amrex::MultiFab fine_mask
 
amrex::Vector< amrex::Realdz_min
 
int sampler_interval = -1
 
amrex::Real sampler_per = -1.0
 
std::unique_ptr< SampleDatadata_sampler = nullptr
 
amrex::Vector< std::unique_ptr< std::fstream > > datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
 
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
 
amrex::Vector< std::string > datalogname
 
amrex::Vector< std::string > der_datalogname
 
amrex::Vector< std::string > tot_e_datalogname
 
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
 
amrex::Vector< std::string > sampleptlogname
 
amrex::Vector< amrex::IntVect > samplepoint
 
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
 
amrex::Vector< std::string > samplelinelogname
 
amrex::Vector< amrex::IntVect > sampleline
 
amrex::Vector< std::unique_ptr< eb_ > > eb
 

Static Private Attributes

static int last_plot3d_file_step_1 = -1
 
static int last_plot3d_file_step_2 = -1
 
static int last_plot2d_file_step_1 = -1
 
static int last_plot2d_file_step_2 = -1
 
static int last_check_file_step = -1
 
static int last_subvol_step = -1
 
static amrex::Real last_plot3d_file_time_1 = 0.0
 
static amrex::Real last_plot3d_file_time_2 = 0.0
 
static amrex::Real last_plot2d_file_time_1 = 0.0
 
static amrex::Real last_plot2d_file_time_2 = 0.0
 
static amrex::Real last_check_file_time = 0.0
 
static amrex::Real last_subvol_time = 0.0
 
static bool plot_file_on_restart = true
 
static amrex::Real start_time = 0.0
 
static amrex::Real stop_time = std::numeric_limits<amrex::Real>::max()
 
static amrex::Real cfl = 0.8
 
static amrex::Real sub_cfl = 1.0
 
static amrex::Real init_shrink = 1.0
 
static amrex::Real change_max = 1.1
 
static amrex::Real dt_max_initial = 2.0e100
 
static amrex::Real dt_max = 1.0e9
 
static int fixed_mri_dt_ratio = 0
 
static SolverChoice solverChoice
 
static int verbose = 0
 
static int mg_verbose = 0
 
static bool use_fft = false
 
static int sum_interval = -1
 
static int pert_interval = -1
 
static amrex::Real sum_per = -1.0
 
static PlotFileType plotfile3d_type_1 = PlotFileType::None
 
static PlotFileType plotfile3d_type_2 = PlotFileType::None
 
static PlotFileType plotfile2d_type_1 = PlotFileType::None
 
static PlotFileType plotfile2d_type_2 = PlotFileType::None
 
static StateInterpType interpolation_type
 
static std::string sponge_type
 
static amrex::Vector< amrex::Vector< std::string > > nc_init_file = {{""}}
 
static std::string nc_bdy_file
 
static std::string nc_low_file
 
static int output_1d_column = 0
 
static int column_interval = -1
 
static amrex::Real column_per = -1.0
 
static amrex::Real column_loc_x = 0.0
 
static amrex::Real column_loc_y = 0.0
 
static std::string column_file_name = "column_data.nc"
 
static int output_bndry_planes = 0
 
static int bndry_output_planes_interval = -1
 
static amrex::Real bndry_output_planes_per = -1.0
 
static amrex::Real bndry_output_planes_start_time = 0.0
 
static int input_bndry_planes = 0
 
static int ng_dens_hse
 
static int ng_pres_hse
 
static amrex::Vector< amrex::AMRErrorTag > ref_tags
 
static amrex::Real startCPUTime = 0.0
 
static amrex::Real previousCPUTimeUsed = 0.0
 

Detailed Description

Main class in ERF code, instantiated from main.cpp

Constructor & Destructor Documentation

◆ ERF() [1/3]

ERF::ERF ( )
122 {
123  int fix_random_seed = 0;
124  ParmParse pp("erf"); pp.query("fix_random_seed", fix_random_seed);
125  // Note that the value of 1024UL is not significant -- the point here is just to set the
126  // same seed for all MPI processes for the purpose of regression testing
127  if (fix_random_seed) {
128  Print() << "Fixing the random seed" << std::endl;
129  InitRandom(1024UL);
130  }
131 
132  ERF_shared();
133 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real pp(amrex::Real y)
Definition: ERF_MicrophysicsUtils.H:230
void ERF_shared()
Definition: ERF.cpp:136
Here is the call graph for this function:

◆ ~ERF()

ERF::~ERF ( )
overridedefault

◆ ERF() [2/3]

ERF::ERF ( ERF &&  )
deletenoexcept

◆ ERF() [3/3]

ERF::ERF ( const ERF other)
delete

Member Function Documentation

◆ Advance()

void ERF::Advance ( int  lev,
amrex::Real  time,
amrex::Real  dt_lev,
int  iteration,
int  ncycle 
)
private
24 {
25  BL_PROFILE("ERF::Advance()");
26 
27  // We must swap the pointers so the previous step's "new" is now this step's "old"
28  std::swap(vars_old[lev], vars_new[lev]);
29 
30  MultiFab& S_old = vars_old[lev][Vars::cons];
31  MultiFab& S_new = vars_new[lev][Vars::cons];
32 
33  MultiFab& U_old = vars_old[lev][Vars::xvel];
34  MultiFab& V_old = vars_old[lev][Vars::yvel];
35  MultiFab& W_old = vars_old[lev][Vars::zvel];
36 
37  MultiFab& U_new = vars_new[lev][Vars::xvel];
38  MultiFab& V_new = vars_new[lev][Vars::yvel];
39  MultiFab& W_new = vars_new[lev][Vars::zvel];
40 
41  // We need to set these because otherwise in the first call to erf_advance we may
42  // read uninitialized data on ghost values in setting the bc's on the velocities
43  U_new.setVal(1.e34,U_new.nGrowVect());
44  V_new.setVal(1.e34,V_new.nGrowVect());
45  W_new.setVal(1.e34,W_new.nGrowVect());
46 
47  // Do error checking for negative (rho theta) here
48  if (solverChoice.anelastic[lev] != 1) {
50  }
51 
52  //
53  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
54  // If lev == 0 we have already FillPatched this in ERF::TimeStep
55  //
56  if (lev > 0) {
57  FillPatchFineLevel(lev, time, {&S_old, &U_old, &V_old, &W_old},
58  {&S_old, &rU_old[lev], &rV_old[lev], &rW_old[lev]},
59  base_state[lev], base_state[lev]);
60  }
61 
62  //
63  // So we must convert the fillpatched to momenta, including the ghost values
64  //
65  VelocityToMomentum(U_old, rU_old[lev].nGrowVect(),
66  V_old, rV_old[lev].nGrowVect(),
67  W_old, rW_old[lev].nGrowVect(),
68  S_old, rU_old[lev], rV_old[lev], rW_old[lev],
69  Geom(lev).Domain(),
71 
72  // Update the inflow perturbation update time and amplitude
73  if (solverChoice.pert_type == PerturbationType::Source ||
74  solverChoice.pert_type == PerturbationType::Direct ||
75  solverChoice.pert_type == PerturbationType::CPM)
76  {
77  turbPert.calc_tpi_update(lev, dt_lev, U_old, V_old, S_old);
78  }
79 
80  // If PerturbationType::Direct or CPM is selected, directly add the computed perturbation
81  // on the conserved field
82  if (solverChoice.pert_type == PerturbationType::Direct ||
83  solverChoice.pert_type == PerturbationType::CPM)
84  {
85  auto m_ixtype = S_old.boxArray().ixType(); // Conserved term
86  for (MFIter mfi(S_old,TileNoZ()); mfi.isValid(); ++mfi) {
87  Box bx = mfi.tilebox();
88  const Array4<Real> &cell_data = S_old.array(mfi);
89  const Array4<const Real> &pert_cell = turbPert.pb_cell[lev].array(mfi);
90  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cell_data, pert_cell);
91  }
92  }
93 
94  // configure SurfaceLayer params if needed
95  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
96  if (m_SurfaceLayer) {
97  IntVect ng = Theta_prim[lev]->nGrowVect();
98  MultiFab::Copy( *Theta_prim[lev], S_old, RhoTheta_comp, 0, 1, ng);
99  MultiFab::Divide(*Theta_prim[lev], S_old, Rho_comp , 0, 1, ng);
100  if (solverChoice.moisture_type != MoistureType::None) {
101  ng = Qv_prim[lev]->nGrowVect();
102 
103  MultiFab::Copy( *Qv_prim[lev], S_old, RhoQ1_comp, 0, 1, ng);
104  MultiFab::Divide(*Qv_prim[lev], S_old, Rho_comp , 0, 1, ng);
105 
106  if (solverChoice.moisture_indices.qr > -1) {
107  MultiFab::Copy( *Qr_prim[lev], S_old, solverChoice.moisture_indices.qr, 0, 1, ng);
108  MultiFab::Divide(*Qr_prim[lev], S_old, Rho_comp , 0, 1, ng);
109  } else {
110  Qr_prim[lev]->setVal(0.0);
111  }
112  }
113  // NOTE: std::swap above causes the field ptrs to be out of date.
114  // Reassign the field ptrs for MAC avg computation.
115  m_SurfaceLayer->update_mac_ptrs(lev, vars_old, Theta_prim, Qv_prim, Qr_prim);
116  m_SurfaceLayer->update_pblh(lev, vars_old, z_phys_cc[lev].get(),
118  m_SurfaceLayer->update_fluxes(lev, time, S_old, z_phys_nd[lev]);
119  }
120  }
121 
122 #if defined(ERF_USE_WINDFARM)
123  // **************************************************************************************
124  // Update the windfarm sources
125  // **************************************************************************************
126  if (solverChoice.windfarm_type != WindFarmType::None) {
127  advance_windfarm(Geom(lev), dt_lev, S_old,
128  U_old, V_old, W_old, vars_windfarm[lev],
129  Nturb[lev], SMark[lev], time);
130  }
131 
132 #endif
133 
134  // **************************************************************************************
135  // Weather data interpolation
136  // **************************************************************************************
137  if(solverChoice.init_type == InitType::HindCast and
140  }
141 
142  // **************************************************************************************
143  // Update the radiation sources
144  // **************************************************************************************
145  advance_radiation(lev, S_new, dt_lev);
146 
147  const BoxArray& ba = S_old.boxArray();
148  const DistributionMapping& dm = S_old.DistributionMap();
149 
150  int nvars = S_old.nComp();
151 
152  // Source array for conserved cell-centered quantities -- this will be filled
153  // in the call to make_sources in ERF_TI_slow_rhs_pre.H
154  MultiFab cc_source(ba,dm,nvars,1); cc_source.setVal(0.0);
155 
156  // Source arrays for momenta -- these will be filled
157  // in the call to make_mom_sources in ERF_TI_slow_rhs_pre.H
158  BoxArray ba_x(ba); ba_x.surroundingNodes(0);
159  MultiFab xmom_source(ba_x,dm,1,1); xmom_source.setVal(0.0);
160 
161  BoxArray ba_y(ba); ba_y.surroundingNodes(1);
162  MultiFab ymom_source(ba_y,dm,1,1); ymom_source.setVal(0.0);
163 
164  BoxArray ba_z(ba); ba_z.surroundingNodes(2);
165  MultiFab zmom_source(ba_z,dm,1,1); zmom_source.setVal(0.0);
166  MultiFab buoyancy(ba_z,dm,1,1); buoyancy.setVal(0.0);
167 
168  amrex::Vector<MultiFab> state_old;
169  amrex::Vector<MultiFab> state_new;
170 
171  // **************************************************************************************
172  // Here we define state_old and state_new which are to be advanced
173  // **************************************************************************************
174  // Initial solution
175  // Note that "old" and "new" here are relative to each RK stage.
176  state_old.push_back(MultiFab(S_old , amrex::make_alias, 0, nvars)); // cons
177  state_old.push_back(MultiFab(rU_old[lev], amrex::make_alias, 0, 1)); // xmom
178  state_old.push_back(MultiFab(rV_old[lev], amrex::make_alias, 0, 1)); // ymom
179  state_old.push_back(MultiFab(rW_old[lev], amrex::make_alias, 0, 1)); // zmom
180 
181  // Final solution
182  // state_new at the end of the last RK stage holds the t^{n+1} data
183  state_new.push_back(MultiFab(S_new , amrex::make_alias, 0, nvars)); // cons
184  state_new.push_back(MultiFab(rU_new[lev], amrex::make_alias, 0, 1)); // xmom
185  state_new.push_back(MultiFab(rV_new[lev], amrex::make_alias, 0, 1)); // ymom
186  state_new.push_back(MultiFab(rW_new[lev], amrex::make_alias, 0, 1)); // zmom
187 
188 #ifdef ERF_USE_SHOC
189  // **************************************************************************************
190  // Update the "old" state using SHOC
191  // **************************************************************************************
192  if (solverChoice.use_shoc) {
193  compute_shoc_tendencies(lev, S_old, U_old, V_old, W_old, dt_lev);
194  }
195 #endif
196 
197  // **************************************************************************************
198  // Update the dycore
199  // **************************************************************************************
200  advance_dycore(lev, state_old, state_new,
201  U_old, V_old, W_old,
202  U_new, V_new, W_new,
203  cc_source, xmom_source, ymom_source, zmom_source, buoyancy,
204  Geom(lev), dt_lev, time);
205 
206  // **************************************************************************************
207  // Update the microphysics (moisture)
208  // **************************************************************************************
210  advance_microphysics(lev, S_new, dt_lev, iteration, time);
211  }
212 
213  // **************************************************************************************
214  // Update the land surface model
215  // **************************************************************************************
216  advance_lsm(lev, S_new, U_new, V_new, dt_lev);
217 
218 #ifdef ERF_USE_PARTICLES
219  // **************************************************************************************
220  // Update the particle positions
221  // **************************************************************************************
222  evolveTracers( lev, dt_lev, vars_new, z_phys_nd );
223 #endif
224 
225  // ***********************************************************************************************
226  // Impose domain boundary conditions here so that in FillPatching the fine data we won't
227  // need to re-fill these
228  // ***********************************************************************************************
229  if (lev < finest_level) {
230  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
232  0,vars_new[lev][Vars::cons].nComp(),
233  vars_new[lev][Vars::cons].nGrowVect(),time,BCVars::cons_bc,true);
234  (*physbcs_u[lev])(vars_new[lev][Vars::xvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
235  ngvect_vels,time,BCVars::xvel_bc,true);
236  (*physbcs_v[lev])(vars_new[lev][Vars::yvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
237  ngvect_vels,time,BCVars::yvel_bc,true);
238  (*physbcs_w[lev])(vars_new[lev][Vars::zvel], vars_new[lev][Vars::xvel], vars_new[lev][Vars::yvel],
239  ngvect_vels,time,BCVars::zvel_bc,true);
240  }
241 
242  // **************************************************************************************
243  // Register old and new coarse data if we are at a level less than the finest level
244  // **************************************************************************************
245  if (lev < finest_level) {
246  if (cf_width > 0) {
247  // We must fill the ghost cells of these so that the parallel copy works correctly
248  state_old[IntVars::cons].FillBoundary(geom[lev].periodicity());
249  state_new[IntVars::cons].FillBoundary(geom[lev].periodicity());
250  FPr_c[lev].RegisterCoarseData({&state_old[IntVars::cons], &state_new[IntVars::cons]},
251  {time, time+dt_lev});
252  }
253 
254  if (cf_width >= 0) {
255  // We must fill the ghost cells of these so that the parallel copy works correctly
256  state_old[IntVars::xmom].FillBoundary(geom[lev].periodicity());
257  state_new[IntVars::xmom].FillBoundary(geom[lev].periodicity());
258  FPr_u[lev].RegisterCoarseData({&state_old[IntVars::xmom], &state_new[IntVars::xmom]},
259  {time, time+dt_lev});
260 
261  state_old[IntVars::ymom].FillBoundary(geom[lev].periodicity());
262  state_new[IntVars::ymom].FillBoundary(geom[lev].periodicity());
263  FPr_v[lev].RegisterCoarseData({&state_old[IntVars::ymom], &state_new[IntVars::ymom]},
264  {time, time+dt_lev});
265 
266  state_old[IntVars::zmom].FillBoundary(geom[lev].periodicity());
267  state_new[IntVars::zmom].FillBoundary(geom[lev].periodicity());
268  FPr_w[lev].RegisterCoarseData({&state_old[IntVars::zmom], &state_new[IntVars::zmom]},
269  {time, time+dt_lev});
270  }
271 
272  //
273  // Now create a MultiFab that holds (S_new - S_old) / dt from the coarse level interpolated
274  // on to the coarse/fine boundary at the fine resolution
275  //
276  Interpolater* mapper_f = &face_cons_linear_interp;
277 
278  // PhysBCFunctNoOp null_bc;
279  // MultiFab tempx(vars_new[lev+1][Vars::xvel].boxArray(),vars_new[lev+1][Vars::xvel].DistributionMap(),1,0);
280  // tempx.setVal(0.0);
281  // xmom_crse_rhs[lev+1].setVal(0.0);
282  // FPr_u[lev].FillSet(tempx , time , null_bc, domain_bcs_type);
283  // FPr_u[lev].FillSet(xmom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
284  // MultiFab::Subtract(xmom_crse_rhs[lev+1],tempx,0,0,1,IntVect{0});
285  // xmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
286 
287  // MultiFab tempy(vars_new[lev+1][Vars::yvel].boxArray(),vars_new[lev+1][Vars::yvel].DistributionMap(),1,0);
288  // tempy.setVal(0.0);
289  // ymom_crse_rhs[lev+1].setVal(0.0);
290  // FPr_v[lev].FillSet(tempy , time , null_bc, domain_bcs_type);
291  // FPr_v[lev].FillSet(ymom_crse_rhs[lev+1], time+dt_lev, null_bc, domain_bcs_type);
292  // MultiFab::Subtract(ymom_crse_rhs[lev+1],tempy,0,0,1,IntVect{0});
293  // ymom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
294 
295  MultiFab temp_state(zmom_crse_rhs[lev+1].boxArray(),zmom_crse_rhs[lev+1].DistributionMap(),1,0);
296  InterpFromCoarseLevel(temp_state, IntVect{0}, IntVect{0}, state_old[IntVars::zmom], 0, 0, 1,
297  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
298  InterpFromCoarseLevel(zmom_crse_rhs[lev+1], IntVect{0}, IntVect{0}, state_new[IntVars::zmom], 0, 0, 1,
299  geom[lev], geom[lev+1], refRatio(lev), mapper_f, domain_bcs_type, BCVars::zvel_bc);
300  MultiFab::Subtract(zmom_crse_rhs[lev+1],temp_state,0,0,1,IntVect{0});
301  zmom_crse_rhs[lev+1].mult(1.0/dt_lev,0,1,0);
302  }
303 
304  // ***********************************************************************************************
305  // Update the time averaged velocities if they are requested
306  // ***********************************************************************************************
308  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(), U_new, V_new, W_new);
309  }
310 }
void check_for_negative_theta(amrex::MultiFab &S_old)
Definition: ERF_Advance.cpp:313
@ nvars
Definition: ERF_DataStruct.H:91
#define Rho_comp
Definition: ERF_IndexDefines.H:36
#define RhoTheta_comp
Definition: ERF_IndexDefines.H:37
#define RhoQ1_comp
Definition: ERF_IndexDefines.H:42
@ surface_layer
AMREX_FORCE_INLINE amrex::IntVect TileNoZ()
Definition: ERF_TileNoZ.H:11
void Time_Avg_Vel_atCC(const Real &dt, Real &t_avg_cnt, MultiFab *vel_t_avg, MultiFab &xvel, MultiFab &yvel, MultiFab &zvel)
Definition: ERF_TimeAvgVel.cpp:9
void VelocityToMomentum(const amrex::MultiFab &xvel_in, const amrex::IntVect &xvel_ngrow, const amrex::MultiFab &yvel_in, const amrex::IntVect &yvel_ngrow, const amrex::MultiFab &zvel_in, const amrex::IntVect &zvel_ngrow, const amrex::MultiFab &cons_in, amrex::MultiFab &xmom_out, amrex::MultiFab &ymom_out, amrex::MultiFab &zmom_out, const amrex::Box &domain, const amrex::Vector< amrex::BCRec > &domain_bcs_type_h, const amrex::MultiFab *c_vfrac=nullptr)
amrex::Vector< amrex::MultiFab > rU_new
Definition: ERF.H:809
amrex::Vector< ERFFillPatcher > FPr_u
Definition: ERF.H:863
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_new
Definition: ERF.H:781
amrex::Vector< ERFFillPatcher > FPr_v
Definition: ERF.H:864
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_cons > > physbcs_cons
Definition: ERF.H:796
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc
Definition: ERF.H:888
static SolverChoice solverChoice
Definition: ERF.H:1100
amrex::Vector< ERFFillPatcher > FPr_c
Definition: ERF.H:862
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vel_t_avg
Definition: ERF.H:788
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_w > > physbcs_w
Definition: ERF.H:799
amrex::Vector< amrex::MultiFab > base_state
Definition: ERF.H:919
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qv_prim
Definition: ERF.H:804
amrex::Vector< amrex::MultiFab > rV_new
Definition: ERF.H:811
amrex::Vector< amrex::BCRec > domain_bcs_type
Definition: ERF.H:935
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Qr_prim
Definition: ERF.H:805
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_u > > physbcs_u
Definition: ERF.H:797
amrex::Vector< amrex::Real > t_avg_cnt
Definition: ERF.H:789
void FillPatchFineLevel(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, const amrex::MultiFab &old_base_state, const amrex::MultiFab &new_base_state, bool fillset=true, bool cons_only=false)
Definition: ERF_FillPatch.cpp:20
amrex::Vector< amrex::MultiFab > rU_old
Definition: ERF.H:808
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Theta_prim
Definition: ERF.H:803
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_v > > physbcs_v
Definition: ERF.H:798
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd
Definition: ERF.H:887
void advance_dycore(int level, amrex::Vector< amrex::MultiFab > &state_old, amrex::Vector< amrex::MultiFab > &state_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, amrex::MultiFab &xmom_src, amrex::MultiFab &ymom_src, amrex::MultiFab &zmom_src, amrex::MultiFab &buoyancy, amrex::Geometry fine_geom, amrex::Real dt, amrex::Real time)
Definition: ERF_AdvanceDycore.cpp:38
amrex::Vector< amrex::MultiFab > rW_new
Definition: ERF.H:813
amrex::Vector< amrex::MultiFab > zmom_crse_rhs
Definition: ERF.H:817
void advance_lsm(int lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, const amrex::Real &dt_advance)
Definition: ERF_AdvanceLSM.cpp:5
TurbulentPerturbation turbPert
Definition: ERF.H:1103
amrex::Vector< amrex::MultiFab > rW_old
Definition: ERF.H:812
std::unique_ptr< SurfaceLayer > m_SurfaceLayer
Definition: ERF.H:1263
amrex::Vector< ERFFillPatcher > FPr_w
Definition: ERF.H:865
amrex::Vector< amrex::Real > dt
Definition: ERF.H:775
void advance_radiation(int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance)
Definition: ERF_AdvanceRadiation.cpp:5
void advance_microphysics(int lev, amrex::MultiFab &cons_in, const amrex::Real &dt_advance, const int &iteration, const amrex::Real &time)
Definition: ERF_AdvanceMicrophysics.cpp:5
int cf_width
Definition: ERF.H:860
void WeatherDataInterpolation(const amrex::Real time)
Definition: ERF_WeatherDataInterpolation.cpp:497
amrex::GpuArray< ERF_BC, AMREX_SPACEDIM *2 > phys_bc_type
Definition: ERF.H:948
amrex::Vector< amrex::MultiFab > rV_old
Definition: ERF.H:810
amrex::Vector< amrex::Vector< amrex::MultiFab > > vars_old
Definition: ERF.H:782
@ zvel_bc
Definition: ERF_IndexDefines.H:89
@ yvel_bc
Definition: ERF_IndexDefines.H:88
@ cons_bc
Definition: ERF_IndexDefines.H:76
@ xvel_bc
Definition: ERF_IndexDefines.H:87
@ ymom
Definition: ERF_IndexDefines.H:160
@ cons
Definition: ERF_IndexDefines.H:158
@ zmom
Definition: ERF_IndexDefines.H:161
@ xmom
Definition: ERF_IndexDefines.H:159
@ ng
Definition: ERF_Morrison.H:48
@ xvel
Definition: ERF_IndexDefines.H:141
@ cons
Definition: ERF_IndexDefines.H:140
@ zvel
Definition: ERF_IndexDefines.H:143
@ yvel
Definition: ERF_IndexDefines.H:142
int qr
Definition: ERF_DataStruct.H:103
bool use_shoc
Definition: ERF_DataStruct.H:895
bool hindcast_lateral_forcing
Definition: ERF_DataStruct.H:954
bool moisture_tight_coupling
Definition: ERF_DataStruct.H:932
amrex::Vector< int > anelastic
Definition: ERF_DataStruct.H:831
MoistureType moisture_type
Definition: ERF_DataStruct.H:911
PerturbationType pert_type
Definition: ERF_DataStruct.H:901
WindFarmType windfarm_type
Definition: ERF_DataStruct.H:912
static InitType init_type
Definition: ERF_DataStruct.H:800
MoistureComponentIndices moisture_indices
Definition: ERF_DataStruct.H:930
bool time_avg_vel
Definition: ERF_DataStruct.H:898
amrex::Vector< amrex::MultiFab > pb_cell
Definition: ERF_TurbPertStruct.H:640
void calc_tpi_update(const int lev, const amrex::Real dt, amrex::MultiFab &mf_xvel, amrex::MultiFab &mf_yvel, amrex::MultiFab &mf_cons)
Definition: ERF_TurbPertStruct.H:223
void apply_tpi(const int &lev, const amrex::Box &vbx, const int &comp, const amrex::IndexType &m_ixtype, const amrex::Array4< amrex::Real > &src_arr, const amrex::Array4< amrex::Real const > &pert_cell)
Definition: ERF_TurbPertStruct.H:324
Here is the call graph for this function:

◆ advance_dycore()

void ERF::advance_dycore ( int  level,
amrex::Vector< amrex::MultiFab > &  state_old,
amrex::Vector< amrex::MultiFab > &  state_new,
amrex::MultiFab &  xvel_old,
amrex::MultiFab &  yvel_old,
amrex::MultiFab &  zvel_old,
amrex::MultiFab &  xvel_new,
amrex::MultiFab &  yvel_new,
amrex::MultiFab &  zvel_new,
amrex::MultiFab &  source,
amrex::MultiFab &  xmom_src,
amrex::MultiFab &  ymom_src,
amrex::MultiFab &  zmom_src,
amrex::MultiFab &  buoyancy,
amrex::Geometry  fine_geom,
amrex::Real  dt,
amrex::Real  time 
)

Function that advances the solution at one level for a single time step – this sets up the multirate time integrator and calls the integrator's advance function

Parameters
[in]levellevel of refinement (coarsest level is 0)
[in]state_oldold-time conserved variables
[in]state_newnew-time conserved variables
[in]xvel_oldold-time x-component of velocity
[in]yvel_oldold-time y-component of velocity
[in]zvel_oldold-time z-component of velocity
[in]xvel_newnew-time x-component of velocity
[in]yvel_newnew-time y-component of velocity
[in]zvel_newnew-time z-component of velocity
[in]cc_srcsource term for conserved variables
[in]xmom_srcsource term for x-momenta
[in]ymom_srcsource term for y-momenta
[in]zmom_srcsource term for z-momenta
[in]fine_geomcontainer for geometry information at current level
[in]dt_advancetime step for this time advance
[in]old_timeold time for this time advance
48 {
49  BL_PROFILE_VAR("erf_advance_dycore()",erf_advance_dycore);
50 
51  const Box& domain = fine_geom.Domain();
52 
56 
57  MultiFab r_hse (base_state[level], make_alias, BaseState::r0_comp , 1);
58  MultiFab p_hse (base_state[level], make_alias, BaseState::p0_comp , 1);
59  MultiFab pi_hse(base_state[level], make_alias, BaseState::pi0_comp, 1);
60 
61  // These pointers are used in the MRI utility functions
62  MultiFab* r0 = &r_hse;
63  MultiFab* p0 = &p_hse;
64  MultiFab* pi0 = &pi_hse;
65 
66  Real* dptr_rhotheta_src = solverChoice.custom_rhotheta_forcing ? d_rhotheta_src[level].data() : nullptr;
67  Real* dptr_rhoqt_src = solverChoice.custom_moisture_forcing ? d_rhoqt_src[level].data() : nullptr;
68  Real* dptr_wbar_sub = solverChoice.custom_w_subsidence ? d_w_subsid[level].data() : nullptr;
69 
70  // Turbulent Perturbation Pointer
71  //Real* dptr_rhotheta_src = solverChoice.pert_type ? d_rhotheta_src[level].data() : nullptr;
72 
73  Vector<Real*> d_rayleigh_ptrs_at_lev;
74  d_rayleigh_ptrs_at_lev.resize(Rayleigh::nvars);
75  d_rayleigh_ptrs_at_lev[Rayleigh::ubar] = solverChoice.rayleigh_damp_U ? d_rayleigh_ptrs[level][Rayleigh::ubar].data() : nullptr;
76  d_rayleigh_ptrs_at_lev[Rayleigh::vbar] = solverChoice.rayleigh_damp_V ? d_rayleigh_ptrs[level][Rayleigh::vbar].data() : nullptr;
77  d_rayleigh_ptrs_at_lev[Rayleigh::wbar] = solverChoice.rayleigh_damp_W ? d_rayleigh_ptrs[level][Rayleigh::wbar].data() : nullptr;
78  d_rayleigh_ptrs_at_lev[Rayleigh::thetabar] = solverChoice.rayleigh_damp_T ? d_rayleigh_ptrs[level][Rayleigh::thetabar].data() : nullptr;
79 
80  Vector<Real*> d_sponge_ptrs_at_lev;
81  if(sc.sponge_type=="input_sponge")
82  {
83  d_sponge_ptrs_at_lev.resize(Sponge::nvars_sponge);
84  d_sponge_ptrs_at_lev[Sponge::ubar_sponge] = d_sponge_ptrs[level][Sponge::ubar_sponge].data();
85  d_sponge_ptrs_at_lev[Sponge::vbar_sponge] = d_sponge_ptrs[level][Sponge::vbar_sponge].data();
86  }
87 
88  bool l_use_terrain_fitted_coords = (solverChoice.mesh_type != MeshType::ConstantDz);
89  bool l_use_kturb = tc.use_kturb;
90  bool l_use_diff = ( (dc.molec_diff_type != MolecDiffType::None) ||
91  l_use_kturb );
92  bool l_use_moisture = ( solverChoice.moisture_type != MoistureType::None );
93  bool l_implicit_substepping = ( solverChoice.substepping_type[level] == SubsteppingType::Implicit );
94 
95  const bool use_SurfLayer = (m_SurfaceLayer != nullptr);
96  const MultiFab* z_0 = (use_SurfLayer) ? m_SurfaceLayer->get_z0(level) : nullptr;
97 
98  const BoxArray& ba = state_old[IntVars::cons].boxArray();
99  const BoxArray& ba_z = zvel_old.boxArray();
100  const DistributionMapping& dm = state_old[IntVars::cons].DistributionMap();
101 
102  int num_prim = state_old[IntVars::cons].nComp() - 1;
103 
104  MultiFab S_prim (ba , dm, num_prim, state_old[IntVars::cons].nGrowVect());
105  MultiFab pi_stage (ba , dm, 1, state_old[IntVars::cons].nGrowVect());
106  MultiFab fast_coeffs(ba_z, dm, 5, 0);
107  MultiFab* eddyDiffs = eddyDiffs_lev[level].get();
108  MultiFab* SmnSmn = SmnSmn_lev[level].get();
109 
110  // **************************************************************************************
111  // Compute strain for use in slow RHS and Smagorinsky model
112  // **************************************************************************************
113  {
114  BL_PROFILE("erf_advance_strain");
115  if (l_use_diff) {
116 
117  const BCRec* bc_ptr_h = domain_bcs_type.data();
118  const GpuArray<Real, AMREX_SPACEDIM> dxInv = fine_geom.InvCellSizeArray();
119 
120 #ifdef _OPENMP
121 #pragma omp parallel if (Gpu::notInLaunchRegion())
122 #endif
123  for ( MFIter mfi(state_new[IntVars::cons],TileNoZ()); mfi.isValid(); ++mfi)
124  {
125  Box bxcc = mfi.growntilebox(IntVect(1,1,0));
126  Box tbxxy = mfi.tilebox(IntVect(1,1,0),IntVect(1,1,0));
127  Box tbxxz = mfi.tilebox(IntVect(1,0,1),IntVect(1,1,0));
128  Box tbxyz = mfi.tilebox(IntVect(0,1,1),IntVect(1,1,0));
129 
130  if (bxcc.smallEnd(2) != domain.smallEnd(2)) {
131  bxcc.growLo(2,1);
132  tbxxy.growLo(2,1);
133  tbxxz.growLo(2,1);
134  tbxyz.growLo(2,1);
135  }
136 
137  if (bxcc.bigEnd(2) != domain.bigEnd(2)) {
138  bxcc.growHi(2,1);
139  tbxxy.growHi(2,1);
140  tbxxz.growHi(2,1);
141  tbxyz.growHi(2,1);
142  }
143 
144  const Array4<const Real> & u = xvel_old.array(mfi);
145  const Array4<const Real> & v = yvel_old.array(mfi);
146  const Array4<const Real> & w = zvel_old.array(mfi);
147 
148  Array4<Real> tau11 = Tau[level][TauType::tau11].get()->array(mfi);
149  Array4<Real> tau22 = Tau[level][TauType::tau22].get()->array(mfi);
150  Array4<Real> tau33 = Tau[level][TauType::tau33].get()->array(mfi);
151  Array4<Real> tau12 = Tau[level][TauType::tau12].get()->array(mfi);
152  Array4<Real> tau13 = Tau[level][TauType::tau13].get()->array(mfi);
153  Array4<Real> tau23 = Tau[level][TauType::tau23].get()->array(mfi);
154 
155  Array4<Real> tau21 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau21].get()->array(mfi) : Array4<Real>{};
156  Array4<Real> tau31 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau31].get()->array(mfi) : Array4<Real>{};
157  Array4<Real> tau32 = l_use_terrain_fitted_coords ? Tau[level][TauType::tau32].get()->array(mfi) : Array4<Real>{};
158  const Array4<const Real>& z_nd = z_phys_nd[level]->const_array(mfi);
159 
160  const Array4<const Real> mf_mx = mapfac[level][MapFacType::m_x]->const_array(mfi);
161  const Array4<const Real> mf_ux = mapfac[level][MapFacType::u_x]->const_array(mfi);
162  const Array4<const Real> mf_vx = mapfac[level][MapFacType::v_x]->const_array(mfi);
163  const Array4<const Real> mf_my = mapfac[level][MapFacType::m_y]->const_array(mfi);
164  const Array4<const Real> mf_uy = mapfac[level][MapFacType::u_y]->const_array(mfi);
165  const Array4<const Real> mf_vy = mapfac[level][MapFacType::v_y]->const_array(mfi);
166 
167  if (solverChoice.mesh_type == MeshType::StretchedDz) {
168  ComputeStrain_S(bxcc, tbxxy, tbxxz, tbxyz, domain,
169  u, v, w,
170  tau11, tau22, tau33,
171  tau12, tau21,
172  tau13, tau31,
173  tau23, tau32,
174  stretched_dz_d[level], dxInv,
175  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h);
176  } else if (l_use_terrain_fitted_coords) {
177  ComputeStrain_T(bxcc, tbxxy, tbxxz, tbxyz, domain,
178  u, v, w,
179  tau11, tau22, tau33,
180  tau12, tau21,
181  tau13, tau31,
182  tau23, tau32,
183  z_nd, detJ_cc[level]->const_array(mfi), dxInv,
184  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h);
185  } else {
186  ComputeStrain_N(bxcc, tbxxy, tbxxz, tbxyz, domain,
187  u, v, w,
188  tau11, tau22, tau33,
189  tau12, tau13, tau23,
190  dxInv,
191  mf_mx, mf_ux, mf_vx, mf_my, mf_uy, mf_vy, bc_ptr_h);
192  }
193  } // mfi
194  } // l_use_diff
195  } // profile
196 
197 #include "ERF_TI_utils.H"
198 
199  // Additional SFS quantities, calculated once per timestep
200  MultiFab* Hfx1 = SFS_hfx1_lev[level].get();
201  MultiFab* Hfx2 = SFS_hfx2_lev[level].get();
202  MultiFab* Hfx3 = SFS_hfx3_lev[level].get();
203  MultiFab* Q1fx1 = SFS_q1fx1_lev[level].get();
204  MultiFab* Q1fx2 = SFS_q1fx2_lev[level].get();
205  MultiFab* Q1fx3 = SFS_q1fx3_lev[level].get();
206  MultiFab* Q2fx3 = SFS_q2fx3_lev[level].get();
207  MultiFab* Diss = SFS_diss_lev[level].get();
208 
209  // *************************************************************************
210  // Calculate cell-centered eddy viscosity & diffusivities
211  //
212  // Notes -- we fill all the data in ghost cells before calling this so
213  // that we can fill the eddy viscosity in the ghost regions and
214  // not have to call a boundary filler on this data itself
215  //
216  // LES - updates both horizontal and vertical eddy viscosity components
217  // PBL - only updates vertical eddy viscosity components so horizontal
218  // components come from the LES model or are left as zero.
219  // *************************************************************************
220  if (l_use_kturb)
221  {
222  // NOTE: state_new transfers to state_old for PBL (due to ptr swap in advance)
223  const BCRec* bc_ptr_h = domain_bcs_type.data();
224  ComputeTurbulentViscosity(dt_advance, xvel_old, yvel_old,Tau[level],
225  state_old[IntVars::cons],
226  *walldist[level].get(),
227  *eddyDiffs, *Hfx1, *Hfx2, *Hfx3, *Diss, // to be updated
228  fine_geom, mapfac[level],
229  z_phys_nd[level], solverChoice,
230  m_SurfaceLayer, z_0, l_use_terrain_fitted_coords,
231  l_use_moisture, level, bc_ptr_h);
232  }
233 
234  // ***********************************************************************************************
235  // Update user-defined source terms -- these are defined once per time step (not per RK stage)
236  // ***********************************************************************************************
238  prob->update_rhotheta_sources(old_time,
239  h_rhotheta_src[level], d_rhotheta_src[level],
240  fine_geom, z_phys_cc[level]);
241  }
242 
244  prob->update_rhoqt_sources(old_time,
245  h_rhoqt_src[level], d_rhoqt_src[level],
246  fine_geom, z_phys_cc[level]);
247  }
248 
250  prob->update_geostrophic_profile(old_time,
251  h_u_geos[level], d_u_geos[level],
252  h_v_geos[level], d_v_geos[level],
253  fine_geom, z_phys_cc[level]);
254  }
255 
257  prob->update_w_subsidence(old_time,
258  h_w_subsid[level], d_w_subsid[level],
259  fine_geom, z_phys_nd[level]);
260  }
261 
262  // ***********************************************************************************************
263  // Convert old velocity available on faces to old momentum on faces to be used in time integration
264  // ***********************************************************************************************
265  MultiFab density(state_old[IntVars::cons], make_alias, Rho_comp, 1);
266 
267  //
268  // This is an optimization since we won't need more than one ghost
269  // cell of momentum in the integrator if not using numerical diffusion
270  //
271  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : xvel_old.nGrowVect();
272  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : yvel_old.nGrowVect();
273  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : zvel_old.nGrowVect();
274 
275  VelocityToMomentum(xvel_old, ngu, yvel_old, ngv, zvel_old, ngw, density,
276  state_old[IntVars::xmom],
277  state_old[IntVars::ymom],
278  state_old[IntVars::zmom],
279  domain, domain_bcs_type);
280 
281  MultiFab::Copy(xvel_new,xvel_old,0,0,1,xvel_old.nGrowVect());
282  MultiFab::Copy(yvel_new,yvel_old,0,0,1,yvel_old.nGrowVect());
283  MultiFab::Copy(zvel_new,zvel_old,0,0,1,zvel_old.nGrowVect());
284 
285  bool fast_only = false;
286  bool vel_and_mom_synced = true;
287 
288  apply_bcs(state_old, old_time,
289  state_old[IntVars::cons].nGrow(), state_old[IntVars::xmom].nGrow(),
290  fast_only, vel_and_mom_synced);
291  cons_to_prim(state_old[IntVars::cons], state_old[IntVars::cons].nGrow());
292 
293  // ***********************************************************************************************
294  // Define a new MultiFab that holds q_total and fill it by summing the moisture components --
295  // to be used in buoyancy calculation and as part of the inertial weighting in the
296  // ***********************************************************************************************
297  MultiFab qt(grids[level], dmap[level], 1, 1);
298  qt.setVal(0.0);
299 
300 #include "ERF_TI_no_substep_fun.H"
301 #include "ERF_TI_substep_fun.H"
302 #include "ERF_TI_slow_rhs_pre.H"
303 #include "ERF_TI_slow_rhs_post.H"
304 
305  // ***************************************************************************************
306  // Setup the integrator and integrate for a single timestep
307  // **************************************************************************************
308  MRISplitIntegrator<Vector<MultiFab> >& mri_integrator = *mri_integrator_mem[level];
309 
310  // Define rhs and 'post update' utility function that is called after calculating
311  // any state data (e.g. at RK stages or at the end of a timestep)
312  mri_integrator.set_slow_rhs_pre(slow_rhs_fun_pre);
313  mri_integrator.set_slow_rhs_post(slow_rhs_fun_post);
314 
315  mri_integrator.set_fast_rhs(fast_rhs_fun);
317  mri_integrator.set_no_substep(no_substep_fun);
318 
319  mri_integrator.advance(state_old, state_new, old_time, dt_advance);
320 
321  if (verbose) Print() << "Done with advance_dycore at level " << level << std::endl;
322 }
void ComputeStrain_N(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau13, Array4< Real > &tau23, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr)
Definition: ERF_ComputeStrain_N.cpp:28
void ComputeStrain_S(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau21, Array4< Real > &tau13, Array4< Real > &tau31, Array4< Real > &tau23, Array4< Real > &tau32, const Gpu::DeviceVector< Real > &stretched_dz_d, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr)
Definition: ERF_ComputeStrain_S.cpp:36
void ComputeStrain_T(Box bxcc, Box tbxxy, Box tbxxz, Box tbxyz, Box domain, const Array4< const Real > &u, const Array4< const Real > &v, const Array4< const Real > &w, Array4< Real > &tau11, Array4< Real > &tau22, Array4< Real > &tau33, Array4< Real > &tau12, Array4< Real > &tau21, Array4< Real > &tau13, Array4< Real > &tau31, Array4< Real > &tau23, Array4< Real > &tau32, const Array4< const Real > &z_nd, const Array4< const Real > &detJ, const GpuArray< Real, AMREX_SPACEDIM > &dxInv, const Array4< const Real > &mf_mx, const Array4< const Real > &mf_ux, const Array4< const Real > &mf_vx, const Array4< const Real > &mf_my, const Array4< const Real > &mf_uy, const Array4< const Real > &mf_vy, const BCRec *bc_ptr)
Definition: ERF_ComputeStrain_T.cpp:36
void ComputeTurbulentViscosity(Real dt, const MultiFab &xvel, const MultiFab &yvel, Vector< std::unique_ptr< MultiFab >> &Tau_lev, MultiFab &cons_in, const MultiFab &wdist, MultiFab &eddyViscosity, MultiFab &Hfx1, MultiFab &Hfx2, MultiFab &Hfx3, MultiFab &Diss, const Geometry &geom, Vector< std::unique_ptr< MultiFab >> &mapfac, const std::unique_ptr< MultiFab > &z_phys_nd, const SolverChoice &solverChoice, std::unique_ptr< SurfaceLayer > &SurfLayer, const MultiFab *z_0, const bool &use_terrain_fitted_coords, const bool &use_moisture, int level, const BCRec *bc_ptr, bool vert_only)
Definition: ERF_ComputeTurbulentViscosity.cpp:591
@ tau12
Definition: ERF_DataStruct.H:30
@ tau23
Definition: ERF_DataStruct.H:30
@ tau33
Definition: ERF_DataStruct.H:30
@ tau22
Definition: ERF_DataStruct.H:30
@ tau11
Definition: ERF_DataStruct.H:30
@ tau32
Definition: ERF_DataStruct.H:30
@ tau31
Definition: ERF_DataStruct.H:30
@ tau21
Definition: ERF_DataStruct.H:30
@ tau13
Definition: ERF_DataStruct.H:30
@ ubar
Definition: ERF_DataStruct.H:91
@ wbar
Definition: ERF_DataStruct.H:91
@ vbar
Definition: ERF_DataStruct.H:91
@ thetabar
Definition: ERF_DataStruct.H:91
@ nvars_sponge
Definition: ERF_DataStruct.H:96
@ vbar_sponge
Definition: ERF_DataStruct.H:96
@ ubar_sponge
Definition: ERF_DataStruct.H:96
@ v_x
Definition: ERF_DataStruct.H:22
@ u_y
Definition: ERF_DataStruct.H:23
@ v_y
Definition: ERF_DataStruct.H:23
@ m_y
Definition: ERF_DataStruct.H:23
@ u_x
Definition: ERF_DataStruct.H:22
@ m_x
Definition: ERF_DataStruct.H:22
amrex::Real Real
Definition: ERF_ShocInterface.H:19
auto no_substep_fun
Definition: ERF_TI_no_substep_fun.H:4
auto slow_rhs_fun_post
Definition: ERF_TI_slow_rhs_post.H:3
auto slow_rhs_fun_pre
Definition: ERF_TI_slow_rhs_pre.H:6
auto fast_rhs_fun
Definition: ERF_TI_substep_fun.H:6
auto apply_bcs
Definition: ERF_TI_utils.H:71
auto cons_to_prim
Definition: ERF_TI_utils.H:4
amrex::Vector< std::unique_ptr< amrex::MultiFab > > walldist
Definition: ERF.H:911
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > mapfac
Definition: ERF.H:914
amrex::Vector< std::unique_ptr< MRISplitIntegrator< amrex::Vector< amrex::MultiFab > > > > mri_integrator_mem
Definition: ERF.H:791
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhotheta_src
Definition: ERF.H:1212
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx3_lev
Definition: ERF.H:880
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx3_lev
Definition: ERF.H:878
amrex::Vector< amrex::Vector< amrex::Real > > h_w_subsid
Definition: ERF.H:1217
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx1_lev
Definition: ERF.H:878
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc
Definition: ERF.H:890
amrex::Vector< std::unique_ptr< amrex::MultiFab > > eddyDiffs_lev
Definition: ERF.H:869
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_sponge_ptrs
Definition: ERF.H:1245
amrex::Vector< amrex::Vector< amrex::Real > > h_rhoqt_src
Definition: ERF.H:1214
amrex::Vector< long > dt_mri_ratio
Definition: ERF.H:776
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > Tau
Definition: ERF.H:868
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q2fx3_lev
Definition: ERF.H:881
static int verbose
Definition: ERF.H:1135
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx2_lev
Definition: ERF.H:880
std::unique_ptr< ProblemBase > prob
Definition: ERF.H:763
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > stretched_dz_d
Definition: ERF.H:917
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_diss_lev
Definition: ERF.H:879
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_v_geos
Definition: ERF.H:1224
amrex::Vector< amrex::Vector< amrex::Real > > h_v_geos
Definition: ERF.H:1223
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_rhoqt_src
Definition: ERF.H:1215
amrex::Vector< amrex::Vector< amrex::Real > > h_rhotheta_src
Definition: ERF.H:1211
amrex::Vector< amrex::Vector< amrex::Real > > h_u_geos
Definition: ERF.H:1220
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SmnSmn_lev
Definition: ERF.H:870
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_u_geos
Definition: ERF.H:1221
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > d_w_subsid
Definition: ERF.H:1218
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_hfx2_lev
Definition: ERF.H:878
static int fixed_mri_dt_ratio
Definition: ERF.H:1010
amrex::Vector< amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > > d_rayleigh_ptrs
Definition: ERF.H:1242
amrex::Vector< std::unique_ptr< amrex::MultiFab > > SFS_q1fx1_lev
Definition: ERF.H:880
Definition: ERF_MRI.H:16
void set_slow_rhs_pre(std::function< void(T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const int)> F)
Definition: ERF_MRI.H:133
void set_no_substep(std::function< void(T &, T &, T &, amrex::Real, amrex::Real, int)> F)
Definition: ERF_MRI.H:159
void set_fast_rhs(std::function< void(int, int, int, T &, const T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const amrex::Real)> F)
Definition: ERF_MRI.H:142
void set_slow_fast_timestep_ratio(const int timestep_ratio=1)
Definition: ERF_MRI.H:149
amrex::Real advance(T &S_old, T &S_new, amrex::Real time, const amrex::Real time_step)
Definition: ERF_MRI.H:169
void set_slow_rhs_post(std::function< void(T &, T &, T &, T &, T &, const amrex::Real, const amrex::Real, const amrex::Real, const int)> F)
Definition: ERF_MRI.H:137
@ pi0_comp
Definition: ERF_IndexDefines.H:65
@ p0_comp
Definition: ERF_IndexDefines.H:64
@ r0_comp
Definition: ERF_IndexDefines.H:63
@ qt
Definition: ERF_Kessler.H:27
real(c_double), parameter p0
Definition: ERF_module_model_constants.F90:40
Definition: ERF_DiffStruct.H:19
MolecDiffType molec_diff_type
Definition: ERF_DiffStruct.H:84
bool rayleigh_damp_T
Definition: ERF_DataStruct.H:850
static MeshType mesh_type
Definition: ERF_DataStruct.H:815
bool rayleigh_damp_V
Definition: ERF_DataStruct.H:848
DiffChoice diffChoice
Definition: ERF_DataStruct.H:824
bool custom_rhotheta_forcing
Definition: ERF_DataStruct.H:881
bool custom_w_subsidence
Definition: ERF_DataStruct.H:883
bool rayleigh_damp_U
Definition: ERF_DataStruct.H:847
bool custom_geostrophic_profile
Definition: ERF_DataStruct.H:884
amrex::Vector< SubsteppingType > substepping_type
Definition: ERF_DataStruct.H:830
bool use_num_diff
Definition: ERF_DataStruct.H:904
bool custom_moisture_forcing
Definition: ERF_DataStruct.H:882
amrex::Vector< TurbChoice > turbChoice
Definition: ERF_DataStruct.H:826
bool rayleigh_damp_W
Definition: ERF_DataStruct.H:849
SpongeChoice spongeChoice
Definition: ERF_DataStruct.H:825
Definition: ERF_SpongeStruct.H:15
std::string sponge_type
Definition: ERF_SpongeStruct.H:58
Definition: ERF_TurbStruct.H:41
bool use_kturb
Definition: ERF_TurbStruct.H:381
Here is the call graph for this function:

◆ advance_lsm()

void ERF::advance_lsm ( int  lev,
amrex::MultiFab &  cons_in,
amrex::MultiFab &  xvel_in,
amrex::MultiFab &  yvel_in,
const amrex::Real dt_advance 
)
10 {
11  if (solverChoice.lsm_type != LandSurfaceType::None) {
12  if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
13  lsm.Advance(lev, cons_in, xvel_in, yvel_in, SFS_hfx3_lev[lev].get(), SFS_q1fx3_lev[lev].get(), dt_advance, istep[lev]);
14  } else {
15  lsm.Advance(lev, dt_advance);
16  }
17  }
18 }
LandSurface lsm
Definition: ERF.H:835
amrex::Vector< int > istep
Definition: ERF.H:769
void Advance(const int &lev, amrex::MultiFab &cons_in, amrex::MultiFab &xvel_in, amrex::MultiFab &yvel_in, amrex::MultiFab *hfx3_out, amrex::MultiFab *qfx3_out, const amrex::Real &dt_advance, const int &nstep)
Definition: ERF_LandSurface.H:52
LandSurfaceType lsm_type
Definition: ERF_DataStruct.H:914

◆ advance_microphysics()

void ERF::advance_microphysics ( int  lev,
amrex::MultiFab &  cons_in,
const amrex::Real dt_advance,
const int &  iteration,
const amrex::Real time 
)
10 {
11  if (solverChoice.moisture_type != MoistureType::None) {
12  micro->Update_Micro_Vars_Lev(lev, cons);
13  micro->Advance(lev, dt_advance, iteration, time, solverChoice, vars_new, z_phys_nd, phys_bc_type);
14  micro->Update_State_Vars_Lev(lev, cons);
15  }
16 }
std::unique_ptr< Microphysics > micro
Definition: ERF.H:819

◆ advance_radiation()

void ERF::advance_radiation ( int  lev,
amrex::MultiFab &  cons_in,
const amrex::Real dt_advance 
)
8 {
9  if (solverChoice.rad_type != RadiationType::None) {
10 #ifdef ERF_USE_NETCDF
11  MultiFab *lat_ptr = lat_m[lev].get();
12  MultiFab *lon_ptr = lon_m[lev].get();
13 #else
14  MultiFab *lat_ptr = nullptr;
15  MultiFab *lon_ptr = nullptr;
16 #endif
17  // RRTMGP inputs names and pointers
18  Vector<std::string> lsm_input_names = rad[lev]->get_lsm_input_varnames();
19  Vector<MultiFab*> lsm_input_ptrs(lsm_input_names.size(),nullptr);
20  for (int i(0); i<lsm_input_ptrs.size(); ++i) {
21  int varIdx = lsm.Get_DataIdx(lev,lsm_input_names[i]);
22  lsm_input_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
23  }
24 
25  // RRTMGP output names and pointers
26  Vector<std::string> lsm_output_names = rad[lev]->get_lsm_output_varnames();
27  Vector<MultiFab*> lsm_output_ptrs(lsm_output_names.size(),nullptr);
28  for (int i(0); i<lsm_output_ptrs.size(); ++i) {
29  int varIdx = lsm.Get_DataIdx(lev,lsm_output_names[i]);
30  lsm_output_ptrs[i] = lsm.Get_Data_Ptr(lev,varIdx);
31  }
32 
33  // Enter radiation class driver
34  amrex::Real time_for_rad = t_new[lev] + start_time;
35  rad[lev]->Run(lev, istep[lev], time_for_rad, dt_advance,
36  cons.boxArray(), geom[lev], &(cons),
37  sw_lw_fluxes[lev].get(), solar_zenith[lev].get(),
38  lsm_input_ptrs, lsm_output_ptrs,
39  qheating_rates[lev].get(), z_phys_nd[lev].get() ,
40  lat_ptr, lon_ptr);
41  }
42 }
static amrex::Real start_time
Definition: ERF.H:990
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sw_lw_fluxes
Definition: ERF.H:853
amrex::Vector< std::unique_ptr< IRadiation > > rad
Definition: ERF.H:841
amrex::Vector< amrex::Real > t_new
Definition: ERF.H:773
amrex::Vector< std::unique_ptr< amrex::MultiFab > > solar_zenith
Definition: ERF.H:854
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lon_m
Definition: ERF.H:726
amrex::Vector< std::unique_ptr< amrex::MultiFab > > lat_m
Definition: ERF.H:726
amrex::Vector< std::unique_ptr< amrex::MultiFab > > qheating_rates
Definition: ERF.H:842
int Get_DataIdx(const int &lev, std::string &varname)
Definition: ERF_LandSurface.H:101
amrex::MultiFab * Get_Data_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:83
RadiationType rad_type
Definition: ERF_DataStruct.H:915

◆ appendPlotVariables()

void ERF::appendPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
185 {
186  ParmParse pp(pp_prefix);
187 
188  Vector<std::string> plot_var_names(0);
189  if (pp.contains(pp_plot_var_names.c_str())) {
190  std::string nm;
191  int nPltVars = pp.countval(pp_plot_var_names.c_str());
192  for (int i = 0; i < nPltVars; i++) {
193  pp.get(pp_plot_var_names.c_str(), nm, i);
194  // Add the named variable to our list of plot variables
195  // if it is not already in the list
196  if (!containerHasElement(plot_var_names, nm)) {
197  plot_var_names.push_back(nm);
198  }
199  }
200  }
201 
202  Vector<std::string> tmp_plot_names(0);
203 #ifdef ERF_USE_PARTICLES
204  Vector<std::string> particle_mesh_plot_names;
205  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
206  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
207  std::string tmp(particle_mesh_plot_names[i]);
208  if (containerHasElement(plot_var_names, tmp) ) {
209  tmp_plot_names.push_back(tmp);
210  }
211  }
212 #endif
213 
214  {
215  Vector<std::string> microphysics_plot_names;
216  micro->GetPlotVarNames(microphysics_plot_names);
217  if (microphysics_plot_names.size() > 0) {
218  static bool first_call = true;
219  if (first_call) {
220  Print() << getEnumNameString(solverChoice.moisture_type)
221  << ": the following additional variables are available to plot:\n";
222  for (int i = 0; i < microphysics_plot_names.size(); i++) {
223  Print() << " " << microphysics_plot_names[i] << "\n";
224  }
225  first_call = false;
226  }
227  for (auto& plot_name : microphysics_plot_names) {
228  if (containerHasElement(plot_var_names, plot_name)) {
229  tmp_plot_names.push_back(plot_name);
230  }
231  }
232  }
233  }
234 
235  for (int i = 0; i < tmp_plot_names.size(); i++) {
236  a_plot_var_names.push_back( tmp_plot_names[i] );
237  }
238 
239  // Finally, check to see if we found all the requested variables
240  for (const auto& plot_name : plot_var_names) {
241  if (!containerHasElement(a_plot_var_names, plot_name)) {
242  if (amrex::ParallelDescriptor::IOProcessor()) {
243  Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
244  }
245  }
246  }
247 }
bool containerHasElement(const V &iterable, const T &query)
Definition: ERF_Container.H:5
std::string pp_prefix
Definition: ERF.H:499
Here is the call graph for this function:

◆ AverageDown()

void ERF::AverageDown ( )
private
17 {
18  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
19 
20  int src_comp, num_comp;
21  for (int lev = finest_level-1; lev >= 0; --lev)
22  {
23  // If anelastic we don't average down rho because rho == rho0.
24  if (solverChoice.anelastic[lev]) {
25  src_comp = 1;
26  } else {
27  src_comp = 0;
28  }
29  num_comp = vars_new[0][Vars::cons].nComp() - src_comp;
30  AverageDownTo(lev,src_comp,num_comp);
31  }
32 }
void AverageDownTo(int crse_lev, int scomp, int ncomp)
Definition: ERF_AverageDown.cpp:36
CouplingType coupling_type
Definition: ERF_DataStruct.H:910

◆ AverageDownTo()

void ERF::AverageDownTo ( int  crse_lev,
int  scomp,
int  ncomp 
)
37 {
38  if (solverChoice.anelastic[crse_lev]) {
39  AMREX_ALWAYS_ASSERT(scomp == 1);
40  } else {
41  AMREX_ALWAYS_ASSERT(scomp == 0);
42  }
43 
44  AMREX_ALWAYS_ASSERT(ncomp == vars_new[crse_lev][Vars::cons].nComp() - scomp);
45  AMREX_ALWAYS_ASSERT(solverChoice.coupling_type == CouplingType::TwoWay);
46 
47  // ******************************************************************************************
48  // First do cell-centered quantities
49  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
50  // m is the map scale factor at cell centers
51  // Here we multiply (rho S) by detJ and divide (rho S) by m^2 before average down
52  // ******************************************************************************************
53  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
54  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
55  const Box& bx = mfi.tilebox();
56  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
57  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
58  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
59  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
60  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
61  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
62  {
63  cons_arr(i,j,k,scomp+n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
64  });
65  } else {
66  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
67  {
68  cons_arr(i,j,k,scomp+n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
69  });
70  }
71  } // mfi
72  } // lev
73 
74  int fine_lev = crse_lev+1;
75 
76  if (interpolation_type == StateInterpType::Perturbational) {
77  // Make the fine rho and (rho theta) be perturbational
78  MultiFab::Divide(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
79  Rho_comp,RhoTheta_comp,1,IntVect{0});
80  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
81  BaseState::r0_comp,Rho_comp,1,IntVect{0});
82  MultiFab::Subtract(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
83  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
84 
85  // Make the crse rho and (rho theta) be perturbational
86  MultiFab::Divide(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
87  Rho_comp,RhoTheta_comp,1,IntVect{0});
88  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
89  BaseState::r0_comp,Rho_comp,1,IntVect{0});
90  MultiFab::Subtract(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
91  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
92  }
93 
94  if (SolverChoice::terrain_type != TerrainType::EB) {
95  average_down(vars_new[crse_lev+1][Vars::cons],vars_new[crse_lev ][Vars::cons],
96  scomp, ncomp, refRatio(crse_lev));
97  } else {
98  const auto dx = geom[fine_lev].CellSize();
99  const Real cell_vol = dx[0]*dx[1]*dx[2];
100  const BoxArray& ba = vars_new[fine_lev][IntVars::cons].boxArray();
101  const DistributionMapping& dm = vars_new[fine_lev][IntVars::cons].DistributionMap();
102  MultiFab vol_fine(ba, dm, 1, 0);
103  vol_fine.setVal(cell_vol);
104  EB_average_down(vars_new[fine_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
105  *detJ_cc[fine_lev], vol_fine,
106  scomp, ncomp, refRatio(crse_lev));
107  }
108 
109  if (interpolation_type == StateInterpType::Perturbational) {
110  // Restore the fine data to what it was
111  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
112  BaseState::r0_comp,Rho_comp,1,IntVect{0});
113  MultiFab::Add(vars_new[fine_lev][Vars::cons],base_state[fine_lev],
114  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
115  MultiFab::Multiply(vars_new[fine_lev][Vars::cons],vars_new[fine_lev][Vars::cons],
116  Rho_comp,RhoTheta_comp,1,IntVect{0});
117 
118  // Make the crse data be full values not perturbational
119  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
120  BaseState::r0_comp,Rho_comp,1,IntVect{0});
121  MultiFab::Add(vars_new[crse_lev][Vars::cons],base_state[crse_lev],
122  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
123  MultiFab::Multiply(vars_new[crse_lev][Vars::cons],vars_new[crse_lev][Vars::cons],
124  Rho_comp,RhoTheta_comp,1,IntVect{0});
125  }
126 
127  vars_new[crse_lev][Vars::cons].FillBoundary(geom[crse_lev].periodicity());
128 
129  // ******************************************************************************************
130  // Here we multiply (rho S) by m^2 and divide by detJ after average down
131  // ******************************************************************************************
132  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
133  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
134  const Box& bx = mfi.tilebox();
135  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
136  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
137  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
138  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
139  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
140  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
141  {
142  cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
143  });
144  } else { // MeshType::ConstantDz
145  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
146  {
147  cons_arr(i,j,k,scomp+n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
148  });
149  }
150  } // mfi
151  } // lev
152 
153  // Fill EB covered cells by old values
154  // (This won't be needed because EB_average_down copyies the covered value.)
155  if (SolverChoice::terrain_type == TerrainType::EB) {
156  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
157  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
158  const Box& bx = mfi.tilebox();
159  const Array4< Real> cons_new = vars_new[lev][Vars::cons].array(mfi);
160  const Array4<const Real> cons_old = vars_old[lev][Vars::cons].array(mfi);
161  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
162  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
163  {
164  if (detJ_arr(i,j,k) == 0.0) {
165  cons_new(i,j,k,scomp+n) = cons_old(i,j,k,scomp+n);
166  }
167  });
168  } // mfi
169  } // lev
170  }
171 
172  // ******************************************************************************************
173  // Now average down momenta.
174  // Note that vars_new holds velocities not momenta, but we want to do conservative
175  // averaging so we first convert to momentum, then average down, then convert
176  // back to velocities -- only on the valid region
177  // ******************************************************************************************
178  for (int lev = crse_lev; lev <= crse_lev+1; lev++)
179  {
180  // FillBoundary for density so we can go back and forth between velocity and momentum
181  vars_new[lev][Vars::cons].FillBoundary(geom[lev].periodicity());
182 
183  if (SolverChoice::terrain_type != TerrainType::EB) {
184  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
185  vars_new[lev][Vars::yvel], IntVect(0,0,0),
186  vars_new[lev][Vars::zvel], IntVect(0,0,0),
187  vars_new[lev][Vars::cons],
188  rU_new[lev],
189  rV_new[lev],
190  rW_new[lev],
191  Geom(lev).Domain(),
193  } else {
194  const MultiFab& c_vfrac = (get_eb(lev).get_const_factory())->getVolFrac();
195 
196  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect(0,0,0),
197  vars_new[lev][Vars::yvel], IntVect(0,0,0),
198  vars_new[lev][Vars::zvel], IntVect(0,0,0),
199  vars_new[lev][Vars::cons],
200  rU_new[lev],
201  rV_new[lev],
202  rW_new[lev],
203  Geom(lev).Domain(),
205  &c_vfrac);
206  }
207  }
208 
209  if (SolverChoice::terrain_type != TerrainType::EB) {
210  average_down_faces(rU_new[crse_lev+1], rU_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
211  average_down_faces(rV_new[crse_lev+1], rV_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
212  average_down_faces(rW_new[crse_lev+1], rW_new[crse_lev], refRatio(crse_lev), geom[crse_lev]);
213  } else {
214  EB_average_down_faces({&rU_new[crse_lev+1], &rV_new[crse_lev+1], &rW_new[crse_lev+1]},
215  {&rU_new[crse_lev], &rV_new[crse_lev], &rW_new[crse_lev]},
216  refRatio(crse_lev), 0);
217  }
218 
219  for (int lev = crse_lev; lev <= crse_lev+1; lev++) {
220  if (SolverChoice::terrain_type != TerrainType::EB) {
222  vars_new[lev][Vars::yvel],
223  vars_new[lev][Vars::zvel],
224  vars_new[lev][Vars::cons],
225  rU_new[lev],
226  rV_new[lev],
227  rW_new[lev],
228  Geom(lev).Domain(),
230  } else {
231  const MultiFab& c_vfrac = (get_eb(lev).get_const_factory())->getVolFrac();
232 
234  vars_new[lev][Vars::yvel],
235  vars_new[lev][Vars::zvel],
236  vars_new[lev][Vars::cons],
237  rU_new[lev],
238  rV_new[lev],
239  rW_new[lev],
240  Geom(lev).Domain(),
242  &c_vfrac);
243  }
244  }
245 }
void MomentumToVelocity(MultiFab &xvel, MultiFab &yvel, MultiFab &zvel, const MultiFab &density, const MultiFab &xmom_in, const MultiFab &ymom_in, const MultiFab &zmom_in, const Box &domain, const Vector< BCRec > &domain_bcs_type_h, const MultiFab *c_vfrac)
Definition: ERF_MomentumToVelocity.cpp:25
eb_ const & get_eb(int lev) const noexcept
Definition: ERF.H:1553
static StateInterpType interpolation_type
Definition: ERF.H:1150
const std::unique_ptr< amrex::EBFArrayBoxFactory > & get_const_factory() const noexcept
Definition: ERF_EB.H:46
@ th0_comp
Definition: ERF_IndexDefines.H:66
static TerrainType terrain_type
Definition: ERF_DataStruct.H:806
Here is the call graph for this function:

◆ build_fine_mask()

MultiFab & ERF::build_fine_mask ( int  level)

Helper function for constructing a fine mask, that is, a MultiFab masking coarser data at a lower level by zeroing out covered cells in the fine mask MultiFab we compute.

Parameters
levelFine level index which masks underlying coarser data
711 {
712  // Mask for zeroing covered cells
713  AMREX_ASSERT(level > 0);
714 
715  const BoxArray& cba = grids[level-1];
716  const DistributionMapping& cdm = dmap[level-1];
717 
718  // TODO -- we should make a vector of these a member of ERF class
719  fine_mask.define(cba, cdm, 1, 0, MFInfo());
720  fine_mask.setVal(1.0);
721 
722  BoxArray fba = grids[level];
723  iMultiFab ifine_mask = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
724 
725  const auto fma = fine_mask.arrays();
726  const auto ifma = ifine_mask.arrays();
727  ParallelFor(fine_mask, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
728  {
729  fma[bno](i,j,k) = ifma[bno](i,j,k);
730  });
731 
732  return fine_mask;
733 }
amrex::MultiFab fine_mask
Definition: ERF.H:1277

◆ ClearLevel()

void ERF::ClearLevel ( int  lev)
override
678 {
679  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
680  vars_new[lev][var_idx].clear();
681  vars_old[lev][var_idx].clear();
682  }
683 
684  base_state[lev].clear();
685 
686  rU_new[lev].clear();
687  rU_old[lev].clear();
688  rV_new[lev].clear();
689  rV_old[lev].clear();
690  rW_new[lev].clear();
691  rW_old[lev].clear();
692 
693  if (lev > 0) {
694  zmom_crse_rhs[lev].clear();
695  }
696 
698  pp_inc[lev].clear();
699  }
700 
701  // Clears the integrator memory
702  mri_integrator_mem[lev].reset();
703 
704  // Clears the physical boundary condition routines
705  physbcs_cons[lev].reset();
706  physbcs_u[lev].reset();
707  physbcs_v[lev].reset();
708  physbcs_w[lev].reset();
709  physbcs_base[lev].reset();
710 
711  // Clears the flux register array
712  advflux_reg[lev]->reset();
713 }
amrex::Vector< amrex::MultiFab > pp_inc
Definition: ERF.H:793
amrex::Vector< amrex::YAFluxRegister * > advflux_reg
Definition: ERF.H:930
amrex::Vector< std::unique_ptr< ERFPhysBCFunct_base > > physbcs_base
Definition: ERF.H:800
@ NumTypes
Definition: ERF_IndexDefines.H:144
bool project_initial_velocity
Definition: ERF_DataStruct.H:873

◆ cloud_fraction()

Real ERF::cloud_fraction ( amrex::Real  time)
452 {
453  BL_PROFILE("ERF::cloud_fraction()");
454 
455  int lev = 0;
456  // This holds all of qc
457  MultiFab qc(vars_new[lev][Vars::cons],make_alias,RhoQ2_comp,1);
458 
459  int direction = 2; // z-direction
460  Box const& domain = geom[lev].Domain();
461 
462  auto const& qc_arr = qc.const_arrays();
463 
464  // qc_2d is an BaseFab<int> holding the max value over the column
465  auto qc_2d = ReduceToPlane<ReduceOpMax,int>(direction, domain, qc,
466  [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) -> int
467  {
468  if (qc_arr[box_no](i,j,k) > 0) {
469  return 1;
470  } else {
471  return 0;
472  }
473  });
474 
475  auto* p = qc_2d.dataPtr();
476 
477  Long numpts = qc_2d.numPts();
478 
479  AMREX_ASSERT(numpts < Long(std::numeric_limits<int>::max));
480 
481 #if 1
482  if (ParallelDescriptor::UseGpuAwareMpi()) {
483  ParallelDescriptor::ReduceIntMax(p,static_cast<int>(numpts));
484  } else {
485  Gpu::PinnedVector<int> hv(numpts);
486  Gpu::copyAsync(Gpu::deviceToHost, p, p+numpts, hv.data());
487  Gpu::streamSynchronize();
488  ParallelDescriptor::ReduceIntMax(hv.data(),static_cast<int>(numpts));
489  Gpu::copyAsync(Gpu::hostToDevice, hv.data(), hv.data()+numpts, p);
490  }
491 
492  // Sum over component 0
493  Long num_cloudy = qc_2d.template sum<RunOn::Device>(0);
494 
495 #else
496  //
497  // We need this if we allow domain decomposition in the vertical
498  // but for now we leave it commented out
499  //
500  Long num_cloudy = Reduce::Sum<Long>(numpts,
501  [=] AMREX_GPU_DEVICE (Long i) -> Long {
502  if (p[i] == 1) {
503  return 1;
504  } else {
505  return 0;
506  }
507  });
508  ParallelDescriptor::ReduceLongSum(num_cloudy);
509 #endif
510 
511  Real num_total = qc_2d.box().d_numPts();
512 
513  Real cloud_frac = num_cloudy / num_total;
514 
515  return cloud_frac;
516 }
#define RhoQ2_comp
Definition: ERF_IndexDefines.H:43
@ qc
Definition: ERF_SatAdj.H:36

◆ compute_divergence()

void ERF::compute_divergence ( int  lev,
amrex::MultiFab &  rhs,
amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM >  rho0_u_const,
amrex::Geometry const &  geom_at_lev 
)

Project the single-level velocity field to enforce incompressibility Note that the level may or may not be level 0.

11 {
12  BL_PROFILE("ERF::compute_divergence()");
13 
14  auto dxInv = geom_at_lev.InvCellSizeArray();
15 
16  // ****************************************************************************
17  // Compute divergence which will form RHS
18  // Note that we replace "rho0w" with the contravariant momentum, Omega
19  // ****************************************************************************
20  if (solverChoice.terrain_type == TerrainType::EB)
21  {
22  bool already_on_centroids = true;
23  EB_computeDivergence(rhs, rho0_u_const, geom_at_lev, already_on_centroids);
24  }
25  else if (SolverChoice::mesh_type == MeshType::ConstantDz)
26  {
27  computeDivergence(rhs, rho0_u_const, geom_at_lev);
28  }
29  else
30  {
31  for ( MFIter mfi(rhs,TilingIfNotGPU()); mfi.isValid(); ++mfi)
32  {
33  Box bx = mfi.tilebox();
34  const Array4<Real const>& rho0u_arr = rho0_u_const[0]->const_array(mfi);
35  const Array4<Real const>& rho0v_arr = rho0_u_const[1]->const_array(mfi);
36  const Array4<Real const>& rho0w_arr = rho0_u_const[2]->const_array(mfi);
37  const Array4<Real >& rhs_arr = rhs.array(mfi);
38 
39  const Array4<Real const>& mf_mx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
40  const Array4<Real const>& mf_my = mapfac[lev][MapFacType::m_y]->const_array(mfi);
41 
42  if (SolverChoice::mesh_type == MeshType::StretchedDz) {
43  Real* stretched_dz_d_ptr = stretched_dz_d[lev].data();
44  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
45  Real dz = stretched_dz_d_ptr[k];
46  Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
47  rhs_arr(i,j,k) = mfsq * ( (rho0u_arr(i+1,j ,k ) - rho0u_arr(i,j,k)) * dxInv[0]
48  +(rho0v_arr(i ,j+1,k ) - rho0v_arr(i,j,k)) * dxInv[1]
49  +(rho0w_arr(i ,j ,k+1) - rho0w_arr(i,j,k)) / dz );
50  });
51 
52  } else {
53 
54  //
55  // Note we compute the divergence using "rho0w" == Omega
56  //
57  const Array4<Real const>& ax_arr = ax[lev]->const_array(mfi);
58  const Array4<Real const>& ay_arr = ay[lev]->const_array(mfi);
59  const Array4<Real const>& dJ_arr = detJ_cc[lev]->const_array(mfi);
60  //
61  // az == 1 for terrain-fitted coordinates
62  //
63  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
64  {
65  Real mfsq = mf_mx(i,j,0) * mf_my(i,j,0);
66  rhs_arr(i,j,k) = mfsq * ( (ax_arr(i+1,j,k)*rho0u_arr(i+1,j,k) - ax_arr(i,j,k)*rho0u_arr(i,j,k)) * dxInv[0]
67  +(ay_arr(i,j+1,k)*rho0v_arr(i,j+1,k) - ay_arr(i,j,k)*rho0v_arr(i,j,k)) * dxInv[1]
68  +( rho0w_arr(i,j,k+1) - rho0w_arr(i,j,k)) * dxInv[2] ) / dJ_arr(i,j,k);
69  });
70  }
71  } // mfi
72  }
73 }
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax
Definition: ERF.H:891
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay
Definition: ERF.H:892

◆ ComputeDt()

void ERF::ComputeDt ( int  step = -1)
private

Function that calls estTimeStep for each level

12 {
13  Vector<Real> dt_tmp(finest_level+1);
14 
15  for (int lev = 0; lev <= finest_level; ++lev)
16  {
17  dt_tmp[lev] = estTimeStep(lev, dt_mri_ratio[lev]);
18  }
19 
20  ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
21 
22  Real dt_0 = dt_tmp[0];
23  int n_factor = 1;
24  for (int lev = 0; lev <= finest_level; ++lev) {
25  dt_tmp[lev] = amrex::min(dt_tmp[lev], change_max*dt[lev]);
26  n_factor *= nsubsteps[lev];
27  dt_0 = amrex::min(dt_0, n_factor*dt_tmp[lev]);
28  if (step == 0){
29  dt_0 *= init_shrink;
30  if (verbose && init_shrink != 1.0) {
31  Print() << "Timestep 0: shrink initial dt at level " << lev << " by " << init_shrink << std::endl;
32  }
33  }
34  }
35  // Limit dt's by the value of stop_time.
36  const Real eps = 1.e-3*dt_0;
37  if (t_new[0] + dt_0 > stop_time - eps) {
38  dt_0 = stop_time - t_new[0];
39  }
40 
41  dt[0] = dt_0;
42  for (int lev = 1; lev <= finest_level; ++lev) {
43  dt[lev] = dt[lev-1] / nsubsteps[lev];
44  }
45 }
amrex::Real estTimeStep(int lev, long &dt_fast_ratio) const
Definition: ERF_ComputeTimestep.cpp:54
static amrex::Real stop_time
Definition: ERF.H:991
amrex::Vector< int > nsubsteps
Definition: ERF.H:770
static amrex::Real init_shrink
Definition: ERF.H:1002
static amrex::Real change_max
Definition: ERF.H:1003

◆ ComputeGhostCells()

static AMREX_FORCE_INLINE int ERF::ComputeGhostCells ( const SolverChoice sc)
inlinestaticprivate
1284  {
1285  int ngrow = 0;
1286 
1287  if (sc.use_num_diff)
1288  {
1289  ngrow = 3;
1290  } else {
1291  if (
1298  { ngrow = 3; }
1299  else if (
1306  { ngrow = 3; }
1307  else if (
1316  { ngrow = 3; }
1317  else if (
1326  { ngrow = 4; }
1327  else
1328  {
1329  if (sc.terrain_type == TerrainType::EB){
1330  ngrow = 3;
1331  } else {
1332  ngrow = 2;
1333  }
1334  }
1335  }
1336 
1337  return ngrow;
1338  }
@ Centered_6th
AdvType moistscal_horiz_adv_type
Definition: ERF_AdvStruct.H:399
AdvType dycore_vert_adv_type
Definition: ERF_AdvStruct.H:396
AdvType moistscal_vert_adv_type
Definition: ERF_AdvStruct.H:400
AdvType dryscal_horiz_adv_type
Definition: ERF_AdvStruct.H:397
AdvType dycore_horiz_adv_type
Definition: ERF_AdvStruct.H:395
AdvType dryscal_vert_adv_type
Definition: ERF_AdvStruct.H:398
AdvChoice advChoice
Definition: ERF_DataStruct.H:823

◆ Construct_ERFFillPatchers()

void ERF::Construct_ERFFillPatchers ( int  lev)
private
2441 {
2442  auto& fine_new = vars_new[lev];
2443  auto& crse_new = vars_new[lev-1];
2444  auto& ba_fine = fine_new[Vars::cons].boxArray();
2445  auto& ba_crse = crse_new[Vars::cons].boxArray();
2446  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2447  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2448 
2449  int ncomp = vars_new[lev][Vars::cons].nComp();
2450 
2451  FPr_c.emplace_back(ba_fine, dm_fine, geom[lev] ,
2452  ba_crse, dm_crse, geom[lev-1],
2453  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2454  FPr_u.emplace_back(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2455  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2456  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2457  FPr_v.emplace_back(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2458  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2459  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2460  FPr_w.emplace_back(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2461  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2462  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2463 }
int cf_set_width
Definition: ERF.H:861

◆ CreateForecastStateMultiFabs()

void ERF::CreateForecastStateMultiFabs ( amrex::Vector< amrex::Vector< amrex::MultiFab >> &  forecast_state)
250 {
251 
252  forecast_state.resize(max_level+1);
253  for (int lev = 0; lev < max_level+1; ++lev) {
254  forecast_state[lev].resize(vars_new[lev].size()+1);
255  for (int comp = 0; comp < vars_new[lev].size(); ++comp) {
256  const MultiFab& src = vars_new[lev][comp];
257  forecast_state[lev][comp].define(src.boxArray(), src.DistributionMap(),
258  src.nComp(), src.nGrow());
259  }
260  int comp = vars_new[lev].size();
261  const MultiFab& src = vars_new[lev][0];
262  forecast_state[lev][comp].define(src.boxArray(), src.DistributionMap(),
263  2, src.nGrow());
264  }
265 }

◆ CreateWeatherDataGeomBoxArrayDistMap()

void ERF::CreateWeatherDataGeomBoxArrayDistMap ( const std::string &  filename,
amrex::Geometry &  geom_weather,
amrex::BoxArray &  nba,
amrex::DistributionMapping &  dm 
)
159 {
160  Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
161  Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
162 
163  ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
164  xvec_h, yvec_h, zvec_h, rho_h,
165  uvel_h, vvel_h, wvel_h,
166  theta_h, qv_h, qc_h, qr_h);
167 
168  const auto prob_lo_erf = geom[0].ProbLoArray();
169  const auto prob_hi_erf = geom[0].ProbHiArray();
170  const auto dx_erf = geom[0].CellSizeArray();
171 
172  if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
173  amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
174  }
175  if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
176  amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
177  }
178  if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
179  amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
180  }
181  if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
182  amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
183  }
184 
185  // Number of cells
186  int nx_cells = xvec_h.size()-1;
187  int ny_cells = yvec_h.size()-1;
188 
189  const amrex::Geometry& geom0 = geom[0]; // or whatever your Geometry vector is called
190  const amrex::Box& domainBox = geom0.Domain();
191  const amrex::IntVect& domainSize = domainBox.size(); // Number of cells in each direction
192  int nz_cells = domainSize[2];
193 
194  IntVect dom_lo(0, 0, 0);
195  IntVect dom_hi(nx_cells-1, ny_cells-1, nz_cells-1);
196  Box domain(dom_lo, dom_hi);
197 
198  const amrex::Real* prob_hi = geom0.ProbHi();
199 
200  // Define the extents of the physical domain box
201  RealBox real_box({xvec_h[0], yvec_h[0], zvec_h[0]}, {xvec_h[nx_cells], yvec_h[ny_cells], prob_hi[2]});
202 
203  int coord = 0; // Cartesian
204  Array<int, AMREX_SPACEDIM> is_periodic{0, 0, 0}; // non-periodic
205 
206  geom_weather.define(domain, real_box, coord, is_periodic);
207 
208  BoxArray ba(domain);
209  ba.maxSize(64);
210  nba = amrex::convert(ba, IntVect::TheNodeVector()); // nodal in all directions
211 
212  // Create DistributionMapping
213  dm = DistributionMapping(nba);
214  }
const auto & dom_hi
Definition: ERF_DiffSetup.H:10
const auto & dom_lo
Definition: ERF_DiffSetup.H:9
void ReadCustomBinaryIC(const std::string filename, amrex::Vector< amrex::Real > &latvec_h, amrex::Vector< amrex::Real > &lonvec_h, amrex::Vector< amrex::Real > &xvec_h, amrex::Vector< amrex::Real > &yvec_h, amrex::Vector< amrex::Real > &zvec_h, amrex::Vector< amrex::Real > &rho_h, amrex::Vector< amrex::Real > &uvel_h, amrex::Vector< amrex::Real > &vvel_h, amrex::Vector< amrex::Real > &wvel_h, amrex::Vector< amrex::Real > &theta_h, amrex::Vector< amrex::Real > &qv_h, amrex::Vector< amrex::Real > &qc_h, amrex::Vector< amrex::Real > &qr_h)
Definition: ERF_ReadCustomBinaryIC.H:12
Here is the call graph for this function:

◆ DataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DataLog ( int  i)
inlineprivate
1349  {
1350  return *datalog[i];
1351  }
amrex::Vector< std::unique_ptr< std::fstream > > datalog
Definition: ERF.H:1525

◆ DataLogName()

std::string ERF::DataLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith datalog file.

1541 { return datalogname[i]; }
amrex::Vector< std::string > datalogname
Definition: ERF.H:1528

◆ Define_ERFFillPatchers()

void ERF::Define_ERFFillPatchers ( int  lev)
private
2467 {
2468  auto& fine_new = vars_new[lev];
2469  auto& crse_new = vars_new[lev-1];
2470  auto& ba_fine = fine_new[Vars::cons].boxArray();
2471  auto& ba_crse = crse_new[Vars::cons].boxArray();
2472  auto& dm_fine = fine_new[Vars::cons].DistributionMap();
2473  auto& dm_crse = crse_new[Vars::cons].DistributionMap();
2474 
2475  int ncomp = fine_new[Vars::cons].nComp();
2476 
2477  FPr_c[lev-1].Define(ba_fine, dm_fine, geom[lev] ,
2478  ba_crse, dm_crse, geom[lev-1],
2479  -cf_width, -cf_set_width, ncomp, &cell_cons_interp);
2480  FPr_u[lev-1].Define(convert(ba_fine, IntVect(1,0,0)), dm_fine, geom[lev] ,
2481  convert(ba_crse, IntVect(1,0,0)), dm_crse, geom[lev-1],
2482  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2483  FPr_v[lev-1].Define(convert(ba_fine, IntVect(0,1,0)), dm_fine, geom[lev] ,
2484  convert(ba_crse, IntVect(0,1,0)), dm_crse, geom[lev-1],
2485  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2486  FPr_w[lev-1].Define(convert(ba_fine, IntVect(0,0,1)), dm_fine, geom[lev] ,
2487  convert(ba_crse, IntVect(0,0,1)), dm_crse, geom[lev-1],
2488  -cf_width, -cf_set_width, 1, &face_cons_linear_interp);
2489 }

◆ DerDataLog()

AMREX_FORCE_INLINE std::ostream& ERF::DerDataLog ( int  i)
inlineprivate
1356  {
1357  return *der_datalog[i];
1358  }
amrex::Vector< std::unique_ptr< std::fstream > > der_datalog
Definition: ERF.H:1526

◆ DerDataLogName()

std::string ERF::DerDataLogName ( int  i) const
inlineprivatenoexcept
1542 { return der_datalogname[i]; }
amrex::Vector< std::string > der_datalogname
Definition: ERF.H:1529

◆ derive_diag_profiles()

void ERF::derive_diag_profiles ( amrex::Real  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
205 {
206  // We assume that this is always called at level 0
207  int lev = 0;
208 
209  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
210  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
211  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, uw, vv, vw, ww, uth, vth, wth,
212  // 0 1 2 3 4 5 6 7 8 9 10 11 12 13
213  // thth, uiuiu, uiuiv, uiuiw, p, pu, pv, pw, qv, qc, qr, wqv, wqc, wqr,
214  // 14 15 16 17 18 19 20 21 22 23 24 25 26 27
215  // qi, qs, qg, wthv
216  // 28 29 30 31
217  MultiFab mf_out(grids[lev], dmap[lev], 32, 0);
218 
219  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
220 
221  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
222  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
223  MultiFab w_cc(mf_vels, make_alias, 2, 1); // w at cell centers
224 
225  average_face_to_cellcenter(mf_vels,0,
226  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
227 
228  int zdir = 2;
229  auto domain = geom[0].Domain();
230 
231  // Sum in the horizontal plane
232  h_avg_u = sumToLine(mf_vels ,0,1,domain,zdir);
233  h_avg_v = sumToLine(mf_vels ,1,1,domain,zdir);
234  h_avg_w = sumToLine(mf_vels ,2,1,domain,zdir);
235 
236  int hu_size = h_avg_u.size();
237 
238  // Divide by the total number of cells we are averaging over
239  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
240  for (int k = 0; k < hu_size; ++k) {
241  h_avg_u[k] /= area_z; h_avg_v[k] /= area_z; h_avg_w[k] /= area_z;
242  }
243 
244  Gpu::DeviceVector<Real> d_avg_u(hu_size, Real(0.0));
245  Gpu::DeviceVector<Real> d_avg_v(hu_size, Real(0.0));
246  Gpu::DeviceVector<Real> d_avg_w(hu_size, Real(0.0));
247 
248 #if 0
249  auto* avg_u_ptr = d_avg_u.data();
250  auto* avg_v_ptr = d_avg_v.data();
251  auto* avg_w_ptr = d_avg_w.data();
252 #endif
253 
254  Gpu::copy(Gpu::hostToDevice, h_avg_u.begin(), h_avg_u.end(), d_avg_u.begin());
255  Gpu::copy(Gpu::hostToDevice, h_avg_v.begin(), h_avg_v.end(), d_avg_v.begin());
256  Gpu::copy(Gpu::hostToDevice, h_avg_w.begin(), h_avg_w.end(), d_avg_w.begin());
257 
258  int nvars = vars_new[lev][Vars::cons].nComp();
259  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
260 
261  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
262 
263  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
264 
265  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
266  {
267  const Box& bx = mfi.tilebox();
268  const Array4<Real>& fab_arr = mf_out.array(mfi);
269  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
270  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
271  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
272  const Array4<Real>& cons_arr = mf_cons.array(mfi);
273  const Array4<Real>& p0_arr = p_hse.array(mfi);
274  const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
275  Array4<const Real>{};
276 
277  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
278  {
279  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
280  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
281  fab_arr(i, j, k, 1) = theta;
282  Real ksgs = 0.0;
283  if (l_use_KE) {
284  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
285  }
286  fab_arr(i, j, k, 2) = ksgs;
287 #if 1
288  if (l_use_kturb) {
289  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
290  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
291  } else {
292  fab_arr(i, j, k, 3) = 0.0;
293  fab_arr(i, j, k, 4) = 0.0;
294  }
295 #else
296  // Here we hijack the "Kturb" variable name to print out the resolved kinetic energy
297  Real upert = u_cc_arr(i,j,k) - avg_u_ptr[k];
298  Real vpert = v_cc_arr(i,j,k) - avg_v_ptr[k];
299  Real wpert = w_cc_arr(i,j,k) - avg_w_ptr[k];
300  fab_arr(i, j, k, 3) = 0.5 * (upert*upert + vpert*vpert + wpert*wpert);
301 #endif
302  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
303  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
304  fab_arr(i, j, k, 7) = u_cc_arr(i,j,k) * w_cc_arr(i,j,k); // u*w
305  fab_arr(i, j, k, 8) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
306  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * w_cc_arr(i,j,k); // v*w
307  fab_arr(i, j, k,10) = w_cc_arr(i,j,k) * w_cc_arr(i,j,k); // w*w
308  fab_arr(i, j, k,11) = u_cc_arr(i,j,k) * theta; // u*th
309  fab_arr(i, j, k,12) = v_cc_arr(i,j,k) * theta; // v*th
310  fab_arr(i, j, k,13) = w_cc_arr(i,j,k) * theta; // w*th
311  fab_arr(i, j, k,14) = theta * theta; // th*th
312 
313  // if the number of fields is changed above, then be sure to update
314  // the following def!
315  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,8) + fab_arr(i,j,k,10);
316  fab_arr(i, j, k,15) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
317  fab_arr(i, j, k,16) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
318  fab_arr(i, j, k,17) = uiui * w_cc_arr(i,j,k); // (ui*ui)*w
319 
320  if (!use_moisture) {
321  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
322  p -= p0_arr(i,j,k);
323  fab_arr(i, j, k,18) = p; // p
324  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
325  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
326  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
327  fab_arr(i, j, k,22) = 0.; // qv
328  fab_arr(i, j, k,23) = 0.; // qc
329  fab_arr(i, j, k,24) = 0.; // qr
330  fab_arr(i, j, k,25) = 0.; // w*qv
331  fab_arr(i, j, k,26) = 0.; // w*qc
332  fab_arr(i, j, k,27) = 0.; // w*qr
333  fab_arr(i, j, k,28) = 0.; // qi
334  fab_arr(i, j, k,29) = 0.; // qs
335  fab_arr(i, j, k,30) = 0.; // qg
336  fab_arr(i, j, k,31) = 0.; // w*thv
337  }
338  });
339  } // mfi
340 
341  if (use_moisture)
342  {
343  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
344 
345  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
346  {
347  const Box& bx = mfi.tilebox();
348  const Array4<Real>& fab_arr = mf_out.array(mfi);
349  const Array4<Real>& cons_arr = mf_cons.array(mfi);
350  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
351  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
352  const Array4<Real>& w_cc_arr = w_cc.array(mfi);
353  const Array4<Real>& p0_arr = p_hse.array(mfi);
354 
355  int rhoqr_comp = solverChoice.moisture_indices.qr;
356 
357  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
358  {
359  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
360  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
361  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
362  Real(0.0);
363  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
364 
365  p -= p0_arr(i,j,k);
366  fab_arr(i, j, k,18) = p; // p
367  fab_arr(i, j, k,19) = p * u_cc_arr(i,j,k); // p*u
368  fab_arr(i, j, k,20) = p * v_cc_arr(i,j,k); // p*v
369  fab_arr(i, j, k,21) = p * w_cc_arr(i,j,k); // p*w
370  fab_arr(i, j, k,22) = qv; // qv
371  fab_arr(i, j, k,23) = qc; // qc
372  fab_arr(i, j, k,24) = qr; // qr
373  fab_arr(i, j, k,25) = w_cc_arr(i,j,k) * qv; // w*qv
374  fab_arr(i, j, k,26) = w_cc_arr(i,j,k) * qc; // w*qc
375  fab_arr(i, j, k,27) = w_cc_arr(i,j,k) * qr; // w*qr
376  if (n_qstate_moist > 3) {
377  fab_arr(i, j, k,28) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
378  fab_arr(i, j, k,29) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
379  fab_arr(i, j, k,30) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
380  } else {
381  fab_arr(i, j, k,28) = 0.0; // qi
382  fab_arr(i, j, k,29) = 0.0; // qs
383  fab_arr(i, j, k,30) = 0.0; // qg
384  }
385  Real ql = qc + qr;
386  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
387  Real thv = theta * (1 + 0.61*qv - ql);
388  fab_arr(i, j, k,31) = w_cc_arr(i,j,k) * thv; // w*thv
389  });
390  } // mfi
391  } // use_moisture
392 
393  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
394  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
395  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
396  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
397  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
398  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
399  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
400  h_avg_uw = sumToLine(mf_out, 7,1,domain,zdir);
401  h_avg_vv = sumToLine(mf_out, 8,1,domain,zdir);
402  h_avg_vw = sumToLine(mf_out, 9,1,domain,zdir);
403  h_avg_ww = sumToLine(mf_out,10,1,domain,zdir);
404  h_avg_uth = sumToLine(mf_out,11,1,domain,zdir);
405  h_avg_vth = sumToLine(mf_out,12,1,domain,zdir);
406  h_avg_wth = sumToLine(mf_out,13,1,domain,zdir);
407  h_avg_thth = sumToLine(mf_out,14,1,domain,zdir);
408  h_avg_uiuiu = sumToLine(mf_out,15,1,domain,zdir);
409  h_avg_uiuiv = sumToLine(mf_out,16,1,domain,zdir);
410  h_avg_uiuiw = sumToLine(mf_out,17,1,domain,zdir);
411  h_avg_p = sumToLine(mf_out,18,1,domain,zdir);
412  h_avg_pu = sumToLine(mf_out,19,1,domain,zdir);
413  h_avg_pv = sumToLine(mf_out,20,1,domain,zdir);
414  h_avg_pw = sumToLine(mf_out,21,1,domain,zdir);
415  h_avg_qv = sumToLine(mf_out,22,1,domain,zdir);
416  h_avg_qc = sumToLine(mf_out,23,1,domain,zdir);
417  h_avg_qr = sumToLine(mf_out,24,1,domain,zdir);
418  h_avg_wqv = sumToLine(mf_out,25,1,domain,zdir);
419  h_avg_wqc = sumToLine(mf_out,26,1,domain,zdir);
420  h_avg_wqr = sumToLine(mf_out,27,1,domain,zdir);
421  h_avg_qi = sumToLine(mf_out,28,1,domain,zdir);
422  h_avg_qs = sumToLine(mf_out,29,1,domain,zdir);
423  h_avg_qg = sumToLine(mf_out,30,1,domain,zdir);
424  h_avg_wthv = sumToLine(mf_out,31,1,domain,zdir);
425 
426  // Divide by the total number of cells we are averaging over
427  int h_avg_u_size = static_cast<int>(h_avg_u.size());
428  for (int k = 0; k < h_avg_u_size; ++k) {
429  h_avg_rho[k] /= area_z;
430  h_avg_ksgs[k] /= area_z;
431  h_avg_Kmv[k] /= area_z;
432  h_avg_Khv[k] /= area_z;
433  h_avg_th[k] /= area_z;
434  h_avg_thth[k] /= area_z;
435  h_avg_uu[k] /= area_z;
436  h_avg_uv[k] /= area_z;
437  h_avg_uw[k] /= area_z;
438  h_avg_vv[k] /= area_z;
439  h_avg_vw[k] /= area_z;
440  h_avg_ww[k] /= area_z;
441  h_avg_uth[k] /= area_z;
442  h_avg_vth[k] /= area_z;
443  h_avg_wth[k] /= area_z;
444  h_avg_uiuiu[k] /= area_z;
445  h_avg_uiuiv[k] /= area_z;
446  h_avg_uiuiw[k] /= area_z;
447  h_avg_p[k] /= area_z;
448  h_avg_pu[k] /= area_z;
449  h_avg_pv[k] /= area_z;
450  h_avg_pw[k] /= area_z;
451  h_avg_qv[k] /= area_z;
452  h_avg_qc[k] /= area_z;
453  h_avg_qr[k] /= area_z;
454  h_avg_wqv[k] /= area_z;
455  h_avg_wqc[k] /= area_z;
456  h_avg_wqr[k] /= area_z;
457  h_avg_qi[k] /= area_z;
458  h_avg_qs[k] /= area_z;
459  h_avg_qg[k] /= area_z;
460  h_avg_wthv[k] /= area_z;
461  }
462 
463 #if 0
464  // Here we print the integrated total kinetic energy as computed in the 1D profile above
465  Real sum = 0.;
466  Real dz = geom[0].ProbHi(2) / static_cast<Real>(h_avg_u_size);
467  for (int k = 0; k < h_avg_u_size; ++k) {
468  sum += h_avg_kturb[k] * h_avg_rho[k] * dz;
469  }
470  amrex::Print() << "ITKE " << time << " " << sum << " using " << h_avg_u_size << " " << dz << std::endl;
471 #endif
472 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getPgivenRTh(const amrex::Real rhotheta, const amrex::Real qv=0.)
Definition: ERF_EOS.H:81
#define RhoQ3_comp
Definition: ERF_IndexDefines.H:44
#define RhoQ6_comp
Definition: ERF_IndexDefines.H:47
#define RhoQ5_comp
Definition: ERF_IndexDefines.H:46
#define RhoKE_comp
Definition: ERF_IndexDefines.H:38
@ Theta_v
Definition: ERF_IndexDefines.H:176
@ Mom_v
Definition: ERF_IndexDefines.H:175
@ theta
Definition: ERF_MM5.H:20
@ qv
Definition: ERF_Kessler.H:28
Here is the call graph for this function:

◆ derive_diag_profiles_stag()

void ERF::derive_diag_profiles_stag ( amrex::Real  time,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_u,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_v,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_w,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_rho,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_th,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ksgs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Kmv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_Khv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqc,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wqr,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qi,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qs,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_qg,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ww,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_uth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_vth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_thth,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_ku,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_kw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_p,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pu,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pv,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_pw,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_wthv 
)

Computes the profiles for diagnostic quantities at staggered heights.

Parameters
h_avg_uProfile for x-velocity on Host
h_avg_vProfile for y-velocity on Host
h_avg_wProfile for z-velocity on Host
h_avg_rhoProfile for density on Host
h_avg_thProfile for potential temperature on Host
h_avg_ksgsProfile for Kinetic Energy on Host
h_avg_uuProfile for x-velocity squared on Host
h_avg_uvProfile for x-velocity * y-velocity on Host
h_avg_uwProfile for x-velocity * z-velocity on Host
h_avg_vvProfile for y-velocity squared on Host
h_avg_vwProfile for y-velocity * z-velocity on Host
h_avg_wwProfile for z-velocity squared on Host
h_avg_uthProfile for x-velocity * potential temperature on Host
h_avg_uiuiuProfile for u_i*u_i*u triple product on Host
h_avg_uiuivProfile for u_i*u_i*v triple product on Host
h_avg_uiuiwProfile for u_i*u_i*w triple product on Host
h_avg_pProfile for pressure perturbation on Host
h_avg_puProfile for pressure perturbation * x-velocity on Host
h_avg_pvProfile for pressure perturbation * y-velocity on Host
h_avg_pwProfile for pressure perturbation * z-velocity on Host
311 {
312  // We assume that this is always called at level 0
313  int lev = 0;
314 
315  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
316  bool l_use_KE = solverChoice.turbChoice[lev].use_tke;
317  // Note: "uiui" == u_i*u_i = u*u + v*v + w*w
318  // This will hold rho, theta, ksgs, Kmh, Kmv, uu, uv, vv, uth, vth,
319  // indices: 0 1 2 3 4 5 6 7 8 9
320  // thth, uiuiu, uiuiv, p, pu, pv, qv, qc, qr, qi, qs, qg
321  // 10 11 12 13 14 15 16 17 18 19 20 21
322  MultiFab mf_out(grids[lev], dmap[lev], 22, 0);
323 
324  // This will hold uw, vw, ww, wth, uiuiw, pw, wqv, wqc, wqr, wthv
325  // indices: 0 1 2 3 4 5 6 7 8 9
326  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 10, 0);
327 
328  // This is only used to average u and v; w is not averaged to cell centers
329  MultiFab mf_vels(grids[lev], dmap[lev], 2, 0);
330 
331  MultiFab u_cc(mf_vels, make_alias, 0, 1); // u at cell centers
332  MultiFab v_cc(mf_vels, make_alias, 1, 1); // v at cell centers
333  MultiFab w_fc(vars_new[lev][Vars::zvel], make_alias, 0, 1); // w at face centers (staggered)
334 
335  int zdir = 2;
336  auto domain = geom[0].Domain();
337  Box stag_domain = domain;
338  stag_domain.convert(IntVect(0,0,1));
339 
340  int nvars = vars_new[lev][Vars::cons].nComp();
341  MultiFab mf_cons(vars_new[lev][Vars::cons], make_alias, 0, nvars);
342 
343  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
344 
345  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
346 
347  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
348  {
349  const Box& bx = mfi.tilebox();
350  const Array4<Real>& fab_arr = mf_out.array(mfi);
351  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
352  const Array4<Real>& u_arr = vars_new[lev][Vars::xvel].array(mfi);
353  const Array4<Real>& v_arr = vars_new[lev][Vars::yvel].array(mfi);
354  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
355  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
356  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
357  const Array4<Real>& cons_arr = mf_cons.array(mfi);
358  const Array4<Real>& p0_arr = p_hse.array(mfi);
359  const Array4<const Real>& eta_arr = (l_use_kturb) ? eddyDiffs_lev[lev]->const_array(mfi) :
360  Array4<const Real>{};
361 
362  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
363  {
364  u_cc_arr(i,j,k) = 0.5 * (u_arr(i,j,k) + u_arr(i+1,j ,k));
365  v_cc_arr(i,j,k) = 0.5 * (v_arr(i,j,k) + v_arr(i ,j+1,k));
366 
367  Real theta = cons_arr(i,j,k,RhoTheta_comp) / cons_arr(i,j,k,Rho_comp);
368  fab_arr(i, j, k, 0) = cons_arr(i,j,k,Rho_comp);
369  fab_arr(i, j, k, 1) = theta;
370  Real ksgs = 0.0;
371  if (l_use_KE) {
372  ksgs = cons_arr(i,j,k,RhoKE_comp) / cons_arr(i,j,k,Rho_comp);
373  }
374  fab_arr(i, j, k, 2) = ksgs;
375  if (l_use_kturb) {
376  fab_arr(i, j, k, 3) = eta_arr(i,j,k,EddyDiff::Mom_v); // Kmv
377  fab_arr(i, j, k, 4) = eta_arr(i,j,k,EddyDiff::Theta_v); // Khv
378  } else {
379  fab_arr(i, j, k, 3) = 0.0;
380  fab_arr(i, j, k, 4) = 0.0;
381  }
382  fab_arr(i, j, k, 5) = u_cc_arr(i,j,k) * u_cc_arr(i,j,k); // u*u
383  fab_arr(i, j, k, 6) = u_cc_arr(i,j,k) * v_cc_arr(i,j,k); // u*v
384  fab_arr(i, j, k, 7) = v_cc_arr(i,j,k) * v_cc_arr(i,j,k); // v*v
385  fab_arr(i, j, k, 8) = u_cc_arr(i,j,k) * theta; // u*th
386  fab_arr(i, j, k, 9) = v_cc_arr(i,j,k) * theta; // v*th
387  fab_arr(i, j, k,10) = theta * theta; // th*th
388 
389  Real wcc = 0.5 * (w_fc_arr(i,j,k) + w_fc_arr(i,j,k+1));
390 
391  // if the number of fields is changed above, then be sure to update
392  // the following def!
393  Real uiui = fab_arr(i,j,k,5) + fab_arr(i,j,k,7) + wcc*wcc;
394  fab_arr(i, j, k,11) = uiui * u_cc_arr(i,j,k); // (ui*ui)*u
395  fab_arr(i, j, k,12) = uiui * v_cc_arr(i,j,k); // (ui*ui)*v
396 
397  if (!use_moisture) {
398  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp));
399  p -= p0_arr(i,j,k);
400  fab_arr(i, j, k,13) = p; // p
401  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
402  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
403  fab_arr(i, j, k,16) = 0.; // qv
404  fab_arr(i, j, k,17) = 0.; // qc
405  fab_arr(i, j, k,18) = 0.; // qr
406  fab_arr(i, j, k,19) = 0.; // qi
407  fab_arr(i, j, k,20) = 0.; // qs
408  fab_arr(i, j, k,21) = 0.; // qg
409  }
410  });
411 
412  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
413  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
414  {
415  // average to z faces (first to cell centers, then in z)
416  Real uface = 0.25 * ( u_arr(i ,j,k) + u_arr(i ,j,k-1)
417  + u_arr(i+1,j,k) + u_arr(i+1,j,k-1));
418  Real vface = 0.25 * ( v_arr(i,j ,k) + v_arr(i,j ,k-1)
419  + v_arr(i,j+1,k) + v_arr(i,j+1,k-1));
420  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
421  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
422  Real thface = 0.5*(theta0 + theta1);
423  fab_arr_stag(i,j,k,0) = uface * w_fc_arr(i,j,k); // u*w
424  fab_arr_stag(i,j,k,1) = vface * w_fc_arr(i,j,k); // v*w
425  fab_arr_stag(i,j,k,2) = w_fc_arr(i,j,k) * w_fc_arr(i,j,k); // w*w
426  fab_arr_stag(i,j,k,3) = thface * w_fc_arr(i,j,k); // th*w
427  Real uiui = uface*uface + vface*vface + fab_arr_stag(i,j,k,2);
428  fab_arr_stag(i,j,k,4) = uiui * w_fc_arr(i,j,k); // (ui*ui)*w
429  if (!use_moisture) {
430  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp)) - p0_arr(i,j,k );
431  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp)) - p0_arr(i,j,k-1);
432  Real pface = 0.5 * (p0 + p1);
433  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
434  fab_arr_stag(i,j,k,6) = 0.; // w*qv
435  fab_arr_stag(i,j,k,7) = 0.; // w*qc
436  fab_arr_stag(i,j,k,8) = 0.; // w*qr
437  fab_arr_stag(i,j,k,9) = 0.; // w*thv
438  }
439  });
440 
441  } // mfi
442 
443  if (use_moisture)
444  {
445  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
446 
447  for ( MFIter mfi(mf_cons,TilingIfNotGPU()); mfi.isValid(); ++mfi)
448  {
449  const Box& bx = mfi.tilebox();
450  const Array4<Real>& fab_arr = mf_out.array(mfi);
451  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
452  const Array4<Real>& cons_arr = mf_cons.array(mfi);
453  const Array4<Real>& u_cc_arr = u_cc.array(mfi);
454  const Array4<Real>& v_cc_arr = v_cc.array(mfi);
455  const Array4<Real>& w_fc_arr = w_fc.array(mfi);
456  const Array4<Real>& p0_arr = p_hse.array(mfi);
457 
458  int rhoqr_comp = solverChoice.moisture_indices.qr;
459 
460  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
461  {
462  Real qv = cons_arr(i,j,k,RhoQ1_comp) / cons_arr(i,j,k,Rho_comp);
463  Real qc = cons_arr(i,j,k,RhoQ2_comp) / cons_arr(i,j,k,Rho_comp);
464  Real qr = (rhoqr_comp > -1) ? cons_arr(i,j,k,rhoqr_comp) / cons_arr(i,j,k,Rho_comp) :
465  Real(0.0);
466  Real p = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
467 
468  p -= p0_arr(i,j,k);
469  fab_arr(i, j, k,13) = p; // p
470  fab_arr(i, j, k,14) = p * u_cc_arr(i,j,k); // p*u
471  fab_arr(i, j, k,15) = p * v_cc_arr(i,j,k); // p*v
472  fab_arr(i, j, k,16) = qv; // qv
473  fab_arr(i, j, k,17) = qc; // qc
474  fab_arr(i, j, k,18) = qr; // qr
475  if (n_qstate_moist > 3) { // SAM model
476  fab_arr(i, j, k,19) = cons_arr(i,j,k,RhoQ3_comp) / cons_arr(i,j,k,Rho_comp); // qi
477  fab_arr(i, j, k,20) = cons_arr(i,j,k,RhoQ5_comp) / cons_arr(i,j,k,Rho_comp); // qs
478  fab_arr(i, j, k,21) = cons_arr(i,j,k,RhoQ6_comp) / cons_arr(i,j,k,Rho_comp); // qg
479  } else {
480  fab_arr(i, j, k,19) = 0.0; // qi
481  fab_arr(i, j, k,20) = 0.0; // qs
482  fab_arr(i, j, k,21) = 0.0; // qg
483  }
484  });
485 
486  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
487  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
488  {
489  Real qv0 = cons_arr(i,j,k ,RhoQ1_comp) / cons_arr(i,j,k ,Rho_comp);
490  Real qv1 = cons_arr(i,j,k-1,RhoQ1_comp) / cons_arr(i,j,k-1,Rho_comp);
491  Real qc0 = cons_arr(i,j,k ,RhoQ2_comp) / cons_arr(i,j,k ,Rho_comp);
492  Real qc1 = cons_arr(i,j,k-1,RhoQ2_comp) / cons_arr(i,j,k-1,Rho_comp);
493  Real qr0 = (rhoqr_comp > -1) ? cons_arr(i,j,k ,RhoQ3_comp) / cons_arr(i,j,k ,Rho_comp) :
494  Real(0.0);
495  Real qr1 = (rhoqr_comp > -1) ? cons_arr(i,j,k-1,RhoQ3_comp) / cons_arr(i,j,k-1,Rho_comp) :
496  Real(0.0);
497  Real qvface = 0.5 * (qv0 + qv1);
498  Real qcface = 0.5 * (qc0 + qc1);
499  Real qrface = 0.5 * (qr0 + qr1);
500 
501  Real p0 = getPgivenRTh(cons_arr(i, j, k , RhoTheta_comp), qv0) - p0_arr(i,j,k );
502  Real p1 = getPgivenRTh(cons_arr(i, j, k-1, RhoTheta_comp), qv1) - p0_arr(i,j,k-1);
503  Real pface = 0.5 * (p0 + p1);
504 
505  Real theta0 = cons_arr(i,j,k ,RhoTheta_comp) / cons_arr(i,j,k ,Rho_comp);
506  Real theta1 = cons_arr(i,j,k-1,RhoTheta_comp) / cons_arr(i,j,k-1,Rho_comp);
507  Real thface = 0.5*(theta0 + theta1);
508  Real ql = qcface + qrface;
509  Real thv = thface * (1 + 0.61*qvface - ql);
510 
511  fab_arr_stag(i,j,k,5) = pface * w_fc_arr(i,j,k); // p*w
512  fab_arr_stag(i,j,k,6) = qvface * w_fc_arr(i,j,k); // w*qv
513  fab_arr_stag(i,j,k,7) = qcface * w_fc_arr(i,j,k); // w*qc
514  fab_arr_stag(i,j,k,8) = qrface * w_fc_arr(i,j,k); // w*qr
515  fab_arr_stag(i,j,k,9) = thv * w_fc_arr(i,j,k); // w*thv
516  });
517  } // mfi
518  } // use_moisture
519 
520  // Sum in the horizontal plane
521  h_avg_u = sumToLine(u_cc,0,1, domain,zdir);
522  h_avg_v = sumToLine(v_cc,0,1, domain,zdir);
523  h_avg_w = sumToLine(w_fc,0,1,stag_domain,zdir);
524 
525  h_avg_rho = sumToLine(mf_out, 0,1,domain,zdir);
526  h_avg_th = sumToLine(mf_out, 1,1,domain,zdir);
527  h_avg_ksgs = sumToLine(mf_out, 2,1,domain,zdir);
528  h_avg_Kmv = sumToLine(mf_out, 3,1,domain,zdir);
529  h_avg_Khv = sumToLine(mf_out, 4,1,domain,zdir);
530  h_avg_uu = sumToLine(mf_out, 5,1,domain,zdir);
531  h_avg_uv = sumToLine(mf_out, 6,1,domain,zdir);
532  h_avg_vv = sumToLine(mf_out, 7,1,domain,zdir);
533  h_avg_uth = sumToLine(mf_out, 8,1,domain,zdir);
534  h_avg_vth = sumToLine(mf_out, 9,1,domain,zdir);
535  h_avg_thth = sumToLine(mf_out,10,1,domain,zdir);
536  h_avg_uiuiu = sumToLine(mf_out,11,1,domain,zdir);
537  h_avg_uiuiv = sumToLine(mf_out,12,1,domain,zdir);
538  h_avg_p = sumToLine(mf_out,13,1,domain,zdir);
539  h_avg_pu = sumToLine(mf_out,14,1,domain,zdir);
540  h_avg_pv = sumToLine(mf_out,15,1,domain,zdir);
541  h_avg_qv = sumToLine(mf_out,16,1,domain,zdir);
542  h_avg_qc = sumToLine(mf_out,17,1,domain,zdir);
543  h_avg_qr = sumToLine(mf_out,18,1,domain,zdir);
544  h_avg_qi = sumToLine(mf_out,19,1,domain,zdir);
545  h_avg_qs = sumToLine(mf_out,20,1,domain,zdir);
546  h_avg_qg = sumToLine(mf_out,21,1,domain,zdir);
547 
548  h_avg_uw = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
549  h_avg_vw = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
550  h_avg_ww = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
551  h_avg_wth = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
552  h_avg_uiuiw = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
553  h_avg_pw = sumToLine(mf_out_stag,5,1,stag_domain,zdir);
554  h_avg_wqv = sumToLine(mf_out_stag,6,1,stag_domain,zdir);
555  h_avg_wqc = sumToLine(mf_out_stag,7,1,stag_domain,zdir);
556  h_avg_wqr = sumToLine(mf_out_stag,8,1,stag_domain,zdir);
557  h_avg_wthv = sumToLine(mf_out_stag,9,1,stag_domain,zdir);
558 
559  // Divide by the total number of cells we are averaging over
560  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
561  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
562  for (int k = 0; k < unstag_size; ++k) {
563  h_avg_u[k] /= area_z;
564  h_avg_v[k] /= area_z;
565  h_avg_rho[k] /= area_z;
566  h_avg_ksgs[k] /= area_z;
567  h_avg_Kmv[k] /= area_z;
568  h_avg_Khv[k] /= area_z;
569  h_avg_th[k] /= area_z;
570  h_avg_thth[k] /= area_z;
571  h_avg_uu[k] /= area_z;
572  h_avg_uv[k] /= area_z;
573  h_avg_vv[k] /= area_z;
574  h_avg_uth[k] /= area_z;
575  h_avg_vth[k] /= area_z;
576  h_avg_uiuiu[k] /= area_z;
577  h_avg_uiuiv[k] /= area_z;
578  h_avg_p[k] /= area_z;
579  h_avg_pu[k] /= area_z;
580  h_avg_pv[k] /= area_z;
581  h_avg_qv[k] /= area_z;
582  h_avg_qc[k] /= area_z;
583  h_avg_qr[k] /= area_z;
584  h_avg_qi[k] /= area_z;
585  h_avg_qs[k] /= area_z;
586  h_avg_qg[k] /= area_z;
587  }
588 
589  for (int k = 0; k < unstag_size+1; ++k) { // staggered heights
590  h_avg_w[k] /= area_z;
591  h_avg_uw[k] /= area_z;
592  h_avg_vw[k] /= area_z;
593  h_avg_ww[k] /= area_z;
594  h_avg_wth[k] /= area_z;
595  h_avg_uiuiw[k] /= area_z;
596  h_avg_pw[k] /= area_z;
597  h_avg_wqv[k] /= area_z;
598  h_avg_wqc[k] /= area_z;
599  h_avg_wqr[k] /= area_z;
600  h_avg_wthv[k] /= area_z;
601  }
602 }
const Box zbx
Definition: ERF_DiffSetup.H:23
Here is the call graph for this function:

◆ derive_stress_profiles()

void ERF::derive_stress_profiles ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
480 {
481  int lev = 0;
482 
483  // This will hold the stress tensor components
484  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
485 
486  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
487 
488  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
489 
490  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
491  {
492  const Box& bx = mfi.tilebox();
493  const Array4<Real>& fab_arr = mf_out.array(mfi);
494 
495  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
496 
497  // NOTE: These are from the last RK stage...
498  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
499  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
500  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
501  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
502  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
503  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
504 
505  // These should be re-calculated during ERF_slow_rhs_post
506  // -- just vertical SFS kinematic heat flux for now
507  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
508  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
509  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
510  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
511  Array4<const Real>{};
512  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
513  Array4<const Real>{};
514  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
515 
516  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
517  {
518  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
519  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
520  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
521  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
522  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
523  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
524  fab_arr(i, j, k, 2) = ( tau13_arr(i,j,k ) + tau13_arr(i+1,j,k )
525  + tau13_arr(i,j,k+1) + tau13_arr(i+1,j,k+1) )
526  / ( rho_arr(i,j,k ) + rho_arr(i+1,j,k )
527  + rho_arr(i,j,k+1) + rho_arr(i+1,j,k+1) );
528  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
529  fab_arr(i, j, k, 4) = ( tau23_arr(i,j,k ) + tau23_arr(i,j+1,k )
530  + tau23_arr(i,j,k+1) + tau23_arr(i,j+1,k+1) )
531  / ( rho_arr(i,j,k ) + rho_arr(i,j+1,k )
532  + rho_arr(i,j,k+1) + rho_arr(i,j+1,k+1) );
533  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
534  fab_arr(i, j, k, 6) = 0.5 * ( hfx3_arr(i,j,k) + hfx3_arr(i,j,k+1) ) / rho_arr(i,j,k);
535  fab_arr(i, j, k, 7) = (l_use_moist) ? 0.5 * ( q1fx3_arr(i,j,k) + q1fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
536  fab_arr(i, j, k, 8) = (l_use_moist) ? 0.5 * ( q2fx3_arr(i,j,k) + q2fx3_arr(i,j,k+1) ) / rho_arr(i,j,k) : 0.0;
537  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
538  });
539  }
540 
541  int zdir = 2;
542  auto domain = geom[0].Domain();
543 
544  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
545  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
546  h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
547  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
548  h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
549  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
550  h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
551  h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
552  h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
553  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
554 
555  int ht_size = h_avg_tau11.size();
556 
557  // Divide by the total number of cells we are averaging over
558  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
559  for (int k = 0; k < ht_size; ++k) {
560  h_avg_tau11[k] /= area_z;
561  h_avg_tau12[k] /= area_z;
562  h_avg_tau13[k] /= area_z;
563  h_avg_tau22[k] /= area_z;
564  h_avg_tau23[k] /= area_z;
565  h_avg_tau33[k] /= area_z;
566  h_avg_hfx3[k] /= area_z;
567  h_avg_q1fx3[k] /= area_z;
568  h_avg_q2fx3[k] /= area_z;
569  h_avg_diss[k] /= area_z;
570  }
571 }

◆ derive_stress_profiles_stag()

void ERF::derive_stress_profiles_stag ( amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau11,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau12,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau13,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau22,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau23,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_tau33,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_hfx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q1fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_q2fx3,
amrex::Gpu::HostVector< amrex::Real > &  h_avg_diss 
)
610 {
611  int lev = 0;
612 
613  // This will hold the stress tensor components
614  MultiFab mf_out(grids[lev], dmap[lev], 10, 0);
615 
616  // This will hold Tau13 and Tau23
617  MultiFab mf_out_stag(convert(grids[lev], IntVect(0,0,1)), dmap[lev], 5, 0);
618 
619  MultiFab mf_rho(vars_new[lev][Vars::cons], make_alias, 0, 1);
620 
621  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
622 
623  for ( MFIter mfi(mf_out,TilingIfNotGPU()); mfi.isValid(); ++mfi)
624  {
625  const Box& bx = mfi.tilebox();
626  const Array4<Real>& fab_arr = mf_out.array(mfi);
627  const Array4<Real>& fab_arr_stag = mf_out_stag.array(mfi);
628 
629  const Array4<const Real>& rho_arr = mf_rho.const_array(mfi);
630 
631  // NOTE: These are from the last RK stage...
632  const Array4<const Real>& tau11_arr = Tau[lev][TauType::tau11]->const_array(mfi);
633  const Array4<const Real>& tau12_arr = Tau[lev][TauType::tau12]->const_array(mfi);
634  const Array4<const Real>& tau13_arr = Tau[lev][TauType::tau13]->const_array(mfi);
635  const Array4<const Real>& tau22_arr = Tau[lev][TauType::tau22]->const_array(mfi);
636  const Array4<const Real>& tau23_arr = Tau[lev][TauType::tau23]->const_array(mfi);
637  const Array4<const Real>& tau33_arr = Tau[lev][TauType::tau33]->const_array(mfi);
638 
639  // These should be re-calculated during ERF_slow_rhs_post
640  // -- just vertical SFS kinematic heat flux for now
641  //const Array4<const Real>& hfx1_arr = SFS_hfx1_lev[lev]->const_array(mfi);
642  //const Array4<const Real>& hfx2_arr = SFS_hfx2_lev[lev]->const_array(mfi);
643  const Array4<const Real>& hfx3_arr = SFS_hfx3_lev[lev]->const_array(mfi);
644  const Array4<const Real>& q1fx3_arr = (l_use_moist) ? SFS_q1fx3_lev[lev]->const_array(mfi) :
645  Array4<const Real>{};
646  const Array4<const Real>& q2fx3_arr = (l_use_moist) ? SFS_q2fx3_lev[lev]->const_array(mfi) :
647  Array4<const Real>{};
648  const Array4<const Real>& diss_arr = SFS_diss_lev[lev]->const_array(mfi);
649 
650  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
651  {
652  // rho averaging should follow Diffusion/ERF_ComputeStress_*.cpp
653  fab_arr(i, j, k, 0) = tau11_arr(i,j,k) / rho_arr(i,j,k);
654  fab_arr(i, j, k, 1) = ( tau12_arr(i,j ,k) + tau12_arr(i+1,j ,k)
655  + tau12_arr(i,j+1,k) + tau12_arr(i+1,j+1,k) )
656  / ( rho_arr(i,j ,k) + rho_arr(i+1,j ,k)
657  + rho_arr(i,j+1,k) + rho_arr(i+1,j+1,k) );
658  fab_arr(i, j, k, 3) = tau22_arr(i,j,k) / rho_arr(i,j,k);
659  fab_arr(i, j, k, 5) = tau33_arr(i,j,k) / rho_arr(i,j,k);
660  fab_arr(i, j, k, 9) = diss_arr(i,j,k) / rho_arr(i,j,k);
661  });
662 
663  const Box& zbx = mfi.tilebox(IntVect(0,0,1));
664  ParallelFor(zbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
665  {
666  Real rho_face = 0.5 * (rho_arr(i,j,k-1) + rho_arr(i,j,k));
667  // average from edge to face center
668  fab_arr_stag(i,j,k,0) = 0.5*(tau13_arr(i,j,k) + tau13_arr(i+1,j ,k)) / rho_face;
669  fab_arr_stag(i,j,k,1) = 0.5*(tau23_arr(i,j,k) + tau23_arr(i ,j+1,k)) / rho_face;
670 
671  fab_arr_stag(i,j,k,2) = hfx3_arr(i,j,k) / rho_face;
672  fab_arr_stag(i,j,k,3) = (l_use_moist) ? q1fx3_arr(i,j,k) / rho_face : 0.0;
673  fab_arr_stag(i,j,k,4) = (l_use_moist) ? q2fx3_arr(i,j,k) / rho_face : 0.0;
674  });
675  }
676 
677  int zdir = 2;
678  auto domain = geom[0].Domain();
679  Box stag_domain = domain;
680  stag_domain.convert(IntVect(0,0,1));
681 
682  h_avg_tau11 = sumToLine(mf_out,0,1,domain,zdir);
683  h_avg_tau12 = sumToLine(mf_out,1,1,domain,zdir);
684 // h_avg_tau13 = sumToLine(mf_out,2,1,domain,zdir);
685  h_avg_tau22 = sumToLine(mf_out,3,1,domain,zdir);
686 // h_avg_tau23 = sumToLine(mf_out,4,1,domain,zdir);
687  h_avg_tau33 = sumToLine(mf_out,5,1,domain,zdir);
688 // h_avg_hfx3 = sumToLine(mf_out,6,1,domain,zdir);
689 // h_avg_q1fx3 = sumToLine(mf_out,7,1,domain,zdir);
690 // h_avg_q2fx3 = sumToLine(mf_out,8,1,domain,zdir);
691  h_avg_diss = sumToLine(mf_out,9,1,domain,zdir);
692 
693  h_avg_tau13 = sumToLine(mf_out_stag,0,1,stag_domain,zdir);
694  h_avg_tau23 = sumToLine(mf_out_stag,1,1,stag_domain,zdir);
695  h_avg_hfx3 = sumToLine(mf_out_stag,2,1,stag_domain,zdir);
696  h_avg_q1fx3 = sumToLine(mf_out_stag,3,1,stag_domain,zdir);
697  h_avg_q2fx3 = sumToLine(mf_out_stag,4,1,stag_domain,zdir);
698 
699  int ht_size = h_avg_tau11.size(); // _un_staggered
700 
701  // Divide by the total number of cells we are averaging over
702  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
703  for (int k = 0; k < ht_size; ++k) {
704  h_avg_tau11[k] /= area_z;
705  h_avg_tau12[k] /= area_z;
706  h_avg_tau13[k] /= area_z;
707  h_avg_tau22[k] /= area_z;
708  h_avg_tau23[k] /= area_z;
709  h_avg_tau33[k] /= area_z;
710  h_avg_hfx3[k] /= area_z;
711  h_avg_q1fx3[k] /= area_z;
712  h_avg_q2fx3[k] /= area_z;
713  h_avg_diss[k] /= area_z;
714  }
715  // staggered heights
716  h_avg_tau13[ht_size] /= area_z;
717  h_avg_tau23[ht_size] /= area_z;
718  h_avg_hfx3[ht_size] /= area_z;
719  h_avg_q1fx3[ht_size] /= area_z;
720  h_avg_q2fx3[ht_size] /= area_z;
721 }

◆ derive_upwp()

void ERF::derive_upwp ( amrex::Vector< amrex::Real > &  h_havg)

◆ EBFactory()

amrex::EBFArrayBoxFactory const& ERF::EBFactory ( int  lev) const
inlineprivatenoexcept
1559  {
1560  return *(eb[lev]->get_const_factory());
1561  }
amrex::Vector< std::unique_ptr< eb_ > > eb
Definition: ERF.H:1551

Referenced by WriteMyEBSurface().

Here is the caller graph for this function:

◆ erf_enforce_hse()

void ERF::erf_enforce_hse ( int  lev,
amrex::MultiFab &  dens,
amrex::MultiFab &  pres,
amrex::MultiFab &  pi,
amrex::MultiFab &  th,
amrex::MultiFab &  qv,
std::unique_ptr< amrex::MultiFab > &  z_cc 
)

Enforces hydrostatic equilibrium when using terrain.

Parameters
[in]levInteger specifying the current level
[out]densMultiFab storing base state density
[out]presMultiFab storing base state pressure
[out]piMultiFab storing base state Exner function
[in]z_ccPointer to MultiFab storing cell centered z-coordinates
163 {
164  Real l_gravity = solverChoice.gravity;
165  bool l_use_terrain = (solverChoice.mesh_type != MeshType::ConstantDz);
166 
167  const auto geomdata = geom[lev].data();
168  const Real dz = geomdata.CellSize(2);
169 
170  for ( MFIter mfi(dens, TileNoZ()); mfi.isValid(); ++mfi )
171  {
172  // Create a flat box with same horizontal extent but only one cell in vertical
173  const Box& tbz = mfi.nodaltilebox(2);
174  int klo = tbz.smallEnd(2);
175  int khi = tbz.bigEnd(2);
176 
177  // Note we only grow by 1 because that is how big z_cc is.
178  Box b2d = tbz; // Copy constructor
179  b2d.grow(0,1);
180  b2d.grow(1,1);
181  b2d.setRange(2,0);
182 
183  // Intersect this box with the domain
184  Box zdomain = convert(geom[lev].Domain(),tbz.ixType());
185  b2d &= zdomain;
186 
187  // We integrate to the first cell (and below) by using rho in this cell
188  // If gravity == 0 this is constant pressure
189  // If gravity != 0, hence this is a wall, this gives gp0 = dens[0] * gravity
190  // (dens_hse*gravity would also be dens[0]*gravity because we use foextrap for rho at k = -1)
191  // Note ng_pres_hse = 1
192 
193  // We start by assuming pressure on the ground is p_0 (in ERF_Constants.H)
194  // Note that gravity is positive
195 
196  Array4<Real> rho_arr = dens.array(mfi);
197  Array4<Real> pres_arr = pres.array(mfi);
198  Array4<Real> pi_arr = pi.array(mfi);
199  Array4<Real> th_arr = theta.array(mfi);
200  Array4<Real> zcc_arr;
201  if (l_use_terrain) {
202  zcc_arr = z_cc->array(mfi);
203  }
204 
205  const Real rdOcp = solverChoice.rdOcp;
206 
207  ParallelFor(b2d, [=] AMREX_GPU_DEVICE (int i, int j, int)
208  {
209  // Set value at surface from Newton iteration for rho
210  if (klo == 0)
211  {
212  // Physical height of the terrain at cell center
213  Real hz;
214  if (l_use_terrain) {
215  hz = zcc_arr(i,j,klo);
216  } else {
217  hz = 0.5*dz;
218  }
219 
220  pres_arr(i,j,klo) = p_0 - hz * rho_arr(i,j,klo) * l_gravity;
221  pi_arr(i,j,klo) = getExnergivenP(pres_arr(i,j,klo), rdOcp);
222  th_arr(i,j,klo) = getRhoThetagivenP(pres_arr(i,j,klo)) / rho_arr(i,j,klo);
223 
224  //
225  // Set ghost cell with dz and rho at boundary
226  // (We will set the rest of the ghost cells in the boundary condition routine)
227  //
228  pres_arr(i,j,klo-1) = p_0 + hz * rho_arr(i,j,klo) * l_gravity;
229  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
230  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
231 
232  } else {
233 
234  // If level > 0 and klo > 0, we need to use the value of pres_arr(i,j,klo-1) which was
235  // filled from FillPatch-ing it.
236  Real dz_loc;
237  if (l_use_terrain) {
238  dz_loc = (zcc_arr(i,j,klo) - zcc_arr(i,j,klo-1));
239  } else {
240  dz_loc = dz;
241  }
242 
243  Real dens_interp = 0.5*(rho_arr(i,j,klo) + rho_arr(i,j,klo-1));
244  pres_arr(i,j,klo) = pres_arr(i,j,klo-1) - dz_loc * dens_interp * l_gravity;
245 
246  pi_arr(i,j,klo ) = getExnergivenP(pres_arr(i,j,klo ), rdOcp);
247  th_arr(i,j,klo ) = getRhoThetagivenP(pres_arr(i,j,klo )) / rho_arr(i,j,klo );
248 
249  pi_arr(i,j,klo-1) = getExnergivenP(pres_arr(i,j,klo-1), rdOcp);
250  th_arr(i,j,klo-1) = getRhoThetagivenP(pres_arr(i,j,klo-1)) / rho_arr(i,j,klo-1);
251  }
252 
253  Real dens_interp;
254  if (l_use_terrain) {
255  for (int k = klo+1; k <= khi; k++) {
256  Real dz_loc = (zcc_arr(i,j,k) - zcc_arr(i,j,k-1));
257  dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
258  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz_loc * dens_interp * l_gravity;
259  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
260  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
261  }
262  } else {
263  for (int k = klo+1; k <= khi; k++) {
264  dens_interp = 0.5*(rho_arr(i,j,k) + rho_arr(i,j,k-1));
265  pres_arr(i,j,k) = pres_arr(i,j,k-1) - dz * dens_interp * l_gravity;
266  pi_arr(i,j,k) = getExnergivenP(pres_arr(i,j,k), rdOcp);
267  th_arr(i,j,k) = getRhoThetagivenP(pres_arr(i,j,k)) / rho_arr(i,j,k);
268  }
269  }
270  });
271 
272  } // mfi
273 
274  dens.FillBoundary(geom[lev].periodicity());
275  pres.FillBoundary(geom[lev].periodicity());
276  pi.FillBoundary(geom[lev].periodicity());
277  theta.FillBoundary(geom[lev].periodicity());
278  qv.FillBoundary(geom[lev].periodicity());
279 }
constexpr amrex::Real p_0
Definition: ERF_Constants.H:18
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getRhoThetagivenP(const amrex::Real p, const amrex::Real qv=0.0)
Definition: ERF_EOS.H:172
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getExnergivenP(const amrex::Real P, const amrex::Real rdOcp)
Definition: ERF_EOS.H:141
@ pres
Definition: ERF_Kessler.H:25
real(c_double), parameter, private pi
Definition: ERF_module_mp_morr_two_moment.F90:100
amrex::Real rdOcp
Definition: ERF_DataStruct.H:866
amrex::Real gravity
Definition: ERF_DataStruct.H:864
Here is the call graph for this function:

◆ ERF_shared()

void ERF::ERF_shared ( )
137 {
138  if (ParallelDescriptor::IOProcessor()) {
139  const char* erf_hash = buildInfoGetGitHash(1);
140  const char* amrex_hash = buildInfoGetGitHash(2);
141  const char* buildgithash = buildInfoGetBuildGitHash();
142  const char* buildgitname = buildInfoGetBuildGitName();
143 
144  if (strlen(erf_hash) > 0) {
145  Print() << "\n"
146  << "ERF git hash: " << erf_hash << "\n";
147  }
148  if (strlen(amrex_hash) > 0) {
149  Print() << "AMReX git hash: " << amrex_hash << "\n";
150  }
151  if (strlen(buildgithash) > 0) {
152  Print() << buildgitname << " git hash: " << buildgithash << "\n";
153  }
154 
155  Print() << "\n";
156  }
157 
158  int nlevs_max = max_level + 1;
159 
160 #ifdef ERF_USE_WINDFARM
161  Nturb.resize(nlevs_max);
162  vars_windfarm.resize(nlevs_max);
163  SMark.resize(nlevs_max);
164 #endif
165 
166  qheating_rates.resize(nlevs_max);
167  sw_lw_fluxes.resize(nlevs_max);
168  solar_zenith.resize(nlevs_max);
169 
170  // NOTE: size lsm before readparams (chooses the model at all levels)
171  lsm.ReSize(nlevs_max);
172  lsm_data.resize(nlevs_max);
173  lsm_flux.resize(nlevs_max);
174 
175  // NOTE: size canopy model before readparams (if file exists, we construct)
176  m_forest_drag.resize(nlevs_max);
177  for (int lev = 0; lev <= max_level; ++lev) { m_forest_drag[lev] = nullptr;}
178 
179  ReadParameters();
180  initializeMicrophysics(nlevs_max);
181 
182 #ifdef ERF_USE_WINDFARM
183  initializeWindFarm(nlevs_max);
184 #endif
185 
186 #ifdef ERF_USE_SHOC
187  shoc_interface.resize(nlevs_max);
188  if (solverChoice.use_shoc) {
189  for (int lev = 0; lev <= max_level; ++lev) {
190  shoc_interface[lev] = std::make_unique<SHOCInterface>(lev, solverChoice);
191  }
192  }
193 #endif
194 
195  rad.resize(nlevs_max);
196  for (int lev = 0; lev <= max_level; ++lev) {
197  if (solverChoice.rad_type == RadiationType::RRTMGP) {
198 #ifdef ERF_USE_RRTMGP
199  rad[lev] = std::make_unique<Radiation>(lev, solverChoice);
200  // pass radiation datalog frequency to model - RRTMGP needs to know when to save data for profiles
201  rad[lev]->setDataLogFrequency(rad_datalog_int);
202 #endif
203  } else if (solverChoice.rad_type != RadiationType::None) {
204  Abort("Don't know this radiation model!");
205  }
206  }
207 
208  const std::string& pv3d_1 = "plot_vars_1" ; setPlotVariables(pv3d_1,plot3d_var_names_1);
209  const std::string& pv3d_2 = "plot_vars_2" ; setPlotVariables(pv3d_2,plot3d_var_names_2);
210  const std::string& pv2d_1 = "plot2d_vars_1"; setPlotVariables(pv2d_1,plot2d_var_names_1);
211  const std::string& pv2d_2 = "plot2d_vars_2"; setPlotVariables(pv2d_2,plot2d_var_names_2);
212 
213  // This is only used when we have mesh_type == MeshType::StretchedDz
214  stretched_dz_h.resize(nlevs_max);
215  stretched_dz_d.resize(nlevs_max);
216 
217  // Initialize staggered vertical levels for grid stretching or terrain, and
218  // to simplify Rayleigh damping layer calculations.
219  zlevels_stag.resize(max_level+1);
223  geom,
224  refRatio(),
227  solverChoice.dz0);
228 
229  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
230  SolverChoice::mesh_type == MeshType::VariableDz) {
231  int nz = geom[0].Domain().length(2) + 1; // staggered
232  if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
233  Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
234  << " does not match highest requested z level " << zlevels_stag[0][nz-1]
235  << std::endl;
236  }
237  if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > 1.0e-4) {
238  Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
239  << " does not match lowest requested level " << zlevels_stag[0][0]
240  << std::endl;
241  }
242 
243  // Redefine the problem domain here?
244  }
245 
246  // Get lo/hi indices for massflux calc
248  if (solverChoice.mesh_type == MeshType::ConstantDz) {
249  const Real massflux_zlo = solverChoice.const_massflux_layer_lo - geom[0].ProbLo(2);
250  const Real massflux_zhi = solverChoice.const_massflux_layer_hi - geom[0].ProbLo(2);
251  const Real dz = geom[0].CellSize(2);
252  if (massflux_zlo == -1e34) {
253  solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
254  } else {
255  solverChoice.massflux_klo = static_cast<int>(std::ceil(massflux_zlo / dz - 0.5));
256  }
257  if (massflux_zhi == 1e34) {
258  solverChoice.massflux_khi = geom[0].Domain().bigEnd(2);
259  } else {
260  solverChoice.massflux_khi = static_cast<int>(std::floor(massflux_zhi / dz - 0.5));
261  }
262  } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
263  const Real massflux_zlo = solverChoice.const_massflux_layer_lo;
264  const Real massflux_zhi = solverChoice.const_massflux_layer_hi;
265  solverChoice.massflux_klo = geom[0].Domain().smallEnd(2);
266  solverChoice.massflux_khi = geom[0].Domain().bigEnd(2) + 1;
267  for (int k=0; k <= geom[0].Domain().bigEnd(2)+1; ++k) {
268  if (zlevels_stag[0][k] <= massflux_zlo) solverChoice.massflux_klo = k;
269  if (zlevels_stag[0][k] <= massflux_zhi) solverChoice.massflux_khi = k;
270  }
271  } else { // solverChoice.mesh_type == MeshType::VariableDz
272  Error("Const massflux with variable dz not supported -- planar averages are on k rather than constant-z planes");
273  }
274 
275  Print() << "Constant mass flux based on k in ["
276  << solverChoice.massflux_klo << ", " << solverChoice.massflux_khi << "]" << std::endl;
277  }
278 
279  prob = amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
280 
281  // Geometry on all levels has been defined already.
282 
283  // No valid BoxArray and DistributionMapping have been defined.
284  // But the arrays for them have been resized.
285 
286  istep.resize(nlevs_max, 0);
287  nsubsteps.resize(nlevs_max, 1);
288  for (int lev = 1; lev <= max_level; ++lev) {
289  nsubsteps[lev] = MaxRefRatio(lev-1);
290  }
291 
292  t_new.resize(nlevs_max, 0.0);
293  t_old.resize(nlevs_max, -1.e100);
294  dt.resize(nlevs_max, std::min(1.e100,dt_max_initial));
295  dt_mri_ratio.resize(nlevs_max, 1);
296 
297  vars_new.resize(nlevs_max);
298  vars_old.resize(nlevs_max);
299  gradp.resize(nlevs_max);
300 
301  // We resize this regardless in order to pass it without error
302  pp_inc.resize(nlevs_max);
303 
304  rU_new.resize(nlevs_max);
305  rV_new.resize(nlevs_max);
306  rW_new.resize(nlevs_max);
307 
308  rU_old.resize(nlevs_max);
309  rV_old.resize(nlevs_max);
310  rW_old.resize(nlevs_max);
311 
312  // xmom_crse_rhs.resize(nlevs_max);
313  // ymom_crse_rhs.resize(nlevs_max);
314  zmom_crse_rhs.resize(nlevs_max);
315 
316  for (int lev = 0; lev < nlevs_max; ++lev) {
317  vars_new[lev].resize(Vars::NumTypes);
318  vars_old[lev].resize(Vars::NumTypes);
319  gradp[lev].resize(AMREX_SPACEDIM);
320  }
321 
322  // Time integrator
323  mri_integrator_mem.resize(nlevs_max);
324 
325  // Physical boundary conditions
326  physbcs_cons.resize(nlevs_max);
327  physbcs_u.resize(nlevs_max);
328  physbcs_v.resize(nlevs_max);
329  physbcs_w.resize(nlevs_max);
330  physbcs_base.resize(nlevs_max);
331 
332  // Planes to hold Dirichlet values at boundaries
333  xvel_bc_data.resize(nlevs_max);
334  yvel_bc_data.resize(nlevs_max);
335  zvel_bc_data.resize(nlevs_max);
336  th_bc_data.resize(nlevs_max);
337 
338  advflux_reg.resize(nlevs_max);
339 
340  // Stresses
341  Tau.resize(nlevs_max);
342  SFS_hfx1_lev.resize(nlevs_max); SFS_hfx2_lev.resize(nlevs_max); SFS_hfx3_lev.resize(nlevs_max);
343  SFS_diss_lev.resize(nlevs_max);
344  SFS_q1fx1_lev.resize(nlevs_max); SFS_q1fx2_lev.resize(nlevs_max); SFS_q1fx3_lev.resize(nlevs_max);
345  SFS_q2fx3_lev.resize(nlevs_max);
346  eddyDiffs_lev.resize(nlevs_max);
347  SmnSmn_lev.resize(nlevs_max);
348 
349  // Sea surface temps
350  sst_lev.resize(nlevs_max);
351  tsk_lev.resize(nlevs_max);
352  lmask_lev.resize(nlevs_max);
353 
354  // Metric terms
355  z_phys_nd.resize(nlevs_max);
356  z_phys_cc.resize(nlevs_max);
357  detJ_cc.resize(nlevs_max);
358  ax.resize(nlevs_max);
359  ay.resize(nlevs_max);
360  az.resize(nlevs_max);
361 
362  z_phys_nd_new.resize(nlevs_max);
363  detJ_cc_new.resize(nlevs_max);
364 
365  z_phys_nd_src.resize(nlevs_max);
366  z_phys_cc_src.resize(nlevs_max);
367  detJ_cc_src.resize(nlevs_max);
368  ax_src.resize(nlevs_max);
369  ay_src.resize(nlevs_max);
370  az_src.resize(nlevs_max);
371 
372  z_t_rk.resize(nlevs_max);
373 
374  terrain_blanking.resize(nlevs_max);
375 
376  // Wall distance
377  walldist.resize(nlevs_max);
378 
379  // BoxArrays to make MultiFabs needed to convert WRFBdy data
380  ba1d.resize(nlevs_max);
381  ba2d.resize(nlevs_max);
382 
383  // MultiFabs needed to convert WRFBdy data
384  mf_PSFC.resize(nlevs_max);
385 
386  // Map factors
387  mapfac.resize(nlevs_max);
388 
389  // Thin immersed body
390  xflux_imask.resize(nlevs_max);
391  yflux_imask.resize(nlevs_max);
392  zflux_imask.resize(nlevs_max);
393  //overset_imask.resize(nlevs_max);
394  thin_xforce.resize(nlevs_max);
395  thin_yforce.resize(nlevs_max);
396  thin_zforce.resize(nlevs_max);
397 
398  // Base state
399  base_state.resize(nlevs_max);
400  base_state_new.resize(nlevs_max);
401 
402  // Wave coupling data
403  Hwave.resize(nlevs_max);
404  Lwave.resize(nlevs_max);
405  for (int lev = 0; lev < max_level; ++lev)
406  {
407  Hwave[lev] = nullptr;
408  Lwave[lev] = nullptr;
409  }
410  Hwave_onegrid.resize(nlevs_max);
411  Lwave_onegrid.resize(nlevs_max);
412  for (int lev = 0; lev < max_level; ++lev)
413  {
414  Hwave_onegrid[lev] = nullptr;
415  Lwave_onegrid[lev] = nullptr;
416  }
417 
418  // Theta prim for MOST
419  Theta_prim.resize(nlevs_max);
420 
421  // Qv prim for MOST
422  Qv_prim.resize(nlevs_max);
423 
424  // Qr prim for MOST
425  Qr_prim.resize(nlevs_max);
426 
427  // Time averaged velocity field
428  vel_t_avg.resize(nlevs_max);
429  t_avg_cnt.resize(nlevs_max);
430 
431  // Size lat long arrays and default to null pointers
432  lat_m.resize(nlevs_max);
433  lon_m.resize(nlevs_max);
434  for (int lev = 0; lev < max_level; ++lev) {
435  lat_m[lev] = nullptr;
436  lon_m[lev] = nullptr;
437  }
438 
439  // Variable coriolis
440  sinPhi_m.resize(nlevs_max);
441  cosPhi_m.resize(nlevs_max);
442  for (int lev = 0; lev < max_level; ++lev) {
443  sinPhi_m[lev] = nullptr;
444  cosPhi_m[lev] = nullptr;
445  }
446 
447  // Initialize tagging criteria for mesh refinement
449 
450  for (int lev = 0; lev < max_level; ++lev)
451  {
452  Print() << "Refinement ratio at level " << lev+1 << " set to be " <<
453  ref_ratio[lev][0] << " " << ref_ratio[lev][1] << " " << ref_ratio[lev][2] << std::endl;
454  }
455 
456  // We will create each of these in MakeNewLevelFromScratch
457  eb.resize(max_level+1);
458  for (int lev = 0; lev < max_level + 1; lev++){
459  eb[lev] = std::make_unique<eb_>();
460  }
461 
462  //
463  // Construct the EB data structures and store in a separate class
464  //
465  // This is needed before initializing level MultiFabs
466  if ( solverChoice.terrain_type == TerrainType::EB ||
467  solverChoice.terrain_type == TerrainType::ImmersedForcing)
468  {
469  std::string geometry ="terrain";
470  ParmParse pp("eb2");
471  pp.queryAdd("geometry", geometry);
472 
473  bool build_coarse_level_by_coarsening(false);
474  // Note this just needs to be an integer > number of V-cycles one might use
475  int max_coarsening_level = ( solverChoice.terrain_type == TerrainType::EB &&
477  solverChoice.anelastic[0] == 1) ) ? 100 : 0;
478  if (geometry == "terrain") {
479  Box terrain_bx(surroundingNodes(geom[max_level].Domain())); terrain_bx.grow(3);
480  FArrayBox terrain_fab(makeSlab(terrain_bx,2,0),1);
481  Real dummy_time = 0.0;
482  prob->init_terrain_surface(geom[max_level], terrain_fab, dummy_time);
483  TerrainIF implicit_fun(terrain_fab, geom[max_level], stretched_dz_d[max_level]);
484  auto gshop = EB2::makeShop(implicit_fun);
485  amrex::EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level, build_coarse_level_by_coarsening);
486  } else if (geometry == "box") {
487  RealArray box_lo{0.0, 0.0, 0.0};
488  RealArray box_hi{0.0, 0.0, 0.0};
489  pp.query("box_lo", box_lo);
490  pp.query("box_hi", box_hi);
491  EB2::BoxIF implicit_fun(box_lo, box_hi, false);
492  auto gshop = EB2::makeShop(implicit_fun);
493  amrex::EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level, build_coarse_level_by_coarsening);
494  } else if (geometry == "sphere") {
495  auto ProbLoArr = geom[max_level].ProbLoArray();
496  auto ProbHiArr = geom[max_level].ProbHiArray();
497  const Real xcen = 0.5 * (ProbLoArr[0] + ProbHiArr[0]);
498  const Real ycen = 0.5 * (ProbLoArr[1] + ProbHiArr[1]);
499  RealArray sphere_center = {xcen, ycen, 0.0};
500  EB2::SphereIF implicit_fun(0.5, sphere_center, false);
501  auto gshop = EB2::makeShop(implicit_fun);
502  amrex::EB2::Build(gshop, geom[max_level], max_level, max_coarsening_level, build_coarse_level_by_coarsening);
503  }
504  }
505 }
void init_zlevels(Vector< Vector< Real >> &zlevels_stag, Vector< Vector< Real >> &stretched_dz_h, Vector< Gpu::DeviceVector< Real >> &stretched_dz_d, Vector< Geometry > const &geom, Vector< IntVect > const &ref_ratio, const Real grid_stretching_ratio, const Real zsurf, const Real dz0)
Definition: ERF_InitZLevels.cpp:11
std::unique_ptr< ProblemBase > amrex_probinit(const amrex_real *problo, const amrex_real *probhi) AMREX_ATTRIBUTE_WEAK
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave_onegrid
Definition: ERF.H:925
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_yforce
Definition: ERF.H:958
void setPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:23
amrex::Vector< amrex::BoxArray > ba2d
Definition: ERF.H:1183
amrex::Vector< amrex::Vector< amrex::MultiFab > > gradp
Definition: ERF.H:785
void ReadParameters()
Definition: ERF.cpp:1869
amrex::Vector< std::unique_ptr< amrex::MultiFab > > mf_PSFC
Definition: ERF.H:1188
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_src
Definition: ERF.H:895
amrex::Vector< amrex::MultiFab > base_state_new
Definition: ERF.H:920
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az
Definition: ERF.H:893
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::iMultiFab > > > lmask_lev
Definition: ERF.H:875
amrex::Vector< std::unique_ptr< amrex::MultiFab > > terrain_blanking
Definition: ERF.H:908
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_nd_new
Definition: ERF.H:902
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_zforce
Definition: ERF.H:959
amrex::Vector< std::string > plot3d_var_names_2
Definition: ERF.H:1051
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > sst_lev
Definition: ERF.H:873
amrex::Vector< std::string > plot2d_var_names_1
Definition: ERF.H:1052
amrex::Vector< std::unique_ptr< amrex::MultiFab > > thin_xforce
Definition: ERF.H:957
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > th_bc_data
Definition: ERF.H:740
amrex::Vector< amrex::Real > t_old
Definition: ERF.H:774
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_t_rk
Definition: ERF.H:905
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave_onegrid
Definition: ERF.H:926
amrex::Vector< std::unique_ptr< ForestDrag > > m_forest_drag
Definition: ERF.H:1264
amrex::Vector< amrex::BoxArray > ba1d
Definition: ERF.H:1182
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > xvel_bc_data
Definition: ERF.H:737
int rad_datalog_int
Definition: ERF.H:857
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_src
Definition: ERF.H:897
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ay_src
Definition: ERF.H:899
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > yflux_imask
Definition: ERF.H:952
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_flux
Definition: ERF.H:839
amrex::Vector< std::string > plot3d_var_names_1
Definition: ERF.H:1050
void refinement_criteria_setup()
Definition: ERF_Tagging.cpp:223
amrex::Vector< std::string > plot2d_var_names_2
Definition: ERF.H:1053
amrex::Vector< std::unique_ptr< amrex::MultiFab > > sinPhi_m
Definition: ERF.H:728
amrex::Vector< std::unique_ptr< amrex::MultiFab > > ax_src
Definition: ERF.H:898
amrex::Vector< std::unique_ptr< amrex::MultiFab > > z_phys_cc_src
Definition: ERF.H:896
amrex::Vector< amrex::Vector< amrex::Real > > zlevels_stag
Definition: ERF.H:884
amrex::Vector< amrex::Vector< amrex::MultiFab * > > lsm_data
Definition: ERF.H:837
amrex::Vector< amrex::Vector< amrex::Real > > stretched_dz_h
Definition: ERF.H:916
amrex::Vector< std::unique_ptr< amrex::MultiFab > > az_src
Definition: ERF.H:900
static amrex::Real dt_max_initial
Definition: ERF.H:1004
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Lwave
Definition: ERF.H:924
amrex::Vector< std::unique_ptr< amrex::MultiFab > > cosPhi_m
Definition: ERF.H:728
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > zflux_imask
Definition: ERF.H:953
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > zvel_bc_data
Definition: ERF.H:739
amrex::Vector< std::unique_ptr< amrex::MultiFab > > detJ_cc_new
Definition: ERF.H:903
amrex::Vector< amrex::Gpu::DeviceVector< amrex::Real > > yvel_bc_data
Definition: ERF.H:738
amrex::Vector< std::unique_ptr< amrex::MultiFab > > Hwave
Definition: ERF.H:923
amrex::Vector< std::unique_ptr< amrex::iMultiFab > > xflux_imask
Definition: ERF.H:951
amrex::Vector< amrex::Vector< std::unique_ptr< amrex::MultiFab > > > tsk_lev
Definition: ERF.H:874
void initializeMicrophysics(const int &)
Definition: ERF.cpp:1650
void ReSize(const int &nlev)
Definition: ERF_LandSurface.H:24
Definition: ERF_EBIFTerrain.H:14
const char * buildInfoGetGitHash(int i)
amrex::Real dz0
Definition: ERF_DataStruct.H:871
amrex::Real const_massflux_layer_lo
Definition: ERF_DataStruct.H:948
amrex::Real const_massflux_v
Definition: ERF_DataStruct.H:946
int massflux_klo
Definition: ERF_DataStruct.H:950
amrex::Real grid_stretching_ratio
Definition: ERF_DataStruct.H:869
amrex::Real const_massflux_u
Definition: ERF_DataStruct.H:945
amrex::Real zsurf
Definition: ERF_DataStruct.H:870
amrex::Real const_massflux_layer_hi
Definition: ERF_DataStruct.H:949
int massflux_khi
Definition: ERF_DataStruct.H:951
Here is the call graph for this function:

◆ ErrorEst()

void ERF::ErrorEst ( int  lev,
amrex::TagBoxArray &  tags,
amrex::Real  time,
int  ngrow 
)
override

Function to tag cells for refinement – this overrides the pure virtual function in AmrCore

Parameters
[in]levclevel of refinement at which we tag cells (0 is coarsest level)
[out]tagsarray of tagged cells
[in]timecurrent time
21 {
22  const int clearval = TagBox::CLEAR;
23  const int tagval = TagBox::SET;
24 
25 
26 #ifdef ERF_USE_NETCDF
27  if (solverChoice.init_type == InitType::WRFInput) {
28  int ratio;
29  Box subdomain;
30  if (!nc_init_file[levc+1].empty()) {
31  amrex::Print() << "WRFIinput file to read: " << nc_init_file[levc+1][0] << std::endl;
32  subdomain = read_subdomain_from_wrfinput(levc, nc_init_file[levc+1][0], ratio);
33  amrex::Print() << " WRFInput subdomain at level " << levc+1 << " is " << subdomain << std::endl;
34  }
35 
36  if ( (ratio != ref_ratio[levc][0]) || (ratio != ref_ratio[levc][1]) ) {
37  amrex::Print() << "File " << nc_init_file[levc+1][0] << " has refinement ratio = " << ratio << std::endl;
38  amrex::Print() << "The inputs file has refinement ratio = " << ref_ratio[levc] << std::endl;
39  amrex::Abort("These must be the same -- please edit your inputs file and try again.");
40  }
41 
42  if ( (ref_ratio[levc][2]) != 1) {
43  amrex::Abort("The ref_ratio specified in the inputs file must have 1 in the z direction; please use ref_ratio_vect rather than ref_ratio");
44  }
45 
46  subdomain.coarsen(IntVect(ratio,ratio,1));
47 
48  // We assume there is only one subdomain at levc; otherwise we don't know
49  // which one is the parent of the fine region we are trying to create
50  AMREX_ALWAYS_ASSERT(subdomains[levc].size() == 1);
51 
52  Box coarser_level(subdomains[levc][0].minimalBox());
53  subdomain.shift(coarser_level.smallEnd());
54 
55  if (verbose > 0) {
56  amrex::Print() << " Crse subdomain to be tagged is" << subdomain << std::endl;
57  }
58 
59  Box new_fine(subdomain); new_fine.refine(IntVect(ratio,ratio,1));
60  num_boxes_at_level[levc+1] = 1;
61  boxes_at_level[levc+1].push_back(new_fine);
62 
63  for (MFIter mfi(tags); mfi.isValid(); ++mfi) {
64  auto tag_arr = tags.array(mfi); // Get device-accessible array
65 
66  Box bx = mfi.validbox(); bx &= subdomain;
67 
68  if (!bx.isEmpty()) {
69  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
70  tag_arr(i,j,k) = TagBox::SET;
71  });
72  }
73  }
74  return;
75  }
76 #endif
77 
78  //
79  // Make sure the ghost cells of the level we are tagging at are filled
80  // in case we take differences that require them
81  // NOTE: We are Fillpatching only the cell-centered variables here
82  //
83  MultiFab& S_new = vars_new[levc][Vars::cons];
84  MultiFab& U_new = vars_new[levc][Vars::xvel];
85  MultiFab& V_new = vars_new[levc][Vars::yvel];
86  MultiFab& W_new = vars_new[levc][Vars::zvel];
87  //
88  if (levc == 0) {
89  FillPatchCrseLevel(levc, time, {&S_new, &U_new, &V_new, &W_new});
90  } else {
91  FillPatchFineLevel(levc, time, {&S_new, &U_new, &V_new, &W_new},
92  {&S_new, &rU_new[levc], &rV_new[levc], &rW_new[levc]},
93  base_state[levc], base_state[levc],
94  false, true);
95  }
96 
97  for (int j=0; j < ref_tags.size(); ++j)
98  {
99  //
100  // This mf must have ghost cells because we may take differences between adjacent values
101  //
102  std::unique_ptr<MultiFab> mf = std::make_unique<MultiFab>(grids[levc], dmap[levc], 1, 1);
103 
104  // This allows dynamic refinement based on the value of the density
105  if (ref_tags[j].Field() == "density")
106  {
107  MultiFab::Copy(*mf,vars_new[levc][Vars::cons],Rho_comp,0,1,1);
108 
109  // This allows dynamic refinement based on the value of qv
110  } else if ( ref_tags[j].Field() == "qv" ) {
111  MultiFab::Copy( *mf, vars_new[levc][Vars::cons], RhoQ1_comp, 0, 1, 1);
112  MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
113 
114 
115  // This allows dynamic refinement based on the value of qc
116  } else if (ref_tags[j].Field() == "qc" ) {
117  MultiFab::Copy( *mf, vars_new[levc][Vars::cons], RhoQ2_comp, 0, 1, 1);
118  MultiFab::Divide(*mf, vars_new[levc][Vars::cons], Rho_comp, 0, 1, 1);
119 
120  // This allows dynamic refinement based on the value of the z-component of vorticity
121  } else if (ref_tags[j].Field() == "vorticity" ) {
122  Vector<MultiFab> mf_cc_vel(1);
123  mf_cc_vel[0].define(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(1,1,1));
124  average_face_to_cellcenter(mf_cc_vel[0],0,Array<const MultiFab*,3>{&U_new, &V_new, &W_new});
125 
126  // Impose bc's at domain boundaries at all levels
127  FillBdyCCVels(mf_cc_vel,levc);
128 
129  mf->setVal(0.);
130 
131  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
132  {
133  const Box& bx = mfi.tilebox();
134  auto& dfab = (*mf)[mfi];
135  auto& sfab = mf_cc_vel[0][mfi];
136  derived::erf_dervortz(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
137  }
138 
139  // This allows dynamic refinement based on the value of the scalar/theta
140  } else if ( (ref_tags[j].Field() == "scalar" ) ||
141  (ref_tags[j].Field() == "theta" ) )
142  {
143  for (MFIter mfi(*mf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
144  {
145  const Box& bx = mfi.growntilebox();
146  auto& dfab = (*mf)[mfi];
147  auto& sfab = vars_new[levc][Vars::cons][mfi];
148  if (ref_tags[j].Field() == "scalar") {
149  derived::erf_derscalar(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
150  } else if (ref_tags[j].Field() == "theta") {
151  derived::erf_dertheta(bx, dfab, 0, 1, sfab, Geom(levc), time, nullptr, levc);
152  }
153  } // mfi
154  // This allows dynamic refinement based on the value of the density
155  } else if ( (SolverChoice::terrain_type == TerrainType::ImmersedForcing) &&
156  (ref_tags[j].Field() == "terrain_blanking") )
157  {
158  MultiFab::Copy(*mf,*terrain_blanking[levc],0,0,1,1);
159  } else if (ref_tags[j].Field() == "velmag") {
160  mf->setVal(0.0);
161  ParmParse pp(pp_prefix);
162  Vector<std::string> refinement_indicators;
163  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
164  Real velmag_threshold = 1e10;
165  for (int i=0; i<refinement_indicators.size(); ++i)
166  {
167  if(refinement_indicators[i]=="hurricane_tracker"){
168  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
169  ParmParse ppr(ref_prefix);
170  ppr.get("value_greater",velmag_threshold);
171  break;
172  }
173  }
174  HurricaneTracker(levc, U_new, V_new, W_new, velmag_threshold, false, &tags);
175 #ifdef ERF_USE_PARTICLES
176  } else {
177  //
178  // This allows dynamic refinement based on the number of particles per cell
179  //
180  // Note that we must count all the particles in levels both at and above the current,
181  // since otherwise, e.g., if the particles are all at level 1, counting particles at
182  // level 0 will not trigger refinement when regridding so level 1 will disappear,
183  // then come back at the next regridding
184  //
185  const auto& particles_namelist( particleData.getNames() );
186  mf->setVal(0.0);
187  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++)
188  {
189  std::string tmp_string(particles_namelist[i]+"_count");
190  IntVect rr = IntVect::TheUnitVector();
191  if (ref_tags[j].Field() == tmp_string) {
192  for (int lev = levc; lev <= finest_level; lev++)
193  {
194  MultiFab temp_dat(grids[lev], dmap[lev], 1, 0); temp_dat.setVal(0);
195  particleData[particles_namelist[i]]->IncrementWithTotal(temp_dat, lev);
196 
197  MultiFab temp_dat_crse(grids[levc], dmap[levc], 1, 0); temp_dat_crse.setVal(0);
198 
199  if (lev == levc) {
200  MultiFab::Copy(*mf, temp_dat, 0, 0, 1, 0);
201  } else {
202  for (int d = 0; d < AMREX_SPACEDIM; d++) {
203  rr[d] *= ref_ratio[levc][d];
204  }
205  average_down(temp_dat, temp_dat_crse, 0, 1, rr);
206  MultiFab::Add(*mf, temp_dat_crse, 0, 0, 1, 0);
207  }
208  }
209  }
210  }
211 #endif
212  }
213 
214  ref_tags[j](tags,mf.get(),clearval,tagval,time,levc,geom[levc]);
215  } // loop over j
216 }
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
Definition: ERF.H:767
void FillBdyCCVels(amrex::Vector< amrex::MultiFab > &mf_cc_vel, int levc=0)
Definition: ERF_FillBdyCCVels.cpp:11
void HurricaneTracker(int lev, const amrex::MultiFab &U_new, const amrex::MultiFab &V_new, const amrex::MultiFab &W_new, const amrex::Real velmag_threshold, const bool is_track_io, amrex::TagBoxArray *tags=nullptr)
Definition: ERF_Tagging.cpp:451
void FillPatchCrseLevel(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, bool cons_only=false)
Definition: ERF_FillPatch.cpp:282
static amrex::Vector< amrex::Vector< std::string > > nc_init_file
Definition: ERF.H:1156
amrex::Vector< amrex::Vector< amrex::BoxArray > > subdomains
Definition: ERF.H:1271
static amrex::Vector< amrex::AMRErrorTag > ref_tags
Definition: ERF.H:1269
amrex::Vector< int > num_boxes_at_level
Definition: ERF.H:765
void erf_derscalar(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:165
void erf_dervortz(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:256
void erf_dertheta(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:144
real(c_double), private rr
Definition: ERF_module_mp_morr_two_moment.F90:223
Here is the call graph for this function:

◆ estTimeStep()

Real ERF::estTimeStep ( int  level,
long &  dt_fast_ratio 
) const

Function that calls estTimeStep for each level

Parameters
[in]levellevel of refinement (coarsest level i 0)
[out]dt_fast_ratioratio of slow to fast time step
55 {
56  BL_PROFILE("ERF::estTimeStep()");
57 
58  Real estdt_comp = 1.e20;
59  Real estdt_lowM = 1.e20;
60 
61  // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation
62  const int nxc = geom[0].Domain().length(0);
63  const int nyc = geom[0].Domain().length(1);
64 
65  auto const dxinv = geom[level].InvCellSizeArray();
66  auto const dzinv = 1.0 / dz_min[level];
67 
68  MultiFab const& S_new = vars_new[level][Vars::cons];
69 
70  MultiFab ccvel(grids[level],dmap[level],3,0);
71 
72  average_face_to_cellcenter(ccvel,0,
73  Array<const MultiFab*,3>{&vars_new[level][Vars::xvel],
74  &vars_new[level][Vars::yvel],
75  &vars_new[level][Vars::zvel]});
76 
77  int l_implicit_substepping = (solverChoice.substepping_type[level] == SubsteppingType::Implicit);
78  int l_anelastic = solverChoice.anelastic[level];
79 
80  Real estdt_comp_inv;
81 
82  if (solverChoice.terrain_type == TerrainType::EB)
83  {
84  const eb_& eb_lev = get_eb(level);
85  const MultiFab& detJ = (eb_lev.get_const_factory())->getVolFrac();
86 
87  estdt_comp_inv = ReduceMax(S_new, ccvel, detJ, 0,
88  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
89  Array4<Real const> const& s,
90  Array4<Real const> const& u,
91  Array4<Real const> const& vf) -> Real
92  {
93  Real new_comp_dt = -1.e100;
94  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
95  {
96  if (vf(i,j,k) > 0.)
97  {
98  const Real rho = s(i, j, k, Rho_comp);
99  const Real rhotheta = s(i, j, k, RhoTheta_comp);
100 
101  // NOTE: even when moisture is present,
102  // we only use the partial pressure of the dry air
103  // to compute the soundspeed
104  Real pressure = getPgivenRTh(rhotheta);
105  Real c = std::sqrt(Gamma * pressure / rho);
106 
107  // If we are doing implicit acoustic substepping, then the z-direction does not contribute
108  // to the computation of the time step
109  if (l_implicit_substepping) {
110  if ((nxc > 1) && (nyc==1)) {
111  // 2-D in x-z
112  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
113  } else if ((nyc > 1) && (nxc==1)) {
114  // 2-D in y-z
115  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
116  } else {
117  // 3-D or SCM
118  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
119  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
120  }
121 
122  // If we are not doing implicit acoustic substepping, then the z-direction contributes
123  // to the computation of the time step
124  } else {
125  if (nxc > 1 && nyc > 1) {
126  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
127  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
128  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
129  } else if (nxc > 1) {
130  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
131  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
132  } else if (nyc > 1) {
133  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
134  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
135  } else {
136  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
137  }
138 
139  }
140  }
141  });
142  return new_comp_dt;
143  });
144 
145  } else {
146  estdt_comp_inv = ReduceMax(S_new, ccvel, 0,
147  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
148  Array4<Real const> const& s,
149  Array4<Real const> const& u) -> Real
150  {
151  Real new_comp_dt = -1.e100;
152  amrex::Loop(b, [=,&new_comp_dt] (int i, int j, int k) noexcept
153  {
154  {
155  const Real rho = s(i, j, k, Rho_comp);
156  const Real rhotheta = s(i, j, k, RhoTheta_comp);
157 
158  // NOTE: even when moisture is present,
159  // we only use the partial pressure of the dry air
160  // to compute the soundspeed
161  Real pressure = getPgivenRTh(rhotheta);
162  Real c = std::sqrt(Gamma * pressure / rho);
163 
164  // If we are doing implicit acoustic substepping, then the z-direction does not contribute
165  // to the computation of the time step
166  if (l_implicit_substepping) {
167  if ((nxc > 1) && (nyc==1)) {
168  // 2-D in x-z
169  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt);
170  } else if ((nyc > 1) && (nxc==1)) {
171  // 2-D in y-z
172  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
173  } else {
174  // 3-D or SCM
175  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
176  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt);
177  }
178 
179  // If we are not doing implicit acoustic substepping, then the z-direction contributes
180  // to the computation of the time step
181  } else {
182  if (nxc > 1 && nyc > 1) {
183  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
184  ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
185  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
186  } else if (nxc > 1) {
187  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]),
188  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
189  } else if (nyc > 1) {
190  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]),
191  ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
192  } else {
193  new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt);
194  }
195 
196  }
197  }
198  });
199  return new_comp_dt;
200  });
201  } // not EB
202 
203  ParallelDescriptor::ReduceRealMax(estdt_comp_inv);
204  estdt_comp = cfl / estdt_comp_inv;
205 
206  Real estdt_lowM_inv = ReduceMax(ccvel, 0,
207  [=] AMREX_GPU_HOST_DEVICE (Box const& b,
208  Array4<Real const> const& u) -> Real
209  {
210  Real new_lm_dt = -1.e100;
211  Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
212  {
213  new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
214  ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
215  ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
216  });
217  return new_lm_dt;
218  });
219 
220  ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
221  if (estdt_lowM_inv > 0.0_rt)
222  estdt_lowM = cfl / estdt_lowM_inv;
223 
224  if (verbose) {
225  if (fixed_dt[level] <= 0.0) {
226  Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " <<
227  1.0/dxinv[0] << " " << 1.0/dxinv[1] << " " << dz_min[level] << std::endl;
228  Print() << "Compressible dt at level " << level << ": " << estdt_comp << std::endl;
229  if (estdt_lowM_inv > 0.0_rt) {
230  Print() << "Anelastic dt at level " << level << ": " << estdt_lowM << std::endl;
231  } else {
232  Print() << "Anelastic dt at level " << level << ": undefined " << std::endl;
233  }
234  }
235 
236  if (fixed_dt[level] > 0.0) {
237  Print() << "Based on cfl of 1.0 " << std::endl;
238  Print() << "Compressible dt at level " << level << " would be: " << estdt_comp/cfl << std::endl;
239  if (estdt_lowM_inv > 0.0_rt) {
240  Print() << "Anelastic dt at level " << level << " would be: " << estdt_lowM/cfl << std::endl;
241  } else {
242  Print() << "Anelastic dt at level " << level << " would be undefined " << std::endl;
243  }
244  Print() << "Fixed dt at level " << level << " is: " << fixed_dt[level] << std::endl;
245  if (fixed_fast_dt[level] > 0.0) {
246  Print() << "Fixed fast dt at level " << level << " is: " << fixed_fast_dt[level] << std::endl;
247  }
248  }
249  }
250 
251  if (solverChoice.substepping_type[level] != SubsteppingType::None) {
252  if (fixed_dt[level] > 0. && fixed_fast_dt[level] > 0.) {
253  dt_fast_ratio = static_cast<long>( fixed_dt[level] / fixed_fast_dt[level] );
254  } else if (fixed_dt[level] > 0.) {
255  // Max CFL_c = 1.0 for substeps by default, but we enforce a min of 4 substeps
256  auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
257  dt_fast_ratio = static_cast<long>( std::max(fixed_dt[level]/dt_sub_max,4.) );
258  } else {
259  // auto dt_sub_max = (estdt_comp/cfl * sub_cfl);
260  // dt_fast_ratio = static_cast<long>( std::max(estdt_comp/dt_sub_max,4.) );
261  dt_fast_ratio = static_cast<long>( std::max(cfl / sub_cfl, 4.) );
262  }
263 
264  // Force time step ratio to be an even value
266  if ( dt_fast_ratio%2 != 0) dt_fast_ratio += 1;
267  } else {
268  if ( dt_fast_ratio%6 != 0) {
269  Print() << "mri_dt_ratio = " << dt_fast_ratio
270  << " not divisible by 6 for N/3 substeps in stage 1" << std::endl;
271  dt_fast_ratio = static_cast<int>(std::ceil(dt_fast_ratio/6.0) * 6);
272  }
273  }
274 
275  if (verbose) {
276  Print() << "smallest even ratio is: " << dt_fast_ratio << std::endl;
277  }
278  } // if substepping, either explicit or implicit
279 
280  if (fixed_dt[level] > 0.0) {
281  return fixed_dt[level];
282  } else {
283  // Anelastic (substepping is not allowed)
284  if (l_anelastic) {
285 
286  // Make sure that timestep is less than the dt_max
287  estdt_lowM = amrex::min(estdt_lowM, dt_max);
288 
289  // On the first timestep enforce dt_max_initial
290  if(istep[level] == 0){
291  return amrex::min(dt_max_initial, estdt_lowM);
292  }
293  else{
294  return estdt_lowM;
295  }
296 
297 
298  // Compressible with or without substepping
299  } else {
300  return estdt_comp;
301  }
302  }
303 }
constexpr amrex::Real Gamma
Definition: ERF_Constants.H:19
amrex::Vector< amrex::Real > dz_min
Definition: ERF.H:1279
amrex::Vector< amrex::Real > fixed_dt
Definition: ERF.H:1008
static amrex::Real dt_max
Definition: ERF.H:1005
amrex::Vector< amrex::Real > fixed_fast_dt
Definition: ERF.H:1009
static amrex::Real cfl
Definition: ERF.H:1000
static amrex::Real sub_cfl
Definition: ERF.H:1001
Definition: ERF_EB.H:13
@ rho
Definition: ERF_Kessler.H:22
int force_stage1_single_substep
Definition: ERF_DataStruct.H:828
Here is the call graph for this function:

◆ Evolve()

void ERF::Evolve ( )
512 {
513  BL_PROFILE_VAR("ERF::Evolve()", evolve);
514 
515  Real cur_time = t_new[0];
516 
517  // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
518  // for finer levels (with or without subcycling)
519  for (int step = istep[0]; step < max_step && start_time+cur_time < stop_time; ++step)
520  {
521  if (use_datetime) {
522  Print() << "\n" << getTimestamp(start_time+cur_time, datetime_format)
523  << " (" << cur_time << " s elapsed)" << std::endl;
524  }
525  Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
526 
527  ComputeDt(step);
528 
529  // Make sure we have read enough of the boundary plane data to make it through this timestep
530  if (input_bndry_planes)
531  {
532  m_r2d->read_input_files(cur_time,dt[0],m_bc_extdir_vals);
533  }
534 
535 #ifdef ERF_USE_PARTICLES
536  // We call this every time step with the knowledge that the particles may be
537  // initialized at a later time than the simulation start time.
538  // The ParticleContainer carries a "start time" so the initialization will happen
539  // only when a) time > start_time, and b) particles have not yet been initialized
540  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,cur_time);
541 #endif
542 
543  int lev = 0;
544  int iteration = 1;
545  timeStep(lev, cur_time, iteration);
546 
547  cur_time += dt[0];
548 
549  Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
550  << " DT = " << dt[0] << std::endl;
551 
552  post_timestep(step, cur_time, dt[0]);
553 
554  if (writeNow(cur_time, step+1, m_plot3d_int_1, m_plot3d_per_1, dt[0], last_plot3d_file_time_1)) {
555  last_plot3d_file_step_1 = step+1;
558  }
559  if (writeNow(cur_time, step+1, m_plot3d_int_2, m_plot3d_per_2, dt[0], last_plot3d_file_time_2)) {
560  last_plot3d_file_step_2 = step+1;
563  }
564 
565  if (writeNow(cur_time, step+1, m_plot2d_int_1, m_plot2d_per_1, dt[0], last_plot2d_file_time_1)) {
566  last_plot2d_file_step_1 = step+1;
569  }
570 
571  if (writeNow(cur_time, step+1, m_plot2d_int_2, m_plot2d_per_2, dt[0], last_plot2d_file_time_2)) {
572  last_plot2d_file_step_2 = step+1;
575  }
576 
577  if (writeNow(cur_time, step+1, m_subvol_int, m_subvol_per, dt[0], last_subvol_time)) {
578  last_subvol_step = step+1;
579  WriteSubvolume();
581  }
582 
583  if (writeNow(cur_time, step+1, m_check_int, m_check_per, dt[0], last_check_file_time)) {
584  last_check_file_step = step+1;
587  }
588 
589 #ifdef AMREX_MEM_PROFILING
590  {
591  std::ostringstream ss;
592  ss << "[STEP " << step+1 << "]";
593  MemProfiler::report(ss.str());
594  }
595 #endif
596 
597  if (cur_time >= stop_time - 1.e-6*dt[0]) break;
598  }
599 
600  // Write plotfiles at final time
601  if ( (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.) && istep[0] > last_plot3d_file_step_1 ) {
604  }
605  if ( (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.) && istep[0] > last_plot3d_file_step_2) {
608  }
609  if ( (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.) && istep[0] > last_plot2d_file_step_1 ) {
612  }
613  if ( (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.) && istep[0] > last_plot2d_file_step_2) {
616  }
617  if ( (m_subvol_int > 0 || m_subvol_per > 0.) && istep[0] > last_subvol_step) {
618  WriteSubvolume();
620  }
621 
622  if ( (m_check_int > 0 || m_check_per > 0.) && istep[0] > last_check_file_step) {
625  }
626 
627  BL_PROFILE_VAR_STOP(evolve);
628 }
AMREX_FORCE_INLINE std::string getTimestamp(const amrex::Real epoch_real, const std::string &datetime_format)
Definition: ERF_EpochTime.H:72
static int last_check_file_step
Definition: ERF.H:965
static int last_subvol_step
Definition: ERF.H:966
int max_step
Definition: ERF.H:987
static amrex::Real last_plot2d_file_time_2
Definition: ERF.H:971
amrex::Real m_plot2d_per_1
Definition: ERF.H:1034
static amrex::Real last_plot2d_file_time_1
Definition: ERF.H:970
static int last_plot2d_file_step_2
Definition: ERF.H:964
int m_subvol_int
Definition: ERF.H:1031
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_extdir_vals
Definition: ERF.H:942
void WriteSubvolume()
Definition: ERF_WriteSubvolume.cpp:9
static amrex::Real last_plot3d_file_time_2
Definition: ERF.H:969
int m_plot2d_int_2
Definition: ERF.H:1030
int m_plot3d_int_1
Definition: ERF.H:1027
static int last_plot3d_file_step_2
Definition: ERF.H:962
void post_timestep(int nstep, amrex::Real time, amrex::Real dt_lev)
Definition: ERF.cpp:632
amrex::Real m_subvol_per
Definition: ERF.H:1036
amrex::Real m_plot2d_per_2
Definition: ERF.H:1035
amrex::Real m_check_per
Definition: ERF.H:1048
int m_check_int
Definition: ERF.H:1047
static int input_bndry_planes
Definition: ERF.H:1205
static amrex::Real last_subvol_time
Definition: ERF.H:973
void Write2DPlotFile(int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:1859
const std::string datetime_format
Definition: ERF.H:994
bool use_datetime
Definition: ERF.H:993
void ComputeDt(int step=-1)
Definition: ERF_ComputeTimestep.cpp:11
amrex::Real m_plot3d_per_2
Definition: ERF.H:1033
static PlotFileType plotfile3d_type_2
Definition: ERF.H:1146
static PlotFileType plotfile2d_type_2
Definition: ERF.H:1148
bool writeNow(const amrex::Real cur_time, const int nstep, const int plot_int, const amrex::Real plot_per, const amrex::Real dt_0, amrex::Real &last_file_time)
Definition: ERF.cpp:2517
int m_plot2d_int_1
Definition: ERF.H:1029
void WriteCheckpointFile() const
Definition: ERF_Checkpoint.cpp:26
void Write3DPlotFile(int which, PlotFileType plotfile_type, amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:263
static int last_plot2d_file_step_1
Definition: ERF.H:963
amrex::Real m_plot3d_per_1
Definition: ERF.H:1032
std::unique_ptr< ReadBndryPlanes > m_r2d
Definition: ERF.H:1262
static amrex::Real last_check_file_time
Definition: ERF.H:972
static int last_plot3d_file_step_1
Definition: ERF.H:961
static amrex::Real last_plot3d_file_time_1
Definition: ERF.H:968
static PlotFileType plotfile2d_type_1
Definition: ERF.H:1147
static PlotFileType plotfile3d_type_1
Definition: ERF.H:1145
int m_plot3d_int_2
Definition: ERF.H:1028
void timeStep(int lev, amrex::Real time, int iteration)
Definition: ERF_TimeStep.cpp:17

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ fill_from_bndryregs()

void ERF::fill_from_bndryregs ( const amrex::Vector< amrex::MultiFab * > &  mfs,
amrex::Real  time 
)
14 {
15  //
16  // We now assume that if we read in on one face, we read in on all faces
17  //
18  AMREX_ALWAYS_ASSERT(m_r2d);
19 
20  int lev = 0;
21  const Box& domain = geom[lev].Domain();
22 
23  const auto& dom_lo = lbound(domain);
24  const auto& dom_hi = ubound(domain);
25 
26  Vector<std::unique_ptr<PlaneVector>>& bndry_data = m_r2d->interp_in_time(time);
27 
28  const BCRec* bc_ptr = domain_bcs_type_d.data();
29 
30  // xlo: ori = 0
31  // ylo: ori = 1
32  // zlo: ori = 2
33  // xhi: ori = 3
34  // yhi: ori = 4
35  // zhi: ori = 5
36  const auto& bdatxlo = (*bndry_data[0])[lev].const_array();
37  const auto& bdatylo = (*bndry_data[1])[lev].const_array();
38  const auto& bdatxhi = (*bndry_data[3])[lev].const_array();
39  const auto& bdatyhi = (*bndry_data[4])[lev].const_array();
40 
41  int bccomp;
42 
43  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx)
44  {
45  MultiFab& mf = *mfs[var_idx];
46  const int icomp = 0;
47  const int ncomp = mf.nComp();
48 
49  if (var_idx == Vars::xvel) {
50  bccomp = BCVars::xvel_bc;
51  } else if (var_idx == Vars::yvel) {
52  bccomp = BCVars::yvel_bc;
53  } else if (var_idx == Vars::zvel) {
54  bccomp = BCVars::zvel_bc;
55  } else if (var_idx == Vars::cons) {
56  bccomp = BCVars::cons_bc;
57  }
58 
59 #ifdef AMREX_USE_OMP
60 #pragma omp parallel if (Gpu::notInLaunchRegion())
61 #endif
62  for (MFIter mfi(mf); mfi.isValid(); ++mfi)
63  {
64  const Array4<Real>& dest_arr = mf.array(mfi);
65  Box bx = mfi.growntilebox();
66 
67  // x-faces
68  {
69  Box bx_xlo(bx); bx_xlo.setBig(0,dom_lo.x-1);
70  if (var_idx == Vars::xvel) bx_xlo.setBig(0,dom_lo.x);
71 
72  Box bx_xhi(bx); bx_xhi.setSmall(0,dom_hi.x+1);
73  if (var_idx == Vars::xvel) bx_xhi.setSmall(0,dom_hi.x);
74 
75  ParallelFor(
76  bx_xlo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
77  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
78  BCVars::RhoScalar_bc_comp : icomp+n;
79  if (bc_ptr[bc_comp].lo(0) == ERFBCType::ext_dir_ingested) {
80  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
81  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
82  dest_arr(i,j,k,icomp+n) = bdatxlo(dom_lo.x-1,jb,kb,bccomp+n);
83  }
84  },
85  bx_xhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
86  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
87  BCVars::RhoScalar_bc_comp : icomp+n;
88  if (bc_ptr[bc_comp].hi(0) == ERFBCType::ext_dir_ingested) {
89  int jb = std::min(std::max(j,dom_lo.y),dom_hi.y);
90  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
91  dest_arr(i,j,k,icomp+n) = bdatxhi(dom_hi.x+1,jb,kb,bccomp+n);
92  }
93  }
94  );
95  } // x-faces
96 
97  // y-faces
98  {
99  Box bx_ylo(bx); bx_ylo.setBig (1,dom_lo.y-1);
100  if (var_idx == Vars::yvel) bx_ylo.setBig(1,dom_lo.y);
101 
102  Box bx_yhi(bx); bx_yhi.setSmall(1,dom_hi.y+1);
103  if (var_idx == Vars::yvel) bx_yhi.setSmall(1,dom_hi.y);
104 
105  ParallelFor(
106  bx_ylo, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
107  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
108  BCVars::RhoScalar_bc_comp : icomp+n;
109  if (bc_ptr[bc_comp].lo(1) == ERFBCType::ext_dir_ingested) {
110  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
111  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
112  dest_arr(i,j,k,icomp+n) = bdatylo(ib,dom_lo.y-1,kb,bccomp+n);
113  }
114  },
115  bx_yhi, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) {
116  int bc_comp = (icomp+n >= RhoScalar_comp && icomp+n < RhoScalar_comp+NSCALARS) ?
117  BCVars::RhoScalar_bc_comp : icomp+n;
118  if (bc_ptr[bc_comp].hi(1) == ERFBCType::ext_dir_ingested) {
119  int ib = std::min(std::max(i,dom_lo.x),dom_hi.x);
120  int kb = std::min(std::max(k,dom_lo.z),dom_hi.z);
121  dest_arr(i,j,k,icomp+n) = bdatyhi(ib,dom_hi.y+1,kb,bccomp+n);
122  }
123  }
124  );
125  } // y-faces
126  } // mf
127  } // var_idx
128 }
#define RhoScalar_comp
Definition: ERF_IndexDefines.H:40
#define NSCALARS
Definition: ERF_IndexDefines.H:16
amrex::Gpu::DeviceVector< amrex::BCRec > domain_bcs_type_d
Definition: ERF.H:936
@ RhoScalar_bc_comp
Definition: ERF_IndexDefines.H:80
@ ext_dir_ingested
Definition: ERF_IndexDefines.H:212

◆ fill_rhs()

void ERF::fill_rhs ( amrex::MultiFab &  rhs_mf,
const amrex::MultiFab &  state_mf,
amrex::Real  time,
const amrex::Geometry &  geom 
)
private

◆ FillBdyCCVels()

void ERF::FillBdyCCVels ( amrex::Vector< amrex::MultiFab > &  mf_cc_vel,
int  levc = 0 
)
12 {
13  // Impose bc's at domain boundaries
14  for (int ilev(0); ilev < mf_cc_vel.size(); ++ilev)
15  {
16  int lev = ilev + levc;
17  Box domain(Geom(lev).Domain());
18 
19  int ihi = domain.bigEnd(0);
20  int jhi = domain.bigEnd(1);
21  int khi = domain.bigEnd(2);
22 
23  // Impose periodicity first
24  mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
25 
26  int jper = (Geom(lev).isPeriodic(1));
27  int kper = (Geom(lev).isPeriodic(2));
28 
29  for (MFIter mfi(mf_cc_vel[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
30  {
31  const Box& bx = mfi.tilebox();
32  const Array4<Real>& vel_arr = mf_cc_vel[lev].array(mfi);
33 
34  if (!Geom(lev).isPeriodic(0)) {
35  // Low-x side
36  if (bx.smallEnd(0) <= domain.smallEnd(0)) {
37  Real multn = ( (phys_bc_type[0] == ERF_BC::slip_wall ) ||
39  (phys_bc_type[0] == ERF_BC::symmetry ) ) ? -1. : 1.;
40  Real multt = (phys_bc_type[0] == ERF_BC::no_slip_wall) ? -1. : 1.;
41  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
42  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
43  {
44  vel_arr(-1,j,k,0) = multn*vel_arr(0,j,k,0); // u
45  vel_arr(-1,j,k,1) = multt*vel_arr(0,j,k,1); // v
46  vel_arr(-1,j,k,2) = multt*vel_arr(0,j,k,2); // w
47  });
48  }
49 
50  // High-x side
51  if (bx.bigEnd(0) >= domain.bigEnd(0)) {
52  Real multn = ( (phys_bc_type[3] == ERF_BC::slip_wall ) ||
54  (phys_bc_type[3] == ERF_BC::symmetry ) ) ? -1. : 1.;
55  Real multt = (phys_bc_type[3] == ERF_BC::no_slip_wall) ? -1. : 1.;
56  Box gbx(bx); gbx.grow(1,jper); gbx.grow(2,kper);
57  ParallelFor(makeSlab(gbx,0,0), [=] AMREX_GPU_DEVICE(int , int j, int k) noexcept
58  {
59  vel_arr(ihi+1,j,k,0) = multn*vel_arr(ihi,j,k,0); // u
60  vel_arr(ihi+1,j,k,1) = multt*vel_arr(ihi,j,k,1); // v
61  vel_arr(ihi+1,j,k,2) = multt*vel_arr(ihi,j,k,2); // w
62  });
63  }
64  } // !periodic
65 
66  if (!Geom(lev).isPeriodic(1)) {
67  // Low-y side
68  if (bx.smallEnd(1) <= domain.smallEnd(1)) {
69  Real multn = ( (phys_bc_type[1] == ERF_BC::slip_wall ) ||
71  (phys_bc_type[1] == ERF_BC::symmetry ) ) ? -1. : 1.;
72  Real multt = (phys_bc_type[1] == ERF_BC::no_slip_wall) ? -1. : 1.;
73  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
74  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
75  {
76  vel_arr(i,-1,k,0) = multt*vel_arr(i,0,k,0); // u
77  vel_arr(i,-1,k,1) = multn*vel_arr(i,0,k,1); // u
78  vel_arr(i,-1,k,2) = multt*vel_arr(i,0,k,2); // w
79  });
80  }
81 
82  // High-y side
83  if (bx.bigEnd(1) >= domain.bigEnd(1)) {
84  Real multn = ( (phys_bc_type[4] == ERF_BC::slip_wall ) ||
86  (phys_bc_type[4] == ERF_BC::symmetry ) ) ? -1. : 1.;
87  Real multt = (phys_bc_type[4] == ERF_BC::no_slip_wall) ? -1. : 1.;
88  Box gbx(bx); gbx.grow(0,1); gbx.grow(2,kper);
89  ParallelFor(makeSlab(gbx,1,0), [=] AMREX_GPU_DEVICE(int i, int , int k) noexcept
90  {
91  vel_arr(i,jhi+1,k,0) = multt*vel_arr(i,jhi,k,0); // u
92  vel_arr(i,jhi+1,k,1) = multn*vel_arr(i,jhi,k,1); // v
93  vel_arr(i,jhi+1,k,2) = multt*vel_arr(i,jhi,k,2); // w
94  });
95  }
96  } // !periodic
97 
98  if (!Geom(lev).isPeriodic(2)) {
99  // Low-z side
100  if (bx.smallEnd(2) <= domain.smallEnd(2)) {
101  Real multn = ( (phys_bc_type[2] == ERF_BC::slip_wall ) ||
103  (phys_bc_type[2] == ERF_BC::symmetry ) ) ? -1. : 1.;
104  Real multt = (phys_bc_type[2] == ERF_BC::no_slip_wall) ? -1. : 1.;
105  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
106  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
107  {
108  vel_arr(i,j,-1,0) = multt*vel_arr(i,j,0,0); // u
109  vel_arr(i,j,-1,1) = multt*vel_arr(i,j,0,1); // v
110  vel_arr(i,j,-1,2) = multn*vel_arr(i,j,0,2); // w
111  });
112  }
113 
114  // High-z side
115  if (bx.bigEnd(2) >= domain.bigEnd(2)) {
116  Real multn = ( (phys_bc_type[5] == ERF_BC::slip_wall ) ||
118  (phys_bc_type[5] == ERF_BC::symmetry ) ) ? -1. : 1.;
119  Real multt = (phys_bc_type[5] == ERF_BC::no_slip_wall) ? -1. : 1.;
120  Box gbx(bx); gbx.grow(0,1); gbx.grow(1,1);
121  ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept
122  {
123  vel_arr(i,j,khi+1,0) = multt*vel_arr(i,j,khi,0); // u
124  vel_arr(i,j,khi+1,1) = multt*vel_arr(i,j,khi,1); // v
125  vel_arr(i,j,khi+1,2) = multn*vel_arr(i,j,khi,2); // w
126  });
127  }
128  } // !periodic
129  } // MFIter
130 
131  // Impose periodicity again
132  mf_cc_vel[lev].FillBoundary(geom[lev].periodicity());
133  } // lev
134 }
@ no_slip_wall

◆ FillCoarsePatch()

void ERF::FillCoarsePatch ( int  lev,
amrex::Real  time 
)
private
22 {
23  BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
24  AMREX_ASSERT(lev > 0);
25 
26  //
27  //****************************************************************************************************************
28  // First fill velocities and density at the COARSE level so we can convert velocity to momenta at the COARSE level
29  //****************************************************************************************************************
30  //
31  bool cons_only = false;
32  if (lev == 1) {
33  FillPatchCrseLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
34  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
35  cons_only);
36  } else {
37  FillPatchFineLevel(lev-1, time, {&vars_new[lev-1][Vars::cons], &vars_new[lev-1][Vars::xvel],
38  &vars_new[lev-1][Vars::yvel], &vars_new[lev-1][Vars::zvel]},
39  {&vars_new[lev-1][Vars::cons],
40  &rU_new[lev-1], &rV_new[lev-1], &rW_new[lev-1]},
41  base_state[lev-1], base_state[lev-1],
42  false, cons_only);
43  }
44 
45  //
46  // ************************************************
47  // Convert velocity to momentum at the COARSE level
48  // ************************************************
49  //
50  VelocityToMomentum(vars_new[lev-1][Vars::xvel], IntVect{0},
51  vars_new[lev-1][Vars::yvel], IntVect{0},
52  vars_new[lev-1][Vars::zvel], IntVect{0},
53  vars_new[lev-1][Vars::cons],
54  rU_new[lev-1],
55  rV_new[lev-1],
56  rW_new[lev-1],
57  Geom(lev).Domain(),
59  //
60  // *****************************************************************
61  // Interpolate all cell-centered variables from coarse to fine level
62  // *****************************************************************
63  //
64  Interpolater* mapper_c = &cell_cons_interp;
65  Interpolater* mapper_f = &face_cons_linear_interp;
66 
67  //
68  //************************************************************************************************
69  // Interpolate cell-centered data from coarse to fine level
70  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
71  // ************************************************************************************************
72  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
73  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
74 
75  InterpFromCoarseLevel(vars_new[lev ][Vars::cons], ngvect_cons, IntVect(0,0,0),
76  vars_new[lev-1][Vars::cons], 0, 0, ncomp_cons,
77  geom[lev-1], geom[lev],
78  refRatio(lev-1), mapper_c, domain_bcs_type, BCVars::cons_bc);
79 
80  // ***************************************************************************
81  // Physical bc's for cell centered variables at domain boundary
82  // ***************************************************************************
84  0,ncomp_cons,ngvect_cons,time,BCVars::cons_bc,true);
85 
86  //
87  //************************************************************************************************
88  // Interpolate x-momentum from coarse to fine level
89  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
90  // ************************************************************************************************
91  //
92  InterpFromCoarseLevel(rU_new[lev], IntVect{0}, IntVect{0}, rU_new[lev-1], 0, 0, 1,
93  geom[lev-1], geom[lev],
94  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::xvel_bc);
95 
96  //
97  //************************************************************************************************
98  // Interpolate y-momentum from coarse to fine level
99  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
100  // ************************************************************************************************
101  //
102  InterpFromCoarseLevel(rV_new[lev], IntVect{0}, IntVect{0}, rV_new[lev-1], 0, 0, 1,
103  geom[lev-1], geom[lev],
104  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::yvel_bc);
105 
106  //************************************************************************************************
107  // Interpolate z-momentum from coarse to fine level
108  // with InterpFromCoarseLevel which ASSUMES that all ghost cells at lev-1 have already been filled
109  // ************************************************************************************************
110  InterpFromCoarseLevel(rW_new[lev], IntVect{0}, IntVect{0}, rW_new[lev-1], 0, 0, 1,
111  geom[lev-1], geom[lev],
112  refRatio(lev-1), mapper_f, domain_bcs_type, BCVars::zvel_bc);
113  //
114  // *********************************************************
115  // After interpolation of momentum, convert back to velocity
116  // *********************************************************
117  //
118  for (int which_lev = lev-1; which_lev <= lev; which_lev++)
119  {
121  vars_new[which_lev][Vars::yvel],
122  vars_new[which_lev][Vars::zvel],
123  vars_new[which_lev][Vars::cons],
124  rU_new[which_lev],
125  rV_new[which_lev],
126  rW_new[which_lev],
127  Geom(lev).Domain(),
129  }
130 
131  // ***************************************************************************
132  // Physical bc's at domain boundary
133  // ***************************************************************************
134  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
135 
137  ngvect_vels,time,BCVars::xvel_bc,true);
139  ngvect_vels,time,BCVars::yvel_bc,true);
141  ngvect_vels,time,BCVars::zvel_bc,true);
142 
143  // ***************************************************************************
144  // Since lev > 0 here we don't worry about m_r2d or wrfbdy data
145  // ***************************************************************************
146 }
void FillCoarsePatch(int lev, amrex::Real time)
Definition: ERF_FillCoarsePatch.cpp:21
Here is the call graph for this function:

◆ FillIntermediatePatch()

void ERF::FillIntermediatePatch ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
int  ng_cons,
int  ng_vel,
bool  cons_only,
int  icomp_cons,
int  ncomp_cons 
)
private
33 {
34  BL_PROFILE_VAR("FillIntermediatePatch()",FillIntermediatePatch);
35  Interpolater* mapper;
36 
37  PhysBCFunctNoOp null_bc;
38 
39  //
40  // ***************************************************************************
41  // The first thing we do is interpolate the momenta on the "valid" faces of
42  // the fine grids (where the interface is coarse/fine not fine/fine) -- this
43  // will not be over-written by interpolation below because the FillPatch
44  // operators see these as valid faces. But we must have these interpolated
45  // values in the fine data before we call FillPatchTwoLevels.
46  //
47  // Also -- note that we might be filling values by interpolation at physical boundaries
48  // here but that's ok because we will overwrite those values when we impose
49  // the physical bc's below
50  // ***************************************************************************
51  if (lev>0) {
52  if (cf_set_width > 0) {
53  // We note that mfs_vel[Vars::cons] and mfs_mom[Vars::cons] are in fact the same pointer
54  FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
55  }
56  if ( !cons_only && (cf_set_width >= 0) ) {
57  FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
58  FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
59  FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
60  }
61  }
62 
63  // amrex::Print() << "LEVEL " << lev << " CONS ONLY " << cons_only <<
64  // " ICOMP NCOMP " << icomp_cons << " " << ncomp_cons << " NGHOST " << ng_cons << std::endl;
65 
66  if (!cons_only) {
67  AMREX_ALWAYS_ASSERT(mfs_mom.size() == IntVars::NumTypes);
68  AMREX_ALWAYS_ASSERT(mfs_vel.size() == Vars::NumTypes);
69  }
70 
71  // Enforce no penetration for thin immersed body
72  if (!cons_only) {
73  // Enforce no penetration for thin immersed body
74  if (xflux_imask[lev]) {
75  ApplyMask(*mfs_mom[IntVars::xmom], *xflux_imask[lev]);
76  }
77  if (yflux_imask[lev]) {
78  ApplyMask(*mfs_mom[IntVars::ymom], *yflux_imask[lev]);
79  }
80  if (zflux_imask[lev]) {
81  ApplyMask(*mfs_mom[IntVars::zmom], *zflux_imask[lev]);
82  }
83  }
84 
85  //
86  // We now start working on conserved quantities + VELOCITY
87  //
88  if (lev == 0)
89  {
90  // We don't do anything here because we will call the physbcs routines below,
91  // which calls FillBoundary and fills other domain boundary conditions
92  // Physical boundaries will be filled below
93 
94  if (!cons_only)
95  {
96  // ***************************************************************************
97  // We always come in to this call with updated momenta but we need to create updated velocity
98  // in order to impose the rest of the bc's
99  // ***************************************************************************
100  // This only fills VALID region of velocity
101  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
102  *mfs_vel[Vars::cons],
103  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
104  Geom(lev).Domain(), domain_bcs_type);
105  }
106  }
107  else
108  {
109  //
110  // We must fill a temporary then copy it back so we don't double add/subtract
111  //
112  MultiFab mf(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
113  mfs_vel[Vars::cons]->nComp() ,mfs_vel[Vars::cons]->nGrowVect());
114  //
115  // Set all components to 1.789e19, then copy just the density from *mfs_vel[Vars::cons]
116  //
117  mf.setVal(1.789e19);
118  MultiFab::Copy(mf,*mfs_vel[Vars::cons],Rho_comp,Rho_comp,1,mf.nGrowVect());
119 
120  Vector<MultiFab*> fmf = {mfs_vel[Vars::cons],mfs_vel[Vars::cons]};
121  Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
122  Vector<Real> ctime = {t_old[lev-1], t_new[lev-1]};
123  Vector<Real> ftime = {time,time};
124 
125  if (interpolation_type == StateInterpType::Perturbational)
126  {
127  if (icomp_cons+ncomp_cons > 1)
128  {
129  // Divide (rho theta) by rho to get theta
130  MultiFab::Divide(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
131 
132  // Subtract theta_0 from theta
133  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
134 
135  if (!amrex::almostEqual(time,ctime[1])) {
136  MultiFab::Divide(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
137  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
138  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
139  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
140  }
141  if (!amrex::almostEqual(time,ctime[0])) {
142  MultiFab::Divide(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
143  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
144  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
145  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
146  }
147  }
148 
149  // Subtract rho_0 from rho before we interpolate -- note we only subtract
150  // on valid region of mf since the ghost cells will be filled below
151  if (icomp_cons == 0)
152  {
153  MultiFab::Subtract(*mfs_vel[Vars::cons],base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{0});
154 
155  if (!amrex::almostEqual(time,ctime[1])) {
156  MultiFab::Subtract(vars_old[lev-1][Vars::cons], base_state[lev-1],
157  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
158  }
159  if (!amrex::almostEqual(time,ctime[0])) {
160  MultiFab::Subtract(vars_new[lev-1][Vars::cons], base_state[lev-1],
161  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
162  }
163  }
164  } // interpolation_type == StateInterpType::Perturbational
165 
166  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
167  mapper = &cell_cons_interp;
168  FillPatchTwoLevels(mf, IntVect{ng_cons}, IntVect(0,0,0),
169  time, cmf, ctime, fmf, ftime,
170  icomp_cons, icomp_cons, ncomp_cons, geom[lev-1], geom[lev],
171  refRatio(lev-1), mapper, domain_bcs_type,
172  icomp_cons);
173 
174  if (interpolation_type == StateInterpType::Perturbational)
175  {
176  if (icomp_cons == 0)
177  {
178  // Restore the coarse values to what they were
179  if (!amrex::almostEqual(time,ctime[1])) {
180  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
181  BaseState::r0_comp,Rho_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
182  }
183  if (!amrex::almostEqual(time,ctime[0])) {
184  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
185  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
186  }
187 
188  // Set values in the cells outside the domain boundary so that we can do the Add
189  // without worrying about uninitialized values outside the domain -- these
190  // will be filled in the physbcs call
191  mf.setDomainBndry(1.234e20,Rho_comp,1,geom[lev]);
192 
193  // Add rho_0 back to rho after we interpolate -- on all the valid + ghost region
194  MultiFab::Add(mf, base_state[lev],BaseState::r0_comp,Rho_comp,1,IntVect{ng_cons});
195  }
196 
197  if (icomp_cons+ncomp_cons > 1)
198  {
199  // Add theta_0 to theta
200  if (!amrex::almostEqual(time,ctime[1])) {
201  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
202  BaseState::th0_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
203  MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
204  Rho_comp,RhoTheta_comp,1,vars_old[lev-1][Vars::cons].nGrowVect());
205  }
206  if (!amrex::almostEqual(time,ctime[0])) {
207  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
208  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
209  MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
210  Rho_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
211  }
212 
213  // Multiply theta by rho to get (rho theta)
214  MultiFab::Multiply(*mfs_vel[Vars::cons],*mfs_vel[Vars::cons],Rho_comp,RhoTheta_comp,1,IntVect{0});
215 
216  // Add theta_0 to theta
217  MultiFab::Add(*mfs_vel[Vars::cons],base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
218 
219  // Add theta_0 back to theta
220  MultiFab::Add(mf,base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,IntVect{ng_cons});
221 
222  // Multiply (theta) by rho to get (rho theta)
223  MultiFab::Multiply(mf,mf,Rho_comp,RhoTheta_comp,1,IntVect{ng_cons});
224  }
225  } // interpolation_type == StateInterpType::Perturbational
226 
227  // Impose physical bc's on fine data (note time and 0 are not used)
228  // Note that we do this after the FillPatch because imposing physical bc's on fine ghost
229  // cells that need to be filled from coarse requires that we have done the interpolation first
230  bool do_fb = true; bool do_terrain_adjustment = false;
231  (*physbcs_cons[lev])(mf,*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
232  icomp_cons,ncomp_cons,IntVect{ng_cons},time,BCVars::cons_bc,
233  do_fb, do_terrain_adjustment);
234 
235  // Make sure to only copy back the components we worked on
236  MultiFab::Copy(*mfs_vel[Vars::cons],mf,icomp_cons,icomp_cons,ncomp_cons,IntVect{ng_cons});
237 
238  // *****************************************************************************************
239 
240  if (!cons_only)
241  {
242  // ***************************************************************************
243  // We always come in to this call with updated momenta but we need to create updated velocity
244  // in order to impose the rest of the bc's
245  // ***************************************************************************
246  // This only fills VALID region of velocity
247  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
248  *mfs_vel[Vars::cons],
249  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
250  Geom(lev).Domain(), domain_bcs_type);
251 
252  mapper = &face_cons_linear_interp;
253 
254  //
255  // NOTE: All interpolation here happens on velocities not momenta;
256  // note we only do the interpolation and FillBoundary here,
257  // physical bc's are imposed later
258  //
259  // NOTE: This will only fill velocity from coarse grid *outside* the fine grids
260  // unlike the FillSet calls above which filled momenta on the coarse/fine bdy
261  //
262 
263  MultiFab& mfu = *mfs_vel[Vars::xvel];
264 
265  fmf = {&mfu,&mfu};
266  cmf = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
267 
268  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
269  FillPatchTwoLevels(mfu, IntVect{ng_vel}, IntVect(0,0,0),
270  time, cmf, ctime, fmf, ftime,
271  0, 0, 1, geom[lev-1], geom[lev],
272  refRatio(lev-1), mapper, domain_bcs_type,
274 
275  // *****************************************************************************************
276 
277  MultiFab& mfv = *mfs_vel[Vars::yvel];
278 
279  fmf = {&mfv,&mfv};
280  cmf = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
281 
282  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
283  FillPatchTwoLevels(mfv, IntVect{ng_vel}, IntVect(0,0,0),
284  time, cmf, ctime, fmf, ftime,
285  0, 0, 1, geom[lev-1], geom[lev],
286  refRatio(lev-1), mapper, domain_bcs_type,
288 
289  // *****************************************************************************************
290 
291  MultiFab& mfw = *mfs_vel[Vars::zvel];
292 
293  fmf = {&mfw,&mfw};
294  cmf = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
295 
296  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
297  FillPatchTwoLevels(mfw, IntVect{ng_vel}, IntVect(0,0,0),
298  time, cmf, ctime, fmf, ftime,
299  0, 0, 1, geom[lev-1], geom[lev],
300  refRatio(lev-1), mapper, domain_bcs_type,
302  } // !cons_only
303  } // lev > 0
304 
305  // ***************************************************************************
306  // Physical bc's at domain boundary
307  // ***************************************************************************
308  IntVect ngvect_cons = IntVect(ng_cons,ng_cons,ng_cons);
309  IntVect ngvect_vels = IntVect(ng_vel ,ng_vel ,ng_vel);
310 
311  bool do_fb = true;
312 
313 #ifdef ERF_USE_NETCDF
314  // We call this here because it is an ERF routine
315  if (solverChoice.use_real_bcs && (lev==0)) {
317  fill_from_realbdy_upwind(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
318  } else {
319  fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
320  }
321  do_fb = false;
322  }
323 #endif
324 
325  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
326 
327  // We call this even if use_real_bcs is true because these will fill the vertical bcs
328  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
329  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
330  if (!cons_only) {
331  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
332  ngvect_vels,time,BCVars::xvel_bc, do_fb);
333  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
334  ngvect_vels,time,BCVars::yvel_bc, do_fb);
335  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
336  ngvect_vels,time,BCVars::zvel_bc, do_fb);
337  }
338  // ***************************************************************************
339 
340  // We always come in to this call with momenta so we need to leave with momenta!
341  // We need to make sure we convert back on all ghost cells/faces because this is
342  // how velocity from fine-fine copies (as well as physical and interpolated bcs) will be filled
343  if (!cons_only)
344  {
345  IntVect ngu = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::xvel]->nGrowVect();
346  IntVect ngv = (!solverChoice.use_num_diff) ? IntVect(1,1,1) : mfs_vel[Vars::yvel]->nGrowVect();
347  IntVect ngw = (!solverChoice.use_num_diff) ? IntVect(1,1,0) : mfs_vel[Vars::zvel]->nGrowVect();
348 
349  VelocityToMomentum(*mfs_vel[Vars::xvel], ngu,
350  *mfs_vel[Vars::yvel], ngv,
351  *mfs_vel[Vars::zvel], ngw,
352  *mfs_vel[Vars::cons],
353  *mfs_mom[IntVars::xmom], *mfs_mom[IntVars::ymom], *mfs_mom[IntVars::zmom],
354  Geom(lev).Domain(),
356  }
357 
358  // NOTE: There are not FillBoundary calls here for the following reasons:
359  // Removal of the FillBoundary (FB) calls has bee completed for the following reasons:
360  //
361  // 1. physbc_cons is called before VelocityToMomentum and a FB is completed in that functor.
362  // Therefore, the conserved CC vars have their inter-rank ghost cells filled and then their
363  // domain ghost cells filled from the BC operations. We should not call FB on this MF again.
364  //
365  // 2. physbc_u/v/w is also called before VelocityToMomentum and a FB is completed those functors.
366  // Furthermore, VelocityToMomentum operates on a growntilebox so we exit that routine with momentum
367  // filled everywhere---i.e., physbc_u/v/w fills velocity ghost cells (inter-rank and domain)
368  // and then V2M does the conversion to momenta everywhere; so there is again no need to do a FB on momenta.
369 }
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:399
void fill_from_bndryregs(const amrex::Vector< amrex::MultiFab * > &mfs, amrex::Real time)
Definition: ERF_BoundaryConditionsBndryReg.cpp:13
void FillIntermediatePatch(int lev, amrex::Real time, const amrex::Vector< amrex::MultiFab * > &mfs_vel, const amrex::Vector< amrex::MultiFab * > &mfs_mom, int ng_cons, int ng_vel, bool cons_only, int icomp_cons, int ncomp_cons)
Definition: ERF_FillIntermediatePatch.cpp:28
@ NumTypes
Definition: ERF_IndexDefines.H:162
static bool upwind_real_bcs
Definition: ERF_DataStruct.H:812
static bool use_real_bcs
Definition: ERF_DataStruct.H:809
Here is the call graph for this function:

◆ FillPatchCrseLevel()

void ERF::FillPatchCrseLevel ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
bool  cons_only = false 
)
private
285 {
286  BL_PROFILE_VAR("ERF::FillPatchCrseLevel()",ERF_FillPatchCrseLevel);
287 
288  AMREX_ALWAYS_ASSERT(lev == 0);
289 
290  IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
291  IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
292 
293  Vector<Real> ftime = {t_old[lev], t_new[lev]};
294 
295  //
296  // Below we call FillPatchSingleLevel which does NOT fill ghost cells outside the domain
297  //
298 
299  Vector<MultiFab*> fmf;
300  Vector<MultiFab*> fmf_u;
301  Vector<MultiFab*> fmf_v;
302  Vector<MultiFab*> fmf_w;
303 
304  if (amrex::almostEqual(time,ftime[0])) {
305  fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
306  } else if (amrex::almostEqual(time,ftime[1])) {
307  fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
308  } else {
309  fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
310  }
311 
312  const int ncomp = mfs_vel[Vars::cons]->nComp();
313 
314  FillPatchSingleLevel(*mfs_vel[Vars::cons], ngvect_cons, time, fmf, IntVect(0,0,0), ftime,
315  0, 0, ncomp, geom[lev]);
316 
317  if (!cons_only) {
318  if (amrex::almostEqual(time,ftime[0])) {
319  fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
320  fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
321  fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
322  } else if (amrex::almostEqual(time,ftime[1])) {
323  fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
324  fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
325  fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
326  } else {
327  fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
328  fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
329  fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
330  }
331  FillPatchSingleLevel(*mfs_vel[Vars::xvel], ngvect_vels, time, fmf_u,
332  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
333 
334  FillPatchSingleLevel(*mfs_vel[Vars::yvel], ngvect_vels, time, fmf_v,
335  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
336 
337  FillPatchSingleLevel(*mfs_vel[Vars::zvel], ngvect_vels, time, fmf_w,
338  IntVect(0,0,0), ftime, 0, 0, 1, geom[lev]);
339  } // !cons_only
340 
341  // ***************************************************************************
342  // Physical bc's at domain boundary
343  // ***************************************************************************
344  int icomp_cons = 0;
345  int ncomp_cons = mfs_vel[Vars::cons]->nComp();
346 
347  bool do_fb = true;
348 
349 #ifdef ERF_USE_NETCDF
350  // We call this here because it is an ERF routine
351  if(solverChoice.use_real_bcs && (lev==0)) {
353  fill_from_realbdy_upwind(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
354  } else {
355  fill_from_realbdy(mfs_vel,time,cons_only,icomp_cons,ncomp_cons,ngvect_cons,ngvect_vels);
356  }
357  do_fb = false;
358  }
359 #endif
360 
361  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
362 
363  // We call this even if use_real_bcs is true because these will fill the vertical bcs
364  // Note that we call FillBoundary inside the physbcs call
365  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
366  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
367  if (!cons_only) {
368  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
369  ngvect_vels,time,BCVars::xvel_bc, do_fb);
370  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
371  ngvect_vels,time,BCVars::yvel_bc, do_fb);
372  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
373  ngvect_vels,time,BCVars::zvel_bc, do_fb);
374  }
375 }

◆ FillPatchFineLevel()

void ERF::FillPatchFineLevel ( int  lev,
amrex::Real  time,
const amrex::Vector< amrex::MultiFab * > &  mfs_vel,
const amrex::Vector< amrex::MultiFab * > &  mfs_mom,
const amrex::MultiFab &  old_base_state,
const amrex::MultiFab &  new_base_state,
bool  fillset = true,
bool  cons_only = false 
)
private
26 {
27  BL_PROFILE_VAR("ERF::FillPatchFineLevel()",ERF_FillPatchFineLevel);
28 
29  AMREX_ALWAYS_ASSERT(lev > 0);
30 
31  Interpolater* mapper = nullptr;
32 
33  PhysBCFunctNoOp null_bc;
34 
35  //
36  // ***************************************************************************
37  // The first thing we do is interpolate the momenta on the "valid" faces of
38  // the fine grids (where the interface is coarse/fine not fine/fine) -- this
39  // will not be over-written below because the FillPatch operators see these as
40  // valid faces.
41  //
42  // Note that we interpolate momentum not velocity, but all the other boundary
43  // conditions are imposed on velocity, so we convert to momentum here then
44  // convert back.
45  // ***************************************************************************
46  if (fillset) {
47  if (cf_set_width > 0) {
48  FPr_c[lev-1].FillSet(*mfs_vel[Vars::cons], time, null_bc, domain_bcs_type);
49  }
50  if (cf_set_width >= 0 && !cons_only) {
51  VelocityToMomentum(*mfs_vel[Vars::xvel], IntVect{0},
52  *mfs_vel[Vars::yvel], IntVect{0},
53  *mfs_vel[Vars::zvel], IntVect{0},
54  *mfs_vel[Vars::cons],
55  *mfs_mom[IntVars::xmom],
56  *mfs_mom[IntVars::ymom],
57  *mfs_mom[IntVars::zmom],
58  Geom(lev).Domain(),
60 
61  FPr_u[lev-1].FillSet(*mfs_mom[IntVars::xmom], time, null_bc, domain_bcs_type);
62  FPr_v[lev-1].FillSet(*mfs_mom[IntVars::ymom], time, null_bc, domain_bcs_type);
63  FPr_w[lev-1].FillSet(*mfs_mom[IntVars::zmom], time, null_bc, domain_bcs_type);
64 
65  MomentumToVelocity(*mfs_vel[Vars::xvel], *mfs_vel[Vars::yvel], *mfs_vel[Vars::zvel],
66  *mfs_vel[Vars::cons],
67  *mfs_mom[IntVars::xmom],
68  *mfs_mom[IntVars::ymom],
69  *mfs_mom[IntVars::zmom],
70  Geom(lev).Domain(),
72  }
73  }
74 
75  IntVect ngvect_cons = mfs_vel[Vars::cons]->nGrowVect();
76  IntVect ngvect_vels = mfs_vel[Vars::xvel]->nGrowVect();
77 
78  Vector<Real> ftime = {t_old[lev ], t_new[lev ]};
79  Vector<Real> ctime = {t_old[lev-1], t_new[lev-1]};
80 
81  amrex::Real small_dt = 1.e-8 * (ftime[1] - ftime[0]);
82 
83  Vector<MultiFab*> fmf;
84  if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
85  fmf = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::cons]};
86  } else if (amrex::almostEqual(time,ftime[1])) {
87  fmf = {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::cons]};
88  } else {
89  fmf = {&vars_old[lev][Vars::cons], &vars_new[lev][Vars::cons]};
90  }
91  Vector<MultiFab*> cmf = {&vars_old[lev-1][Vars::cons], &vars_new[lev-1][Vars::cons]};
92 
93  // We must fill a temporary then copy it back so we don't double add/subtract
94  MultiFab mf_c(mfs_vel[Vars::cons]->boxArray(),mfs_vel[Vars::cons]->DistributionMap(),
95  mfs_vel[Vars::cons]->nComp() ,mfs_vel[Vars::cons]->nGrowVect());
96 
97  mapper = &cell_cons_interp;
98 
99  if (interpolation_type == StateInterpType::Perturbational)
100  {
101  // Divide (rho theta) by rho to get theta (before we subtract rho0 from rho!)
102  if (!amrex::almostEqual(time,ctime[1])) {
103  MultiFab::Divide(vars_old[lev-1][Vars::cons],vars_old[lev-1][Vars::cons],
104  Rho_comp,RhoTheta_comp,1,ngvect_cons);
105  MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
106  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
107  MultiFab::Subtract(vars_old[lev-1][Vars::cons],base_state[lev-1],
108  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
109  }
110  if (!amrex::almostEqual(time,ctime[0])) {
111  MultiFab::Divide(vars_new[lev-1][Vars::cons],vars_new[lev-1][Vars::cons],
112  Rho_comp,RhoTheta_comp,1,ngvect_cons);
113  MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
114  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
115  MultiFab::Subtract(vars_new[lev-1][Vars::cons],base_state[lev-1],
116  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
117  }
118 
119  if (!amrex::almostEqual(time,ftime[1])) {
120  MultiFab::Divide(vars_old[lev ][Vars::cons],vars_old[lev ][Vars::cons],
121  Rho_comp,RhoTheta_comp,1,IntVect{0});
122  MultiFab::Subtract(vars_old[lev ][Vars::cons],old_base_state,
123  BaseState::r0_comp,Rho_comp,1,IntVect{0});
124  MultiFab::Subtract(vars_old[lev ][Vars::cons],old_base_state,
125  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
126  }
127  if (!amrex::almostEqual(time,ftime[0])) {
128  MultiFab::Divide(vars_new[lev ][Vars::cons],vars_new[lev ][Vars::cons],
129  Rho_comp,RhoTheta_comp,1,IntVect{0});
130  MultiFab::Subtract(vars_new[lev ][Vars::cons],old_base_state,
131  BaseState::r0_comp,Rho_comp,1,IntVect{0});
132  MultiFab::Subtract(vars_new[lev ][Vars::cons],old_base_state,
133  BaseState::th0_comp,RhoTheta_comp,1,IntVect{0});
134  }
135  }
136 
137  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
138  FillPatchTwoLevels(mf_c, ngvect_cons, IntVect(0,0,0),
139  time, cmf, ctime, fmf, ftime,
140  0, 0, mf_c.nComp(), geom[lev-1], geom[lev],
141  refRatio(lev-1), mapper, domain_bcs_type,
143 
144  if (interpolation_type == StateInterpType::Perturbational)
145  {
146  // Restore the coarse values to what they were
147  if (!amrex::almostEqual(time,ctime[1])) {
148  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
149  BaseState::r0_comp,Rho_comp,1,ngvect_cons);
150  MultiFab::Add(vars_old[lev-1][Vars::cons], base_state[lev-1],
151  BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
152  MultiFab::Multiply(vars_old[lev-1][Vars::cons], vars_old[lev-1][Vars::cons],
153  Rho_comp,RhoTheta_comp,1,ngvect_cons);
154  }
155  if (!amrex::almostEqual(time,ctime[0])) {
156  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
157  BaseState::r0_comp,Rho_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
158  MultiFab::Add(vars_new[lev-1][Vars::cons], base_state[lev-1],
159  BaseState::th0_comp,RhoTheta_comp,1,vars_new[lev-1][Vars::cons].nGrowVect());
160  MultiFab::Multiply(vars_new[lev-1][Vars::cons], vars_new[lev-1][Vars::cons],
161  Rho_comp,RhoTheta_comp,1,ngvect_cons);
162  }
163 
164  if (!amrex::almostEqual(time,ftime[1])) {
165  MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev ],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
166  MultiFab::Add(vars_old[lev][Vars::cons],base_state[lev ],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
167  MultiFab::Multiply(vars_old[lev][Vars::cons], vars_old[lev][Vars::cons],
168  Rho_comp,RhoTheta_comp,1,ngvect_cons);
169  }
170  if (!amrex::almostEqual(time,ftime[0])) {
171  MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::r0_comp,Rho_comp,1,ngvect_cons);
172  MultiFab::Add(vars_new[lev][Vars::cons], base_state[lev],BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
173  MultiFab::Multiply(vars_new[lev][Vars::cons], vars_new[lev][Vars::cons],
174  Rho_comp,RhoTheta_comp,1,ngvect_cons);
175  }
176 
177  // Set values in the cells outside the domain boundary so that we can do the Add
178  // without worrying about uninitialized values outside the domain -- these
179  // will be filled in the physbcs call
180  mf_c.setDomainBndry(1.234e20,0,2,geom[lev]); // Do both rho and (rho theta) together
181 
182  // Add rho_0 back to rho and theta_0 back to theta
183  MultiFab::Add(mf_c, new_base_state,BaseState::r0_comp,Rho_comp,1,ngvect_cons);
184  MultiFab::Add(mf_c, new_base_state,BaseState::th0_comp,RhoTheta_comp,1,ngvect_cons);
185 
186  // Multiply (theta) by rho to get (rho theta)
187  MultiFab::Multiply(mf_c,mf_c,Rho_comp,RhoTheta_comp,1,ngvect_cons);
188  }
189 
190  MultiFab::Copy(*mfs_vel[Vars::cons],mf_c,0,0,mf_c.nComp(),mf_c.nGrowVect());
191 
192  // ***************************************************************************************
193 
194  if (!cons_only)
195  {
196  mapper = &face_cons_linear_interp;
197 
198  MultiFab& mf_u = *mfs_vel[Vars::xvel];
199  MultiFab& mf_v = *mfs_vel[Vars::yvel];
200  MultiFab& mf_w = *mfs_vel[Vars::zvel];
201 
202  Vector<MultiFab*> fmf_u; Vector<MultiFab*> fmf_v; Vector<MultiFab*> fmf_w;
203  Vector<MultiFab*> cmf_u; Vector<MultiFab*> cmf_v; Vector<MultiFab*> cmf_w;
204 
205  // **********************************************************************
206 
207  if ( amrex::almostEqual(time,ftime[0]) || (time-ftime[0]) < small_dt ) {
208  fmf_u = {&vars_old[lev][Vars::xvel], &vars_old[lev][Vars::xvel]};
209  fmf_v = {&vars_old[lev][Vars::yvel], &vars_old[lev][Vars::yvel]};
210  fmf_w = {&vars_old[lev][Vars::zvel], &vars_old[lev][Vars::zvel]};
211  } else if ( amrex::almostEqual(time,ftime[1]) ) {
212  fmf_u = {&vars_new[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
213  fmf_v = {&vars_new[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
214  fmf_w = {&vars_new[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
215  } else {
216  fmf_u = {&vars_old[lev][Vars::xvel], &vars_new[lev][Vars::xvel]};
217  fmf_v = {&vars_old[lev][Vars::yvel], &vars_new[lev][Vars::yvel]};
218  fmf_w = {&vars_old[lev][Vars::zvel], &vars_new[lev][Vars::zvel]};
219  }
220  cmf_u = {&vars_old[lev-1][Vars::xvel], &vars_new[lev-1][Vars::xvel]};
221  cmf_v = {&vars_old[lev-1][Vars::yvel], &vars_new[lev-1][Vars::yvel]};
222  cmf_w = {&vars_old[lev-1][Vars::zvel], &vars_new[lev-1][Vars::zvel]};
223 
224  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
225  FillPatchTwoLevels(mf_u, ngvect_vels, IntVect(0,0,0),
226  time, cmf_u, ctime, fmf_u, ftime,
227  0, 0, 1, geom[lev-1], geom[lev],
228  refRatio(lev-1), mapper, domain_bcs_type,
230 
231  FillPatchTwoLevels(mf_v, ngvect_vels, IntVect(0,0,0),
232  time, cmf_v, ctime, fmf_v, ftime,
233  0, 0, 1, geom[lev-1], geom[lev],
234  refRatio(lev-1), mapper, domain_bcs_type,
236 
237  // We put these here because these may be used in constructing omega outside the
238  // domain when fillpatching w
239  bool do_fb = true;
240  (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
241  ngvect_vels,time,BCVars::xvel_bc, do_fb);
242  (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
243  ngvect_vels,time,BCVars::yvel_bc, do_fb);
244 
245  // **********************************************************************
246 
247  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
248  FillPatchTwoLevels(mf_w, ngvect_vels, IntVect(0,0,0),
249  time, cmf_w, ctime, fmf_w, ftime,
250  0, 0, 1, geom[lev-1], geom[lev],
251  refRatio(lev-1), mapper, domain_bcs_type,
253  } // !cons_only
254 
255  // ***************************************************************************
256  // Physical bc's at domain boundary
257  // ***************************************************************************
258  int icomp_cons = 0;
259  int ncomp_cons = mfs_vel[Vars::cons]->nComp();
260 
261  bool do_fb = true;
262 
263  if (m_r2d) fill_from_bndryregs(mfs_vel,time);
264 
265  // We call this even if use_real_bcs is true because these will fill the vertical bcs
266  // Note that we call FillBoundary inside the physbcs call
267  (*physbcs_cons[lev])(*mfs_vel[Vars::cons],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
268  icomp_cons,ncomp_cons,ngvect_cons,time,BCVars::cons_bc, do_fb);
269  if (!cons_only) {
270  // Note that we need to fill u and v in the case of terrain because we will use
271  // these in the call of WFromOmega in lateral ghost cells of the fine grid
272  // (*physbcs_u[lev])(*mfs_vel[Vars::xvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
273  // ngvect_vels,time,BCVars::xvel_bc, do_fb);
274  // (*physbcs_v[lev])(*mfs_vel[Vars::yvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
275  // ngvect_vels,time,BCVars::yvel_bc, do_fb);
276  (*physbcs_w[lev])(*mfs_vel[Vars::zvel],*mfs_vel[Vars::xvel],*mfs_vel[Vars::yvel],
277  ngvect_vels,time,BCVars::zvel_bc, do_fb);
278  }
279 }
Here is the call graph for this function:

◆ FillWeatherDataMultiFab()

void ERF::FillWeatherDataMultiFab ( const std::string &  filename,
const amrex::Geometry &  geom_weather,
const amrex::BoxArray &  nba,
const amrex::DistributionMapping &  dm,
amrex::Vector< amrex::MultiFab > &  weather_forecast_data 
)
443 {
444 
445  Vector<Real> latvec_h, lonvec_h, xvec_h, yvec_h, zvec_h;
446  Vector<Real> rho_h, uvel_h, vvel_h, wvel_h, theta_h, qv_h, qc_h, qr_h;
447 
448  ReadCustomBinaryIC(filename, latvec_h, lonvec_h,
449  xvec_h, yvec_h, zvec_h, rho_h,
450  uvel_h, vvel_h, wvel_h,
451  theta_h, qv_h, qc_h, qr_h);
452 
453  const auto prob_lo_erf = geom[0].ProbLoArray();
454  const auto prob_hi_erf = geom[0].ProbHiArray();
455  const auto dx_erf = geom[0].CellSizeArray();
456 
457  if(prob_lo_erf[0] < xvec_h.front() + 4*dx_erf[0]){
458  amrex::Abort("The xlo value of the domain has to be greater than " + std::to_string(xvec_h.front() + 4*dx_erf[0]));
459  }
460  if(prob_hi_erf[0] > xvec_h.back() - 4*dx_erf[0]){
461  amrex::Abort("The xhi value of the domain has to be less than " + std::to_string(xvec_h.back() - 4*dx_erf[0]));
462  }
463  if(prob_lo_erf[1] < yvec_h.front() + 4*dx_erf[1]){
464  amrex::Abort("The ylo value of the domain has to be greater than " + std::to_string(yvec_h.front() + 4*dx_erf[1]));
465  }
466  if(prob_hi_erf[1] > yvec_h.back() - 4*dx_erf[1]){
467  amrex::Abort("The yhi value of the domain has to be less than " + std::to_string(yvec_h.back() - 4*dx_erf[1]));
468  }
469 
470  // Number of cells
471  int nx_cells = xvec_h.size()-1;
472  int ny_cells = yvec_h.size()-1;
473 
474  const amrex::Geometry& geom0 = geom[0]; // or whatever your Geometry vector is called
475  const amrex::Box& domainBox = geom0.Domain();
476  const amrex::IntVect& domainSize = domainBox.size(); // Number of cells in each direction
477  int nz_cells = domainSize[2];
478 
479 
480  int ncomp = 10;
481  int ngrow = 0;
482 
483  int n_time = 1; // or however many time slices you want
484  weather_forecast_data.resize(n_time);
485  MultiFab& weather_mf = weather_forecast_data[0];
486  weather_mf.define(nba, dm, ncomp, ngrow);
487 
488  fill_weather_data_multifab(weather_mf, geom_weather, nx_cells+1, ny_cells+1, nz_cells+1,
489  latvec_h, lonvec_h, zvec_h,
490  rho_h,uvel_h, vvel_h, wvel_h,
491  theta_h, qv_h, qc_h, qr_h);
492 
493  //PlotMultiFab(weather_mf, geom_weather, "plt_coarse_weather", MultiFabType::NC);
494 }
void fill_weather_data_multifab(MultiFab &mf, const Geometry &geom_weather, const int nx, const int ny, const int nz, const Vector< Real > &latvec_h, const Vector< Real > &lonvec_h, const Vector< Real > &zvec_h, const Vector< Real > &rho_h, const Vector< Real > &uvel_h, const Vector< Real > &vvel_h, const Vector< Real > &wvel_h, const Vector< Real > &theta_h, const Vector< Real > &qv_h, const Vector< Real > &qc_h, const Vector< Real > &qr_h)
Definition: ERF_WeatherDataInterpolation.cpp:18
Here is the call graph for this function:

◆ get_eb()

eb_ const& ERF::get_eb ( int  lev) const
inlineprivatenoexcept
1553  {
1554  AMREX_ASSERT(lev >= 0 && lev < eb.size() && eb[lev] != nullptr);
1555  return *eb[lev];
1556  }

◆ get_projection_bc()

Array< LinOpBCType, AMREX_SPACEDIM > ERF::get_projection_bc ( amrex::Orientation::Side  side) const
noexcept
18 {
19  amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> r;
20  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
21  if (geom[0].isPeriodic(dir)) {
22  r[dir] = LinOpBCType::Periodic;
23  } else {
24  auto bc_type = domain_bc_type[Orientation(dir,side)];
25  if (bc_type == "Outflow") {
26  r[dir] = LinOpBCType::Dirichlet;
27  } else
28  {
29  r[dir] = LinOpBCType::Neumann;
30  }
31  }
32  }
33  return r;
34 }
amrex::Array< std::string, 2 *AMREX_SPACEDIM > domain_bc_type
Definition: ERF.H:939

◆ getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* ERF::getAdvFluxReg ( int  lev)
inlineprivate
1342  {
1343  return advflux_reg[lev];
1344  }

◆ getCPUTime()

static amrex::Real ERF::getCPUTime ( )
inlinestaticprivate
1434  {
1435  int numCores = amrex::ParallelDescriptor::NProcs();
1436 #ifdef _OPENMP
1437  numCores = numCores * omp_get_max_threads();
1438 #endif
1439 
1440  amrex::Real T =
1441  numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
1443 
1444  return T;
1445  }
static amrex::Real previousCPUTimeUsed
Definition: ERF.H:1430
static amrex::Real startCPUTime
Definition: ERF.H:1429
@ T
Definition: ERF_IndexDefines.H:110

◆ GotoNextLine()

void ERF::GotoNextLine ( std::istream &  is)
staticprivate

Utility to skip to next line in Header file input stream.

17 {
18  constexpr std::streamsize bl_ignore_max { 100000 };
19  is.ignore(bl_ignore_max, '\n');
20 }

◆ HurricaneTracker()

void ERF::HurricaneTracker ( int  lev,
const amrex::MultiFab &  U_new,
const amrex::MultiFab &  V_new,
const amrex::MultiFab &  W_new,
const amrex::Real  velmag_threshold,
const bool  is_track_io,
amrex::TagBoxArray *  tags = nullptr 
)
458 {
459  const auto dx = geom[levc].CellSizeArray();
460  const auto prob_lo = geom[levc].ProbLoArray();
461 
462  const int ncomp = AMREX_SPACEDIM; // Number of components (3 for 3D)
463 
464  Gpu::DeviceVector<Real> d_coords(3, 0.0); // Initialize to -1
465  Real* d_coords_ptr = d_coords.data(); // Get pointer to device vector
466  Gpu::DeviceVector<int> d_found(1,0);
467  int* d_found_ptr = d_found.data();
468 
469  MultiFab mf_cc_vel(grids[levc], dmap[levc], AMREX_SPACEDIM, IntVect(0,0,0));
470  average_face_to_cellcenter(mf_cc_vel,0,{AMREX_D_DECL(&U_new,&V_new,&W_new)},0);
471 
472  // Loop through MultiFab using MFIter
473  for (MFIter mfi(mf_cc_vel); mfi.isValid(); ++mfi) {
474  const Box& box = mfi.validbox(); // Get the valid box for the current MFIter
475  const Array4<const Real>& vel_arr = mf_cc_vel.const_array(mfi); // Get the array for this MFIter
476 
477  // ParallelFor loop to check velocity magnitudes on the GPU
478  amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
479  // Access velocity components using ncomp
480  Real magnitude = 0.0; // Initialize magnitude
481 
482  for (int comp = 0; comp < ncomp; ++comp) {
483  Real vel = vel_arr(i, j, k, comp); // Access the component for each (i, j, k)
484  magnitude += vel * vel; // Sum the square of the components
485  }
486 
487  magnitude = std::sqrt(magnitude)*3.6; // Calculate magnitude
488  Real x = prob_lo[0] + (i + 0.5) * dx[0];
489  Real y = prob_lo[1] + (j + 0.5) * dx[1];
490  Real z = prob_lo[2] + (k + 0.5) * dx[2];
491 
492  // Check if magnitude exceeds threshold
493  if (z < 2.0e3 && magnitude > velmag_threshold) {
494  // Use atomic operations to set found flag and store coordinates
495  Gpu::Atomic::Add(&d_found_ptr[0], 1); // Mark as found
496 
497  // Store coordinates
498  Gpu::Atomic::Add(&d_coords_ptr[0],x); // Store x index
499  Gpu::Atomic::Add(&d_coords_ptr[1],y); // Store x index
500  Gpu::Atomic::Add(&d_coords_ptr[2],z); // Store x index
501  }
502  });
503  }
504 
505  // Synchronize to ensure all threads complete their execution
506  amrex::Gpu::streamSynchronize(); // Wait for all GPU threads to finish
507 
508  Vector<int> h_found(1,0);
509  Gpu::copy(Gpu::deviceToHost, d_found.begin(), d_found.end(), h_found.begin());
510  ParallelAllReduce::Sum(h_found.data(),
511  h_found.size(),
512  ParallelContext::CommunicatorAll());
513 
514  Real eye_x, eye_y;
515  // Broadcast coordinates if found
516  if (h_found[0] > 0) {
517  Vector<Real> h_coords(3,-1e10);
518  Gpu::copy(Gpu::deviceToHost, d_coords.begin(), d_coords.end(), h_coords.begin());
519 
520  ParallelAllReduce::Sum(h_coords.data(),
521  h_coords.size(),
522  ParallelContext::CommunicatorAll());
523 
524  eye_x = h_coords[0]/h_found[0];
525  eye_y = h_coords[1]/h_found[0];
526 
527  // Data structure to hold the hurricane track for I/O
528  if (amrex::ParallelDescriptor::IOProcessor() and is_track_io) {
529  hurricane_track_xy.push_back({eye_x, eye_y});
530  }
531 
532  if(is_track_io) {
533  return;
534  }
535 
536  Real rad_tag = 3e5*std::pow(2, max_level-1-levc);
537 
538  for (MFIter mfi(*tags); mfi.isValid(); ++mfi) {
539  TagBox& tag = (*tags)[mfi];
540  auto tag_arr = tag.array(); // Get device-accessible array
541 
542  const Box& tile_box = mfi.tilebox(); // The box for this tile
543 
544  ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
545  // Compute cell center coordinates
546  Real x = prob_lo[0] + (i + 0.5) * dx[0];
547  Real y = prob_lo[1] + (j + 0.5) * dx[1];
548 
549  Real dist = std::sqrt((x - eye_x)*(x - eye_x) + (y - eye_y)*(y - eye_y));
550 
551  if (dist < rad_tag) {
552  tag_arr(i,j,k) = TagBox::SET;
553  }
554  });
555  }
556  }
557 }
amrex::Vector< std::array< amrex::Real, 2 > > hurricane_track_xy
Definition: ERF.H:153

◆ ImposeBCsOnPhi()

void ERF::ImposeBCsOnPhi ( int  lev,
amrex::MultiFab &  phi,
const amrex::Box &  subdomain 
)

Impose bc's on the pressure that comes out of the solve

13 {
14  BL_PROFILE("ERF::ImposeBCsOnPhi()");
15 
16  auto const sub_lo = lbound(subdomain);
17  auto const sub_hi = ubound(subdomain);
18 
19  auto const dom_lo = lbound(geom[lev].Domain());
20  auto const dom_hi = ubound(geom[lev].Domain());
21 
22  phi.setBndry(1.e25);
23  phi.FillBoundary(geom[lev].periodicity());
24 
25  // ****************************************************************************
26  // Impose bc's on pprime
27  // ****************************************************************************
28 #ifdef _OPENMP
29 #pragma omp parallel if (Gpu::notInLaunchRegion())
30 #endif
31  for (MFIter mfi(phi,TilingIfNotGPU()); mfi.isValid(); ++mfi)
32  {
33  Array4<Real> const& pp_arr = phi.array(mfi);
34  Box const& bx = mfi.tilebox();
35  auto const bx_lo = lbound(bx);
36  auto const bx_hi = ubound(bx);
37 
38  auto bc_type_xlo = domain_bc_type[Orientation(0,Orientation::low)];
39  auto bc_type_xhi = domain_bc_type[Orientation(0,Orientation::high)];
40  auto bc_type_ylo = domain_bc_type[Orientation(1,Orientation::low)];
41  auto bc_type_yhi = domain_bc_type[Orientation(1,Orientation::high)];
42  auto bc_type_zhi = domain_bc_type[Orientation(2,Orientation::high)];
43 
44  if ( (bx_lo.x == dom_lo.x) && (bc_type_xlo == "Outflow" || bc_type_xlo == "Open") ) {
45  ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
46  {
47  pp_arr(i-1,j,k) = -pp_arr(i,j,k);
48  });
49  } else if (bx_lo.x == sub_lo.x) {
50  ParallelFor(makeSlab(bx,0,sub_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
51  {
52  pp_arr(i-1,j,k) = pp_arr(i,j,k);
53  });
54  }
55 
56  if ( (bx_hi.x == dom_hi.x) && (bc_type_xhi == "Outflow" || bc_type_xhi == "Open") ) {
57  ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
58  {
59  pp_arr(i+1,j,k) = -pp_arr(i,j,k);
60  });
61  } else if (bx_hi.x == sub_hi.x) {
62  ParallelFor(makeSlab(bx,0,sub_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
63  {
64  pp_arr(i+1,j,k) = pp_arr(i,j,k);
65  });
66  }
67 
68  if ( (bx_lo.y == dom_lo.y) && (bc_type_ylo == "Outflow" || bc_type_ylo == "Open") ) {
69  ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
70  {
71  pp_arr(i,j-1,k) = -pp_arr(i,j,k);
72  });
73  } else if (bx_lo.y == sub_lo.y) {
74  ParallelFor(makeSlab(bx,1,sub_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
75  {
76  pp_arr(i,j-1,k) = pp_arr(i,j,k);
77  });
78  }
79 
80  if ( (bx_hi.y == dom_hi.y) && (bc_type_yhi == "Outflow" || bc_type_yhi == "Open") ) {
81  ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
82  {
83  pp_arr(i,j+1,k) = -pp_arr(i,j,k);
84  });
85  } else if (bx_hi.y == sub_hi.y) {
86  ParallelFor(makeSlab(bx,1,sub_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
87  {
88  pp_arr(i,j+1,k) = pp_arr(i,j,k);
89  });
90  }
91 
92  // At low z we are always Neumann whether the box touches the bottom boundary or not
93  Box zbx(bx); zbx.grow(0,1); zbx.grow(1,1); // Grow in x-dir and y-dir because we have filled that above
94  if (bx_lo.z == sub_lo.z) {
95  ParallelFor(makeSlab(zbx,2,dom_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
96  {
97  pp_arr(i,j,k-1) = pp_arr(i,j,k);
98  });
99  }
100 
101  if ( (bx_hi.z == dom_hi.z) && (bc_type_zhi == "Outflow" || bc_type_zhi == "Open") ) {
102  ParallelFor(makeSlab(bx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
103  {
104  pp_arr(i,j,k+1) = -pp_arr(i,j,k);
105  });
106  } else if (bx_hi.z == sub_hi.z) {
107  ParallelFor(makeSlab(bx,2,sub_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
108  {
109  pp_arr(i,j,k+1) = pp_arr(i,j,k);
110  });
111  }
112  } // mfi
113 
114  // Now overwrite with periodic fill outside domain and fine-fine fill inside
115  phi.FillBoundary(geom[lev].periodicity());
116 }

◆ init1DArrays()

void ERF::init1DArrays ( )
private

◆ init_bcs()

void ERF::init_bcs ( )
private

Initializes data structures in the ERF class that specify which boundary conditions we are implementing on each face of the domain.

This function also maps the selected boundary condition types (e.g. Outflow, Inflow, InflowOutflow, Periodic, Dirichlet, ...) to the specific implementation needed for each variable.

Stores this information in both host and device vectors so it is available for GPU kernels.

21 {
22  bool rho_read = false;
23  bool read_prim_theta = true;
24  Vector<Real> cons_dir_init(NBCVAR_max,0.0);
25  cons_dir_init[BCVars::Rho_bc_comp] = 1.0;
26  cons_dir_init[BCVars::RhoTheta_bc_comp] = -1.0;
27  auto f = [this,&rho_read,&read_prim_theta] (std::string const& bcid, Orientation ori)
28  {
29  // These are simply defaults for Dirichlet faces -- they should be over-written below
31  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = -1.0; // It is important to set this negative
32  // because the sign is tested on below
33  for (int n = BCVars::RhoKE_bc_comp; n < BCVars::xvel_bc; n++) {
34  m_bc_extdir_vals[n][ori] = 0.0;
35  }
36 
37  m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.0; // default
40 
41  // These are simply defaults for Neumann gradients -- they should be over-written below
44 
53 
57 
58  std::string pp_text = pp_prefix + "." + bcid;
59  ParmParse pp(pp_text);
60 
61  std::string bc_type_in;
62  if (pp.query("type", bc_type_in) <= 0)
63  {
64  pp_text = bcid;
65  pp = ParmParse(pp_text);
66  pp.query("type", bc_type_in);
67  }
68 
69  std::string bc_type = amrex::toLower(bc_type_in);
70 
71  if (bc_type == "symmetry")
72  {
73  // Print() << bcid << " set to symmetry.\n";
75  domain_bc_type[ori] = "Symmetry";
76  }
77  else if (bc_type == "outflow")
78  {
79  // Print() << bcid << " set to outflow.\n";
81  domain_bc_type[ori] = "Outflow";
82  }
83  else if (bc_type == "open")
84  {
85  // Print() << bcid << " set to open.\n";
86  AMREX_ASSERT_WITH_MESSAGE((ori.coordDir() != 2), "Open boundary not valid on zlo or zhi!");
88  domain_bc_type[ori] = "Open";
89  }
90  else if (bc_type == "ho_outflow")
91  {
93  domain_bc_type[ori] = "HO_Outflow";
94  }
95 
96  else if (bc_type == "inflow" || bc_type == "inflow_outflow")
97  {
98  if (bc_type == "inflow") {
99  // Print() << bcid << " set to inflow.\n";
101  domain_bc_type[ori] = "Inflow";
102  } else {
103  // Print() << bcid << " set to inflow_outflow.\n";
105  domain_bc_type[ori] = "InflowOutflow";
106  }
107 
108  std::vector<Real> v;
109  if (input_bndry_planes && m_r2d->ingested_velocity()) {
113  } else {
114  // Test for input data file if at xlo face
115  std::string dirichlet_file;
116  auto file_exists = pp.query("dirichlet_file", dirichlet_file);
117  if (file_exists) {
118  pp.query("read_prim_theta", read_prim_theta);
119  init_Dirichlet_bc_data(dirichlet_file);
120  } else {
121  pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
122  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
123  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
124  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
125  }
126  }
127 
128  Real rho_in = 0.;
129  if (input_bndry_planes && m_r2d->ingested_density()) {
131  } else {
132  if (!pp.query("density", rho_in)) {
133  amrex::Print() << "Using interior values to set conserved vars" << std::endl;
134  }
135  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
136  }
137 
138  bool th_read = (th_bc_data[0].data()!=nullptr);
139  Real theta_in = 0.;
140  if (input_bndry_planes && m_r2d->ingested_theta()) {
142  } else if (!th_read) {
143  if (rho_in > 0) {
144  pp.get("theta", theta_in);
145  }
146  m_bc_extdir_vals[BCVars::RhoTheta_bc_comp][ori] = rho_in*theta_in;
147  }
148 
149  Real scalar_in = 0.;
150  if (input_bndry_planes && m_r2d->ingested_scalar()) {
152  } else {
153  if (pp.query("scalar", scalar_in))
154  m_bc_extdir_vals[BCVars::RhoScalar_bc_comp][ori] = rho_in*scalar_in;
155  }
156 
157  if (solverChoice.moisture_type != MoistureType::None) {
158  Real qv_in = 0.;
159  if (input_bndry_planes && m_r2d->ingested_q1()) {
161  } else {
162  if (pp.query("qv", qv_in))
163  m_bc_extdir_vals[BCVars::RhoQ1_bc_comp][ori] = rho_in*qv_in;
164  }
165  Real qc_in = 0.;
166  if (input_bndry_planes && m_r2d->ingested_q2()) {
168  } else {
169  if (pp.query("qc", qc_in))
170  m_bc_extdir_vals[BCVars::RhoQ2_bc_comp][ori] = rho_in*qc_in;
171  }
172  }
173 
174  Real KE_in = 0.;
175  if (input_bndry_planes && m_r2d->ingested_KE()) {
177  } else {
178  if (pp.query("KE", KE_in))
179  m_bc_extdir_vals[BCVars::RhoKE_bc_comp][ori] = rho_in*KE_in;
180  }
181  }
182  else if (bc_type == "noslipwall")
183  {
184  // Print() << bcid <<" set to no-slip wall.\n";
186  domain_bc_type[ori] = "NoSlipWall";
187 
188  std::vector<Real> v;
189 
190  // The values of m_bc_extdir_vals default to 0.
191  // But if we find "velocity" in the inputs file, use those values instead.
192  if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
193  {
194  v[ori.coordDir()] = 0.0;
195  m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
196  m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
197  m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
198  }
199 
200  Real rho_in;
201  rho_read = pp.query("density", rho_in);
202  if (rho_read)
203  {
204  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
205  }
206 
207  Real theta_in;
208  if (pp.query("theta", theta_in))
209  {
211  }
212 
213  Real theta_grad_in;
214  if (pp.query("theta_grad", theta_grad_in))
215  {
216  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
217  }
218 
219  Real qv_in;
220  if (pp.query("qv", qv_in))
221  {
223  }
224  }
225  else if (bc_type == "slipwall")
226  {
227  // Print() << bcid <<" set to slip wall.\n";
228 
230  domain_bc_type[ori] = "SlipWall";
231 
232  Real rho_in;
233  rho_read = pp.query("density", rho_in);
234  if (rho_read)
235  {
236  m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] = rho_in;
237  }
238 
239  Real theta_in;
240  if (pp.query("theta", theta_in))
241  {
243  }
244 
245  Real rho_grad_in;
246  if (pp.query("density_grad", rho_grad_in))
247  {
248  m_bc_neumann_vals[BCVars::Rho_bc_comp][ori] = rho_grad_in;
249  }
250 
251  Real theta_grad_in;
252  if (pp.query("theta_grad", theta_grad_in))
253  {
254  m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori] = theta_grad_in;
255  }
256  }
257  else if (bc_type == "surface_layer")
258  {
260  domain_bc_type[ori] = "surface_layer";
261  }
262  else
263  {
265  }
266 
267  if (geom[0].isPeriodic(ori.coordDir())) {
268  domain_bc_type[ori] = "Periodic";
269  if (phys_bc_type[ori] == ERF_BC::undefined)
270  {
272  } else {
273  Abort("Wrong BC type for periodic boundary");
274  }
275  }
276 
277  if (phys_bc_type[ori] == ERF_BC::undefined)
278  {
279  Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
280  Abort("This BC type is unknown");
281  }
282  };
283 
284  f("xlo", Orientation(Direction::x,Orientation::low));
285  f("xhi", Orientation(Direction::x,Orientation::high));
286  f("ylo", Orientation(Direction::y,Orientation::low));
287  f("yhi", Orientation(Direction::y,Orientation::high));
288  f("zlo", Orientation(Direction::z,Orientation::low));
289  f("zhi", Orientation(Direction::z,Orientation::high));
290 
291  // *****************************************************************************
292  //
293  // Here we translate the physical boundary conditions -- one type per face --
294  // into logical boundary conditions for each velocity component
295  //
296  // *****************************************************************************
297  {
298  domain_bcs_type.resize(AMREX_SPACEDIM+NBCVAR_max);
299  domain_bcs_type_d.resize(AMREX_SPACEDIM+NBCVAR_max);
300 
301  for (OrientationIter oit; oit; ++oit) {
302  Orientation ori = oit();
303  int dir = ori.coordDir();
304  Orientation::Side side = ori.faceDir();
305  auto const bct = phys_bc_type[ori];
306  if ( bct == ERF_BC::symmetry )
307  {
308  if (side == Orientation::low) {
309  for (int i = 0; i < AMREX_SPACEDIM; i++) {
311  }
313  } else {
314  for (int i = 0; i < AMREX_SPACEDIM; i++) {
316  }
318  }
319  }
320  else if (bct == ERF_BC::outflow or bct == ERF_BC::ho_outflow )
321  {
322  if (side == Orientation::low) {
323  for (int i = 0; i < AMREX_SPACEDIM; i++) {
325  }
326  if (!solverChoice.anelastic[0]) {
328  }
329  } else {
330  for (int i = 0; i < AMREX_SPACEDIM; i++) {
332  }
333  if (!solverChoice.anelastic[0]) {
335  }
336  }
337  }
338  else if (bct == ERF_BC::open)
339  {
340  if (side == Orientation::low) {
341  for (int i = 0; i < AMREX_SPACEDIM; i++)
343  } else {
344  for (int i = 0; i < AMREX_SPACEDIM; i++)
346  }
347  }
348  else if (bct == ERF_BC::inflow)
349  {
350  if (side == Orientation::low) {
351  for (int i = 0; i < AMREX_SPACEDIM; i++) {
353  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
355  }
356  }
357  } else {
358  for (int i = 0; i < AMREX_SPACEDIM; i++) {
360  if (input_bndry_planes && dir < 2 && m_r2d->ingested_velocity()) {
362  }
363  }
364  }
365  }
366  else if (bct == ERF_BC::inflow_outflow)
367  {
368  if (side == Orientation::low) {
369  for (int i = 0; i < AMREX_SPACEDIM; i++) {
371  }
372  } else {
373  for (int i = 0; i < AMREX_SPACEDIM; i++) {
375  }
376  }
377  }
378  else if (bct == ERF_BC::no_slip_wall)
379  {
380  if (side == Orientation::low) {
381  for (int i = 0; i < AMREX_SPACEDIM; i++) {
383  }
384  } else {
385  for (int i = 0; i < AMREX_SPACEDIM; i++) {
387  }
388  }
389  }
390  else if (bct == ERF_BC::slip_wall)
391  {
392  if (side == Orientation::low) {
393  for (int i = 0; i < AMREX_SPACEDIM; i++) {
395  }
396  // Only normal direction has ext_dir
398 
399  } else {
400  for (int i = 0; i < AMREX_SPACEDIM; i++) {
402  }
403  // Only normal direction has ext_dir
405  }
406  }
407  else if (bct == ERF_BC::periodic)
408  {
409  if (side == Orientation::low) {
410  for (int i = 0; i < AMREX_SPACEDIM; i++) {
412  }
413  } else {
414  for (int i = 0; i < AMREX_SPACEDIM; i++) {
416  }
417  }
418  }
419  else if ( bct == ERF_BC::surface_layer )
420  {
421  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
425  }
426  }
427  }
428 
429  // *****************************************************************************
430  //
431  // Here we translate the physical boundary conditions -- one type per face --
432  // into logical boundary conditions for each cell-centered variable
433  // (including the base state variables)
434  // NOTE: all "scalars" share the same type of boundary condition
435  //
436  // *****************************************************************************
437  {
438  for (OrientationIter oit; oit; ++oit) {
439  Orientation ori = oit();
440  int dir = ori.coordDir();
441  Orientation::Side side = ori.faceDir();
442  auto const bct = phys_bc_type[ori];
443  if ( bct == ERF_BC::symmetry )
444  {
445  if (side == Orientation::low) {
446  for (int i = 0; i < NBCVAR_max; i++) {
448  }
449  } else {
450  for (int i = 0; i < NBCVAR_max; i++) {
452  }
453  }
454  }
455  else if ( bct == ERF_BC::outflow )
456  {
457  if (side == Orientation::low) {
458  for (int i = 0; i < NBCVAR_max; i++) {
460  }
461  } else {
462  for (int i = 0; i < NBCVAR_max; i++) {
464  }
465  }
466  }
467  else if ( bct == ERF_BC::ho_outflow )
468  {
469  if (side == Orientation::low) {
470  for (int i = 0; i < NBCVAR_max; i++) {
472  }
473  } else {
474  for (int i = 0; i < NBCVAR_max; i++) {
476  }
477  }
478  }
479  else if ( bct == ERF_BC::open )
480  {
481  if (side == Orientation::low) {
482  for (int i = 0; i < NBCVAR_max; i++)
484  } else {
485  for (int i = 0; i < NBCVAR_max; i++)
487  }
488  }
489  else if ( bct == ERF_BC::no_slip_wall )
490  {
491  if (side == Orientation::low) {
492  for (int i = 0; i < NBCVAR_max; i++) {
494  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
495  if (rho_read) {
497  } else {
499  }
500  }
501  }
502  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
504  }
505  } else {
506  for (int i = 0; i < NBCVAR_max; i++) {
508  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
509  if (rho_read) {
511  } else {
513  }
514  }
515  }
516  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
518  }
519  }
520  }
521  else if (bct == ERF_BC::slip_wall)
522  {
523  if (side == Orientation::low) {
524  for (int i = 0; i < NBCVAR_max; i++) {
526  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
527  if (rho_read) {
529  } else {
531  }
532  }
533  }
534  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
536  }
537  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
539  }
540  } else {
541  for (int i = 0; i < NBCVAR_max; i++) {
543  if (m_bc_extdir_vals[BCVars::cons_bc+i][ori] != cons_dir_init[BCVars::cons_bc+i]) {
544  if (rho_read) {
546  } else {
548  }
549  }
550  }
551  if (std::abs(m_bc_neumann_vals[BCVars::RhoTheta_bc_comp][ori]) > 0.) {
553  }
554  if (std::abs(m_bc_neumann_vals[BCVars::Rho_bc_comp][ori]) > 0.) {
556  }
557  }
558  }
559  else if (bct == ERF_BC::inflow)
560  {
561  if (side == Orientation::low) {
562  for (int i = 0; i < NBCVAR_max; i++) {
564  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
565  (th_bc_data[0].data() != nullptr))
566  {
567  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setLo(dir, ERFBCType::ext_dir_prim);
568  }
569  else if (input_bndry_planes && dir < 2 && (
570  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
571  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
572  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
573  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
574  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
575  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )) )
576  {
578  }
579  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
581  }
582  }
583  } else {
584  for (int i = 0; i < NBCVAR_max; i++) {
586  if ((BCVars::cons_bc+i == RhoTheta_comp) &&
587  (th_bc_data[0].data() != nullptr))
588  {
589  if (read_prim_theta) domain_bcs_type[BCVars::cons_bc+i].setHi(dir, ERFBCType::ext_dir_prim);
590  }
591  else if (input_bndry_planes && dir < 2 && (
592  ( (BCVars::cons_bc+i == BCVars::Rho_bc_comp) && m_r2d->ingested_density()) ||
593  ( (BCVars::cons_bc+i == BCVars::RhoTheta_bc_comp) && m_r2d->ingested_theta() ) ||
594  ( (BCVars::cons_bc+i == BCVars::RhoKE_bc_comp) && m_r2d->ingested_KE() ) ||
595  ( (BCVars::cons_bc+i == BCVars::RhoScalar_bc_comp) && m_r2d->ingested_scalar() ) ||
596  ( (BCVars::cons_bc+i == BCVars::RhoQ1_bc_comp) && m_r2d->ingested_q1() ) ||
597  ( (BCVars::cons_bc+i == BCVars::RhoQ2_bc_comp) && m_r2d->ingested_q2() )
598  ) )
599  {
601  }
602  else if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
604  }
605  }
606  }
607  }
608  else if (bct == ERF_BC::inflow_outflow )
609  {
610  if (side == Orientation::low) {
611  for (int i = 0; i < NBCVAR_max; i++) {
613  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
615  }
616  }
617  } else {
618  for (int i = 0; i < NBCVAR_max; i++) {
620  if (m_bc_extdir_vals[BCVars::Rho_bc_comp][ori] == 0) {
622  }
623  }
624  }
625  }
626  else if (bct == ERF_BC::periodic)
627  {
628  if (side == Orientation::low) {
629  for (int i = 0; i < NBCVAR_max; i++) {
631  }
632  } else {
633  for (int i = 0; i < NBCVAR_max; i++) {
635  }
636  }
637  }
638  else if ( bct == ERF_BC::surface_layer )
639  {
640  AMREX_ALWAYS_ASSERT(dir == 2 && side == Orientation::low);
641  for (int i = 0; i < NBCVAR_max; i++) {
643  }
644  }
645  }
646  }
647 
648  // NOTE: Gpu:copy is a wrapper to htod_memcpy (GPU) or memcpy (CPU) and is a blocking comm
649  Gpu::copy(Gpu::hostToDevice, domain_bcs_type.begin(), domain_bcs_type.end(), domain_bcs_type_d.begin());
650 }
#define NBCVAR_max
Definition: ERF_IndexDefines.H:29
@ ho_outflow
@ inflow_outflow
void init_Dirichlet_bc_data(const std::string input_file)
Definition: ERF_InitBCs.cpp:652
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NBCVAR_max > m_bc_neumann_vals
Definition: ERF.H:945
@ RhoQ6_bc_comp
Definition: ERF_IndexDefines.H:86
@ RhoQ1_bc_comp
Definition: ERF_IndexDefines.H:81
@ RhoQ4_bc_comp
Definition: ERF_IndexDefines.H:84
@ RhoKE_bc_comp
Definition: ERF_IndexDefines.H:79
@ RhoQ3_bc_comp
Definition: ERF_IndexDefines.H:83
@ RhoTheta_bc_comp
Definition: ERF_IndexDefines.H:78
@ RhoQ2_bc_comp
Definition: ERF_IndexDefines.H:82
@ Rho_bc_comp
Definition: ERF_IndexDefines.H:77
@ RhoQ5_bc_comp
Definition: ERF_IndexDefines.H:85
@ neumann
Definition: ERF_IndexDefines.H:213
@ open
Definition: ERF_IndexDefines.H:215
@ reflect_odd
Definition: ERF_IndexDefines.H:205
@ hoextrap
Definition: ERF_IndexDefines.H:216
@ foextrap
Definition: ERF_IndexDefines.H:208
@ ext_dir
Definition: ERF_IndexDefines.H:209
@ ext_dir_prim
Definition: ERF_IndexDefines.H:211
@ ext_dir_upwind
Definition: ERF_IndexDefines.H:217
@ int_dir
Definition: ERF_IndexDefines.H:206
@ neumann_int
Definition: ERF_IndexDefines.H:214
@ reflect_even
Definition: ERF_IndexDefines.H:207
Here is the call graph for this function:

◆ init_custom()

void ERF::init_custom ( int  lev)
private

Wrapper for custom problem-specific initialization routines that can be defined by the user as they set up a new problem in ERF. This wrapper handles all the overhead of defining the perturbation as well as initializing the random seed if needed.

This wrapper calls a user function to customize initialization on a per-Fab level inside an MFIter loop, so all the MultiFab operations are hidden from the user.

Parameters
levInteger specifying the current level
27 {
28  auto& lev_new = vars_new[lev];
29 
30  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
31  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
32 
33  MultiFab cons_pert(lev_new[Vars::cons].boxArray(), lev_new[Vars::cons].DistributionMap(),
34  lev_new[Vars::cons].nComp() , lev_new[Vars::cons].nGrow());
35  MultiFab xvel_pert(lev_new[Vars::xvel].boxArray(), lev_new[Vars::xvel].DistributionMap(), 1, lev_new[Vars::xvel].nGrowVect());
36  MultiFab yvel_pert(lev_new[Vars::yvel].boxArray(), lev_new[Vars::yvel].DistributionMap(), 1, lev_new[Vars::yvel].nGrowVect());
37  MultiFab zvel_pert(lev_new[Vars::zvel].boxArray(), lev_new[Vars::zvel].DistributionMap(), 1, lev_new[Vars::zvel].nGrowVect());
38 
39  // Default all perturbations to zero
40  cons_pert.setVal(0.);
41  xvel_pert.setVal(0.);
42  yvel_pert.setVal(0.);
43  zvel_pert.setVal(0.);
44 
45 #ifdef _OPENMP
46 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
47 #endif
48  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
49  {
50  const Box &bx = mfi.tilebox();
51  const Box &xbx = mfi.tilebox(IntVect(1,0,0));
52  const Box &ybx = mfi.tilebox(IntVect(0,1,0));
53  const Box &zbx = mfi.tilebox(IntVect(0,0,1));
54 
55  const auto &cons_pert_arr = cons_pert.array(mfi);
56  const auto &xvel_pert_arr = xvel_pert.array(mfi);
57  const auto &yvel_pert_arr = yvel_pert.array(mfi);
58  const auto &zvel_pert_arr = zvel_pert.array(mfi);
59 
60  Array4<Real const> cons_arr = lev_new[Vars::cons].const_array(mfi);
61  Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
62  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
63 
64  // Here we arbitrarily choose the x-oriented map factor -- this should be generalized
65  Array4<Real const> mf_m = mapfac[lev][MapFacType::m_x]->const_array(mfi);
66  Array4<Real const> mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
67  Array4<Real const> mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
68 
69  Array4<Real> r_hse_arr = r_hse.array(mfi);
70  Array4<Real> p_hse_arr = p_hse.array(mfi);
71 
72  prob->init_custom_pert(bx, xbx, ybx, zbx, cons_arr, cons_pert_arr,
73  xvel_pert_arr, yvel_pert_arr, zvel_pert_arr,
74  r_hse_arr, p_hse_arr, z_nd_arr, z_cc_arr,
75  geom[lev].data(), mf_m, mf_u, mf_v,
76  solverChoice);
77  } //mfi
78 
79  // Add problem-specific perturbation to background flow if not doing anelastic with fixed-in-time density
81  MultiFab::Add(lev_new[Vars::cons], cons_pert, Rho_comp, Rho_comp, 1, cons_pert.nGrow());
82  }
83  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoTheta_comp, RhoTheta_comp, 1, cons_pert.nGrow());
84  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoScalar_comp,RhoScalar_comp,NSCALARS, cons_pert.nGrow());
85 
86  // RhoKE is relevant if using Deardorff with LES, k-equation for RANS, or MYNN with PBL
87  if (solverChoice.turbChoice[lev].use_tke) {
88  MultiFab::Add(lev_new[Vars::cons], cons_pert, RhoKE_comp, RhoKE_comp, 1, cons_pert.nGrow());
89  }
90 
91  if (solverChoice.moisture_type != MoistureType::None) {
92  int qstate_size = micro->Get_Qstate_Size();
93  for (int q_offset(0); q_offset<qstate_size; ++q_offset) {
94  int q_idx = RhoQ1_comp+q_offset;
95  MultiFab::Add(lev_new[Vars::cons], cons_pert, q_idx, q_idx, 1, cons_pert.nGrow());
96  }
97  }
98 
99  MultiFab::Add(lev_new[Vars::xvel], xvel_pert, 0, 0, 1, xvel_pert.nGrowVect());
100  MultiFab::Add(lev_new[Vars::yvel], yvel_pert, 0, 0, 1, yvel_pert.nGrowVect());
101  MultiFab::Add(lev_new[Vars::zvel], zvel_pert, 0, 0, 1, zvel_pert.nGrowVect());
102 }
const Box xbx
Definition: ERF_DiffSetup.H:21
const Box ybx
Definition: ERF_DiffSetup.H:22
bool fixed_density
Definition: ERF_DataStruct.H:833
Here is the call graph for this function:

◆ init_Dirichlet_bc_data()

void ERF::init_Dirichlet_bc_data ( const std::string  input_file)
private
653 {
654  const bool use_terrain = (solverChoice.terrain_type != TerrainType::None);
655 
656  // Read the dirichlet_input file
657  Print() << "dirichlet_input file location : " << input_file << std::endl;
658  std::ifstream input_reader(input_file);
659  if (!input_reader.is_open()) {
660  amrex::Abort("Error opening the dirichlet_input file.\n");
661  }
662 
663  Print() << "Successfully opened the dirichlet_input file. Now reading... " << std::endl;
664  std::string line;
665 
666  // Size of Ninp (number of z points in input file)
667  Vector<Real> z_inp_tmp, u_inp_tmp, v_inp_tmp, w_inp_tmp, th_inp_tmp;
668 
669  // Top and bot for domain
670  const int klo = geom[0].Domain().smallEnd()[2];
671  const int khi = geom[0].Domain().bigEnd()[2];
672  const Real zbot = (use_terrain) ? zlevels_stag[0][klo] : geom[0].ProbLo(2);
673  const Real ztop = (use_terrain) ? zlevels_stag[0][khi+1] : geom[0].ProbHi(2);
674 
675  // Flag if theta input
676  Real th_init = -300.0;
677  bool th_read{false};
678 
679  // Add surface
680  z_inp_tmp.push_back(zbot); // height above sea level [m]
681  u_inp_tmp.push_back(0.);
682  v_inp_tmp.push_back(0.);
683  w_inp_tmp.push_back(0.);
684  th_inp_tmp.push_back(th_init);
685 
686  // Read the vertical profile at each given height
687  Real z, u, v, w, th;
688  while(std::getline(input_reader, line)) {
689  std::istringstream iss_z(line);
690 
691  Vector<Real> rval_v;
692  Real rval;
693  while (iss_z >> rval) {
694  rval_v.push_back(rval);
695  }
696  z = rval_v[0];
697  u = rval_v[1];
698  v = rval_v[2];
699  w = rval_v[3];
700 
701  // Format without theta
702  if (rval_v.size() == 4) {
703  if (z == zbot) {
704  u_inp_tmp[0] = u;
705  v_inp_tmp[0] = v;
706  w_inp_tmp[0] = w;
707  } else {
708  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
709  z_inp_tmp.push_back(z);
710  u_inp_tmp.push_back(u);
711  v_inp_tmp.push_back(v);
712  w_inp_tmp.push_back(w);
713  if (z >= ztop) break;
714  }
715  } else if (rval_v.size() == 5) {
716  th_read = true;
717  th = rval_v[4];
718  if (z == zbot) {
719  u_inp_tmp[0] = u;
720  v_inp_tmp[0] = v;
721  w_inp_tmp[0] = w;
722  th_inp_tmp[0] = th;
723  } else {
724  AMREX_ALWAYS_ASSERT(z > z_inp_tmp[z_inp_tmp.size()-1]); // sounding is increasing in height
725  z_inp_tmp.push_back(z);
726  u_inp_tmp.push_back(u);
727  v_inp_tmp.push_back(v);
728  w_inp_tmp.push_back(w);
729  th_inp_tmp.push_back(th);
730  if (z >= ztop) break;
731  }
732  } else {
733  Abort("Unknown inflow file format!");
734  }
735  }
736 
737  // Ensure we set a reasonable theta surface
738  if (th_read) {
739  if (th_inp_tmp[0] == th_init) {
740  Real slope = (th_inp_tmp[2] - th_inp_tmp[1]) / (z_inp_tmp[2] - z_inp_tmp[1]);
741  Real dz = z_inp_tmp[0] - z_inp_tmp[1];
742  th_inp_tmp[0] = slope * dz + th_inp_tmp[1];
743  }
744  }
745 
746  amrex::Print() << "Successfully read and interpolated the dirichlet_input file..." << std::endl;
747  input_reader.close();
748 
749  for (int lev = 0; lev <= max_level; lev++) {
750 
751  const int Nz = geom[lev].Domain().size()[2];
752  const Real dz = geom[lev].CellSize()[2];
753 
754  // Size of Nz (domain grid)
755  Vector<Real> zcc_inp(Nz );
756  Vector<Real> znd_inp(Nz+1);
757  Vector<Real> u_inp(Nz ); xvel_bc_data[lev].resize(Nz ,0.0);
758  Vector<Real> v_inp(Nz ); yvel_bc_data[lev].resize(Nz ,0.0);
759  Vector<Real> w_inp(Nz+1); zvel_bc_data[lev].resize(Nz+1,0.0);
760  Vector<Real> th_inp;
761  if (th_read) {
762  th_inp.resize(Nz);
763  th_bc_data[lev].resize(Nz, 0.0);
764  }
765 
766  // At this point, we have an input from zbot up to
767  // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights
768  const int Ninp = z_inp_tmp.size();
769  for (int k(0); k<Nz; ++k) {
770  zcc_inp[k] = (use_terrain) ? 0.5 * (zlevels_stag[lev][k] + zlevels_stag[lev][k+1])
771  : zbot + (k + 0.5) * dz;
772  znd_inp[k] = (use_terrain) ? zlevels_stag[lev][k+1] : zbot + (k) * dz;
773  u_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), u_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
774  v_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), v_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
775  w_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), znd_inp[k], Ninp);
776  if (th_read) {
777  th_inp[k] = interpolate_1d(z_inp_tmp.dataPtr(), th_inp_tmp.dataPtr(), zcc_inp[k], Ninp);
778  }
779  }
780  znd_inp[Nz] = ztop;
781  w_inp[Nz] = interpolate_1d(z_inp_tmp.dataPtr(), w_inp_tmp.dataPtr(), ztop, Ninp);
782 
783  // Copy host data to the device
784  Gpu::copy(Gpu::hostToDevice, u_inp.begin(), u_inp.end(), xvel_bc_data[lev].begin());
785  Gpu::copy(Gpu::hostToDevice, v_inp.begin(), v_inp.end(), yvel_bc_data[lev].begin());
786  Gpu::copy(Gpu::hostToDevice, w_inp.begin(), w_inp.end(), zvel_bc_data[lev].begin());
787  if (th_read) {
788  Gpu::copy(Gpu::hostToDevice, th_inp.begin(), th_inp.end(), th_bc_data[lev].begin());
789  }
790 
791  // NOTE: These device vectors are passed to the PhysBC constructors when that
792  // class is instantiated in ERF_MakeNewArrays.cpp.
793  } // lev
794 }
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real interpolate_1d(const amrex::Real *alpha, const amrex::Real *beta, const amrex::Real alpha_interp, const int alpha_size)
Definition: ERF_Interpolation_1D.H:12
Here is the call graph for this function:

◆ init_from_hse()

void ERF::init_from_hse ( int  lev)

Initialize the background flow to have the calculated HSE density and rho*theta calculated from the HSE pressure. In general, the hydrostatically balanced density and pressure (r_hse and p_hse from base_state) used here may be calculated through a solver path such as:

ERF::initHSE(lev)

  • call prob->erf_init_dens_hse(...)
    • call Problem::init_isentropic_hse(...), to simultaneously calculate r_hse and p_hse with Newton iteration – assuming constant theta
    • save r_hse
  • call ERF::enforce_hse(...), calculates p_hse from saved r_hse (redundant, but needed because p_hse is not necessarily calculated by the Problem implementation) and pi_hse and th_hse – note: this pressure does not exactly match the p_hse from before because what is calculated by init_isentropic_hse comes from the EOS whereas what is calculated here comes from the hydro- static equation
Parameters
levInteger specifying the current level
33 {
34  auto& lev_new = vars_new[lev];
35 
36  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
37  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp, 1);
38 
39 #ifdef _OPENMP
40 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
41 #endif
42  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi)
43  {
44  const Box &gbx = mfi.growntilebox(1);
45  const Array4<Real >& cons_arr = lev_new[Vars::cons].array(mfi);
46  const Array4<Real const>& r_hse_arr = r_hse.const_array(mfi);
47  const Array4<Real const>& p_hse_arr = p_hse.const_array(mfi);
48 
49  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
50  {
51  cons_arr(i,j,k,Rho_comp) = r_hse_arr(i,j,k);
52  cons_arr(i,j,k,RhoTheta_comp) = getRhoThetagivenP(p_hse_arr(i,j,k));
53  });
54  } //mfi
55 }
Here is the call graph for this function:

◆ init_from_input_sounding()

void ERF::init_from_input_sounding ( int  lev)

High level wrapper for initializing scalar and velocity level data from input sounding data.

Parameters
levInteger specifying the current level
53 {
54  // We only want to read the file once -- here we fill one FArrayBox (per variable) that spans the domain
55  if (lev == 0) {
57  Error("input_sounding file name must be provided via input");
58  }
59 
61 
62  // this will interpolate the input profiles to the nominal height levels
63  // (ranging from 0 to the domain top)
64  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
66  }
67 
68  // this will calculate the hydrostatically balanced density and pressure
69  // profiles following WRF ideal.exe
70  if (solverChoice.sounding_type == SoundingType::Ideal) {
72  } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
73  solverChoice.sounding_type == SoundingType::DryIsentropic) {
74  input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
76  }
77 
78  } else {
79  //
80  // We need to do this interp from coarse level in order to set the values of
81  // the base state inside the domain but outside of the fine region
82  //
83  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
84  //
85  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
86  // have been pre-filled - this includes ghost cells both inside and outside
87  // the domain
88  //
89  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
90  IntVect(0,0,0), // do not fill ghost cells outside the domain
91  base_state[lev-1], 0, 0, base_state[lev].nComp(),
92  geom[lev-1], geom[lev],
93  refRatio(lev-1), &cell_cons_interp,
95 
96  // We need to do this here because the interpolation above may leave corners unfilled
97  // when the corners need to be filled by, for example, reflection of the fine ghost
98  // cell outside the fine region but inide the domain.
99  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
100  }
101 
102  auto& lev_new = vars_new[lev];
103 
104  // updated if sounding is ideal (following WRF) or isentropic
105  const bool l_isentropic = (solverChoice.sounding_type == SoundingType::Isentropic ||
106  solverChoice.sounding_type == SoundingType::DryIsentropic);
107  const bool sounding_ideal_or_isentropic = (solverChoice.sounding_type == SoundingType::Ideal ||
108  l_isentropic);
109  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
110  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
111  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
112  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
113  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
114 
115  const Real l_gravity = solverChoice.gravity;
116  const Real l_rdOcp = solverChoice.rdOcp;
117  const bool l_moist = (solverChoice.moisture_type != MoistureType::None);
118 
119 #ifdef _OPENMP
120 #pragma omp parallel if (Gpu::notInLaunchRegion())
121 #endif
122  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
123  const Box &bx = mfi.tilebox();
124  const auto &cons_arr = lev_new[Vars::cons].array(mfi);
125  const auto &xvel_arr = lev_new[Vars::xvel].array(mfi);
126  const auto &yvel_arr = lev_new[Vars::yvel].array(mfi);
127  const auto &zvel_arr = lev_new[Vars::zvel].array(mfi);
128  Array4<Real> r_hse_arr = r_hse.array(mfi);
129  Array4<Real> p_hse_arr = p_hse.array(mfi);
130  Array4<Real> pi_hse_arr = pi_hse.array(mfi);
131  Array4<Real> th_hse_arr = th_hse.array(mfi);
132  Array4<Real> qv_hse_arr = qv_hse.array(mfi);
133 
134  Array4<Real const> z_cc_arr = (z_phys_cc[lev]) ? z_phys_cc[lev]->const_array(mfi) : Array4<Real const>{};
135  Array4<Real const> z_nd_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) : Array4<Real const>{};
136 
137  if (sounding_ideal_or_isentropic)
138  {
139  // HSE will be initialized here, interpolated from values previously
140  // calculated by calc_rho_p or calc_rho_p_isentropic
142  bx, cons_arr,
143  r_hse_arr, p_hse_arr, pi_hse_arr, th_hse_arr, qv_hse_arr,
144  geom[lev].data(), z_cc_arr,
145  l_gravity, l_rdOcp, l_moist, input_sounding_data,
146  l_isentropic);
147  }
148  else
149  {
150  // This assumes rho_0 = 1.0
151  // HSE will be calculated later with call to initHSE
153  bx, cons_arr,
154  geom[lev].data(), z_cc_arr,
155  l_moist, input_sounding_data);
156  }
157 
159  bx, xvel_arr, yvel_arr, zvel_arr,
160  geom[lev].data(), z_nd_arr,
162 
163  } //mfi
164 }
void init_bx_scalars_from_input_sounding(const Box &bx, Array4< Real > const &state, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const bool &l_moist, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:176
void init_bx_scalars_from_input_sounding_hse(const Box &bx, Array4< Real > const &state, Array4< Real > const &r_hse_arr, Array4< Real > const &p_hse_arr, Array4< Real > const &pi_hse_arr, Array4< Real > const &th_hse_arr, Array4< Real > const &qv_hse_arr, GeometryData const &geomdata, Array4< const Real > const &z_cc_arr, const Real &l_gravity, const Real &l_rdOcp, const bool &l_moist, InputSoundingData const &inputSoundingData, const bool &l_isentropic)
Definition: ERF_InitFromInputSounding.cpp:238
void init_bx_velocities_from_input_sounding(const Box &bx, Array4< Real > const &x_vel, Array4< Real > const &y_vel, Array4< Real > const &z_vel, GeometryData const &geomdata, Array4< const Real > const &z_nd_arr, InputSoundingData const &inputSoundingData)
Definition: ERF_InitFromInputSounding.cpp:374
InputSoundingData input_sounding_data
Definition: ERF.H:731
@ rho0_bc_comp
Definition: ERF_IndexDefines.H:98
@ qv0_comp
Definition: ERF_IndexDefines.H:67
void resize_arrays()
Definition: ERF_InputSoundingData.H:60
int n_sounding_files
Definition: ERF_InputSoundingData.H:398
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag, int itime)
Definition: ERF_InputSoundingData.H:77
amrex::Vector< std::string > input_sounding_file
Definition: ERF_InputSoundingData.H:396
void calc_rho_p(int itime)
Definition: ERF_InputSoundingData.H:176
void calc_rho_p_isentropic(int itime)
Definition: ERF_InputSoundingData.H:262
bool assume_dry
Definition: ERF_InputSoundingData.H:401
static SoundingType sounding_type
Definition: ERF_DataStruct.H:803
Here is the call graph for this function:

◆ init_geo_wind_profile()

void ERF::init_geo_wind_profile ( const std::string  input_file,
amrex::Vector< amrex::Real > &  u_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  u_geos_d,
amrex::Vector< amrex::Real > &  v_geos,
amrex::Gpu::DeviceVector< amrex::Real > &  v_geos_d,
const amrex::Geometry &  lgeom,
const amrex::Vector< amrex::Real > &  zlev_stag 
)
private
17 {
18  const int klo = 0;
19  const int khi = lgeom.Domain().bigEnd()[AMREX_SPACEDIM-1];
20  const amrex::Real dz = lgeom.CellSize()[AMREX_SPACEDIM-1];
21 
22  const bool grid_stretch = (zlev_stag.size() > 0);
23  const Real zbot = (grid_stretch) ? zlev_stag[klo] : lgeom.ProbLo(AMREX_SPACEDIM-1);
24  const Real ztop = (grid_stretch) ? zlev_stag[khi+1] : lgeom.ProbHi(AMREX_SPACEDIM-1);
25 
26  amrex::Print() << "Reading geostrophic wind profile from " << input_file << std::endl;
27  std::ifstream profile_reader(input_file);
28  if(!profile_reader.is_open()) {
29  amrex::Error("Error opening the abl_geo_wind_table\n");
30  }
31 
32  // First, read the input data into temp vectors
33  std::string line;
34  Vector<Real> z_inp, Ug_inp, Vg_inp;
35  Real z, Ug, Vg;
36  amrex::Print() << "z Ug Vg" << std::endl;
37  while(std::getline(profile_reader, line)) {
38  std::istringstream iss(line);
39  iss >> z >> Ug >> Vg;
40  amrex::Print() << z << " " << Ug << " " << Vg << std::endl;
41  z_inp.push_back(z);
42  Ug_inp.push_back(Ug);
43  Vg_inp.push_back(Vg);
44  if (z >= ztop) break;
45  }
46 
47  const int Ninp = z_inp.size();
48  AMREX_ALWAYS_ASSERT(z_inp[0] <= zbot);
49  AMREX_ALWAYS_ASSERT(z_inp[Ninp-1] >= ztop);
50 
51  // Now, interpolate vectors to the cell centers
52  for (int k = 0; k <= khi; k++) {
53  z = (grid_stretch) ? 0.5 * (zlev_stag[k] + zlev_stag[k+1])
54  : zbot + (k + 0.5) * dz;
55  u_geos[k] = interpolate_1d(z_inp.dataPtr(), Ug_inp.dataPtr(), z, Ninp);
56  v_geos[k] = interpolate_1d(z_inp.dataPtr(), Vg_inp.dataPtr(), z, Ninp);
57  }
58 
59  // Copy from host version to device version
60  Gpu::copy(Gpu::hostToDevice, u_geos.begin(), u_geos.end(), u_geos_d.begin());
61  Gpu::copy(Gpu::hostToDevice, v_geos.begin(), v_geos.end(), v_geos_d.begin());
62 
63  profile_reader.close();
64 }
Here is the call graph for this function:

◆ init_only()

void ERF::init_only ( int  lev,
amrex::Real  time 
)
1739 {
1740  t_new[lev] = time;
1741  t_old[lev] = time - 1.e200;
1742 
1743  auto& lev_new = vars_new[lev];
1744  auto& lev_old = vars_old[lev];
1745 
1746  // Loop over grids at this level to initialize our grid data
1747  lev_new[Vars::cons].setVal(0.0); lev_old[Vars::cons].setVal(0.0);
1748  lev_new[Vars::xvel].setVal(0.0); lev_old[Vars::xvel].setVal(0.0);
1749  lev_new[Vars::yvel].setVal(0.0); lev_old[Vars::yvel].setVal(0.0);
1750  lev_new[Vars::zvel].setVal(0.0); lev_old[Vars::zvel].setVal(0.0);
1751 
1752  // Initialize background flow (optional)
1753  if (solverChoice.init_type == InitType::Input_Sounding) {
1754  // The physbc's need the terrain but are needed for initHSE
1755  // We have already made the terrain in the call to init_zphys
1756  // in MakeNewLevelFromScratch
1757  make_physbcs(lev);
1758 
1759  // Now init the base state and the data itself
1761 
1762  // The base state has been initialized by integrating vertically
1763  // through the sounding for ideal (like WRF) or isentropic approaches
1764  if (solverChoice.sounding_type == SoundingType::Ideal ||
1765  solverChoice.sounding_type == SoundingType::Isentropic ||
1766  solverChoice.sounding_type == SoundingType::DryIsentropic) {
1767  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(solverChoice.use_gravity,
1768  "Gravity should be on to be consistent with sounding initialization.");
1769  } else { // SoundingType::ConstantDensity
1770  AMREX_ASSERT_WITH_MESSAGE(!solverChoice.use_gravity,
1771  "Constant density probably doesn't make sense with gravity");
1772  initHSE();
1773  }
1774 
1775 #ifdef ERF_USE_NETCDF
1776  }
1777  else if (solverChoice.init_type == InitType::WRFInput)
1778  {
1779  // The base state is initialized from WRF wrfinput data, output by
1780  // ideal.exe or real.exe
1781 
1782  init_from_wrfinput(lev, *mf_C1H, *mf_C2H, *mf_MUB, *mf_PSFC[lev]);
1783 
1784  if (lev==0) {
1785  if ((start_time > 0) && (start_time != start_bdy_time)) {
1786  Print() << "Ignoring specified start_time="
1787  << std::setprecision(timeprecision) << start_time
1788  << std::endl;
1789  }
1790  }
1791 
1792  start_time = start_bdy_time;
1793 
1794  use_datetime = true;
1795 
1796  // The physbc's need the terrain but are needed for initHSE
1797  if (!solverChoice.use_real_bcs) {
1798  make_physbcs(lev);
1799  }
1800  }
1801  else if (solverChoice.init_type == InitType::NCFile)
1802  {
1803  // The state is initialized by reading from a Netcdf file
1804  init_from_ncfile(lev);
1805 
1806  // The physbc's need the terrain but are needed for initHSE
1807  make_physbcs(lev);
1808  }
1809  else if (solverChoice.init_type == InitType::Metgrid)
1810  {
1811  // The base state is initialized from data output by WPS metgrid;
1812  // we will rebalance after interpolation
1813  init_from_metgrid(lev);
1814 #endif
1815  } else if (solverChoice.init_type == InitType::Uniform) {
1816  // Initialize a uniform background field and base state based on the
1817  // problem-specified reference density and temperature
1818 
1819  // The physbc's need the terrain but are needed for initHSE
1820  make_physbcs(lev);
1821 
1822  init_uniform(lev);
1823  initHSE(lev);
1824  } else {
1825  // No background flow initialization specified, initialize the
1826  // background field to be equal to the base state, calculated from the
1827  // problem-specific erf_init_dens_hse
1828 
1829  // The bc's need the terrain but are needed for initHSE
1830  make_physbcs(lev);
1831 
1832  // We will initialize the state from the background state so must set that first
1833  initHSE(lev);
1834  init_from_hse(lev);
1835  }
1836 
1837  // Add problem-specific flow features
1838  //
1839  // Notes:
1840  // - This calls init_custom_pert that is defined for each problem
1841  // - This may modify the base state
1842  // - The fields set by init_custom_pert are **perturbations** to the
1843  // background flow set based on init_type
1844  if (solverChoice.init_type != InitType::NCFile) {
1845  init_custom(lev);
1846  }
1847 
1848  // Ensure that the face-based data are the same on both sides of a periodic domain.
1849  // The data associated with the lower grid ID is considered the correct value.
1850  lev_new[Vars::xvel].OverrideSync(geom[lev].periodicity());
1851  lev_new[Vars::yvel].OverrideSync(geom[lev].periodicity());
1852  lev_new[Vars::zvel].OverrideSync(geom[lev].periodicity());
1853 
1854  if(solverChoice.spongeChoice.sponge_type == "input_sponge"){
1855  input_sponge(lev);
1856  }
1857 
1858  // Initialize turbulent perturbation
1859  if (solverChoice.pert_type == PerturbationType::Source ||
1860  solverChoice.pert_type == PerturbationType::Direct ||
1861  solverChoice.pert_type == PerturbationType::CPM) {
1862  turbPert_update(lev, 0.);
1863  turbPert_amplitude(lev);
1864  }
1865 }
const int timeprecision
Definition: ERF.H:980
void init_from_input_sounding(int lev)
Definition: ERF_InitFromInputSounding.cpp:52
std::unique_ptr< amrex::MultiFab > mf_MUB
Definition: ERF.H:1186
std::unique_ptr< amrex::MultiFab > mf_C2H
Definition: ERF.H:1185
void init_custom(int lev)
Definition: ERF_InitCustom.cpp:26
void init_from_hse(int lev)
Definition: ERF_InitFromHSE.cpp:32
void initHSE()
Initialize HSE.
Definition: ERF_Init1D.cpp:142
void turbPert_update(const int lev, const amrex::Real dt)
Definition: ERF_InitTurbPert.cpp:12
void input_sponge(int lev)
Definition: ERF_InitSponge.cpp:17
void make_physbcs(int lev)
Definition: ERF_MakeNewArrays.cpp:684
void init_uniform(int lev)
Definition: ERF_InitUniform.cpp:17
std::unique_ptr< amrex::MultiFab > mf_C1H
Definition: ERF.H:1184
void turbPert_amplitude(const int lev)
Definition: ERF_InitTurbPert.cpp:32
bool use_gravity
Definition: ERF_DataStruct.H:843

◆ init_stuff()

void ERF::init_stuff ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm,
amrex::Vector< amrex::MultiFab > &  lev_new,
amrex::Vector< amrex::MultiFab > &  lev_old,
amrex::MultiFab &  tmp_base_state,
std::unique_ptr< amrex::MultiFab > &  tmp_zphys_nd 
)
private
28 {
29  // ********************************************************************************************
30  // Base state holds r_0, pres_0, pi_0, th_0 (in that order)
31  //
32  // Here is where we set 3 ghost cells for the base state!
33  //
34  // ********************************************************************************************
35  tmp_base_state.define(ba,dm,BaseState::num_comps,3);
36  tmp_base_state.setVal(0.);
37 
38  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
39  base_state_new[lev].define(ba,dm,BaseState::num_comps,base_state[lev].nGrowVect());
40  base_state_new[lev].setVal(0.);
41  }
42 
43  // ********************************************************************************************
44  // Allocate terrain arrays
45  // ********************************************************************************************
46 
47  BoxArray ba_nd(ba);
48  ba_nd.surroundingNodes();
49 
50  // NOTE: this is where we actually allocate z_phys_nd -- but here it's called "tmp_zphys_nd"
51  // We need this to be one greater than the ghost cells to handle levels > 0
52 
53  int ngrow = ComputeGhostCells(solverChoice) + 2;
54  tmp_zphys_nd = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
55 
56  z_phys_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
57  init_default_zphys(lev, geom[lev], *tmp_zphys_nd, *z_phys_cc[lev]);
58 
59  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh)
60  {
61  detJ_cc_new[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
62  detJ_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
63 
64  ax_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
65  ay_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
66  az_src[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
67 
68  z_t_rk[lev] = std::make_unique<MultiFab>( convert(ba, IntVect(0,0,1)), dm, 1, 1 );
69 
70  z_phys_nd_new[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
71  z_phys_nd_src[lev] = std::make_unique<MultiFab>(ba_nd,dm,1,IntVect(ngrow,ngrow,ngrow));
72  z_phys_cc_src[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
73  }
74  else
75  {
76  z_phys_nd_new[lev] = nullptr;
77  detJ_cc_new[lev] = nullptr;
78 
79  z_phys_nd_src[lev] = nullptr;
80  z_phys_cc_src[lev] = nullptr;
81  detJ_cc_src[lev] = nullptr;
82 
83  z_t_rk[lev] = nullptr;
84  }
85 
86  if (SolverChoice::terrain_type == TerrainType::ImmersedForcing)
87  {
88  terrain_blanking[lev] = std::make_unique<MultiFab>(ba,dm,1,ngrow);
89  terrain_blanking[lev]->setVal(1.0);
90  }
91 
92  // We use these area arrays regardless of terrain, EB or none of the above
93  detJ_cc[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
94  ax[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(1,0,0)),dm,1,1);
95  ay[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,1,0)),dm,1,1);
96  az[lev] = std::make_unique<MultiFab>(convert(ba,IntVect(0,0,1)),dm,1,1);
97 
98  detJ_cc[lev]->setVal(1.0);
99  ax[lev]->setVal(1.0);
100  ay[lev]->setVal(1.0);
101  az[lev]->setVal(1.0);
102 
103  // ********************************************************************************************
104  // Create wall distance array for RANS modeling
105  // ********************************************************************************************
106  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
107  walldist[lev] = std::make_unique<MultiFab>(ba,dm,1,1);
108  walldist[lev]->setVal(1e23);
109  } else {
110  walldist[lev] = nullptr;
111  }
112 
113  // ********************************************************************************************
114  // These are the persistent containers for the old and new data
115  // ********************************************************************************************
116  int ncomp;
117  if (lev > 0) {
118  ncomp = vars_new[lev-1][Vars::cons].nComp();
119  } else {
120  int n_qstate = micro->Get_Qstate_Size();
121  ncomp = NDRY + NSCALARS + n_qstate;
122  }
123 
124  // ********************************************************************************************
125  // The number of ghost cells for density must be 1 greater than that for velocity
126  // so that we can go back in forth between velocity and momentum on all faces
127  // ********************************************************************************************
128  int ngrow_state = ComputeGhostCells(solverChoice) + 1;
129  int ngrow_vels = ComputeGhostCells(solverChoice);
130 
131  // ********************************************************************************************
132  // New solution data containers
133  // ********************************************************************************************
134  if (solverChoice.terrain_type != TerrainType::EB) {
135  lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state);
136  lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state);
137  } else {
138  // EB: Define the MultiFabs with the EBFactory
139  lev_new[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
140  lev_old[Vars::cons].define(ba, dm, ncomp, ngrow_state, MFInfo(), EBFactory(lev));
141  }
142  lev_new[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
143  lev_old[Vars::xvel].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
144 
145  lev_new[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
146  lev_old[Vars::yvel].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
147 
148  gradp[lev][GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp[lev][GpVars::gpx].setVal(0.);
149  gradp[lev][GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp[lev][GpVars::gpy].setVal(0.);
150  gradp[lev][GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp[lev][GpVars::gpz].setVal(0.);
151 
152  // Note that we need the ghost cells in the z-direction if we are doing any
153  // kind of domain decomposition in the vertical (at level 0 or above)
154  lev_new[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
155  lev_old[Vars::zvel].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
156 
158  pp_inc[lev].define(ba, dm, 1, 1);
159  pp_inc[lev].setVal(0.0);
160  }
161 
162  // ********************************************************************************************
163  // These are just used for scratch in the time integrator but we might as well define them here
164  // ********************************************************************************************
165  if (solverChoice.terrain_type != TerrainType::EB) {
166  rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
167  rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
168 
169  rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
170  rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
171 
172  rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
173  rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels);
174  } else {
175  // EB: Define the MultiFabs with the EBFactory
176  rU_old[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
177  rU_new[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
178 
179  rV_old[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
180  rV_new[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
181 
182  rW_old[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
183  rW_new[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, ngrow_vels, MFInfo(), EBFactory(lev));
184  }
185 
186  if (lev > 0) {
187  //xmom_crse_rhs[lev].define(convert(ba, IntVect(1,0,0)), dm, 1, IntVect{0});
188  //ymom_crse_rhs[lev].define(convert(ba, IntVect(0,1,0)), dm, 1, IntVect{0});
189  zmom_crse_rhs[lev].define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect{0});
190  }
191 
192  // We do this here just so they won't be undefined in the initial FillPatch
193  rU_old[lev].setVal(1.2e21);
194  rV_old[lev].setVal(3.4e22);
195  rW_old[lev].setVal(5.6e23);
196  rU_new[lev].setVal(1.2e21);
197  rV_new[lev].setVal(3.4e22);
198  rW_new[lev].setVal(5.6e23);
199 
200  // ********************************************************************************************
201  // These are just time averaged fields for diagnostics
202  // ********************************************************************************************
203 
204  // NOTE: We are not completing a fillpach call on the time averaged data;
205  // which would copy on intersection and interpolate from coarse.
206  // Therefore, we are restarting the averaging when the ba changes,
207  // this may give poor statistics for dynamic mesh refinement.
208  vel_t_avg[lev] = nullptr;
210  vel_t_avg[lev] = std::make_unique<MultiFab>(ba, dm, 4, 0); // Each vel comp and the mag
211  vel_t_avg[lev]->setVal(0.0);
212  t_avg_cnt[lev] = 0.0;
213  }
214 
215  // ********************************************************************************************
216  // Initialize flux registers whenever we create/re-create a level
217  // ********************************************************************************************
218  if (solverChoice.coupling_type == CouplingType::TwoWay) {
219  if (lev == 0) {
220  advflux_reg[0] = nullptr;
221  } else {
222  int ncomp_reflux = vars_new[0][Vars::cons].nComp();
223  advflux_reg[lev] = new YAFluxRegister(ba , grids[lev-1],
224  dm , dmap[lev-1],
225  geom[lev], geom[lev-1],
226  ref_ratio[lev-1], lev, ncomp_reflux);
227  }
228  }
229 
230  // ********************************************************************************************
231  // Define Theta_prim storage if using surface_layer BC
232  // ********************************************************************************************
233  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
234  Theta_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
235  if (solverChoice.moisture_type != MoistureType::None) {
236  Qv_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
237  Qr_prim[lev] = std::make_unique<MultiFab>(ba,dm,1,IntVect(ngrow_state,ngrow_state,1));
238  } else {
239  Qv_prim[lev] = nullptr;
240  Qr_prim[lev] = nullptr;
241  }
242  } else {
243  Theta_prim[lev] = nullptr;
244  Qv_prim[lev] = nullptr;
245  Qr_prim[lev] = nullptr;
246  }
247 
248  // ********************************************************************************************
249  // Map factors
250  // ********************************************************************************************
251  BoxList bl2d_mf = ba.boxList();
252  for (auto& b : bl2d_mf) {
253  b.setRange(2,0);
254  }
255  BoxArray ba2d_mf(std::move(bl2d_mf));
256 
257  mapfac[lev].resize(MapFacType::num);
258  mapfac[lev][MapFacType::m_x] = std::make_unique<MultiFab>(ba2d_mf,dm,1,3);
259  mapfac[lev][MapFacType::u_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,3);
260  mapfac[lev][MapFacType::v_x] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,3);
261 
262 #if 0
263  // For now we comment this out to avoid CI failures but we will need to re-enable
264  // this if using non-conformal mappings
266  mapfac[lev][MapFacType::m_y] = std::make_unique<MultiFab>(ba2d_mf,dm,1,3);
267  }
269  mapfac[lev][MapFacType::u_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(1,0,0)),dm,1,3);
270  }
272  mapfac[lev][MapFacType::v_y] = std::make_unique<MultiFab>(convert(ba2d_mf,IntVect(0,1,0)),dm,1,3);
273  }
274 #endif
275 
277  for (int i = 0; i < 3; i++) {
278  mapfac[lev][i]->setVal(0.5);
279  }
280  for (int i = 3; i < mapfac[lev].size(); i++) {
281  mapfac[lev][i]->setVal(0.25);
282  }
283  } else {
284  for (int i = 0; i < mapfac[lev].size(); i++) {
285  mapfac[lev][i]->setVal(1.0);
286  }
287  }
288 
289  // ********************************************************************************************
290  // Build 1D BA and 2D BA
291  // ********************************************************************************************
292  BoxList bl1d = ba.boxList();
293  for (auto& b : bl1d) {
294  b.setRange(0,0);
295  b.setRange(1,0);
296  }
297  ba1d[lev] = BoxArray(std::move(bl1d));
298 
299  // Build 2D BA
300  BoxList bl2d = ba.boxList();
301  for (auto& b : bl2d) {
302  b.setRange(2,0);
303  }
304  ba2d[lev] = BoxArray(std::move(bl2d));
305 
306  IntVect ng = vars_new[lev][Vars::cons].nGrowVect();
307 
308  if (lev == 0) {
309  mf_C1H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
310  mf_C2H = std::make_unique<MultiFab>(ba1d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
311  mf_MUB = std::make_unique<MultiFab>(ba2d[lev],dm,1,IntVect(ng[0],ng[1],ng[2]));
312  }
313 
314  mf_PSFC[lev] = std::make_unique<MultiFab>(ba2d[lev],dm,1,ng);
315 
316  //*********************************************************
317  // Variables for Fitch model for windfarm parametrization
318  //*********************************************************
319 #if defined(ERF_USE_WINDFARM)
320  if (solverChoice.windfarm_type == WindFarmType::Fitch){
321  vars_windfarm[lev].define(ba, dm, 5, ngrow_state); // V, dVabsdt, dudt, dvdt, dTKEdt
322  }
323  if (solverChoice.windfarm_type == WindFarmType::EWP){
324  vars_windfarm[lev].define(ba, dm, 3, ngrow_state); // dudt, dvdt, dTKEdt
325  }
326  if (solverChoice.windfarm_type == WindFarmType::SimpleAD) {
327  vars_windfarm[lev].define(ba, dm, 2, ngrow_state);// dudt, dvdt
328  }
329  if (solverChoice.windfarm_type == WindFarmType::GeneralAD) {
330  vars_windfarm[lev].define(ba, dm, 3, ngrow_state);// dudt, dvdt, dwdt
331  }
332  Nturb[lev].define(ba, dm, 1, ngrow_state); // Number of turbines in a cell
333  SMark[lev].define(ba, dm, 2, 1); // Free stream velocity/source term
334  // sampling marker in a cell - 2 components
335 #endif
336 
337 
338 #ifdef ERF_USE_WW3_COUPLING
339  // create a new BoxArray and DistributionMapping for a MultiFab with 1 box
340  BoxArray ba_onegrid(geom[lev].Domain());
341  BoxList bl2d_onegrid = ba_onegrid.boxList();
342  for (auto& b : bl2d_onegrid) {
343  b.setRange(2,0);
344  }
345  BoxArray ba2d_onegrid(std::move(bl2d_onegrid));
346  Vector<int> pmap;
347  pmap.resize(1);
348  pmap[0]=0;
349  DistributionMapping dm_onegrid(ba2d_onegrid);
350  dm_onegrid.define(pmap);
351 
352  Hwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
353  Lwave_onegrid[lev] = std::make_unique<MultiFab>(ba2d_onegrid,dm_onegrid,1,IntVect(1,1,0));
354 
355  BoxList bl2d_wave = ba.boxList();
356  for (auto& b : bl2d_wave) {
357  b.setRange(2,0);
358  }
359  BoxArray ba2d_wave(std::move(bl2d_wave));
360 
361  Hwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
362  Lwave[lev] = std::make_unique<MultiFab>(ba2d_wave,dm,1,IntVect(3,3,0));
363 
364  std::cout<<ba_onegrid<<std::endl;
365  std::cout<<ba2d_onegrid<<std::endl;
366  std::cout<<dm_onegrid<<std::endl;
367 #endif
368 
369 
370  //*********************************************************
371  // Radiation heating source terms
372  //*********************************************************
373  if (solverChoice.rad_type != RadiationType::None || solverChoice.lsm_type != LandSurfaceType::None)
374  {
375  qheating_rates[lev] = std::make_unique<MultiFab>(ba, dm, 2, ngrow_state);
376  qheating_rates[lev]->setVal(0.);
377  }
378 
379  //*********************************************************
380  // Radiation fluxes for coupling to LSM
381  //*********************************************************
382 
383  // NOTE: Finer levels do not need to coincide with the bottom domain boundary
384  // at k=0. We make slabs here with the kmin for a given box. Therefore,
385  // care must be taken before applying these fluxes to an LSM model. For
386 
387  // Radiative fluxes for LSM
388  if (solverChoice.lsm_type != LandSurfaceType::None &&
389  solverChoice.rad_type != RadiationType::None)
390  {
391  BoxList m_bl = ba.boxList();
392  for (auto& b : m_bl) {
393  int kmin = b.smallEnd(2);
394  b.setRange(2,kmin);
395  }
396  BoxArray m_ba(std::move(m_bl));
397 
398  sw_lw_fluxes[lev] = std::make_unique<MultiFab>(m_ba, dm, 5, ngrow_state); // SW direct (2), SW diffuse (2), LW
399  solar_zenith[lev] = std::make_unique<MultiFab>(m_ba, dm, 2, ngrow_state);
400 
401  sw_lw_fluxes[lev]->setVal(0.);
402  solar_zenith[lev]->setVal(0.);
403  }
404 
405  //*********************************************************
406  // Turbulent perturbation region initialization
407  //*********************************************************
408  if (solverChoice.pert_type == PerturbationType::Source ||
409  solverChoice.pert_type == PerturbationType::Direct ||
410  solverChoice.pert_type == PerturbationType::CPM)
411  {
412  amrex::Box bnd_bx = ba.minimalBox();
414  turbPert.init_tpi(lev, bnd_bx.smallEnd(), bnd_bx.bigEnd(), geom[lev].CellSizeArray(),
415  ba, dm, ngrow_state, pp_prefix, refRatio(), max_level);
416  }
417 
418  //
419  // Define the land mask here and set it to all land by default
420  // NOTE: the logic below will BREAK if we have any grids not touching the bottom boundary
421  //
422  {
423  lmask_lev[lev].resize(1);
424  auto ngv = lev_new[Vars::cons].nGrowVect(); ngv[2] = 0;
425  BoxList bl2d_mask = ba.boxList();
426  for (auto& b : bl2d_mask) {
427  b.setRange(2,0);
428  }
429  BoxArray ba2d_mask(std::move(bl2d_mask));
430  lmask_lev[lev][0] = std::make_unique<iMultiFab>(ba2d_mask,dm,1,ngv);
431  lmask_lev[lev][0]->setVal(1);
432  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
433  }
434 
435  // Read in tables needed for windfarm simulations
436  // fill in Nturb multifab - number of turbines in each mesh cell
437  // write out the vtk files for wind turbine location and/or
438  // actuator disks
439  #ifdef ERF_USE_WINDFARM
440  //init_windfarm(lev);
441  #endif
442 }
@ num
Definition: ERF_DataStruct.H:22
#define NDRY
Definition: ERF_IndexDefines.H:13
void init_default_zphys(int, const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc)
Definition: ERF_TerrainMetrics.cpp:15
static AMREX_FORCE_INLINE int ComputeGhostCells(const SolverChoice &sc)
Definition: ERF.H:1283
amrex::EBFArrayBoxFactory const & EBFactory(int lev) const noexcept
Definition: ERF.H:1559
@ num_comps
Definition: ERF_IndexDefines.H:68
@ gpz
Definition: ERF_IndexDefines.H:152
@ gpy
Definition: ERF_IndexDefines.H:151
@ gpx
Definition: ERF_IndexDefines.H:150
bool test_mapfactor
Definition: ERF_DataStruct.H:838
void init_tpi_type(const PerturbationType &pert_type)
Definition: ERF_TurbPertStruct.H:28
void init_tpi(const int lev, const amrex::IntVect &valid_box_lo, const amrex::IntVect &valid_box_hi, const amrex::GpuArray< amrex::Real, 3 > dx, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, const int ngrow_state, std::string pp_prefix, const amrex::Vector< amrex::IntVect > refRatio, const int max_level)
Definition: ERF_TurbPertStruct.H:45
Here is the call graph for this function:

◆ init_thin_body()

void ERF::init_thin_body ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
717 {
718  //********************************************************************************************
719  // Thin immersed body
720  // *******************************************************************************************
721 #if 0
722  if ((solverChoice.advChoice.zero_xflux.size() > 0) ||
723  (solverChoice.advChoice.zero_yflux.size() > 0) ||
724  (solverChoice.advChoice.zero_zflux.size() > 0))
725  {
726  overset_imask[lev] = std::make_unique<iMultiFab>(ba,dm,1,0);
727  overset_imask[lev]->setVal(1); // == value is unknown (to be solved)
728  }
729 #endif
730 
731  if (solverChoice.advChoice.zero_xflux.size() > 0) {
732  amrex::Print() << "Setting up thin immersed body for "
733  << solverChoice.advChoice.zero_xflux.size() << " xfaces" << std::endl;
734  BoxArray ba_xf(ba);
735  ba_xf.surroundingNodes(0);
736  thin_xforce[lev] = std::make_unique<MultiFab>(ba_xf,dm,1,0);
737  thin_xforce[lev]->setVal(0.0);
738  xflux_imask[lev] = std::make_unique<iMultiFab>(ba_xf,dm,1,0);
739  xflux_imask[lev]->setVal(1);
740  for ( MFIter mfi(*xflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
741  {
742  Array4<int> const& imask_arr = xflux_imask[lev]->array(mfi);
743  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
744  Box xbx = mfi.nodaltilebox(0);
745  for (int iv=0; iv < solverChoice.advChoice.zero_xflux.size(); ++iv) {
746  const auto& faceidx = solverChoice.advChoice.zero_xflux[iv];
747  if ((faceidx[0] >= xbx.smallEnd(0)) && (faceidx[0] <= xbx.bigEnd(0)) &&
748  (faceidx[1] >= xbx.smallEnd(1)) && (faceidx[1] <= xbx.bigEnd(1)) &&
749  (faceidx[2] >= xbx.smallEnd(2)) && (faceidx[2] <= xbx.bigEnd(2)))
750  {
751  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
752  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
753  //imask_cell_arr(faceidx[0]-1,faceidx[1],faceidx[2]) = 0;
754  amrex::AllPrint() << " mask xface at " << faceidx << std::endl;
755  }
756  }
757  }
758  } else {
759  thin_xforce[lev] = nullptr;
760  xflux_imask[lev] = nullptr;
761  }
762 
763  if (solverChoice.advChoice.zero_yflux.size() > 0) {
764  amrex::Print() << "Setting up thin immersed body for "
765  << solverChoice.advChoice.zero_yflux.size() << " yfaces" << std::endl;
766  BoxArray ba_yf(ba);
767  ba_yf.surroundingNodes(1);
768  thin_yforce[lev] = std::make_unique<MultiFab>(ba_yf,dm,1,0);
769  thin_yforce[lev]->setVal(0.0);
770  yflux_imask[lev] = std::make_unique<iMultiFab>(ba_yf,dm,1,0);
771  yflux_imask[lev]->setVal(1);
772  for ( MFIter mfi(*yflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
773  {
774  Array4<int> const& imask_arr = yflux_imask[lev]->array(mfi);
775  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
776  Box ybx = mfi.nodaltilebox(1);
777  for (int iv=0; iv < solverChoice.advChoice.zero_yflux.size(); ++iv) {
778  const auto& faceidx = solverChoice.advChoice.zero_yflux[iv];
779  if ((faceidx[0] >= ybx.smallEnd(0)) && (faceidx[0] <= ybx.bigEnd(0)) &&
780  (faceidx[1] >= ybx.smallEnd(1)) && (faceidx[1] <= ybx.bigEnd(1)) &&
781  (faceidx[2] >= ybx.smallEnd(2)) && (faceidx[2] <= ybx.bigEnd(2)))
782  {
783  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
784  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
785  //imask_cell_arr(faceidx[0],faceidx[1]-1,faceidx[2]) = 0;
786  amrex::AllPrint() << " mask yface at " << faceidx << std::endl;
787  }
788  }
789  }
790  } else {
791  thin_yforce[lev] = nullptr;
792  yflux_imask[lev] = nullptr;
793  }
794 
795  if (solverChoice.advChoice.zero_zflux.size() > 0) {
796  amrex::Print() << "Setting up thin immersed body for "
797  << solverChoice.advChoice.zero_zflux.size() << " zfaces" << std::endl;
798  BoxArray ba_zf(ba);
799  ba_zf.surroundingNodes(2);
800  thin_zforce[lev] = std::make_unique<MultiFab>(ba_zf,dm,1,0);
801  thin_zforce[lev]->setVal(0.0);
802  zflux_imask[lev] = std::make_unique<iMultiFab>(ba_zf,dm,1,0);
803  zflux_imask[lev]->setVal(1);
804  for ( MFIter mfi(*zflux_imask[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
805  {
806  Array4<int> const& imask_arr = zflux_imask[lev]->array(mfi);
807  //Array4<int> const& imask_cell_arr = overset_imask[lev]->array(mfi);
808  Box zbx = mfi.nodaltilebox(2);
809  for (int iv=0; iv < solverChoice.advChoice.zero_zflux.size(); ++iv) {
810  const auto& faceidx = solverChoice.advChoice.zero_zflux[iv];
811  if ((faceidx[0] >= zbx.smallEnd(0)) && (faceidx[0] <= zbx.bigEnd(0)) &&
812  (faceidx[1] >= zbx.smallEnd(1)) && (faceidx[1] <= zbx.bigEnd(1)) &&
813  (faceidx[2] >= zbx.smallEnd(2)) && (faceidx[2] <= zbx.bigEnd(2)))
814  {
815  imask_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
816  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]) = 0;
817  //imask_cell_arr(faceidx[0],faceidx[1],faceidx[2]-1) = 0;
818  amrex::AllPrint() << " mask zface at " << faceidx << std::endl;
819  }
820  }
821  }
822  } else {
823  thin_zforce[lev] = nullptr;
824  zflux_imask[lev] = nullptr;
825  }
826 }
amrex::Vector< amrex::IntVect > zero_yflux
Definition: ERF_AdvStruct.H:414
amrex::Vector< amrex::IntVect > zero_xflux
Definition: ERF_AdvStruct.H:413
amrex::Vector< amrex::IntVect > zero_zflux
Definition: ERF_AdvStruct.H:415

◆ init_uniform()

void ERF::init_uniform ( int  lev)
private

Use problem-specific reference density and temperature to set the background state to a uniform value.

Parameters
levInteger specifying the current level
18 {
19  auto& lev_new = vars_new[lev];
20  for (MFIter mfi(lev_new[Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
21  const Box &gbx = mfi.growntilebox(1);
22  const auto &cons_arr = lev_new[Vars::cons].array(mfi);
23  prob->init_uniform(gbx, cons_arr);
24  }
25 }

◆ init_zphys()

void ERF::init_zphys ( int  lev,
amrex::Real  time 
)
536 {
537  if (solverChoice.init_type != InitType::WRFInput && solverChoice.init_type != InitType::Metgrid)
538  {
539  if (lev > 0) {
540  //
541  // First interpolate from coarser level if there is one
542  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
543  // have been pre-filled - this includes ghost cells both inside and outside
544  // the domain
545  //
546  InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
547  IntVect(0,0,0), // do not fill ghost cells outside the domain
548  *z_phys_nd[lev-1], 0, 0, 1,
549  geom[lev-1], geom[lev],
550  refRatio(lev-1), &node_bilinear_interp,
552  }
553 
554  int ngrow = ComputeGhostCells(solverChoice) + 2;
555  Box bx(surroundingNodes(Geom(lev).Domain())); bx.grow(ngrow);
556  FArrayBox terrain_fab(makeSlab(bx,2,0),1);
557 
558  //
559  // If we are using fitted mesh then we use the surface as defined above
560  // If we are not using fitted mesh but are using z_levels, we still need z_phys (for now)
561  // but we need to use a flat terrain for the mesh itself (the EB data has already been made
562  // from the correct terrain)
563  //
564  if (solverChoice.terrain_type != TerrainType::StaticFittedMesh &&
565  solverChoice.terrain_type != TerrainType::MovingFittedMesh) {
566  terrain_fab.template setVal<RunOn::Device>(0.0);
567  } else {
568  //
569  // Fill the values of the terrain height at k=0 only
570  //
571  prob->init_terrain_surface(geom[lev],terrain_fab,time);
572  }
573 
574  for (MFIter mfi(*z_phys_nd[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
575  {
576  Box isect = terrain_fab.box() & (*z_phys_nd[lev])[mfi].box();
577  if (!isect.isEmpty()) {
578  (*z_phys_nd[lev])[mfi].template copy<RunOn::Device>(terrain_fab,isect,0,isect,0,1);
579  }
580  }
581 
583 
584  z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
585 
586  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
587  terrain_blanking[lev]->setVal(1.0);
588  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
589  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
590  }
591 
592  if (lev == 0) {
593  Real zmax = z_phys_nd[0]->max(0,0,false);
594  Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
595  if (rel_diff < 1.e-8) {
596  amrex::Print() << "max of zphys_nd " << zmax << std::endl;
597  amrex::Print() << "max of zlevels " << zlevels_stag[0][zlevels_stag[0].size()-1] << std::endl;
598  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < 1.e-8, "Terrain is taller than domain top!");
599  }
600  } // lev == 0
601 
602  } // init_type
603 }
void make_terrain_fitted_coords(int lev, const Geometry &geom, MultiFab &z_phys_nd, Vector< Real > const &z_levels_h, GpuArray< ERF_BC, AMREX_SPACEDIM *2 > &phys_bc_type)
Definition: ERF_TerrainMetrics.cpp:46
Here is the call graph for this function:

◆ InitData()

void ERF::InitData ( )
819 {
820  BL_PROFILE_VAR("ERF::InitData()", InitData);
821  InitData_pre();
822  InitData_post();
823  BL_PROFILE_VAR_STOP(InitData);
824 }
void InitData_pre()
Definition: ERF.cpp:827
void InitData_post()
Definition: ERF.cpp:892
void InitData()
Definition: ERF.cpp:818

Referenced by main().

Here is the caller graph for this function:

◆ InitData_post()

void ERF::InitData_post ( )
893 {
895  {
896  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level == 0,
897  "Thin immersed body with refinement not currently supported.");
898  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
899  amrex::Print() << "NOTE: Thin immersed body with non-constant dz has not been tested." << std::endl;
900  }
901  }
902 
903  if (!restart_chkfile.empty()) {
904  restart();
905  }
906 
907  //
908  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
909  //
910  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
911  for (int crse_lev = finest_level-1; crse_lev >= 0; crse_lev--) {
912  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
913  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
914  detJ_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
915  z_phys_cc[crse_lev]->FillBoundary(geom[crse_lev].periodicity());
916  }
917  }
918 
919  //
920  // Copy vars_new into vars_old, then use vars_old to fill covered cells in vars_new during AverageDown
921  //
922  if (SolverChoice::terrain_type == TerrainType::EB) {
923  for (int lev = 0; lev <= finest_level; lev++) {
924  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
925  MultiFab::Copy(vars_old[lev][Vars::cons],vars_new[lev][Vars::cons],0,0,ncomp_cons,vars_new[lev][Vars::cons].nGrowVect());
926  }
927  }
928 
929  if (restart_chkfile.empty()) {
930  if (solverChoice.coupling_type == CouplingType::TwoWay) {
931  AverageDown();
932  }
933  }
934 
935 #ifdef ERF_USE_PARTICLES
936  if (restart_chkfile.empty()) {
937  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
939  Warning("Tight coupling has not been tested with Lagrangian microphysics");
940  }
941 
942  for (int lev = 0; lev <= finest_level; lev++) {
943  dynamic_cast<LagrangianMicrophysics&>(*micro).initParticles(z_phys_nd[lev]);
944  }
945  }
946  }
947 #endif
948 
949  if (!restart_chkfile.empty()) { // Restart from a checkpoint
950 
951  // Create the physbc objects for {cons, u, v, w, base state}
952  // We fill the additional base state ghost cells just in case we have read the old format
953  for (int lev(0); lev <= finest_level; ++lev) {
954  make_physbcs(lev);
955  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
956  }
957 
959  for (int lev(0); lev <= finest_level; ++lev) {
960  m_forest_drag[lev]->define_drag_field(grids[lev], dmap[lev], geom[lev],
961  z_phys_cc[lev].get(), z_phys_nd[lev].get());
962  }
963  }
964 
965 #ifdef ERF_USE_NETCDF
966  //
967  // Create the needed bdy_data_xlo etc ... since we don't read it in from checkpoint any more
968  //
970 
971  bdy_time_interval = read_times_from_wrfbdy(nc_bdy_file,
972  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
973  start_bdy_time);
974  Real dT = bdy_time_interval;
975 
976  int n_time_old = static_cast<int>(t_new[0] / dT);
977 
978  int lev = 0;
979  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
980 
981  int ntimes = std::min(n_time_old+3, static_cast<int>(bdy_data_xlo.size()));
982 
983  for (int itime = n_time_old; itime < ntimes; itime++)
984  {
985  read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
986  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
987  real_width);
988  convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
989  *mf_MUB, *mf_C1H, *mf_C2H,
991  geom[lev], use_moist);
992  } // itime
993  } // use_real_bcs && lev == 0
994 #endif
995  } // end restart
996 
997 #ifdef ERF_USE_PARTICLES
998  /* If using a Lagrangian microphysics model, its particle container has now been
999  constructed and initialized (calls to micro->Init). So, add its pointer to
1000  ERF::particleData and remove its name from list of unallocated particle containers. */
1001  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
1002  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
1003  const auto& pc_ptr( dynamic_cast<LagrangianMicrophysics&>(*micro).getParticleContainer() );
1004  particleData.pushBack(pc_name, pc_ptr);
1005  particleData.getNamesUnalloc().remove(pc_name);
1006  }
1007 #endif
1008 
1009  if (input_bndry_planes) {
1010  // Read the "time.dat" file to know what data is available
1011  m_r2d->read_time_file();
1012 
1013  // We haven't populated dt yet, set to 0 to ensure assert doesn't crash
1014  Real dt_dummy = 0.0;
1015  m_r2d->read_input_files(t_new[0],dt_dummy,m_bc_extdir_vals);
1016  }
1017 
1019  {
1020  h_rhotheta_src.resize(max_level+1, Vector<Real>(0));
1021  d_rhotheta_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1022  for (int lev = 0; lev <= finest_level; lev++) {
1023  const int domlen = geom[lev].Domain().length(2);
1024  h_rhotheta_src[lev].resize(domlen, 0.0_rt);
1025  d_rhotheta_src[lev].resize(domlen, 0.0_rt);
1026  prob->update_rhotheta_sources(t_new[0],
1027  h_rhotheta_src[lev], d_rhotheta_src[lev],
1028  geom[lev], z_phys_cc[lev]);
1029  }
1030  }
1031 
1033  {
1034  h_u_geos.resize(max_level+1, Vector<Real>(0));
1035  d_u_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1036  h_v_geos.resize(max_level+1, Vector<Real>(0));
1037  d_v_geos.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1038  for (int lev = 0; lev <= finest_level; lev++) {
1039  const int domlen = geom[lev].Domain().length(2);
1040  h_u_geos[lev].resize(domlen, 0.0_rt);
1041  d_u_geos[lev].resize(domlen, 0.0_rt);
1042  h_v_geos[lev].resize(domlen, 0.0_rt);
1043  d_v_geos[lev].resize(domlen, 0.0_rt);
1045  prob->update_geostrophic_profile(t_new[0],
1046  h_u_geos[lev], d_u_geos[lev],
1047  h_v_geos[lev], d_v_geos[lev],
1048  geom[lev], z_phys_cc[lev]);
1049  } else {
1050  if (SolverChoice::mesh_type == MeshType::VariableDz) {
1051  amrex::Print() << "Note: 1-D geostrophic wind profile input is not defined for real terrain" << std::endl;
1052  }
1054  h_u_geos[lev], d_u_geos[lev],
1055  h_v_geos[lev], d_v_geos[lev],
1056  geom[lev],
1057  zlevels_stag[0]);
1058  }
1059  }
1060  }
1061 
1063  {
1064  h_rhoqt_src.resize(max_level+1, Vector<Real>(0));
1065  d_rhoqt_src.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1066  for (int lev = 0; lev <= finest_level; lev++) {
1067  const int domlen = geom[lev].Domain().length(2);
1068  h_rhoqt_src[lev].resize(domlen, 0.0_rt);
1069  d_rhoqt_src[lev].resize(domlen, 0.0_rt);
1070  prob->update_rhoqt_sources(t_new[0],
1071  h_rhoqt_src[lev], d_rhoqt_src[lev],
1072  geom[lev], z_phys_cc[lev]);
1073  }
1074  }
1075 
1077  {
1078  h_w_subsid.resize(max_level+1, Vector<Real>(0));
1079  d_w_subsid.resize(max_level+1, Gpu::DeviceVector<Real>(0));
1080  for (int lev = 0; lev <= finest_level; lev++) {
1081  const int domlen = geom[lev].Domain().length(2) + 1; // lives on z-faces
1082  h_w_subsid[lev].resize(domlen, 0.0_rt);
1083  d_w_subsid[lev].resize(domlen, 0.0_rt);
1084  prob->update_w_subsidence(t_new[0],
1085  h_w_subsid[lev], d_w_subsid[lev],
1086  geom[lev], z_phys_nd[lev]);
1087  }
1088  }
1089 
1092  {
1093  initRayleigh();
1094  if (solverChoice.init_type == InitType::Input_Sounding)
1095  {
1096  // Overwrite ubar, vbar, and thetabar with input profiles;
1097  // wbar is assumed to be 0. Note: the tau coefficient set by
1098  // prob->erf_init_rayleigh() is still used
1099  bool restarting = (!restart_chkfile.empty());
1100  setRayleighRefFromSounding(restarting);
1101  }
1102  }
1103 
1104  // Read in sponge data from input file
1105  if(solverChoice.spongeChoice.sponge_type == "input_sponge")
1106  {
1107  initSponge();
1108  bool restarting = (!restart_chkfile.empty());
1109  setSpongeRefFromSounding(restarting);
1110  }
1111 
1112  if (solverChoice.pert_type == PerturbationType::Source ||
1113  solverChoice.pert_type == PerturbationType::Direct ||
1114  solverChoice.pert_type == PerturbationType::CPM) {
1115  if (is_it_time_for_action(istep[0], t_new[0], dt[0], pert_interval, -1.)) {
1116  turbPert.debug(t_new[0]);
1117  }
1118  }
1119 
1120  // We only write the file at level 0 for now
1121  if (output_bndry_planes)
1122  {
1123  // Create the WriteBndryPlanes object so we can handle writing of boundary plane data
1124  m_w2d = std::make_unique<WriteBndryPlanes>(grids,geom);
1125 
1126  Real time = 0.;
1127  if (time >= bndry_output_planes_start_time) {
1128  bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
1129  m_w2d->write_planes(0, time, vars_new, is_moist);
1130  }
1131  }
1132 
1133  //
1134  // If we are starting from scratch, we have the option to project the initial velocity field
1135  // regardless of how we initialized.
1136  // pp_inc is used as scratch space here; we zero it out after the projection
1137  //
1138  if (restart_chkfile == "")
1139  {
1141  Real dummy_dt = 1.0;
1142  if (verbose > 0) {
1143  amrex::Print() << "Projecting initial velocity field" << std::endl;
1144  }
1145  for (int lev = 0; lev <= finest_level; ++lev)
1146  {
1147  project_velocity(lev, dummy_dt);
1148  pp_inc[lev].setVal(0.);
1149  gradp[lev][GpVars::gpx].setVal(0.);
1150  gradp[lev][GpVars::gpy].setVal(0.);
1151  gradp[lev][GpVars::gpz].setVal(0.);
1152  }
1153  }
1154  }
1155 
1156  // Copy from new into old just in case
1157  for (int lev = 0; lev <= finest_level; ++lev)
1158  {
1159  auto& lev_new = vars_new[lev];
1160  auto& lev_old = vars_old[lev];
1161 
1162  // ***************************************************************************
1163  // Physical bc's at domain boundary
1164  // ***************************************************************************
1165  IntVect ngvect_cons = vars_new[lev][Vars::cons].nGrowVect();
1166  IntVect ngvect_vels = vars_new[lev][Vars::xvel].nGrowVect();
1167 
1168  int ncomp_cons = lev_new[Vars::cons].nComp();
1169  bool do_fb = true;
1170 
1171 #ifdef ERF_USE_NETCDF
1172  // We call this here because it is an ERF routine
1173  if (solverChoice.use_real_bcs && (lev==0)) {
1174  int icomp_cons = 0;
1175  bool cons_only = false;
1176  Vector<MultiFab*> mfs_vec = {&lev_new[Vars::cons],&lev_new[Vars::xvel],
1177  &lev_new[Vars::yvel],&lev_new[Vars::zvel]};
1179  fill_from_realbdy_upwind(mfs_vec,t_new[lev],cons_only,icomp_cons,
1180  ncomp_cons,ngvect_cons,ngvect_vels);
1181  } else {
1182  fill_from_realbdy(mfs_vec,t_new[lev],cons_only,icomp_cons,
1183  ncomp_cons,ngvect_cons,ngvect_vels);
1184  }
1185  do_fb = false;
1186  }
1187 #endif
1188 
1189  (*physbcs_cons[lev])(lev_new[Vars::cons],lev_new[Vars::xvel],lev_new[Vars::yvel],0,ncomp_cons,
1190  ngvect_cons,t_new[lev],BCVars::cons_bc,do_fb);
1191  ( *physbcs_u[lev])(lev_new[Vars::xvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1192  ngvect_vels,t_new[lev],BCVars::xvel_bc,do_fb);
1193  ( *physbcs_v[lev])(lev_new[Vars::yvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1194  ngvect_vels,t_new[lev],BCVars::yvel_bc,do_fb);
1195  ( *physbcs_w[lev])(lev_new[Vars::zvel],lev_new[Vars::xvel],lev_new[Vars::yvel],
1196  ngvect_vels,t_new[lev],BCVars::zvel_bc,do_fb);
1197 
1198  MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,ncomp_cons,lev_new[Vars::cons].nGrowVect());
1199  MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0, 1,lev_new[Vars::xvel].nGrowVect());
1200  MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0, 1,lev_new[Vars::yvel].nGrowVect());
1201  MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0, 1,lev_new[Vars::zvel].nGrowVect());
1202  }
1203 
1204  // Compute the minimum dz in the domain at each level (to be used for setting the timestep)
1205  dz_min.resize(max_level+1);
1206  for (int lev = 0; lev <= finest_level; ++lev)
1207  {
1208  dz_min[lev] = geom[lev].CellSize(2);
1209  if ( SolverChoice::mesh_type != MeshType::ConstantDz ) {
1210  dz_min[lev] *= (*detJ_cc[lev]).min(0);
1211  }
1212  }
1213 
1214  // We don't need to recompute dt[lev] on restart because we read it in from the checkpoint file.
1215  if (restart_chkfile.empty()) {
1216  ComputeDt();
1217  }
1218 
1219  // Check the viscous limit
1223  Real delta = std::min({geom[finest_level].CellSize(0),
1224  geom[finest_level].CellSize(1),
1225  dz_min[finest_level]});
1226  if (dc.dynamic_viscosity == 0) {
1227  Print() << "Note: Molecular diffusion specified but dynamic_viscosity has not been specified" << std::endl;
1228  } else {
1229  Real nu = dc.dynamic_viscosity / dc.rho0_trans;
1230  Real viscous_limit = 0.5 * delta*delta / nu;
1231  Print() << "Viscous CFL is " << dt[finest_level] / viscous_limit << std::endl;
1232  if (fixed_dt[finest_level] >= viscous_limit) {
1233  Warning("Specified fixed_dt is above the viscous limit");
1234  } else if (dt[finest_level] >= viscous_limit) {
1235  Warning("Adaptive dt based on convective CFL only is above the viscous limit");
1236  }
1237  }
1238  }
1239 
1240  // Fill ghost cells/faces
1241  for (int lev = 0; lev <= finest_level; ++lev)
1242  {
1243  if (lev > 0 && cf_width >= 0) {
1245  }
1246 
1247  auto& lev_new = vars_new[lev];
1248 
1249  //
1250  // Fill boundary conditions -- not sure why we need this here
1251  //
1252  bool fillset = false;
1253  if (lev == 0) {
1254  FillPatchCrseLevel(lev, t_new[lev],
1255  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]});
1256  } else {
1257  FillPatchFineLevel(lev, t_new[lev],
1258  {&lev_new[Vars::cons],&lev_new[Vars::xvel],&lev_new[Vars::yvel],&lev_new[Vars::zvel]},
1259  {&lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
1260  base_state[lev], base_state[lev],
1261  fillset);
1262  }
1263 
1264  //
1265  // We do this here to make sure level (lev-1) boundary conditions are filled
1266  // before we interpolate to level (lev) ghost cells
1267  //
1268  if (lev < finest_level) {
1269  auto& lev_old = vars_old[lev];
1270  MultiFab::Copy(lev_old[Vars::cons],lev_new[Vars::cons],0,0,lev_old[Vars::cons].nComp(),lev_old[Vars::cons].nGrowVect());
1271  MultiFab::Copy(lev_old[Vars::xvel],lev_new[Vars::xvel],0,0,lev_old[Vars::xvel].nComp(),lev_old[Vars::xvel].nGrowVect());
1272  MultiFab::Copy(lev_old[Vars::yvel],lev_new[Vars::yvel],0,0,lev_old[Vars::yvel].nComp(),lev_old[Vars::yvel].nGrowVect());
1273  MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0,lev_old[Vars::zvel].nComp(),lev_old[Vars::zvel].nGrowVect());
1274  }
1275 
1276  //
1277  // We fill the ghost cell values of the base state in case it wasn't done in the initialization
1278  //
1279  base_state[lev].FillBoundary(geom[lev].periodicity());
1280 
1281  // For moving terrain only
1282  if (solverChoice.terrain_type == TerrainType::MovingFittedMesh) {
1283  MultiFab::Copy(base_state_new[lev],base_state[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
1284  base_state_new[lev].FillBoundary(geom[lev].periodicity());
1285  }
1286 
1287  }
1288 
1289  // Allow idealized cases over water, used to set lmask
1290  ParmParse pp("erf");
1291  int is_land;
1292  for (int lev = 0; lev <= finest_level; ++lev)
1293  {
1294  if (pp.query("is_land", is_land, lev)) {
1295  if (is_land == 1) {
1296  amrex::Print() << "Level " << lev << " is land" << std::endl;
1297  } else if (is_land == 0) {
1298  amrex::Print() << "Level " << lev << " is water" << std::endl;
1299  } else {
1300  Error("is_land should be 0 or 1");
1301  }
1302  lmask_lev[lev][0]->setVal(is_land);
1303  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
1304  }
1305  }
1306 
1307  // If lev > 0, we need to fill bc's by interpolation from coarser grid
1308  for (int lev = 1; lev <= finest_level; ++lev)
1309  {
1310  Real time_for_fp = 0.; // This is not actually used
1311  Vector<Real> ftime = {time_for_fp, time_for_fp};
1312  Vector<Real> ctime = {time_for_fp, time_for_fp};
1313  if (lat_m[lev]) {
1314  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1315  Vector<MultiFab*> fmf = {lat_m[lev ].get(), lat_m[lev ].get()};
1316  Vector<MultiFab*> cmf = {lat_m[lev-1].get(), lat_m[lev-1].get()};
1317  IntVect ngv = lat_m[lev]->nGrowVect(); ngv[2] = 0;
1318  Interpolater* mapper = &cell_cons_interp;
1319  FillPatchTwoLevels(*lat_m[lev].get(), ngv, IntVect(0,0,0),
1320  time_for_fp, cmf, ctime, fmf, ftime,
1321  0, 0, 1, geom[lev-1], geom[lev],
1322  refRatio(lev-1), mapper, domain_bcs_type,
1323  BCVars::cons_bc);
1324  }
1325  if (lon_m[lev]) {
1326  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1327  Vector<MultiFab*> fmf = {lon_m[lev ].get(), lon_m[lev ].get()};
1328  Vector<MultiFab*> cmf = {lon_m[lev-1].get(), lon_m[lev-1].get()};
1329  IntVect ngv = lon_m[lev]->nGrowVect(); ngv[2] = 0;
1330  Interpolater* mapper = &cell_cons_interp;
1331  FillPatchTwoLevels(*lon_m[lev].get(), ngv, IntVect(0,0,0),
1332  time_for_fp, cmf, ctime, fmf, ftime,
1333  0, 0, 1, geom[lev-1], geom[lev],
1334  refRatio(lev-1), mapper, domain_bcs_type,
1335  BCVars::cons_bc);
1336  } // lon_m
1337  if (sinPhi_m[lev]) {
1338  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1339  Vector<MultiFab*> fmf = {sinPhi_m[lev ].get(), sinPhi_m[lev ].get()};
1340  Vector<MultiFab*> cmf = {sinPhi_m[lev-1].get(), sinPhi_m[lev-1].get()};
1341  IntVect ngv = sinPhi_m[lev]->nGrowVect(); ngv[2] = 0;
1342  Interpolater* mapper = &cell_cons_interp;
1343  FillPatchTwoLevels(*sinPhi_m[lev].get(), ngv, IntVect(0,0,0),
1344  time_for_fp, cmf, ctime, fmf, ftime,
1345  0, 0, 1, geom[lev-1], geom[lev],
1346  refRatio(lev-1), mapper, domain_bcs_type,
1347  BCVars::cons_bc);
1348  } // sinPhi
1349  if (cosPhi_m[lev]) {
1350  // Call FillPatchTwoLevels which ASSUMES that all ghost cells at lev-1 have already been filled
1351  Vector<MultiFab*> fmf = {cosPhi_m[lev ].get(), cosPhi_m[lev ].get()};
1352  Vector<MultiFab*> cmf = {cosPhi_m[lev-1].get(), cosPhi_m[lev-1].get()};
1353  IntVect ngv = cosPhi_m[lev]->nGrowVect(); ngv[2] = 0;
1354  Interpolater* mapper = &cell_cons_interp;
1355  FillPatchTwoLevels(*cosPhi_m[lev].get(), ngv, IntVect(0,0,0),
1356  time_for_fp, cmf, ctime, fmf, ftime,
1357  0, 0, 1, geom[lev-1], geom[lev],
1358  refRatio(lev-1), mapper, domain_bcs_type,
1359  BCVars::cons_bc);
1360  } // cosPhi
1361  } // lev
1362 
1363 #ifdef ERF_USE_WW3_COUPLING
1364  int my_lev = 0;
1365  amrex::Print() << " About to call send_to_ww3 from ERF.cpp" << std::endl;
1366  send_to_ww3(my_lev);
1367  amrex::Print() << " About to call read_waves from ERF.cpp" << std::endl;
1368  read_waves(my_lev);
1369  // send_to_ww3(my_lev);
1370 #endif
1371 
1372  // Configure SurfaceLayer params if used
1373  // NOTE: we must set up the MOST routine after calling FillPatch
1374  // in order to have lateral ghost cells filled (MOST + terrain interp).
1375  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
1376  {
1377  bool rotate = solverChoice.use_rotate_surface_flux;
1378  if (rotate) {
1379  Print() << "Using surface layer model with stress rotations" << std::endl;
1380  }
1381 
1382  //
1383  // This constructor will make the ABLMost object but not allocate the arrays at each level.
1384  //
1385  m_SurfaceLayer = std::make_unique<SurfaceLayer>(geom, rotate, pp_prefix, Qv_prim,
1387 #ifdef ERF_USE_NETCDF
1388  , bdy_time_interval
1389 #endif
1390  );
1391  // This call will allocate the arrays at each level. If we regrid later, either changing
1392  // the number of level sor just the grids at each existing level, we will call an update routine
1393  // to redefine the internal arrays in m_SurfaceLayer.
1394  int nlevs = geom.size();
1395  for (int lev = 0; lev < nlevs; lev++)
1396  {
1397  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
1398  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
1399  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
1400  mfv_old, Theta_prim[lev], Qv_prim[lev],
1401  Qr_prim[lev], z_phys_nd[lev],
1402  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
1404  sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
1405  }
1406 
1407 
1408  if (restart_chkfile != "") {
1409  // Update surface fields if needed (and available)
1411  }
1412 
1413  // We now configure ABLMost params here so that we can print the averages at t=0
1414  // Note we don't fill ghost cells here because this is just for diagnostics
1415  for (int lev = 0; lev <= finest_level; ++lev)
1416  {
1417  Real time = t_new[lev];
1418  IntVect ng = Theta_prim[lev]->nGrowVect();
1419 
1420  MultiFab::Copy( *Theta_prim[lev], vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, ng);
1421  MultiFab::Divide(*Theta_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1422 
1423  if (solverChoice.moisture_type != MoistureType::None) {
1424  ng = Qv_prim[lev]->nGrowVect();
1425 
1426  MultiFab::Copy( *Qv_prim[lev], vars_new[lev][Vars::cons], RhoQ1_comp, 0, 1, ng);
1427  MultiFab::Divide(*Qv_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1428 
1429  int rhoqr_comp = solverChoice.moisture_indices.qr;
1430  if (rhoqr_comp > -1) {
1431  MultiFab::Copy( *Qr_prim[lev], vars_new[lev][Vars::cons], rhoqr_comp, 0, 1, ng);
1432  MultiFab::Divide(*Qr_prim[lev], vars_new[lev][Vars::cons], Rho_comp, 0, 1, ng);
1433  } else {
1434  Qr_prim[lev]->setVal(0.0);
1435  }
1436  }
1437  m_SurfaceLayer->update_mac_ptrs(lev, vars_new, Theta_prim, Qv_prim, Qr_prim);
1438 
1439  if (restart_chkfile == "") {
1440  // Only do this if starting from scratch; if restarting, then
1441  // we don't want to call update_fluxes multiple times because
1442  // it will change u* and theta* from their previous values
1443  m_SurfaceLayer->update_pblh(lev, vars_new, z_phys_cc[lev].get(),
1445  m_SurfaceLayer->update_fluxes(lev, time, vars_new[lev][Vars::cons], z_phys_nd[lev]);
1446  }
1447  }
1448  } // end if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer)
1449 
1450  // Update micro vars before first plot file
1451  if (solverChoice.moisture_type != MoistureType::None) {
1452  for (int lev = 0; lev <= finest_level; ++lev) micro->Update_Micro_Vars_Lev(lev, vars_new[lev][Vars::cons]);
1453  }
1454 
1455  // Fill time averaged velocities before first plot file
1456  if (solverChoice.time_avg_vel) {
1457  for (int lev = 0; lev <= finest_level; ++lev) {
1458  Time_Avg_Vel_atCC(dt[lev], t_avg_cnt[lev], vel_t_avg[lev].get(),
1459  vars_new[lev][Vars::xvel],
1460  vars_new[lev][Vars::yvel],
1461  vars_new[lev][Vars::zvel]);
1462  }
1463  }
1464 
1465  // check for additional plotting variables that are available after particle containers
1466  // are setup.
1467  const std::string& pv3d_1 = "plot_vars_1" ; appendPlotVariables(pv3d_1,plot3d_var_names_1);
1468  const std::string& pv3d_2 = "plot_vars_2" ; appendPlotVariables(pv3d_2,plot3d_var_names_2);
1469  const std::string& pv2d_1 = "plot2d_vars_1"; appendPlotVariables(pv2d_1,plot2d_var_names_1);
1470  const std::string& pv2d_2 = "plot2d_vars_2"; appendPlotVariables(pv2d_2,plot2d_var_names_2);
1471 
1472  if ( restart_chkfile.empty() && (m_check_int > 0 || m_check_per > 0.) )
1473  {
1477  }
1478 
1479  if ( (restart_chkfile.empty()) ||
1480  (!restart_chkfile.empty() && plot_file_on_restart) )
1481  {
1482  if (m_plot3d_int_1 > 0 || m_plot3d_per_1 > 0.)
1483  {
1487  }
1488  if (m_plot3d_int_2 > 0 || m_plot3d_per_2 > 0.)
1489  {
1493  }
1494  if (m_plot2d_int_1 > 0 || m_plot2d_per_1 > 0.)
1495  {
1499  }
1500  if (m_plot2d_int_2 > 0 || m_plot2d_per_2 > 0.)
1501  {
1505  }
1506  if (m_subvol_int > 0 || m_subvol_per > 0.) {
1507  WriteSubvolume();
1508  last_subvol_step = istep[0];
1510  }
1511  }
1512 
1513  // Set these up here because we need to know which MPI rank "cell" is on...
1514  if (pp.contains("data_log"))
1515  {
1516  int num_datalogs = pp.countval("data_log");
1517  datalog.resize(num_datalogs);
1518  datalogname.resize(num_datalogs);
1519  pp.queryarr("data_log",datalogname,0,num_datalogs);
1520  for (int i = 0; i < num_datalogs; i++) {
1522  }
1523  }
1524 
1525  if (pp.contains("der_data_log"))
1526  {
1527  int num_der_datalogs = pp.countval("der_data_log");
1528  der_datalog.resize(num_der_datalogs);
1529  der_datalogname.resize(num_der_datalogs);
1530  pp.queryarr("der_data_log",der_datalogname,0,num_der_datalogs);
1531  for (int i = 0; i < num_der_datalogs; i++) {
1533  }
1534  }
1535 
1536  if (pp.contains("energy_data_log"))
1537  {
1538  int num_energy_datalogs = pp.countval("energy_data_log");
1539  tot_e_datalog.resize(num_energy_datalogs);
1540  tot_e_datalogname.resize(num_energy_datalogs);
1541  pp.queryarr("energy_data_log",tot_e_datalogname,0,num_energy_datalogs);
1542  for (int i = 0; i < num_energy_datalogs; i++) {
1544  }
1545  }
1546 
1547  if (solverChoice.rad_type != RadiationType::None)
1548  {
1549  // Create data log for radiation model if requested
1550  rad[0]->setupDataLog();
1551  }
1552 
1553 
1554  if (restart_chkfile.empty() && profile_int > 0) {
1555  if (destag_profiles) {
1556  // all variables cell-centered
1558  } else {
1559  // some variables staggered
1561  }
1562  }
1563 
1564  if (pp.contains("sample_point_log") && pp.contains("sample_point"))
1565  {
1566  int lev = 0;
1567 
1568  int num_samplepts = pp.countval("sample_point") / AMREX_SPACEDIM;
1569  if (num_samplepts > 0) {
1570  Vector<int> index; index.resize(num_samplepts*AMREX_SPACEDIM);
1571  samplepoint.resize(num_samplepts);
1572 
1573  pp.queryarr("sample_point",index,0,num_samplepts*AMREX_SPACEDIM);
1574  for (int i = 0; i < num_samplepts; i++) {
1575  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1576  samplepoint[i] = iv;
1577  }
1578  }
1579 
1580  int num_sampleptlogs = pp.countval("sample_point_log");
1581  AMREX_ALWAYS_ASSERT(num_sampleptlogs == num_samplepts);
1582  if (num_sampleptlogs > 0) {
1583  sampleptlog.resize(num_sampleptlogs);
1584  sampleptlogname.resize(num_sampleptlogs);
1585  pp.queryarr("sample_point_log",sampleptlogname,0,num_sampleptlogs);
1586 
1587  for (int i = 0; i < num_sampleptlogs; i++) {
1589  }
1590  }
1591 
1592  }
1593 
1594  if (pp.contains("sample_line_log") && pp.contains("sample_line"))
1595  {
1596  int lev = 0;
1597 
1598  int num_samplelines = pp.countval("sample_line") / AMREX_SPACEDIM;
1599  if (num_samplelines > 0) {
1600  Vector<int> index; index.resize(num_samplelines*AMREX_SPACEDIM);
1601  sampleline.resize(num_samplelines);
1602 
1603  pp.queryarr("sample_line",index,0,num_samplelines*AMREX_SPACEDIM);
1604  for (int i = 0; i < num_samplelines; i++) {
1605  IntVect iv(index[AMREX_SPACEDIM*i+0],index[AMREX_SPACEDIM*i+1],index[AMREX_SPACEDIM*i+2]);
1606  sampleline[i] = iv;
1607  }
1608  }
1609 
1610  int num_samplelinelogs = pp.countval("sample_line_log");
1611  AMREX_ALWAYS_ASSERT(num_samplelinelogs == num_samplelines);
1612  if (num_samplelinelogs > 0) {
1613  samplelinelog.resize(num_samplelinelogs);
1614  samplelinelogname.resize(num_samplelinelogs);
1615  pp.queryarr("sample_line_log",samplelinelogname,0,num_samplelinelogs);
1616 
1617  for (int i = 0; i < num_samplelinelogs; i++) {
1619  }
1620  }
1621 
1622  }
1623 
1628  }
1629 
1630  // Create object to do line and plane sampling if needed
1631  bool do_line = false; bool do_plane = false;
1632  pp.query("do_line_sampling",do_line); pp.query("do_plane_sampling",do_plane);
1633  if (do_line || do_plane) {
1634  data_sampler = std::make_unique<SampleData>(do_line, do_plane);
1635  data_sampler->write_coords(z_phys_cc);
1636  }
1637 
1638  if ( solverChoice.terrain_type == TerrainType::EB ||
1639  solverChoice.terrain_type == TerrainType::ImmersedForcing)
1640  {
1641  bool write_eb_surface = false;
1642  pp.query("write_eb_surface", write_eb_surface);
1643  if (write_eb_surface) WriteMyEBSurface();
1644  }
1645 
1646 }
void initRayleigh()
Initialize Rayleigh damping profiles.
Definition: ERF_InitRayleigh.cpp:14
amrex::Vector< std::string > samplelinelogname
Definition: ERF.H:1537
void setRayleighRefFromSounding(bool restarting)
Set Rayleigh mean profiles from input sounding.
Definition: ERF_InitRayleigh.cpp:55
amrex::Vector< amrex::IntVect > sampleline
Definition: ERF.H:1538
static amrex::Real sum_per
Definition: ERF.H:1142
void setRecordDataInfo(int i, const std::string &filename)
Definition: ERF.H:1447
static bool plot_file_on_restart
Definition: ERF.H:975
amrex::Vector< std::string > lsm_flux_name
Definition: ERF.H:838
void WriteMyEBSurface()
Definition: ERF_EBWriteSurface.cpp:5
void write_1D_profiles_stag(amrex::Real time)
Definition: ERF_Write1DProfiles_stag.cpp:25
void sum_energy_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:318
amrex::Vector< std::unique_ptr< std::fstream > > samplelinelog
Definition: ERF.H:1536
static int sum_interval
Definition: ERF.H:1140
static int pert_interval
Definition: ERF.H:1141
void restart()
Definition: ERF.cpp:1685
void write_1D_profiles(amrex::Real time)
Definition: ERF_Write1DProfiles.cpp:17
int profile_int
Definition: ERF.H:1042
bool destag_profiles
Definition: ERF.H:1043
void appendPlotVariables(const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
Definition: ERF_Plotfile.cpp:184
amrex::Vector< std::string > tot_e_datalogname
Definition: ERF.H:1530
static int output_bndry_planes
Definition: ERF.H:1199
static std::string nc_bdy_file
Definition: ERF.H:1159
void AverageDown()
Definition: ERF_AverageDown.cpp:16
std::unique_ptr< SampleData > data_sampler
Definition: ERF.H:1523
static amrex::Real bndry_output_planes_start_time
Definition: ERF.H:1202
void project_velocity(int lev, amrex::Real dt)
Definition: ERF_PoissonSolve.cpp:10
std::string restart_chkfile
Definition: ERF.H:997
amrex::Vector< std::string > sampleptlogname
Definition: ERF.H:1533
void sum_derived_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:187
void sum_integrated_quantities(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:15
void setRecordDerDataInfo(int i, const std::string &filename)
Definition: ERF.H:1460
amrex::Vector< std::unique_ptr< std::fstream > > sampleptlog
Definition: ERF.H:1532
std::unique_ptr< WriteBndryPlanes > m_w2d
Definition: ERF.H:1261
void init_geo_wind_profile(const std::string input_file, amrex::Vector< amrex::Real > &u_geos, amrex::Gpu::DeviceVector< amrex::Real > &u_geos_d, amrex::Vector< amrex::Real > &v_geos, amrex::Gpu::DeviceVector< amrex::Real > &v_geos_d, const amrex::Geometry &lgeom, const amrex::Vector< amrex::Real > &zlev_stag)
Definition: ERF_InitGeowind.cpp:10
amrex::Vector< std::string > lsm_data_name
Definition: ERF.H:836
void initSponge()
Initialize sponge profiles.
Definition: ERF_InitSponge.cpp:35
amrex::Vector< std::unique_ptr< std::fstream > > tot_e_datalog
Definition: ERF.H:1527
int real_width
Definition: ERF.H:1160
void setRecordEnergyDataInfo(int i, const std::string &filename)
Definition: ERF.H:1473
static bool is_it_time_for_action(int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
Definition: ERF_WriteScalarProfiles.cpp:747
void Construct_ERFFillPatchers(int lev)
Definition: ERF.cpp:2440
void setRecordSampleLineInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1503
void setSpongeRefFromSounding(bool restarting)
Set sponge mean profiles from input sounding.
Definition: ERF_InitSponge.cpp:65
amrex::Vector< amrex::IntVect > samplepoint
Definition: ERF.H:1534
void setRecordSamplePointInfo(int i, int lev, amrex::IntVect &cell, const std::string &filename)
Definition: ERF.H:1486
void ReadCheckpointFileSurfaceLayer()
Definition: ERF_Checkpoint.cpp:1003
static MoistureModelType modelType(const MoistureType a_moisture_type)
query if a specified moisture model is Eulerian or Lagrangian
Definition: ERF_Microphysics.H:86
bool have_zero_flux_faces
Definition: ERF_AdvStruct.H:416
amrex::Real rho0_trans
Definition: ERF_DiffStruct.H:91
amrex::Real dynamic_viscosity
Definition: ERF_DiffStruct.H:96
bool have_geo_wind_profile
Definition: ERF_DataStruct.H:921
std::string abl_geo_wind_table
Definition: ERF_DataStruct.H:920
bool use_rotate_surface_flux
Definition: ERF_DataStruct.H:892
bool do_forest_drag
Definition: ERF_DataStruct.H:942
void debug(amrex::Real)
Definition: ERF_TurbPertStruct.H:613
Here is the call graph for this function:

◆ InitData_pre()

void ERF::InitData_pre ( )
828 {
829  // Initialize the start time for our CPU-time tracker
830  startCPUTime = ParallelDescriptor::second();
831 
832  // Create the ReadBndryPlanes object so we can read boundary plane data
833  // m_r2d is used by init_bcs so we must instantiate this class before
834  if (input_bndry_planes) {
835  Print() << "Defining r2d for the first time " << std::endl;
836  m_r2d = std::make_unique<ReadBndryPlanes>(geom[0], solverChoice.rdOcp);
837  }
838 
839  if (restart_chkfile.empty()) {
840  // Start simulation from the beginning
841  InitFromScratch(0.0);
842  } else {
843  // For initialization this is done in init_only; it is done here for restart
844  init_bcs();
845  }
846 
847  // Verify solver choices
848  for (int lev(0); lev <= max_level; ++lev) {
849  // BC compatibility
850  if ( ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
851  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ||
852  (solverChoice.turbChoice[lev].pbl_type == PBLType::YSU) ||
853  (solverChoice.turbChoice[lev].pbl_type == PBLType::MRF)
854  ) &&
855  phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::surface_layer ) {
856  Abort("MYNN2.5/MYNNEDMF/YSU/MRF PBL Model requires MOST at lower boundary");
857  }
858  if ( (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) &&
859  (solverChoice.turbChoice[lev].Ce_wall > 0) &&
860  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::surface_layer) &&
861  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::slip_wall) &&
862  (phys_bc_type[Orientation(Direction::z,Orientation::low)] != ERF_BC::no_slip_wall) )
863  {
864  Warning("Deardorff LES assumes wall at zlo when applying Ce_wall");
865  }
866 
867  if ( (solverChoice.const_massflux_u != 0) &&
868  (phys_bc_type[Orientation(Direction::x,Orientation::low)] != ERF_BC::periodic ) )
869  {
870  Abort("Constant mass flux (in x) should be used with periodic boundaries");
871  }
872  if ( (solverChoice.const_massflux_v != 0) &&
873  (phys_bc_type[Orientation(Direction::y,Orientation::low)] != ERF_BC::periodic ) )
874  {
875  Abort("Constant mass flux (in y) should be used with periodic boundaries");
876  }
877 
878  // mesoscale diffusion
879  if ((geom[lev].CellSize(0) > 2000.) || (geom[lev].CellSize(1) > 2000.))
880  {
881  if ( (solverChoice.turbChoice[lev].les_type == LESType::Smagorinsky) &&
882  (!solverChoice.turbChoice[lev].smag2d)) {
883  Warning("Should use 2-D Smagorinsky for mesoscale resolution");
884  } else if (solverChoice.turbChoice[lev].les_type == LESType::Deardorff) {
885  Warning("Should not use Deardorff LES for mesoscale resolution");
886  }
887  }
888  }
889 }
void init_bcs()
Definition: ERF_InitBCs.cpp:20

◆ initHSE() [1/2]

void ERF::initHSE ( )
private

Initialize HSE.

143 {
144  for (int lev = 0; lev <= finest_level; lev++)
145  {
146  initHSE(lev);
147  }
148 }

◆ initHSE() [2/2]

void ERF::initHSE ( int  lev)
private

Initialize density and pressure base state in hydrostatic equilibrium.

21 {
22  // This integrates up through column to update p_hse, pi_hse, th_hse;
23  // r_hse is not const b/c FillBoundary is called at the end for r_hse and p_hse
24 
25  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
26  MultiFab p_hse (base_state[lev], make_alias, BaseState::p0_comp, 1);
27  MultiFab pi_hse(base_state[lev], make_alias, BaseState::pi0_comp, 1);
28  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
29  MultiFab qv_hse(base_state[lev], make_alias, BaseState::qv0_comp, 1);
30 
31  bool all_boxes_touch_bottom = true;
32  Box domain(geom[lev].Domain());
33 
34  int icomp = 0; int ncomp = BaseState::num_comps;
35 
36  if (lev == 0) {
37  BoxArray ba(base_state[lev].boxArray());
38  for (int i = 0; i < ba.size(); i++) {
39  if (ba[i].smallEnd(2) != domain.smallEnd(2)) {
40  all_boxes_touch_bottom = false;
41  }
42  }
43  }
44  else
45  {
46  //
47  // We need to do this interp from coarse level in order to set the values of
48  // the base state inside the domain but outside of the fine region
49  //
50  base_state[lev-1].FillBoundary(geom[lev-1].periodicity());
51  //
52  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
53  // have been pre-filled - this includes ghost cells both inside and outside
54  // the domain
55  //
56  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
57  IntVect(0,0,0), // do not fill ghost cells outside the domain
58  base_state[lev-1], icomp, icomp, ncomp,
59  geom[lev-1], geom[lev],
60  refRatio(lev-1), &cell_cons_interp,
62 
63  // We need to do this here because the interpolation above may leave corners unfilled
64  // when the corners need to be filled by, for example, reflection of the fine ghost
65  // cell outside the fine region but inide the domain.
66  (*physbcs_base[lev])(base_state[lev],icomp,ncomp,base_state[lev].nGrowVect());
67  }
68 
69  if (all_boxes_touch_bottom || lev > 0) {
70 
71  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
73  prob->erf_init_dens_hse_moist(r_hse, z_phys_nd[lev], geom[lev]);
74  } else {
75  prob->erf_init_dens_hse(r_hse, z_phys_nd[lev], z_phys_cc[lev], geom[lev]);
76  }
77 
78  erf_enforce_hse(lev, r_hse, p_hse, pi_hse, th_hse, qv_hse, z_phys_cc[lev]);
79 
80  //
81  // Impose physical bc's on the base state
82  //
83  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
84 
85  } else {
86 
87  BoxArray ba_new(domain);
88 
89  ChopGrids2D(ba_new, domain, ParallelDescriptor::NProcs());
90 
91  DistributionMapping dm_new(ba_new);
92 
93  MultiFab new_base_state(ba_new, dm_new, BaseState::num_comps, base_state[lev].nGrowVect());
94  new_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
95  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
96 
97  MultiFab new_r_hse (new_base_state, make_alias, BaseState::r0_comp, 1);
98  MultiFab new_p_hse (new_base_state, make_alias, BaseState::p0_comp, 1);
99  MultiFab new_pi_hse(new_base_state, make_alias, BaseState::pi0_comp, 1);
100  MultiFab new_th_hse(new_base_state, make_alias, BaseState::th0_comp, 1);
101  MultiFab new_qv_hse(new_base_state, make_alias, BaseState::qv0_comp, 1);
102 
103  std::unique_ptr<MultiFab> new_z_phys_cc;
104  std::unique_ptr<MultiFab> new_z_phys_nd;
105  if (solverChoice.mesh_type != MeshType::ConstantDz) {
106  new_z_phys_cc = std::make_unique<MultiFab>(ba_new,dm_new,1,1);
107  new_z_phys_cc->ParallelCopy(*z_phys_cc[lev],0,0,1,1,1);
108 
109  BoxArray ba_new_nd(ba_new);
110  ba_new_nd.surroundingNodes();
111  new_z_phys_nd = std::make_unique<MultiFab>(ba_new_nd,dm_new,1,1);
112  new_z_phys_nd->ParallelCopy(*z_phys_nd[lev],0,0,1,1,1);
113  }
114 
115  // Initial r_hse may or may not be in HSE -- defined in ERF_Prob.cpp
117  prob->erf_init_dens_hse_moist(new_r_hse, new_z_phys_nd, geom[lev]);
118  } else {
119  prob->erf_init_dens_hse(new_r_hse, new_z_phys_nd, new_z_phys_cc, geom[lev]);
120  }
121 
122  erf_enforce_hse(lev, new_r_hse, new_p_hse, new_pi_hse, new_th_hse, new_qv_hse, new_z_phys_cc);
123 
124  //
125  // Impose physical bc's on the base state
126  //
127  (*physbcs_base[lev])(new_base_state,0,new_base_state.nComp(),new_base_state.nGrowVect());
128 
129  // Now copy back into the original arrays
130  base_state[lev].ParallelCopy(new_base_state,0,0,base_state[lev].nComp(),
131  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
132  }
133 
134  //
135  // Impose physical bc's on the base state -- the values outside the fine region
136  // but inside the domain have already been filled in the call above to InterpFromCoarseLevel
137  //
138  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
139 }
void ChopGrids2D(BoxArray &ba, const Box &domain, int target_size)
Definition: ERF_ChopGrids.cpp:21
void erf_enforce_hse(int lev, amrex::MultiFab &dens, amrex::MultiFab &pres, amrex::MultiFab &pi, amrex::MultiFab &th, amrex::MultiFab &qv, std::unique_ptr< amrex::MultiFab > &z_cc)
Definition: ERF_Init1D.cpp:160
bool use_moist_background
Definition: ERF_DataStruct.H:929
Here is the call graph for this function:

◆ initialize_integrator()

void ERF::initialize_integrator ( int  lev,
amrex::MultiFab &  cons_mf,
amrex::MultiFab &  vel_mf 
)
private
663 {
664  const BoxArray& ba(cons_mf.boxArray());
665  const DistributionMapping& dm(cons_mf.DistributionMap());
666 
667  int ncomp_cons = cons_mf.nComp();
668 
669  // Initialize the integrator memory
670  Vector<MultiFab> int_state; // integration state data structure example
671  int_state.push_back(MultiFab(cons_mf, make_alias, 0, ncomp_cons)); // cons
672  int_state.push_back(MultiFab(convert(ba,IntVect(1,0,0)), dm, 1, vel_mf.nGrow())); // xmom
673  int_state.push_back(MultiFab(convert(ba,IntVect(0,1,0)), dm, 1, vel_mf.nGrow())); // ymom
674  int_state.push_back(MultiFab(convert(ba,IntVect(0,0,1)), dm, 1, vel_mf.nGrow())); // zmom
675 
676  mri_integrator_mem[lev] = std::make_unique<MRISplitIntegrator<Vector<MultiFab> > >(int_state);
677  mri_integrator_mem[lev]->setNoSubstepping((solverChoice.substepping_type[lev] == SubsteppingType::None));
678  mri_integrator_mem[lev]->setAnelastic(solverChoice.anelastic[lev]);
679  mri_integrator_mem[lev]->setNcompCons(ncomp_cons);
680  mri_integrator_mem[lev]->setForceFirstStageSingleSubstep(solverChoice.force_stage1_single_substep);
681 }

◆ InitializeFromFile()

void ERF::InitializeFromFile ( )
private

◆ InitializeLevelFromData()

void ERF::InitializeLevelFromData ( int  lev,
const amrex::MultiFab &  initial_data 
)
private

◆ initializeMicrophysics()

void ERF::initializeMicrophysics ( const int &  a_nlevsmax)
private
Parameters
a_nlevsmaxnumber of AMR levels
1651 {
1652  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Eulerian) {
1653 
1654  micro = std::make_unique<EulerianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1655 
1656  } else if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
1657 #ifdef ERF_USE_PARTICLES
1658 
1659  micro = std::make_unique<LagrangianMicrophysics>(a_nlevsmax, solverChoice.moisture_type);
1660  /* Lagrangian microphysics models will have a particle container; it needs to be added
1661  to ERF::particleData */
1662  const auto& pc_name( dynamic_cast<LagrangianMicrophysics&>(*micro).getName() );
1663  /* The particle container has not yet been constructed and initialized, so just add
1664  its name here for now (so that functions to set plotting variables can see it). */
1665  particleData.addName( pc_name );
1666 
1667 #else
1668  Abort("Lagrangian microphysics can be used when compiled with ERF_USE_PARTICLES");
1669 #endif
1670  }
1671 
1672  qmoist.resize(a_nlevsmax);
1673  return;
1674 }
amrex::Vector< amrex::Vector< amrex::MultiFab * > > qmoist
Definition: ERF.H:820
Here is the call graph for this function:

◆ initRayleigh()

void ERF::initRayleigh ( )
private

Initialize Rayleigh damping profiles.

Initialization function for host and device vectors used to store averaged quantities when calculating the effects of Rayleigh Damping.

15 {
16  const int khi = geom[0].Domain().bigEnd(2);
17  solverChoice.rayleigh_ztop = (solverChoice.terrain_type == TerrainType::None) ? geom[0].ProbHi(2) : zlevels_stag[0][khi+1];
18 
19  h_rayleigh_ptrs.resize(max_level+1);
20  d_rayleigh_ptrs.resize(max_level+1);
21 
22  for (int lev = 0; lev <= finest_level; lev++)
23  {
24  // These have 4 components: ubar, vbar, wbar, thetabar
25  h_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
26  d_rayleigh_ptrs[lev].resize(Rayleigh::nvars);
27 
28  const int zlen_rayleigh = geom[lev].Domain().length(2);
29 
30  // Allocate space for these 1D vectors
31  for (int n = 0; n < Rayleigh::nvars; n++) {
32  h_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
33  d_rayleigh_ptrs[lev][n].resize(zlen_rayleigh, 0.0_rt);
34  }
35 
36  // Init the host vectors
37  prob->erf_init_rayleigh(h_rayleigh_ptrs[lev], geom[lev], z_phys_nd[lev], solverChoice.rayleigh_zdamp);
38 
39  // Copy from host vectors to device vectors
40  for (int n = 0; n < Rayleigh::nvars; n++) {
41  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][n].begin(), h_rayleigh_ptrs[lev][n].end(),
42  d_rayleigh_ptrs[lev][n].begin());
43  }
44  }
45 }
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_rayleigh_ptrs
Definition: ERF.H:1236
amrex::Real rayleigh_zdamp
Definition: ERF_DataStruct.H:852
amrex::Real rayleigh_ztop
Definition: ERF_DataStruct.H:853

◆ initSponge()

void ERF::initSponge ( )
private

Initialize sponge profiles.

Initialization function for host and device vectors used to store the effects of sponge Damping.

36 {
37  h_sponge_ptrs.resize(max_level+1);
38  d_sponge_ptrs.resize(max_level+1);
39 
40  for (int lev = 0; lev <= finest_level; lev++)
41  {
42  // These have 2 components: ubar, vbar
45 
46  const int zlen_sponge = geom[lev].Domain().length(2);
47 
48  // Allocate space for these 1D vectors
49  for (int n = 0; n < Sponge::nvars_sponge; n++) {
50  h_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
51  d_sponge_ptrs[lev][n].resize(zlen_sponge, 0.0_rt);
52  }
53 
54  }
55 }
amrex::Vector< amrex::Vector< amrex::Vector< amrex::Real > > > h_sponge_ptrs
Definition: ERF.H:1239

◆ input_sponge()

void ERF::input_sponge ( int  lev)

High level wrapper for sponge x and y velocities level data from input sponge data.

Parameters
levInteger specifying the current level
18 {
19  // We only want to read the file once
20  if (lev == 0) {
22  Error("input_sounding file name must be provided via input");
23 
24  // this will interpolate the input profiles to the nominal height levels
25  // (ranging from 0 to the domain top)
27  }
28 }
InputSpongeData input_sponge_data
Definition: ERF.H:734
void read_from_file(const amrex::Geometry &geom, const amrex::Vector< amrex::Real > &zlevels_stag)
Definition: ERF_InputSpongeData.H:28
std::string input_sponge_file
Definition: ERF_InputSpongeData.H:111

◆ InterpWeatherDataOntoMesh()

void ERF::InterpWeatherDataOntoMesh ( const amrex::Geometry &  geom_weather,
amrex::MultiFab &  weather_forecast_interp,
amrex::Vector< amrex::Vector< amrex::MultiFab >> &  forecast_state 
)
271 {
272 
273  MultiFab& weather_mf = weather_forecast_data;
274  MultiFab& erf_mf_cons = forecast_state[0][Vars::cons];
275  MultiFab& erf_mf_xvel = forecast_state[0][Vars::xvel];
276  MultiFab& erf_mf_yvel = forecast_state[0][Vars::yvel];
277  MultiFab& erf_mf_zvel = forecast_state[0][Vars::zvel];
278  MultiFab& erf_mf_latlon = forecast_state[0][4];
279 
280  erf_mf_cons.setVal(0.0);
281  erf_mf_xvel.setVal(0.0);
282  erf_mf_yvel.setVal(0.0);
283  erf_mf_zvel.setVal(0.0);
284  erf_mf_latlon.setVal(0.0);
285 
286  BoxList bl_erf = erf_mf_cons.boxArray().boxList();
287  BoxList bl_weather = weather_mf.boxArray().boxList();
288 
289  const auto prob_lo_erf = geom[0].ProbLoArray();
290  const auto dx_erf = geom[0].CellSizeArray();
291 
292  for (auto& b : bl_erf) {
293  // You look at the lo corner of b, and find out the lowest cell in
294  // coarse weather data you need for the interpolation. That gives
295  // you the lo corner of the new b. Similarly, you can find out the
296  // hi corner of the new b. For cells outside the coarse_weath_data's
297  // bounding data, it's up to you. You probably want to use a biased
298  // interpolation stencil.
299 
300  // Get the cell indices of the bottom corner and top corner
301  const IntVect& lo_erf = b.smallEnd(); // Lower corner (inclusive)
302  const IntVect& hi_erf = b.bigEnd(); // Upper corner (inclusive)
303 
304  Real x = prob_lo_erf[0] + lo_erf[0] * dx_erf[0];
305  Real y = prob_lo_erf[1] + lo_erf[1] * dx_erf[1];
306  Real z = prob_lo_erf[2] + lo_erf[2] * dx_erf[2];
307 
308  auto idx_lo = find_bound_idx(x, y, z, bl_weather, geom_weather, BoundType::Lo);
309 
310  x = prob_lo_erf[0] + (hi_erf[0]+1) * dx_erf[0];
311  y = prob_lo_erf[1] + (hi_erf[1]+1) * dx_erf[1];
312  z = prob_lo_erf[2] + (hi_erf[2]+1) * dx_erf[2];
313 
314  auto idx_hi = find_bound_idx(x, y, z, bl_weather, geom_weather, BoundType::Hi);
315 
316  b.setSmall(idx_lo);
317  b.setBig(idx_hi);
318  }
319 
320  BoxArray cba(std::move(bl_erf));
321  cba.convert(IndexType::TheNodeType()); // <-- Make it nodal in all directions
322  MultiFab tmp_coarse_data(cba, erf_mf_cons.DistributionMap(), weather_mf.nComp(), 0);
323  tmp_coarse_data.ParallelCopy(weather_mf);
324 
325  //PlotMultiFab(weather_mf, geom_weather, "plt_coarse_weather_par_copy",MultiFabType::NC);
326 
327  const auto prob_lo_weather = geom_weather.ProbLoArray();
328  const auto dx_weather = geom_weather.CellSizeArray();
329 
330  for (MFIter mfi(erf_mf_cons); mfi.isValid(); ++mfi) {
331  const Array4<Real> &fine_cons_arr = erf_mf_cons.array(mfi);
332  const Array4<Real> &fine_xvel_arr = erf_mf_xvel.array(mfi);
333  const Array4<Real> &fine_yvel_arr = erf_mf_yvel.array(mfi);
334  //const Array4<Real> &fine_zvel_arr = erf_mf_zvel.array(mfi);
335  const Array4<Real> &fine_latlon_arr = erf_mf_latlon.array(mfi);
336 
337  const Array4<Real> &crse_arr = tmp_coarse_data.array(mfi);
338 
339  const Box& gbx = mfi.growntilebox(); // tilebox + ghost cells
340 
341  const Box &gtbx = mfi.tilebox(IntVect(1,0,0));
342  const Box &gtby = mfi.tilebox(IntVect(0,1,0));
343  //const Box &gtbz = mfi.tilebox(IntVect(0,0,1));
344 
345  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
346  // Physical location of the fine node
347  Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
348  Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
349  Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
350 
351  Real rho = interpolate_from_coarse(crse_arr, 0, x, y, z, prob_lo_weather.data(), dx_weather.data());
352  Real lat = interpolate_from_coarse(crse_arr, 8, x, y, z, prob_lo_weather.data(), dx_weather.data());
353  Real lon = interpolate_from_coarse(crse_arr, 9, x, y, z, prob_lo_weather.data(), dx_weather.data());
354 
355  fine_cons_arr(i,j,k,Rho_comp) = rho;
356 
357  fine_latlon_arr(i,j,k,0) = lat;
358  fine_latlon_arr(i,j,k,1) = lon;
359  });
360 
361  ParallelFor(gtbx, gtby,
362  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
363  // Physical location of the fine node
364  Real x = prob_lo_erf[0] + i * dx_erf[0];
365  Real y = prob_lo_erf[1] + (j+0.5) * dx_erf[1];
366  Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
367  fine_xvel_arr(i, j, k, 0) = interpolate_from_coarse(crse_arr, 1, x, y, z, prob_lo_weather.data(), dx_weather.data());
368  },
369  [=] AMREX_GPU_DEVICE(int i, int j, int k) {
370  // Physical location of the fine node
371  Real x = prob_lo_erf[0] + (i+0.5) * dx_erf[0];
372  Real y = prob_lo_erf[1] + j * dx_erf[1];
373  Real z = prob_lo_erf[2] + (k+0.5) * dx_erf[2];
374  fine_yvel_arr(i, j, k, 0) = interpolate_from_coarse(crse_arr, 2, x, y, z, prob_lo_weather.data(), dx_weather.data());
375  });
376  }
377 
378  /*Vector<std::string> varnames = {
379  "rho", "uvel", "vvel", "wvel", "theta", "qv", "qc", "qr"
380  }; // Customize variable names
381 
382  Vector<std::string> varnames_cons = {
383  "rho", "rhotheta", "ke", "sc", "rhoqv", "rhoqc", "rhoqr"
384  }; // Customize variable names
385 
386  Vector<std::string> varnames_plot_mf = {
387  "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
388  }; // Customize variable names
389 
390 
391  const Real time = 0.0;
392 
393  std::string pltname = "plt_interp";
394 
395  MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
396  10, 0);
397 
398  plot_mf.setVal(0.0);
399 
400  for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
401  const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
402  const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
403  const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
404  const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
405  const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
406  const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
407 
408  const Box& bx = mfi.validbox();
409 
410  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
411  plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
412  plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
413  plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
414  plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
415  plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
416 
417  plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/2.0;
418  plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/2.0;
419  plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/2.0;
420 
421  plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
422  plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
423  });
424  }
425 
426 
427  WriteSingleLevelPlotfile(
428  pltname,
429  plot_mf,
430  varnames_plot_mf,
431  geom[0],
432  time,
433  0 // level
434  );*/
435 }
AMREX_GPU_DEVICE amrex::Real interpolate_from_coarse(const amrex::Array4< const amrex::Real > &crse, int n, amrex::Real x, amrex::Real y, amrex::Real z, const amrex::Real *prob_lo_crse, const amrex::Real *dx_crse)
Definition: ERF_Interpolation_Bilinear.H:77
IntVect find_bound_idx(const Real &x, const Real &y, const Real &z, const BoxList &bl_weather, const Geometry &geom_weather, BoundType bound_type)
Definition: ERF_WeatherDataInterpolation.cpp:217
Here is the call graph for this function:

◆ is_it_time_for_action()

bool ERF::is_it_time_for_action ( int  nstep,
amrex::Real  time,
amrex::Real  dt,
int  action_interval,
amrex::Real  action_per 
)
static

Helper function which uses the current step number, time, and timestep to determine whether it is time to take an action specified at every interval of timesteps.

Parameters
nstepTimestep number
timeCurrent time
dtlevTimestep for the current level
action_intervalInterval in number of timesteps for taking action
action_perInterval in simulation time for taking action
748 {
749  bool int_test = (action_interval > 0 && nstep % action_interval == 0);
750 
751  bool per_test = false;
752  if (action_per > 0.0) {
753  const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
754  const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
755 
756  if (num_per_old != num_per_new) {
757  per_test = true;
758  }
759  }
760 
761  return int_test || per_test;
762 }

◆ make_eb_box()

void ERF::make_eb_box ( )

◆ make_eb_regular()

void ERF::make_eb_regular ( )

◆ make_physbcs()

void ERF::make_physbcs ( int  lev)
private
685 {
686  if (SolverChoice::mesh_type == MeshType::VariableDz) {
687  AMREX_ALWAYS_ASSERT(z_phys_nd[lev] != nullptr);
688  }
689 
690  physbcs_cons[lev] = std::make_unique<ERFPhysBCFunct_cons> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
692  z_phys_nd[lev], solverChoice.use_real_bcs, th_bc_data[lev].data());
693  physbcs_u[lev] = std::make_unique<ERFPhysBCFunct_u> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
695  z_phys_nd[lev], solverChoice.use_real_bcs, xvel_bc_data[lev].data());
696  physbcs_v[lev] = std::make_unique<ERFPhysBCFunct_v> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
698  z_phys_nd[lev], solverChoice.use_real_bcs, yvel_bc_data[lev].data());
699  physbcs_w[lev] = std::make_unique<ERFPhysBCFunct_w> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
702  solverChoice.use_real_bcs, zvel_bc_data[lev].data());
703  physbcs_base[lev] = std::make_unique<ERFPhysBCFunct_base> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d, z_phys_nd[lev],
704  (solverChoice.terrain_type == TerrainType::MovingFittedMesh));
705 }

◆ make_subdomains()

void ERF::make_subdomains ( const amrex::BoxList &  ba,
amrex::Vector< amrex::BoxArray > &  bins 
)
7 {
8  Vector<BoxList> bins_bl;
9 
10  // Clear out any old bins
11  bins.clear();
12 
13  // Iterate over boxes
14  for (auto bx : bl)
15  {
16  bool added = false;
17 
18  // Try to add box to existing bin
19  for (int j = 0; j < bins_bl.size(); ++j) {
20  BoxList& bin = bins_bl[j];
21  bool touches = false;
22 
23  for (auto& b : bin)
24  {
25  Box gbx(bx); gbx.grow(1);
26  if (gbx.intersects(b)) {
27  touches = true;
28  break;
29  }
30  }
31 
32  if (touches) {
33  bin.push_back(bx);
34  added = true;
35  break;
36  }
37  }
38 
39  // If box couldn't be added to existing bin, create new bin
40  if (!added) {
41  BoxList new_bin;
42  new_bin.push_back(bx);
43  bins_bl.push_back(new_bin);
44  }
45  }
46 
47  // Convert the BoxLists to BoxArrays
48  for (int i = 0; i < bins_bl.size(); ++i) {
49  bins.push_back(BoxArray(bins_bl[i]));
50  }
51 }

◆ MakeDiagnosticAverage()

void ERF::MakeDiagnosticAverage ( amrex::Vector< amrex::Real > &  h_havg,
amrex::MultiFab &  S,
int  n 
)
2395 {
2396  // Get the number of cells in z at level 0
2397  int dir_z = AMREX_SPACEDIM-1;
2398  auto domain = geom[0].Domain();
2399  int size_z = domain.length(dir_z);
2400  int start_z = domain.smallEnd()[dir_z];
2401  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
2402 
2403  // resize the level 0 horizontal average vectors
2404  h_havg.resize(size_z, 0.0_rt);
2405 
2406  // Get the cell centered data and construct sums
2407 #ifdef _OPENMP
2408 #pragma omp parallel if (Gpu::notInLaunchRegion())
2409 #endif
2410  for (MFIter mfi(S); mfi.isValid(); ++mfi) {
2411  const Box& box = mfi.validbox();
2412  const IntVect& se = box.smallEnd();
2413  const IntVect& be = box.bigEnd();
2414 
2415  auto fab_arr = S[mfi].array();
2416 
2417  FArrayBox fab_reduce(box, 1, The_Async_Arena());
2418  auto arr_reduce = fab_reduce.array();
2419 
2420  ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2421  arr_reduce(i, j, k, 0) = fab_arr(i,j,k,n);
2422  });
2423 
2424  for (int k=se[dir_z]; k <= be[dir_z]; ++k) {
2425  Box kbox(box); kbox.setSmall(dir_z,k); kbox.setBig(dir_z,k);
2426  h_havg[k-start_z] += fab_reduce.sum<RunOn::Device>(kbox,0);
2427  }
2428  }
2429 
2430  // combine sums from different MPI ranks
2431  ParallelDescriptor::ReduceRealSum(h_havg.dataPtr(), h_havg.size());
2432 
2433  // divide by the total number of cells we are averaging over
2434  for (int k = 0; k < size_z; ++k) {
2435  h_havg[k] /= area_z;
2436  }
2437 }

◆ MakeEBGeometry()

void ERF::MakeEBGeometry ( )

◆ MakeHorizontalAverages()

void ERF::MakeHorizontalAverages ( )
2289 {
2290  int lev = 0;
2291 
2292  // First, average down all levels (if doing two-way coupling)
2293  if (solverChoice.coupling_type == CouplingType::TwoWay) {
2294  AverageDown();
2295  }
2296 
2297  MultiFab mf(grids[lev], dmap[lev], 5, 0);
2298 
2299  int zdir = 2;
2300  auto domain = geom[0].Domain();
2301 
2302  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
2303  bool is_anelastic = (solverChoice.anelastic[lev] == 1);
2304 
2305  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
2306  const Box& bx = mfi.validbox();
2307  auto fab_arr = mf.array(mfi);
2308  auto const hse_arr = base_state[lev].const_array(mfi);
2309  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
2310  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2311  Real dens = cons_arr(i, j, k, Rho_comp);
2312  fab_arr(i, j, k, 0) = dens;
2313  fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens;
2314  if (!use_moisture) {
2315  if (is_anelastic) {
2316  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
2317  } else {
2318  fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp));
2319  }
2320  }
2321  });
2322  }
2323 
2324  if (use_moisture)
2325  {
2326  for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
2327  const Box& bx = mfi.validbox();
2328  auto fab_arr = mf.array(mfi);
2329  auto const hse_arr = base_state[lev].const_array(mfi);
2330  auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
2331  int ncomp = vars_new[lev][Vars::cons].nComp();
2332 
2333  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
2334  Real dens = cons_arr(i, j, k, Rho_comp);
2335  if (is_anelastic) {
2336  fab_arr(i,j,k,2) = hse_arr(i,j,k,BaseState::p0_comp);
2337  } else {
2338  Real qv = cons_arr(i, j, k, RhoQ1_comp) / dens;
2339  fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv);
2340  }
2341  fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : 0.0);
2342  fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : 0.0);
2343  });
2344  }
2345 
2346  Gpu::HostVector<Real> h_avg_qv = sumToLine(mf,3,1,domain,zdir);
2347  Gpu::HostVector<Real> h_avg_qc = sumToLine(mf,4,1,domain,zdir);
2348  }
2349 
2350  // Sum in the horizontal plane
2351  Gpu::HostVector<Real> h_avg_density = sumToLine(mf,0,1,domain,zdir);
2352  Gpu::HostVector<Real> h_avg_temperature = sumToLine(mf,1,1,domain,zdir);
2353  Gpu::HostVector<Real> h_avg_pressure = sumToLine(mf,2,1,domain,zdir);
2354 
2355  // Divide by the total number of cells we are averaging over
2356  int size_z = domain.length(zdir);
2357  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
2358  int klen = static_cast<int>(h_avg_density.size());
2359 
2360  for (int k = 0; k < klen; ++k) {
2361  h_havg_density[k] /= area_z;
2362  h_havg_temperature[k] /= area_z;
2363  h_havg_pressure[k] /= area_z;
2364  if (solverChoice.moisture_type != MoistureType::None)
2365  {
2366  h_havg_qc[k] /= area_z;
2367  h_havg_qv[k] /= area_z;
2368  }
2369  } // k
2370 
2371  // resize device vectors
2372  d_havg_density.resize(size_z, 0.0_rt);
2373  d_havg_temperature.resize(size_z, 0.0_rt);
2374  d_havg_pressure.resize(size_z, 0.0_rt);
2375 
2376  // copy host vectors to device vectors
2377  Gpu::copy(Gpu::hostToDevice, h_havg_density.begin(), h_havg_density.end(), d_havg_density.begin());
2378  Gpu::copy(Gpu::hostToDevice, h_havg_temperature.begin(), h_havg_temperature.end(), d_havg_temperature.begin());
2379  Gpu::copy(Gpu::hostToDevice, h_havg_pressure.begin(), h_havg_pressure.end(), d_havg_pressure.begin());
2380 
2381  if (solverChoice.moisture_type != MoistureType::None)
2382  {
2383  d_havg_qv.resize(size_z, 0.0_rt);
2384  d_havg_qc.resize(size_z, 0.0_rt);
2385  Gpu::copy(Gpu::hostToDevice, h_havg_qv.begin(), h_havg_qv.end(), d_havg_qv.begin());
2386  Gpu::copy(Gpu::hostToDevice, h_havg_qc.begin(), h_havg_qc.end(), d_havg_qc.begin());
2387  }
2388 }
amrex::Gpu::DeviceVector< amrex::Real > d_havg_temperature
Definition: ERF.H:1254
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qv
Definition: ERF.H:1256
amrex::Vector< amrex::Real > h_havg_pressure
Definition: ERF.H:1249
amrex::Vector< amrex::Real > h_havg_qc
Definition: ERF.H:1251
amrex::Vector< amrex::Real > h_havg_density
Definition: ERF.H:1247
amrex::Gpu::DeviceVector< amrex::Real > d_havg_qc
Definition: ERF.H:1257
amrex::Gpu::DeviceVector< amrex::Real > d_havg_density
Definition: ERF.H:1253
amrex::Vector< amrex::Real > h_havg_temperature
Definition: ERF.H:1248
amrex::Gpu::DeviceVector< amrex::Real > d_havg_pressure
Definition: ERF.H:1255
amrex::Vector< amrex::Real > h_havg_qv
Definition: ERF.H:1250
Here is the call graph for this function:

◆ MakeNewLevelFromCoarse()

void ERF::MakeNewLevelFromCoarse ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
291 {
292  AMREX_ALWAYS_ASSERT(lev > 0);
293 
294  if (verbose) {
295  amrex::Print() <<" NEW BA FROM COARSE AT LEVEL " << lev << " " << ba << std::endl;
296  }
297 
298  //
299  // Grow the subdomains vector and build the subdomains vector at this level
300  //
301  subdomains.resize(lev+1);
302  //
303  // Create subdomains at each level within the domain such that
304  // 1) all boxes in a given subdomain are "connected"
305  // 2) no boxes in a subdomain touch any boxes in any other subdomain
306  //
308  BoxArray dom(geom[lev].Domain());
309  subdomains[lev].push_back(dom);
310  } else {
311  make_subdomains(ba.simplified_list(), subdomains[lev]);
312  }
313 
314  if (lev == 0) init_bcs();
315 
316  //********************************************************************************************
317  // This allocates all kinds of things, including but not limited to: solution arrays,
318  // terrain arrays, metric terms and base state.
319  // *******************************************************************************************
320  init_stuff(lev, ba, dm, vars_new[lev], vars_old[lev], base_state[lev], z_phys_nd[lev]);
321 
322  t_new[lev] = time;
323  t_old[lev] = time - 1.e200;
324 
325  // ********************************************************************************************
326  // Build the data structures for metric quantities used with terrain-fitted coordinates
327  // ********************************************************************************************
328  if ( solverChoice.terrain_type == TerrainType::EB ||
329  solverChoice.terrain_type == TerrainType::ImmersedForcing)
330  {
331  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
332  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
333  if (solverChoice.terrain_type == TerrainType::EB) {
334  eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
335  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
336  eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
337  }
338  }
339  init_zphys(lev, time);
341 
342  //
343  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
344  //
345  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
346  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
347  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
348  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
349  }
350  }
351 
352  // ********************************************************************************************
353  // Build the data structures for canopy model (depends upon z_phys)
354  // ********************************************************************************************
356  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
357  }
358 
359  //********************************************************************************************
360  // Microphysics
361  // *******************************************************************************************
362  int q_size = micro->Get_Qmoist_Size(lev);
363  qmoist[lev].resize(q_size);
364  micro->Define(lev, solverChoice);
365  if (solverChoice.moisture_type != MoistureType::None)
366  {
367  micro->Init(lev, vars_new[lev][Vars::cons],
368  grids[lev], Geom(lev), 0.0,
369  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
370  }
371  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
372  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
373  }
374 
375  //********************************************************************************************
376  // Radiation
377  // *******************************************************************************************
378  if (solverChoice.rad_type != RadiationType::None)
379  {
380  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
381  }
382 
383  // *****************************************************************************************************
384  // Initialize the boundary conditions (after initializing the terrain but before calling
385  // initHSE or FillCoarsePatch)
386  // *****************************************************************************************************
387  make_physbcs(lev);
388 
389  // ********************************************************************************************
390  // Update the base state at this level by interpolation from coarser level
391  // ********************************************************************************************
392  InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
393  IntVect(0,0,0), // do not fill ghost cells outside the domain
394  base_state[lev-1], 0, 0, base_state[lev].nComp(),
395  geom[lev-1], geom[lev],
396  refRatio(lev-1), &cell_cons_interp,
398 
399  // Impose bc's outside the domain
400  (*physbcs_base[lev])(base_state[lev],0,base_state[lev].nComp(),base_state[lev].nGrowVect());
401 
402  // ********************************************************************************************
403  // Build the data structures for calculating diffusive/turbulent terms
404  // ********************************************************************************************
405  update_diffusive_arrays(lev, ba, dm);
406 
407  // ********************************************************************************************
408  // Fill data at the new level by interpolation from the coarser level
409  // Note that internal to FillCoarsePatch we will convert velocity to momentum,
410  // then interpolate momentum, then convert momentum back to velocity
411  // Also note that FillCoarsePatch is hard-wired to act only on lev_new at coarse and fine
412  // ********************************************************************************************
413  FillCoarsePatch(lev, time);
414 
415  // ********************************************************************************************
416  // Initialize the integrator class
417  // ********************************************************************************************
418  dt_mri_ratio[lev] = dt_mri_ratio[lev-1];
420 
421  // ********************************************************************************************
422  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
423  // ********************************************************************************************
424  if (lev > 0 && cf_width >= 0) {
427  }
428 
429  // ********************************************************************************************
430  // Create the SurfaceLayer arrays at this (new) level
431  // ********************************************************************************************
432  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
433  int nlevs = finest_level+1;
434  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
435  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
436  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
437  mfv_old, Theta_prim[lev], Qv_prim[lev],
438  Qr_prim[lev], z_phys_nd[lev],
439  Hwave[lev].get(), Lwave[lev].get(), eddyDiffs_lev[lev].get(),
441  sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
442  }
443 
444 #ifdef ERF_USE_PARTICLES
445  // particleData.Redistribute();
446 #endif
447 }
void update_diffusive_arrays(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewArrays.cpp:445
void initialize_integrator(int lev, amrex::MultiFab &cons_mf, amrex::MultiFab &vel_mf)
Definition: ERF_MakeNewArrays.cpp:662
void make_subdomains(const amrex::BoxList &ba, amrex::Vector< amrex::BoxArray > &bins)
Definition: ERF_MakeSubdomains.cpp:6
void update_terrain_arrays(int lev)
Definition: ERF_MakeNewArrays.cpp:640
void init_zphys(int lev, amrex::Real time)
Definition: ERF_MakeNewArrays.cpp:535
void init_stuff(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &lev_new, amrex::Vector< amrex::MultiFab > &lev_old, amrex::MultiFab &tmp_base_state, std::unique_ptr< amrex::MultiFab > &tmp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:24
void Define_ERFFillPatchers(int lev)
Definition: ERF.cpp:2466

◆ MakeNewLevelFromScratch()

void ERF::MakeNewLevelFromScratch ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
27 {
28  BoxArray ba;
29  DistributionMapping dm;
30  Box domain(Geom(0).Domain());
31  if (lev == 0 && restart_chkfile.empty() &&
32  (max_grid_size[0][0] >= domain.length(0)) &&
33  (max_grid_size[0][1] >= domain.length(1)) &&
34  ba_in.size() != ParallelDescriptor::NProcs())
35  {
36  // We only decompose in z if max_grid_size_z indicates we should
37  bool decompose_in_z = (max_grid_size[0][2] < domain.length(2));
38 
39  ba = ERFPostProcessBaseGrids(Geom(0).Domain(),decompose_in_z);
40  dm = DistributionMapping(ba);
41  } else {
42  ba = ba_in;
43  dm = dm_in;
44  }
45 
46  // ********************************************************************************************
47  // Define grids[lev] to be ba
48  // ********************************************************************************************
49  SetBoxArray(lev, ba);
50 
51  // ********************************************************************************************
52  // Define dmap[lev] to be dm
53  // ********************************************************************************************
54  SetDistributionMap(lev, dm);
55 
56  if (verbose) {
57  amrex::Print() << "BA FROM SCRATCH AT LEVEL " << lev << " " << ba << std::endl;
58  // amrex::Print() <<" SIMPLIFIED BA FROM SCRATCH AT LEVEL " << lev << " " << ba.simplified_list() << std::endl;
59  }
60 
61  subdomains.resize(lev+1);
62  if ( (lev == 0) || (
64  (solverChoice.init_type != InitType::WRFInput) && (solverChoice.init_type != InitType::Metgrid) ) ) {
65  BoxArray dom(geom[lev].Domain());
66  subdomains[lev].push_back(dom);
67  } else {
68  //
69  // Create subdomains at each level within the domain such that
70  // 1) all boxes in a given subdomain are "connected"
71  // 2) no boxes in a subdomain touch any boxes in any other subdomain
72  //
73  make_subdomains(ba.simplified_list(), subdomains[lev]);
74  }
75 
76  if (lev == 0) init_bcs();
77 
78  if ( solverChoice.terrain_type == TerrainType::EB ||
79  solverChoice.terrain_type == TerrainType::ImmersedForcing)
80  {
81  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
82  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
83  if (solverChoice.terrain_type == TerrainType::EB) {
84  eb[lev]->make_all_factories(lev, geom[lev], grids[lev], dmap[lev], eb_level);
85  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
86  eb[lev]->make_cc_factory(lev, geom[lev], grids[lev], dmap[lev], eb_level);
87  }
88  } else {
89  // m_factory[lev] = std::make_unique<FabFactory<FArrayBox>>();
90  }
91 
92  auto& lev_new = vars_new[lev];
93  auto& lev_old = vars_old[lev];
94 
95  //********************************************************************************************
96  // This allocates all kinds of things, including but not limited to: solution arrays,
97  // terrain arrays, metric terms and base state.
98  // *******************************************************************************************
99  init_stuff(lev, ba, dm, lev_new, lev_old, base_state[lev], z_phys_nd[lev]);
100 
101  //********************************************************************************************
102  // Land Surface Model
103  // *******************************************************************************************
104  int lsm_data_size = lsm.Get_Data_Size();
105  int lsm_flux_size = lsm.Get_Flux_Size();
106  lsm_data[lev].resize(lsm_data_size);
107  lsm_data_name.resize(lsm_data_size);
108  lsm_flux[lev].resize(lsm_flux_size);
109  lsm_flux_name.resize(lsm_flux_size);
110  lsm.Define(lev, solverChoice);
111  if (solverChoice.lsm_type != LandSurfaceType::None)
112  {
113  lsm.Init(lev, vars_new[lev][Vars::cons], Geom(lev), 0.0); // dummy dt value
114  }
115  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
116  lsm_data[lev][mvar] = lsm.Get_Data_Ptr(lev,mvar);
117  lsm_data_name[mvar] = lsm.Get_DataName(mvar);
118  }
119  for (int mvar(0); mvar<lsm_flux[lev].size(); ++mvar) {
120  lsm_flux[lev][mvar] = lsm.Get_Flux_Ptr(lev,mvar);
121  lsm_flux_name[mvar] = lsm.Get_FluxName(mvar);
122  }
123 
124 
125 
126  // ********************************************************************************************
127  // Build the data structures for calculating diffusive/turbulent terms
128  // ********************************************************************************************
129  update_diffusive_arrays(lev, ba, dm);
130 
131  // ********************************************************************************************
132  // Build the data structures for holding sea surface temps and skin temps
133  // ********************************************************************************************
134  sst_lev[lev].resize(1); sst_lev[lev][0] = nullptr;
135  tsk_lev[lev].resize(1); tsk_lev[lev][0] = nullptr;
136 
137  // ********************************************************************************************
138  // Thin immersed body
139  // *******************************************************************************************
140  init_thin_body(lev, ba, dm);
141 
142  // ********************************************************************************************
143  // Initialize the integrator class
144  // ********************************************************************************************
145  initialize_integrator(lev, lev_new[Vars::cons],lev_new[Vars::xvel]);
146 
147  // ********************************************************************************************
148  // Initialize the data itself
149  // If (init_type == InitType::WRFInput) then we are initializing terrain and the initial data in
150  // the same call so we must call init_only before update_terrain_arrays
151  // If (init_type != InitType::WRFInput) then we want to initialize the terrain before the initial data
152  // since we may need to use the grid information before constructing
153  // initial idealized data
154  // ********************************************************************************************
155  if (restart_chkfile.empty()) {
156  if ( (solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid) )
157  {
158  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type == TerrainType::StaticFittedMesh);
159  init_only(lev, time);
160  init_zphys(lev, time);
162  make_physbcs(lev);
163  } else {
164  init_zphys(lev, time);
166  // Note that for init_type != InitType::WRFInput and != InitType::Metgrid,
167  // make_physbcs is called inside init_only
168  init_only(lev, time);
169  }
170  } else {
171  // if restarting and nudging from input sounding, load the input sounding files
172  if (lev == 0 && solverChoice.init_type == InitType::Input_Sounding && solverChoice.nudging_from_input_sounding)
173  {
175  Error("input_sounding file name must be provided via input");
176  }
177 
179 
180  // this will interpolate the input profiles to the nominal height levels
181  // (ranging from 0 to the domain top)
182  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
183  input_sounding_data.read_from_file(geom[lev], zlevels_stag[lev], n);
184  }
185 
186  // this will calculate the hydrostatically balanced density and pressure
187  // profiles following WRF ideal.exe
188  if (solverChoice.sounding_type == SoundingType::Ideal) {
190  } else if (solverChoice.sounding_type == SoundingType::Isentropic ||
191  solverChoice.sounding_type == SoundingType::DryIsentropic) {
192  input_sounding_data.assume_dry = (solverChoice.sounding_type == SoundingType::DryIsentropic);
194  }
195  }
196 
197  // We re-create terrain_blanking on restart rather than storing it in the checkpoint
198  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
199  int ngrow = ComputeGhostCells(solverChoice) + 2;
200  terrain_blanking[lev]->setVal(1.0);
201  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, ngrow);
202  terrain_blanking[lev]->FillBoundary(geom[lev].periodicity());
203  }
204  }
205 
206  // Read in tables needed for windfarm simulations
207  // fill in Nturb multifab - number of turbines in each mesh cell
208  // write out the vtk files for wind turbine location and/or
209  // actuator disks
210  #ifdef ERF_USE_WINDFARM
211  init_windfarm(lev);
212  #endif
213 
214  // ********************************************************************************************
215  // Build the data structures for canopy model (depends upon z_phys)
216  // ********************************************************************************************
217  if (restart_chkfile.empty()) {
219  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
220  }
221  }
222 
223  //********************************************************************************************
224  // Create wall distance field for RANS model (depends upon z_phys)
225  // *******************************************************************************************
226  if (solverChoice.turbChoice[lev].rans_type != RANSType::None) {
227  // Handle bottom boundary
228  poisson_wall_dist(lev);
229 
230  // Correct the wall distance for immersed bodies
236  geom[lev],
237  z_phys_cc[lev]);
238  }
239  }
240 
241  //********************************************************************************************
242  // Microphysics
243  // *******************************************************************************************
244  int q_size = micro->Get_Qmoist_Size(lev);
245  qmoist[lev].resize(q_size);
246  micro->Define(lev, solverChoice);
247  if (solverChoice.moisture_type != MoistureType::None)
248  {
249  micro->Init(lev, vars_new[lev][Vars::cons],
250  grids[lev], Geom(lev), 0.0,
251  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
252  }
253  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
254  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
255  }
256 
257  //********************************************************************************************
258  // Radiation
259  // *******************************************************************************************
260  if (solverChoice.rad_type != RadiationType::None)
261  {
262  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
263  }
264 
265  // ********************************************************************************************
266  // If we are making a new level then the FillPatcher for this level hasn't been allocated yet
267  // ********************************************************************************************
268  if (lev > 0 && cf_width >= 0) {
271  }
272 
273 #ifdef ERF_USE_PARTICLES
274  if (restart_chkfile.empty()) {
275  if (lev == 0) {
276  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,time);
277  } else {
278  particleData.Redistribute();
279  }
280  }
281 #endif
282 }
BoxArray ERFPostProcessBaseGrids(const Box &domain, bool decompose_in_z)
Definition: ERF_ChopGrids.cpp:6
void thinbody_wall_dist(std::unique_ptr< MultiFab > &wdist, Vector< IntVect > &xfaces, Vector< IntVect > &yfaces, Vector< IntVect > &zfaces, const Geometry &geomdata, std::unique_ptr< MultiFab > &z_phys_cc)
Definition: ERF_ThinBodyWallDist.cpp:12
void init_only(int lev, amrex::Real time)
Definition: ERF.cpp:1738
void poisson_wall_dist(int lev)
Definition: ERF_PoissonWallDist.cpp:20
void init_thin_body(int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
Definition: ERF_MakeNewLevel.cpp:716
int Get_Data_Size()
Definition: ERF_LandSurface.H:92
std::string Get_DataName(const int &varIdx)
Definition: ERF_LandSurface.H:98
std::string Get_FluxName(const int &varIdx)
Definition: ERF_LandSurface.H:104
amrex::MultiFab * Get_Flux_Ptr(const int &lev, const int &varIdx)
Definition: ERF_LandSurface.H:86
void Init(const int &lev, const amrex::MultiFab &cons_in, const amrex::Geometry &geom, const amrex::Real &dt_advance)
Definition: ERF_LandSurface.H:43
void Define(const int &lev, SolverChoice &sc)
Definition: ERF_LandSurface.H:36
int Get_Flux_Size()
Definition: ERF_LandSurface.H:95
bool nudging_from_input_sounding
Definition: ERF_DataStruct.H:889
Here is the call graph for this function:

◆ MakeVTKFilename()

std::string ERF::MakeVTKFilename ( int  nstep)
574  {
575  // Ensure output directory exists
576  const std::string dir = "Output_HurricaneTracker";
577  if (!fs::exists(dir)) {
578  fs::create_directory(dir);
579  }
580 
581  // Construct filename with zero-padded step
582  std::ostringstream oss;
583  if(nstep==0){
584  oss << dir << "/hurricane_track_" << std::setw(7) << std::setfill('0') << nstep << ".vtk";
585  } else {
586  oss << dir << "/hurricane_track_" << std::setw(7) << std::setfill('0') << nstep+1 << ".vtk";
587  }
588 
589  return oss.str();
590 }

◆ nghost_eb_basic()

static int ERF::nghost_eb_basic ( )
inlinestaticprivate
1564  { return 5; }

◆ nghost_eb_full()

static int ERF::nghost_eb_full ( )
inlinestaticprivate
1571  { return 4; }

◆ nghost_eb_volume()

static int ERF::nghost_eb_volume ( )
inlinestaticprivate
1568  { return 5; }

◆ NumDataLogs()

AMREX_FORCE_INLINE int ERF::NumDataLogs ( )
inlineprivatenoexcept
1363  {
1364  return datalog.size();
1365  }

◆ NumDerDataLogs()

AMREX_FORCE_INLINE int ERF::NumDerDataLogs ( )
inlineprivatenoexcept
1370  {
1371  return der_datalog.size();
1372  }

◆ NumSampleLineLogs()

AMREX_FORCE_INLINE int ERF::NumSampleLineLogs ( )
inlineprivatenoexcept
1399  {
1400  return samplelinelog.size();
1401  }

◆ NumSampleLines()

AMREX_FORCE_INLINE int ERF::NumSampleLines ( )
inlineprivatenoexcept
1425  {
1426  return sampleline.size();
1427  }

◆ NumSamplePointLogs()

AMREX_FORCE_INLINE int ERF::NumSamplePointLogs ( )
inlineprivatenoexcept
1385  {
1386  return sampleptlog.size();
1387  }

◆ NumSamplePoints()

AMREX_FORCE_INLINE int ERF::NumSamplePoints ( )
inlineprivatenoexcept
1412  {
1413  return samplepoint.size();
1414  }

◆ operator=() [1/2]

ERF& ERF::operator= ( const ERF other)
delete

◆ operator=() [2/2]

ERF& ERF::operator= ( ERF &&  other)
deletenoexcept

◆ ParameterSanityChecks()

void ERF::ParameterSanityChecks ( )
private
2222 {
2223  AMREX_ALWAYS_ASSERT(cfl > 0. || fixed_dt[0] > 0.);
2224 
2225  // We don't allow use_real_bcs to be true if init_type is not either InitType::WRFInput or InitType::Metgrid
2226  AMREX_ALWAYS_ASSERT( !solverChoice.use_real_bcs ||
2227  ((solverChoice.init_type == InitType::WRFInput) || (solverChoice.init_type == InitType::Metgrid)) );
2228 
2229  AMREX_ALWAYS_ASSERT(real_width >= 0);
2230  AMREX_ALWAYS_ASSERT(real_set_width >= 0);
2231  AMREX_ALWAYS_ASSERT(real_width >= real_set_width);
2232 
2233  if (cf_width < 0 || cf_set_width < 0 || cf_width < cf_set_width) {
2234  Abort("You must set cf_width >= cf_set_width >= 0");
2235  }
2236  if (max_level > 0 && cf_set_width > 0) {
2237  for (int lev = 1; lev <= max_level; lev++) {
2238  if (cf_set_width%ref_ratio[lev-1][0] != 0 ||
2239  cf_set_width%ref_ratio[lev-1][1] != 0 ||
2240  cf_set_width%ref_ratio[lev-1][2] != 0 ) {
2241  Abort("You must set cf_width to be a multiple of ref_ratio");
2242  }
2243  }
2244  }
2245 
2246  // If fixed_mri_dt_ratio is set, it must be even
2247  if (fixed_mri_dt_ratio > 0 && (fixed_mri_dt_ratio%2 != 0) )
2248  {
2249  Abort("If you specify fixed_mri_dt_ratio, it must be even");
2250  }
2251 
2252  for (int lev = 0; lev <= max_level; lev++)
2253  {
2254  // We ignore fixed_fast_dt if not substepping
2255  if (solverChoice.substepping_type[lev] == SubsteppingType::None) {
2256  fixed_fast_dt[lev] = -1.0;
2257  }
2258 
2259  // If both fixed_dt and fast_dt are specified, their ratio must be an even integer
2260  if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio <= 0)
2261  {
2262  Real eps = 1.e-12;
2263  int ratio = static_cast<int>( ( (1.0+eps) * fixed_dt[lev] ) / fixed_fast_dt[lev] );
2264  if (fixed_dt[lev] / fixed_fast_dt[lev] != ratio)
2265  {
2266  Abort("Ratio of fixed_dt to fixed_fast_dt must be an even integer");
2267  }
2268  }
2269 
2270  // If all three are specified, they must be consistent
2271  if (fixed_dt[lev] > 0. && fixed_fast_dt[lev] > 0. && fixed_mri_dt_ratio > 0)
2272  {
2273  if (fixed_dt[lev] / fixed_fast_dt[lev] != fixed_mri_dt_ratio)
2274  {
2275  Abort("Dt is over-specfied");
2276  }
2277  }
2278  } // lev
2279 
2280  if (solverChoice.coupling_type == CouplingType::TwoWay && cf_width > 0) {
2281  Abort("For two-way coupling you must set cf_width = 0");
2282  }
2283 }
int real_set_width
Definition: ERF.H:1161

◆ PlotFileName()

std::string ERF::PlotFileName ( int  lev) const
private

◆ PlotFileVarNames()

Vector< std::string > ERF::PlotFileVarNames ( amrex::Vector< std::string >  plot_var_names)
staticprivate
252 {
253  Vector<std::string> names;
254 
255  names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
256 
257  return names;
258 
259 }

◆ poisson_wall_dist()

void ERF::poisson_wall_dist ( int  lev)

Calculate wall distances using the Poisson equation

The zlo boundary is assumed to correspond to the land surface. If there are no boundary walls, then the other use case is to calculate wall distances for immersed boundaries (embedded or thin body).

See Tucker, P. G. (2003). Differential equation-based wall distance computation for DES and RANS. Journal of Computational Physics, 190(1), 229–248. https://doi.org/10.1016/S0021-9991(03)00272-9

21 {
22  BL_PROFILE("ERF::poisson_wall_dist()");
23 
24  bool havewall{false};
25  Orientation zlo(Direction::z, Orientation::low);
26  if ( ( phys_bc_type[zlo] == ERF_BC::surface_layer ) ||
27  ( phys_bc_type[zlo] == ERF_BC::no_slip_wall ) )/*||
28  ((phys_bc_type[zlo] == ERF_BC::slip_wall) && (dom_hi.z > dom_lo.z)) )*/
29  {
30  havewall = true;
31  }
32 
33  auto const& geomdata = geom[lev];
34 
35  if (havewall) {
36  if (solverChoice.mesh_type == MeshType::ConstantDz) {
37 // Comment this out to test the wall dist calc in the trivial case:
38 //#if 0
39  Print() << "Directly calculating direct wall distance for constant dz" << std::endl;
40  const Real* prob_lo = geomdata.ProbLo();
41  const Real* dx = geomdata.CellSize();
42  for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
43  const Box& bx = mfi.validbox();
44  auto dist_arr = walldist[lev]->array(mfi);
45  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
46  dist_arr(i, j, k) = prob_lo[2] + (k + 0.5) * dx[2];
47  });
48  }
49  return;
50 //#endif
51  } else if (solverChoice.mesh_type == MeshType::StretchedDz) {
52  // TODO: Handle this trivial case
53  Error("Wall dist calc not implemented with grid stretching yet");
54  } else {
55  // TODO
56  Error("Wall dist calc not implemented over terrain yet");
57  }
58  }
59 
60  Print() << "Calculating Poisson wall distance" << std::endl;
61 
62  // Make sure the solver only sees the levels over which we are solving
63  BoxArray nba = walldist[lev]->boxArray();
64  nba.surroundingNodes();
65  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
66  Vector<BoxArray> ba_tmp; ba_tmp.push_back(nba);
67  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(walldist[lev]->DistributionMap());
68 
69  Vector<MultiFab> rhs;
70  Vector<MultiFab> phi;
71 
72  if (solverChoice.terrain_type == TerrainType::EB) {
73  amrex::Error("Wall dist calc not implemented for EB");
74  } else {
75  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
76  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
77  }
78 
79  rhs[0].setVal(-1.0);
80 
81  // Define an overset mask to set dirichlet nodes on walls
82  iMultiFab mask(ba_tmp[0], dm_tmp[0], 1, 0);
83  Vector<const iMultiFab*> overset_mask = {&mask};
84 
85  auto const dom_lo = lbound(geom[lev].Domain());
86  auto const dom_hi = ubound(geom[lev].Domain());
87 
88  // ****************************************************************************
89  // Initialize phi
90  // (It is essential that we do this in order to fill the corners; this is
91  // used if we include blanking.)
92  // ****************************************************************************
93  phi[0].setVal(0.0);
94 
95  // ****************************************************************************
96  // Interior boundaries are marked with phi=0
97  // ****************************************************************************
98  // Overset mask is 0/1: 1 means the node is an unknown. 0 means it's known.
99  mask.setVal(1);
101  Warning("Poisson distance is inaccurate for bodies in open domains that are small compared to the domain size, skipping");
102  return;
103 #if 0
104  Gpu::DeviceVector<IntVect> xfacelist, yfacelist, zfacelist;
105 
106  xfacelist.resize(solverChoice.advChoice.zero_xflux.size());
107  yfacelist.resize(solverChoice.advChoice.zero_yflux.size());
108  zfacelist.resize(solverChoice.advChoice.zero_zflux.size());
109 
110  if (xfacelist.size() > 0) {
111  Gpu::copy(amrex::Gpu::hostToDevice,
114  xfacelist.begin());
115  Print() << " masking interior xfaces" << std::endl;
116  }
117  if (yfacelist.size() > 0) {
118  Gpu::copy(amrex::Gpu::hostToDevice,
121  yfacelist.begin());
122  Print() << " masking interior yfaces" << std::endl;
123  }
124  if (zfacelist.size() > 0) {
125  Gpu::copy(amrex::Gpu::hostToDevice,
128  zfacelist.begin());
129  Print() << " masking interior zfaces" << std::endl;
130  }
131 
132  for (MFIter mfi(phi[0]); mfi.isValid(); ++mfi) {
133  const Box& bx = mfi.validbox();
134 
135  auto phi_arr = phi[0].array(mfi);
136  auto mask_arr = mask.array(mfi);
137 
138  if (xfacelist.size() > 0) {
139  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
140  for (int iface=0; iface < xfacelist.size(); ++iface) {
141  if ((i == xfacelist[iface][0]) &&
142  (j == xfacelist[iface][1]) &&
143  (k == xfacelist[iface][2]))
144  {
145  mask_arr(i, j , k ) = 0;
146  mask_arr(i, j , k+1) = 0;
147  mask_arr(i, j+1, k ) = 0;
148  mask_arr(i, j+1, k+1) = 0;
149  }
150  }
151  });
152  }
153 
154  if (yfacelist.size() > 0) {
155  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
156  for (int iface=0; iface < yfacelist.size(); ++iface) {
157  if ((i == yfacelist[iface][0]) &&
158  (j == yfacelist[iface][1]) &&
159  (k == yfacelist[iface][2]))
160  {
161  mask_arr(i , j, k ) = 0;
162  mask_arr(i , j, k+1) = 0;
163  mask_arr(i+1, j, k ) = 0;
164  mask_arr(i+1, j, k+1) = 0;
165  }
166  }
167  });
168  }
169 
170  if (zfacelist.size() > 0) {
171  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
172  for (int iface=0; iface < zfacelist.size(); ++iface) {
173  if ((i == xfacelist[iface][0]) &&
174  (j == xfacelist[iface][1]) &&
175  (k == xfacelist[iface][2]))
176  {
177  mask_arr(i , j , k) = 0;
178  mask_arr(i , j+1, k) = 0;
179  mask_arr(i+1, j , k) = 0;
180  mask_arr(i+1, j+1, k) = 0;
181  }
182  }
183  });
184  }
185  }
186 #endif
187  }
188 
189  // ****************************************************************************
190  // Setup BCs, with solid domain boundaries being dirichlet
191  // ****************************************************************************
192  amrex::Array<amrex::LinOpBCType,AMREX_SPACEDIM> bc3d_lo, bc3d_hi;
193  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
194  if (geom[0].isPeriodic(dir)) {
195  bc3d_lo[dir] = LinOpBCType::Periodic;
196  bc3d_hi[dir] = LinOpBCType::Periodic;
197  } else {
198  bc3d_lo[dir] = LinOpBCType::Neumann;
199  bc3d_hi[dir] = LinOpBCType::Neumann;
200  }
201  }
202  if (havewall) {
203  Print() << " Poisson zlo BC is dirichlet" << std::endl;
204  bc3d_lo[2] = LinOpBCType::Dirichlet;
205  }
206  Print() << " bc lo : " << bc3d_lo << std::endl;
207  Print() << " bc hi : " << bc3d_hi << std::endl;
208 
209  if (!solverChoice.advChoice.have_zero_flux_faces && !havewall) {
210  Error("No solid boundaries in the computational domain");
211  }
212 
213  LPInfo info;
214 /* Nodal solver cannot have hidden dimensions */
215 #if 0
216  // Allow a hidden direction if the domain is one cell wide
217  if (dom_lo.x == dom_hi.x) {
218  info.setHiddenDirection(0);
219  Print() << " domain is 2D in yz" << std::endl;
220  } else if (dom_lo.y == dom_hi.y) {
221  info.setHiddenDirection(1);
222  Print() << " domain is 2D in xz" << std::endl;
223  } else if (dom_lo.z == dom_hi.z) {
224  info.setHiddenDirection(2);
225  Print() << " domain is 2D in xy" << std::endl;
226  }
227 #endif
228 
229  // ****************************************************************************
230  // Solve nodal masked Poisson problem with MLMG
231  // TODO: different solver for terrain?
232  // ****************************************************************************
233  const Real reltol = solverChoice.poisson_reltol;
234  const Real abstol = solverChoice.poisson_abstol;
235 
236  Real sigma = 1.0;
237  Vector<EBFArrayBoxFactory const*> factory_vec = { &EBFactory(lev) };
238  MLNodeLaplacian mlpoisson(geom_tmp, ba_tmp, dm_tmp, info, factory_vec, sigma);
239 
240  mlpoisson.setDomainBC(bc3d_lo, bc3d_hi);
241 
242  if (lev > 0) {
243  mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
244  }
245 
246  mlpoisson.setLevelBC(0, nullptr);
247 
248  mlpoisson.setOversetMask(0, mask);
249 
250  // Solve
251  MLMG mlmg(mlpoisson);
252  int max_iter = 100;
253  mlmg.setMaxIter(max_iter);
254 
255  mlmg.setVerbose(mg_verbose);
256  mlmg.setBottomVerbose(0);
257 
258  mlmg.solve(GetVecOfPtrs(phi),
259  GetVecOfConstPtrs(rhs),
260  reltol, abstol);
261 
262  // Now overwrite with periodic fill outside domain and fine-fine fill inside
263  phi[0].FillBoundary(geom[lev].periodicity());
264 
265  // ****************************************************************************
266  // Compute grad(phi) to get distances
267  // - Note that phi is nodal and walldist is cell-centered
268  // - TODO: include terrain metrics for dphi/dz
269  // ****************************************************************************
270  for (MFIter mfi(*walldist[lev]); mfi.isValid(); ++mfi) {
271  const Box& bx = mfi.validbox();
272 
273  const auto invCellSize = geomdata.InvCellSizeArray();
274 
275  auto const& phi_arr = phi[0].const_array(mfi);
276  auto dist_arr = walldist[lev]->array(mfi);
277 
278  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
279  Real dpdx{0}, dpdy{0}, dpdz{0};
280 
281  // dphi/dx
282  if (dom_lo.x != dom_hi.x) {
283  dpdx = 0.25 * invCellSize[0] * (
284  (phi_arr(i+1, j , k ) - phi_arr(i, j , k ))
285  + (phi_arr(i+1, j , k+1) - phi_arr(i, j , k+1))
286  + (phi_arr(i+1, j+1, k ) - phi_arr(i, j+1, k ))
287  + (phi_arr(i+1, j+1, k+1) - phi_arr(i, j+1, k+1)) );
288  }
289 
290  // dphi/dy
291  if (dom_lo.y != dom_hi.y) {
292  dpdy = 0.25 * invCellSize[1] * (
293  (phi_arr(i , j+1, k ) - phi_arr(i , j, k ))
294  + (phi_arr(i , j+1, k+1) - phi_arr(i , j, k+1))
295  + (phi_arr(i+1, j+1, k ) - phi_arr(i+1, j, k ))
296  + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j, k+1)) );
297  }
298 
299  // dphi/dz
300  if (dom_lo.z != dom_hi.z) {
301  dpdz = 0.25 * invCellSize[2] * (
302  (phi_arr(i , j , k+1) - phi_arr(i , j , k))
303  + (phi_arr(i , j+1, k+1) - phi_arr(i , j+1, k))
304  + (phi_arr(i+1, j , k+1) - phi_arr(i+1, j , k))
305  + (phi_arr(i+1, j+1, k+1) - phi_arr(i+1, j+1, k)) );
306  }
307 
308  Real dp_dot_dp = dpdx*dpdx + dpdy*dpdy + dpdz*dpdz;
309  Real phi_avg = 0.125 * (
310  phi_arr(i , j , k ) + phi_arr(i , j , k+1) + phi_arr(i , j+1, k ) + phi_arr(i , j+1, k+1)
311  + phi_arr(i+1, j , k ) + phi_arr(i+1, j , k+1) + phi_arr(i+1, j+1, k ) + phi_arr(i+1, j+1, k+1) );
312  dist_arr(i, j, k) = -std::sqrt(dp_dot_dp) + std::sqrt(dp_dot_dp + 2*phi_avg);
313 
314  // DEBUG: output phi instead
315  //dist_arr(i, j, k) = phi_arr(i, j, k);
316  });
317  }
318 }
if(l_use_mynn &&start_comp<=RhoKE_comp &&end_comp >=RhoKE_comp)
Definition: ERF_DiffQKEAdjustment.H:2
static int mg_verbose
Definition: ERF.H:1136
amrex::Real poisson_reltol
Definition: ERF_DataStruct.H:836
amrex::Real poisson_abstol
Definition: ERF_DataStruct.H:835
Here is the call graph for this function:

◆ post_timestep()

void ERF::post_timestep ( int  nstep,
amrex::Real  time,
amrex::Real  dt_lev 
)
633 {
634  BL_PROFILE("ERF::post_timestep()");
635 
636 #ifdef ERF_USE_PARTICLES
637  particleData.Redistribute();
638 #endif
639 
640  if (solverChoice.coupling_type == CouplingType::TwoWay)
641  {
642  int ncomp = vars_new[0][Vars::cons].nComp();
643  for (int lev = finest_level-1; lev >= 0; lev--)
644  {
645  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
646  // m is the map scale factor at cell centers
647  // Here we pre-divide (rho S) by m^2 before refluxing
648  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
649  const Box& bx = mfi.tilebox();
650  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
651  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
652  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
653  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
654  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
655  {
656  cons_arr(i,j,k,n) /= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
657  });
658  } else {
659  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
660  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
661  {
662  cons_arr(i,j,k,n) *= detJ_arr(i,j,k) / (mfx_arr(i,j,0)*mfy_arr(i,j,0));
663  });
664  }
665  } // mfi
666 
667  // This call refluxes all "slow" cell-centered variables
668  // (i.e. not density or (rho theta) or velocities) from the lev/lev+1 interface onto lev
669  getAdvFluxReg(lev+1)->Reflux(vars_new[lev][Vars::cons], 2, 2, ncomp-2);
670 
671  // Here we multiply (rho S) by m^2 after refluxing
672  for (MFIter mfi(vars_new[lev][Vars::cons], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
673  const Box& bx = mfi.tilebox();
674  const Array4< Real> cons_arr = vars_new[lev][Vars::cons].array(mfi);
675  const Array4<const Real> mfx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
676  const Array4<const Real> mfy_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
677  if (SolverChoice::mesh_type == MeshType::ConstantDz) {
678  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
679  {
680  cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0));
681  });
682  } else {
683  const Array4<const Real> detJ_arr = detJ_cc[lev]->const_array(mfi);
684  ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept
685  {
686  cons_arr(i,j,k,n) *= (mfx_arr(i,j,0)*mfy_arr(i,j,0)) / detJ_arr(i,j,k);
687  });
688  }
689  } // mfi
690 
691  // We need to do this before anything else because refluxing changes the
692  // values of coarse cells underneath fine grids with the assumption they'll
693  // be over-written by averaging down
694  int src_comp;
695  if (solverChoice.anelastic[lev]) {
696  src_comp = 1;
697  } else {
698  src_comp = 0;
699  }
700  int num_comp = ncomp - src_comp;
701  AverageDownTo(lev,src_comp,num_comp);
702  }
703  }
704 
705  if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
708  sum_energy_quantities(time);
709  }
710 
711  if (solverChoice.pert_type == PerturbationType::Source ||
712  solverChoice.pert_type == PerturbationType::Direct ||
713  solverChoice.pert_type == PerturbationType::CPM) {
714  if (is_it_time_for_action(nstep, time, dt_lev0, pert_interval, -1.)) {
715  turbPert.debug(time);
716  }
717  }
718 
719  if (profile_int > 0 && (nstep+1) % profile_int == 0) {
720  if (destag_profiles) {
721  // all variables cell-centered
722  write_1D_profiles(time);
723  } else {
724  // some variables staggered
726  }
727  }
728 
729  if (solverChoice.rad_type != RadiationType::None)
730  {
731  if ( rad_datalog_int > 0 &&
732  (((nstep+1) % rad_datalog_int == 0) || (nstep==0)) ) {
733  if (rad[0]->hasDatalog()) {
734  rad[0]->WriteDataLog(time+start_time);
735  }
736  }
737  }
738 
739  if (output_1d_column) {
740 #ifdef ERF_USE_NETCDF
741  if (is_it_time_for_action(nstep, time, dt_lev0, column_interval, column_per))
742  {
743  int lev_column = 0;
744  for (int lev = finest_level; lev >= 0; lev--)
745  {
746  Real dx_lev = geom[lev].CellSize(0);
747  Real dy_lev = geom[lev].CellSize(1);
748  int i_lev = static_cast<int>(std::floor(column_loc_x / dx_lev));
749  int j_lev = static_cast<int>(std::floor(column_loc_y / dy_lev));
750  if (grids[lev].contains(IntVect(i_lev,j_lev,0))) lev_column = lev;
751  }
752  writeToNCColumnFile(lev_column, column_file_name, column_loc_x, column_loc_y, time);
753  }
754 #else
755  Abort("To output 1D column files ERF must be compiled with NetCDF");
756 #endif
757  }
758 
760  {
763  {
764  bool is_moist = (micro->Get_Qstate_Moist_Size() > 0);
765  m_w2d->write_planes(istep[0], time, vars_new, is_moist);
766  }
767  }
768 
769  // Write plane/line sampler data
770  if (is_it_time_for_action(nstep+1, time, dt_lev0, sampler_interval, sampler_per) && (data_sampler) ) {
771  data_sampler->get_sample_data(geom, vars_new);
772  data_sampler->write_sample_data(t_new, istep, ref_ratio, geom);
773  }
774 
775  // Moving terrain
776  if ( solverChoice.terrain_type == TerrainType::MovingFittedMesh )
777  {
778  for (int lev = finest_level; lev >= 0; lev--)
779  {
780  // Copy z_phs_nd and detJ_cc at end of timestep
781  MultiFab::Copy(*z_phys_nd[lev], *z_phys_nd_new[lev], 0, 0, 1, z_phys_nd[lev]->nGrowVect());
782  MultiFab::Copy( *detJ_cc[lev], *detJ_cc_new[lev], 0, 0, 1, detJ_cc[lev]->nGrowVect());
783  MultiFab::Copy(base_state[lev],base_state_new[lev],0,0,BaseState::num_comps,base_state[lev].nGrowVect());
784 
785  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
786  }
787  }
788 
789  bool is_hurricane_tracker_io=false;
790  ParmParse pp("erf");
791  pp.query("is_hurricane_tracker_io", is_hurricane_tracker_io);
792 
793  if (is_hurricane_tracker_io) {
794  if(nstep == 0 or (nstep+1)%m_plot3d_int_1 == 0){
795  std::string filename = MakeVTKFilename(nstep);
796  Real velmag_threshold = 1e10;
797  pp.query("hurr_track_io_velmag_greater_than", velmag_threshold);
798  if(velmag_threshold==1e10) {
799  Abort("As hurricane tracking IO is active using erf.is_hurricane_tracker_io = true"
800  " there needs to be an input erf.hurr_track_io_velmag_greater_than which specifies the"
801  " magnitude of velocity above which cells will be tagged for refinement.");
802  }
803  int levc=finest_level;
804  MultiFab& U_new = vars_new[levc][Vars::xvel];
805  MultiFab& V_new = vars_new[levc][Vars::yvel];
806  MultiFab& W_new = vars_new[levc][Vars::zvel];
807 
808  HurricaneTracker(levc, U_new, V_new, W_new, velmag_threshold, true);
809  if (ParallelDescriptor::IOProcessor()) {
811  }
812  }
813  }
814 } // post_timestep
void make_zcc(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc)
Definition: ERF_TerrainMetrics.cpp:623
static amrex::Real column_loc_y
Definition: ERF.H:1195
static std::string column_file_name
Definition: ERF.H:1196
AMREX_FORCE_INLINE amrex::YAFluxRegister * getAdvFluxReg(int lev)
Definition: ERF.H:1341
static amrex::Real bndry_output_planes_per
Definition: ERF.H:1201
std::string MakeVTKFilename(int nstep)
Definition: ERF_Write1DProfiles.cpp:574
static amrex::Real column_per
Definition: ERF.H:1193
amrex::Real sampler_per
Definition: ERF.H:1522
static amrex::Real column_loc_x
Definition: ERF.H:1194
static int bndry_output_planes_interval
Definition: ERF.H:1200
int sampler_interval
Definition: ERF.H:1521
static int output_1d_column
Definition: ERF.H:1191
void WriteVTKPolyline(const std::string &filename, amrex::Vector< std::array< amrex::Real, 2 >> &points_xy)
Definition: ERF_Write1DProfiles.cpp:593
static int column_interval
Definition: ERF.H:1192
Here is the call graph for this function:

◆ post_update()

void ERF::post_update ( amrex::MultiFab &  state_mf,
amrex::Real  time,
const amrex::Geometry &  geom 
)
private

◆ print_banner()

void ERF::print_banner ( MPI_Comm  comm,
std::ostream &  out 
)
static
65  : " << msg << std::endl;
66 }
67 
68 void ERF::print_banner (MPI_Comm comm, std::ostream& out)
69 {
70 #ifdef AMREX_USE_MPI
71  int irank = 0;
72  int num_ranks = 1;
73  MPI_Comm_size(comm, &num_ranks);
74  MPI_Comm_rank(comm, &irank);
75 
76  // Only root process does the printing
77  if (irank != 0) return;
78 #else
79  amrex::ignore_unused(comm);
80 #endif
81 
82  auto etime = std::chrono::system_clock::now();
83  auto etimet = std::chrono::system_clock::to_time_t(etime);
84 #ifndef _WIN32
85  char time_buf[64];
86  ctime_r(&etimet, time_buf);
87  const std::string tstamp(time_buf);
88 #else
89  char* time_buf = new char[64];
90  ctime_s(time_buf, 64, &etimet);
91  const std::string tstamp(time_buf);
92 #endif
93 
94  const char* githash1 = amrex::buildInfoGetGitHash(1);
95  const char* githash2 = amrex::buildInfoGetGitHash(2);
96 
97  // clang-format off
98  out << dbl_line
99  << " ERF (https://github.com/erf-model/ERF)"
100  << std::endl << std::endl
101  << " ERF Git SHA :: " << githash1 << std::endl
102  << " AMReX Git SHA :: " << githash2 << std::endl
103  << " AMReX version :: " << amrex::Version() << std::endl << std::endl
104  << " Exec. time :: " << tstamp
105  << " Build time :: " << amrex::buildInfoGetBuildDate() << std::endl
106  << " C++ compiler :: " << amrex::buildInfoGetComp()
107  << " " << amrex::buildInfoGetCompVersion() << std::endl << std::endl
108  << " MPI :: "
109 #ifdef AMREX_USE_MPI
110  << "ON (Num. ranks = " << num_ranks << ")" << std::endl
111 #else
112  << "OFF " << std::endl
113 #endif
114  << " GPU :: "
115 #ifdef AMREX_USE_GPU
116  << "ON "
117 #if defined(AMREX_USE_CUDA)
118  << "(Backend: CUDA)"
119 #elif defined(AMREX_USE_HIP)
120  << "(Backend: HIP)"
121 #elif defined(AMREX_USE_SYCL)
122  << "(Backend: SYCL)"
123 #endif
124  << std::endl
125 #else
126  << "OFF" << std::endl
127 #endif
128  << " OpenMP :: "
129 #ifdef AMREX_USE_OMP
130  << "ON (Num. threads = " << omp_get_max_threads() << ")" << std::endl
131 #else
132  << "OFF" << std::endl
133 #endif
134  << std::endl;
135 
ERF()
Definition: ERF.cpp:121
const char * buildInfoGetBuildDate()
const char * buildInfoGetComp()
const char * buildInfoGetCompVersion()

Referenced by main().

Here is the caller graph for this function:

◆ print_error()

void ERF::print_error ( MPI_Comm  comm,
const std::string &  msg 
)
static
43  :
44  input_file : Input file with simulation settings
45 
46 Optional:
47  param=value : Overrides for parameters during runtime
48 )doc" << std::endl;
49 }
50 
51 void ERF::print_error (MPI_Comm comm, const std::string& msg)
52 {
53 #ifdef AMREX_USE_MPI
54  int irank = 0;
55  int num_ranks = 1;
56  MPI_Comm_size(comm, &num_ranks);
57  MPI_Comm_rank(comm, &irank);
58 

Referenced by main().

Here is the caller graph for this function:

◆ print_summary()

static void ERF::print_summary ( std::ostream &  )
static

◆ print_tpls()

void ERF::print_tpls ( std::ostream &  out)
static
140  ://github.com/erf-model/ERF/blob/development/LICENSE for details. "
141  << dash_line << std::endl;
142  // clang-format on
143 }
144 
145 void ERF::print_tpls (std::ostream& out)
146 {
147  amrex::Vector<std::string> tpls;
148 
149 #ifdef ERF_USE_NETCDF
150  tpls.push_back(std::string("NetCDF ") + NC_VERSION);
151 #endif
152 #ifdef AMREX_USE_SUNDIALS
153  tpls.push_back(std::string("SUNDIALS ") + SUNDIALS_VERSION);
154 #endif
155 
156  if (!tpls.empty()) {
157  out << " Enabled third-party libraries: ";
158  for (const auto& val : tpls) {
static void print_tpls(std::ostream &)
Definition: ERF_ConsoleIO.cpp:137

◆ print_usage()

void ERF::print_usage ( MPI_Comm  comm,
std::ostream &  out 
)
static
27 {
28 #ifdef AMREX_USE_MPI
29  int irank = 0;
30  int num_ranks = 1;
31  MPI_Comm_size(comm, &num_ranks);
32  MPI_Comm_rank(comm, &irank);
33 
34  // Only root process does the printing
35  if (irank != 0) return;
36 #else
37  amrex::ignore_unused(comm);
38 #endif
39 
40  out << R"doc(Usage:
41  ERF3d.*.ex <input_file> [param=value] [param=value] ...

Referenced by main().

Here is the caller graph for this function:

◆ project_momenta()

void ERF::project_momenta ( int  lev,
amrex::Real  dt,
amrex::Vector< amrex::MultiFab > &  vars 
)

Project the single-level momenta to enforce the anelastic constraint Note that the level may or may not be level 0.

42 {
43  BL_PROFILE("ERF::project_momenta()");
44 
45  // Make sure the solver only sees the levels over which we are solving
46  Vector<BoxArray> ba_tmp; ba_tmp.push_back(mom_mf[Vars::cons].boxArray());
47  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap());
48  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
49 
50  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp, 1);
51 
52  Vector<MultiFab> rhs;
53  Vector<MultiFab> phi;
54 
55  if (solverChoice.terrain_type == TerrainType::EB)
56  {
57  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
58  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1, MFInfo(), EBFactory(lev));
59  } else {
60  rhs.resize(1); rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
61  phi.resize(1); phi[0].define(ba_tmp[0], dm_tmp[0], 1, 1);
62  }
63 
64  MultiFab rhs_lev(rhs[0], make_alias, 0, 1);
65 
66  auto dxInv = geom[lev].InvCellSizeArray();
67 
68  // Inflow on an x-face -- note only the normal velocity is used in the projection
69  if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow") {
71  IntVect{1,0,0},t_new[lev],BCVars::xvel_bc,false);
72  }
73 
74  // Inflow on a y-face -- note only the normal velocity is used in the projection
75  if (domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
77  IntVect{0,1,0},t_new[lev],BCVars::yvel_bc,false);
78  }
79 
80  if (domain_bc_type[0] == "Inflow" || domain_bc_type[3] == "Inflow" ||
81  domain_bc_type[1] == "Inflow" || domain_bc_type[4] == "Inflow") {
82  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
83  vars_new[lev][Vars::yvel], IntVect{0},
84  vars_new[lev][Vars::zvel], IntVect{0},
85  vars_new[lev][Vars::cons],
86  mom_mf[IntVars::xmom],
87  mom_mf[IntVars::ymom],
88  mom_mf[IntVars::zmom],
89  Geom(lev).Domain(),
91  }
92 
93  // If !fixed_density, we must convert (rho u) which came in
94  // to (rho0 u) which is what we will project
96  ConvertForProjection(mom_mf[Vars::cons], r_hse,
97  mom_mf[IntVars::xmom],
98  mom_mf[IntVars::ymom],
99  mom_mf[IntVars::zmom],
100  Geom(lev).Domain(),
102  }
103 
104  //
105  // ****************************************************************************
106  // Now convert the rho0w MultiFab to hold Omega rather than rhow
107  // ****************************************************************************
108  //
109  if (solverChoice.mesh_type == MeshType::VariableDz)
110  {
111  for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
112  {
113  const Array4<Real const>& rho0u_arr = mom_mf[IntVars::xmom].const_array(mfi);
114  const Array4<Real const>& rho0v_arr = mom_mf[IntVars::ymom].const_array(mfi);
115  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
116 
117  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
118  const Array4<Real const>& mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
119  const Array4<Real const>& mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
120 
121  //
122  // Define Omega from (rho0 W) but store it in the same array
123  //
124  Box tbz = mfi.nodaltilebox(2);
125  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
126  if (k == 0) {
127  rho0w_arr(i,j,k) = Real(0.0);
128  } else {
129  Real rho0w = rho0w_arr(i,j,k);
130  rho0w_arr(i,j,k) = OmegaFromW(i,j,k,rho0w,
131  rho0u_arr,rho0v_arr,
132  mf_u,mf_v,z_nd,dxInv);
133  }
134  });
135  } // mfi
136  }
137 
138  // ****************************************************************************
139  // Compute divergence which will form RHS
140  // Note that we replace "rho0w" with the contravariant momentum, Omega
141  // ****************************************************************************
142  Array<MultiFab const*, AMREX_SPACEDIM> rho0_u_const;
143  rho0_u_const[0] = &mom_mf[IntVars::xmom];
144  rho0_u_const[1] = &mom_mf[IntVars::ymom];
145  rho0_u_const[2] = &mom_mf[IntVars::zmom];
146 
147  compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
148 
149  if (solverChoice.mesh_type == MeshType::VariableDz) {
150  MultiFab::Multiply(rhs_lev, *detJ_cc[lev], 0, 0, 1, 0);
151  }
152 
153  // Max norm over the entire MultiFab
154  Real rhsnorm = rhs_lev.norm0();
155 
156  if (mg_verbose > 0) {
157  Print() << "Max/L2 norm of divergence before solve at level " << lev << " : " << rhsnorm << " " <<
158  rhs_lev.norm2() << " and sum " << rhs_lev.sum() << std::endl;
159  }
160 
161 
162  if (lev > 0)
163  {
164  Vector<Real> sum; sum.resize(subdomains[lev].size(),Real(0.));
165 
166  for (MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
167  {
168  Box bx = mfi.validbox();
169  for (int i = 0; i < subdomains[lev].size(); ++i) {
170  if (subdomains[lev][i].intersects(bx)) {
171  sum[i] += rhs_lev[mfi.index()].template sum<RunOn::Device>(0);
172  }
173  }
174  }
175  ParallelDescriptor::ReduceRealSum(sum.data(), sum.size());
176 
177  for (int i = 0; i < subdomains[lev].size(); ++i) {
178  sum[i] /= static_cast<Real>(subdomains[lev][i].numPts());
179  }
180 
181  for ( MFIter mfi(rhs_lev); mfi.isValid(); ++mfi)
182  {
183  Box bx = mfi.validbox();
184  for (int i = 0; i < subdomains[lev].size(); ++i) {
185  if (subdomains[lev][i].intersects(bx)) {
186  rhs_lev[mfi.index()].template minus<RunOn::Device>(sum[i]);
187  if (mg_verbose > 1) {
188  amrex::Print() << " Subtracting " << sum[i] << " in " << rhs_lev[mfi.index()].box() << std::endl;
189  }
190  }
191  }
192  }
193  }
194 
195  // ****************************************************************************
196  //
197  // No need to build the solver if RHS == 0
198  //
199  if (rhsnorm <= solverChoice.poisson_abstol) return;
200  // ****************************************************************************
201 
202  // ****************************************************************************
203  // Initialize phi to 0
204  // (It is essential that we do this in order to fill the corners; these are never
205  // used but the Saxpy requires the values to be initialized.)
206  // ****************************************************************************
207  phi[0].setVal(0.0);
208 
209  Real start_step = static_cast<Real>(ParallelDescriptor::second());
210 
211  // ****************************************************************************
212  // Allocate fluxes
213  // ****************************************************************************
214  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
215  fluxes.resize(1);
216  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
217  if (solverChoice.terrain_type == TerrainType::EB) {
218  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
219  } else {
220  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
221  }
222  }
223 
224  // ****************************************************************************
225  // Choose the solver and solve
226  // ****************************************************************************
227 
228  std::map<int,int> index_map;
229 
230  BoxArray ba(grids[lev]);
231 
232  Vector<MultiFab> rhs_sub; rhs_sub.resize(1);
233  Vector<MultiFab> phi_sub; phi_sub.resize(1);
234  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes_sub; fluxes_sub.resize(1);
235 
236  MultiFab ax_sub, ay_sub, znd_sub;
237 
238  for (int i = 0; i < subdomains[lev].size(); ++i)
239  {
240  if (mg_verbose > 0) {
241  amrex::Print() << " Solving in subdomain " << i << " of " << subdomains[lev].size() << " bins at level " << lev << std::endl;
242  }
243 
244  BoxList bl_sub;
245  Vector<int> dm_sub;
246 
247  for (int j = 0; j < ba.size(); j++)
248  {
249  if (subdomains[lev][i].intersects(ba[j]))
250  {
251  // amrex::Print() <<" INTERSECTS I " << i << " " << j << " " << grids[lev][j] << std::endl;
252  //
253  // Note that bl_sub.size() is effectively a counter which is
254  // incremented above
255  //
256  // if (ParallelDescriptor::MyProc() == j) {
257  // }
258  index_map[bl_sub.size()] = j;
259 
260  // amrex::Print() <<" PUSHING BACK " << j << " " << index_map[bl_sub.size()] << std::endl;
261  bl_sub.push_back(grids[lev][j]);
262  dm_sub.push_back(dmap[lev][j]);
263  } // intersects
264 
265  } // loop over ba (j)
266 
267  BoxArray ba_sub(bl_sub);
268 
269  // Define MultiFabs that hold only the data in this particular subdomain
270  rhs_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, rhs[0].nGrowVect(), MFInfo{}.SetAlloc(false));
271  phi_sub[0].define(ba_sub, DistributionMapping(dm_sub), 1, phi[0].nGrowVect(), MFInfo{}.SetAlloc(false));
272 
273  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
274  fluxes_sub[0][idim].define(convert(ba_sub, IntVect::TheDimensionVector(idim)), DistributionMapping(dm_sub), 1,
275  IntVect::TheZeroVector(), MFInfo{}.SetAlloc(false));
276  }
277 
278  // Link the new MultiFabs to the FABs in the original MultiFabs (no copy required)
279  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
280  int orig_index = index_map[mfi.index()];
281  // amrex::Print() << " INDEX " << orig_index << " TO " << mfi.index() << std::endl;
282  rhs_sub[0].setFab(mfi, FArrayBox(rhs[0][orig_index], amrex::make_alias, 0, 1));
283  phi_sub[0].setFab(mfi, FArrayBox(phi[0][orig_index], amrex::make_alias, 0, 1));
284  fluxes_sub[0][0].setFab(mfi,FArrayBox(fluxes[0][0][orig_index], amrex::make_alias, 0, 1));
285  fluxes_sub[0][1].setFab(mfi,FArrayBox(fluxes[0][1][orig_index], amrex::make_alias, 0, 1));
286  fluxes_sub[0][2].setFab(mfi,FArrayBox(fluxes[0][2][orig_index], amrex::make_alias, 0, 1));
287  }
288 
289  if (solverChoice.mesh_type == MeshType::VariableDz) {
290  ax_sub.define(convert(ba_sub,IntVect(1,0,0)), DistributionMapping(dm_sub), 1,
291  ax[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
292  ay_sub.define(convert(ba_sub,IntVect(0,1,0)), DistributionMapping(dm_sub), 1,
293  ay[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
294  znd_sub.define(convert(ba_sub,IntVect(1,1,1)), DistributionMapping(dm_sub), 1,
295  z_phys_nd[lev]->nGrowVect(), MFInfo{}.SetAlloc(false));
296 
297  for (MFIter mfi(rhs_sub[0]); mfi.isValid(); ++mfi) {
298  int orig_index = index_map[mfi.index()];
299  ax_sub.setFab(mfi, FArrayBox((*ax[lev])[orig_index], amrex::make_alias, 0, 1));
300  ay_sub.setFab(mfi, FArrayBox((*ay[lev])[orig_index], amrex::make_alias, 0, 1));
301  znd_sub.setFab(mfi, FArrayBox((*z_phys_nd[lev])[orig_index], amrex::make_alias, 0, 1));
302  }
303  }
304 
305  if (lev > 0) {
306  amrex::Print() << "RHSSUB BA " << rhs_sub[0].boxArray() << std::endl;
307  }
308 
309  // ****************************************************************************
310  // EB
311  // ****************************************************************************
312  if (solverChoice.terrain_type == TerrainType::EB) {
313  solve_with_EB_mlmg(lev, rhs_sub, phi_sub, fluxes_sub);
314  } else {
315 
316  // ****************************************************************************
317  // No terrain or grid stretching
318  // ****************************************************************************
319  if (solverChoice.mesh_type == MeshType::ConstantDz) {
320 #ifdef ERF_USE_FFT
321  if (use_fft) {
322  Box my_region(subdomains[lev][i].minimalBox());
323  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][i].numPts());
324  if (boxes_make_rectangle) {
325  solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
326  } else {
327  amrex::Warning("FFT won't work unless the union of boxes is rectangular: defaulting to MLMG");
328  solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub);
329  }
330  } else {
331  solve_with_mlmg(lev, rhs, phi, fluxes);
332  }
333 #else
334  if (use_fft) {
335  amrex::Warning("You set use_fft=true but didn't build with USE_FFT = TRUE; defaulting to MLMG");
336  }
337  solve_with_mlmg(lev, rhs_sub, phi_sub, fluxes_sub);
338 #endif
339  } // No terrain or grid stretching
340 
341  // ****************************************************************************
342  // Grid stretching (flat terrain)
343  // ****************************************************************************
344  else if (solverChoice.mesh_type == MeshType::StretchedDz) {
345 #ifndef ERF_USE_FFT
346  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT solver");
347 #else
348  Box my_region(subdomains[lev][i].minimalBox());
349  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][i].numPts());
350  if (!boxes_make_rectangle) {
351  amrex::Abort("FFT won't work unless the union of boxes is rectangular");
352  } else {
353  if (!use_fft) {
354  amrex::Warning("Using FFT even though you didn't set use_fft to true; it's the best choice");
355  }
356  solve_with_fft(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0]);
357  }
358 #endif
359  } // grid stretching
360 
361  // ****************************************************************************
362  // General terrain
363  // ****************************************************************************
364  else if (solverChoice.mesh_type == MeshType::VariableDz) {
365 #ifdef ERF_USE_FFT
366  Box my_region(subdomains[lev][i].minimalBox());
367  bool boxes_make_rectangle = (my_region.numPts() == subdomains[lev][i].numPts());
368  if (!boxes_make_rectangle) {
369  amrex::Abort("FFT preconditioner for GMRES won't work unless the union of boxes is rectangular");
370  } else {
371  solve_with_gmres(lev, my_region, rhs_sub[0], phi_sub[0], fluxes_sub[0], ax_sub, ay_sub, znd_sub);
372  }
373 #else
374  amrex::Abort("Rebuild with USE_FFT = TRUE so you can use the FFT preconditioner for GMRES");
375 #endif
376  } // general terrain
377 
378  } // not EB
379 
380  } // loop over subdomains (i)
381 
382  // ****************************************************************************
383  // Print time in solve
384  // ****************************************************************************
385  Real end_step = static_cast<Real>(ParallelDescriptor::second());
386  if (mg_verbose > 0) {
387  amrex::Print() << "Time in solve " << end_step - start_step << std::endl;
388  }
389 
390  // ****************************************************************************
391  // Subtract dt grad(phi) from the momenta (rho0u, rho0v, Omega)
392  // ****************************************************************************
393  MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0);
394  MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0);
395  MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0);
396 
397  // ****************************************************************************
398  // Define gradp from fluxes -- note that fluxes is dt * change in Gp
399  // ****************************************************************************
400  MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
401  MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
402  MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
403 
404  gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
405  gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
406  gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
407 
408  //
409  // This call is only to verify the divergence after the solve
410  // It is important we do this before computing the rho0w_arr from Omega back to rho0w
411  //
412  // ****************************************************************************
413  // THIS IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE
414  // ****************************************************************************
415  //
416  if (mg_verbose > 0)
417  {
418  compute_divergence(lev, rhs_lev, rho0_u_const, geom_tmp[0]);
419 
420  if (solverChoice.mesh_type == MeshType::VariableDz) {
421  MultiFab::Multiply(rhs_lev, *detJ_cc[lev], 0, 0, 1, 0);
422  }
423 
424  Print() << "Max/L2 norm of divergence after solve at level " << lev << " : " << rhs_lev.norm0() << " " <<
425  rhs_lev.norm2() << " and sum " << rhs_lev.sum() << std::endl;
426 
427 #if 0
428  // FOR DEBUGGING ONLY
429  for ( MFIter mfi(rhs_lev,TilingIfNotGPU()); mfi.isValid(); ++mfi)
430  {
431  const Array4<Real const>& rhs_arr = rhs_lev.const_array(mfi);
432  Box bx = mfi.validbox();
433  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
434  if (std::abs(rhs_arr(i,j,k)) > 1.e-10) {
435  amrex::AllPrint() << "RHS AFTER SOLVE AT " <<
436  IntVect(i,j,k) << " " << rhs_arr(i,j,k) << std::endl;
437  }
438  });
439  } // mfi
440 #endif
441 
442  } // mg_verbose
443 
444  //
445  // ****************************************************************************
446  // Now convert the rho0w MultiFab back to holding (rho0w) rather than Omega
447  // ****************************************************************************
448  //
449  if (solverChoice.mesh_type == MeshType::VariableDz)
450  {
451  for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi)
452  {
453  Box tbz = mfi.nodaltilebox(2);
454  const Array4<Real >& rho0u_arr = mom_mf[IntVars::xmom].array(mfi);
455  const Array4<Real >& rho0v_arr = mom_mf[IntVars::ymom].array(mfi);
456  const Array4<Real >& rho0w_arr = mom_mf[IntVars::zmom].array(mfi);
457  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
458  const Array4<Real const>& mf_u = mapfac[lev][MapFacType::u_x]->const_array(mfi);
459  const Array4<Real const>& mf_v = mapfac[lev][MapFacType::v_y]->const_array(mfi);
460  ParallelFor(tbz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept {
461  Real omega = rho0w_arr(i,j,k);
462  rho0w_arr(i,j,k) = WFromOmega(i,j,k,omega,
463  rho0u_arr,rho0v_arr,
464  mf_u,mf_v,z_nd,dxInv);
465  });
466  } // mfi
467  }
468 
469  // If !fixed_density, we must convert (rho0 u) back
470  // to (rho0 u) which is what we will pass back out
472  ConvertForProjection(r_hse, mom_mf[Vars::cons],
473  mom_mf[IntVars::xmom],
474  mom_mf[IntVars::ymom],
475  mom_mf[IntVars::zmom],
476  Geom(lev).Domain(),
478  }
479 
480  // ****************************************************************************
481  // Update pressure variable with phi -- note that phi is dt * change in pressure
482  // ****************************************************************************
483  MultiFab::Saxpy(pp_inc[lev], 1.0/l_dt, phi[0],0,0,1,1);
484 }
void ConvertForProjection(const MultiFab &den_div, const MultiFab &den_mlt, MultiFab &xmom, MultiFab &ymom, MultiFab &zmom, const Box &domain, const Vector< BCRec > &domain_bcs_type_h)
Definition: ERF_ConvertForProjection.cpp:25
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real OmegaFromW(int &i, int &j, int &k, amrex::Real w, const amrex::Array4< const amrex::Real > &u_arr, const amrex::Array4< const amrex::Real > &v_arr, const amrex::Array4< const amrex::Real > &mf_u, const amrex::Array4< const amrex::Real > &mf_v, const amrex::Array4< const amrex::Real > &z_nd, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &dxInv)
Definition: ERF_TerrainMetrics.H:415
AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::Real WFromOmega(int &i, int &j, int &k, amrex::Real omega, const amrex::Array4< const amrex::Real > &u_arr, const amrex::Array4< const amrex::Real > &v_arr, const amrex::Array4< const amrex::Real > &mf_u, const amrex::Array4< const amrex::Real > &mf_v, const amrex::Array4< const amrex::Real > &z_nd, const amrex::GpuArray< amrex::Real, AMREX_SPACEDIM > &dxInv)
Definition: ERF_TerrainMetrics.H:465
static bool use_fft
Definition: ERF.H:1137
void compute_divergence(int lev, amrex::MultiFab &rhs, amrex::Array< amrex::MultiFab const *, AMREX_SPACEDIM > rho0_u_const, amrex::Geometry const &geom_at_lev)
Definition: ERF_ComputeDivergence.cpp:10
void solve_with_mlmg(int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
Definition: ERF_SolveWithMLMG.cpp:40
void solve_with_gmres(int lev, const amrex::Box &subdomain, amrex::MultiFab &rhs, amrex::MultiFab &p, amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &fluxes, amrex::MultiFab &ax_sub, amrex::MultiFab &ay_sub, amrex::MultiFab &znd_sub)
Definition: ERF_SolveWithGMRES.cpp:12
void solve_with_EB_mlmg(int lev, amrex::Vector< amrex::MultiFab > &rhs, amrex::Vector< amrex::MultiFab > &p, amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &fluxes)
Definition: ERF_SolveWithEBMLMG.cpp:19
@ omega
Definition: ERF_Morrison.H:53
Here is the call graph for this function:

◆ project_velocity()

void ERF::project_velocity ( int  lev,
amrex::Real  dt 
)

Project the single-level velocity field to enforce the anelastic constraint Note that the level may or may not be level 0.

11 {
12  BL_PROFILE("ERF::project_velocity()");
13  VelocityToMomentum(vars_new[lev][Vars::xvel], IntVect{0},
14  vars_new[lev][Vars::yvel], IntVect{0},
15  vars_new[lev][Vars::zvel], IntVect{0},
16  vars_new[lev][Vars::cons],
17  rU_new[lev], rV_new[lev], rW_new[lev],
18  Geom(lev).Domain(), domain_bcs_type);
19 
20  Vector<MultiFab> tmp_mom;
21 
22  tmp_mom.push_back(MultiFab(vars_new[lev][Vars::cons],make_alias,0,1));
23  tmp_mom.push_back(MultiFab(rU_new[lev],make_alias,0,1));
24  tmp_mom.push_back(MultiFab(rV_new[lev],make_alias,0,1));
25  tmp_mom.push_back(MultiFab(rW_new[lev],make_alias,0,1));
26 
27  project_momenta(lev, l_dt, tmp_mom);
28 
30  vars_new[lev][Vars::yvel],
31  vars_new[lev][Vars::zvel],
32  vars_new[lev][Vars::cons],
33  rU_new[lev], rV_new[lev], rW_new[lev],
34  Geom(lev).Domain(), domain_bcs_type);
35  }
void project_momenta(int lev, amrex::Real dt, amrex::Vector< amrex::MultiFab > &vars)
Definition: ERF_PoissonSolve.cpp:41
Here is the call graph for this function:

◆ project_velocity_tb()

void ERF::project_velocity_tb ( int  lev,
amrex::Real  dt,
amrex::Vector< amrex::MultiFab > &  vars 
)

Project the single-level velocity field to enforce incompressibility with a thin body

21 {
22  BL_PROFILE("ERF::project_velocity_tb()");
23  AMREX_ALWAYS_ASSERT(solverChoice.mesh_type == MeshType::ConstantDz);
24 
25  // Make sure the solver only sees the levels over which we are solving
26  Vector<BoxArray> ba_tmp; ba_tmp.push_back(vmf[Vars::cons].boxArray());
27  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(vmf[Vars::cons].DistributionMap());
28  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
29 
30  // Use the default settings
31  LPInfo info;
32  std::unique_ptr<MLPoisson> p_mlpoisson;
33 #if 0
34  if (overset_imask[0]) {
35  // Add overset mask around thin body
36  p_mlpoisson = std::make_unique<MLPoisson>(geom, grids, dmap, GetVecOfConstPtrs(overset_imask), info);
37  }
38  else
39 #endif
40  {
41  // Use the default settings
42  p_mlpoisson = std::make_unique<MLPoisson>(geom_tmp, ba_tmp, dm_tmp, info);
43  }
44 
45  auto bclo = get_projection_bc(Orientation::low);
46  auto bchi = get_projection_bc(Orientation::high);
47  bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true;
48  p_mlpoisson->setDomainBC(bclo, bchi);
49 
50  if (lev > 0) {
51  p_mlpoisson->setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
52  }
53 
54  p_mlpoisson->setLevelBC(0, nullptr);
55 
56  Vector<MultiFab> rhs;
57  Vector<MultiFab> phi;
58  Vector<Array<MultiFab,AMREX_SPACEDIM> > fluxes;
59  Vector<Array<MultiFab,AMREX_SPACEDIM> > deltaf; // f^* - f^{n-1}
60  Vector<Array<MultiFab,AMREX_SPACEDIM> > u_plus_dtdf; // u + dt*deltaf
61 
62  // Used to pass array of const MFs to ComputeDivergence
63  Array<MultiFab const*, AMREX_SPACEDIM> u;
64 
65  rhs.resize(1);
66  phi.resize(1);
67  fluxes.resize(1);
68  deltaf.resize(1);
69  u_plus_dtdf.resize(1);
70 
71  rhs[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
72  phi[0].define(ba_tmp[0], dm_tmp[0], 1, 0);
73  rhs[0].setVal(0.0);
74  phi[0].setVal(0.0);
75 
76  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
77  fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
78  u_plus_dtdf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
79 
80  deltaf[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0);
81  deltaf[0][idim].setVal(0.0); // start with f^* == f^{n-1}
82  }
83 
84 #if 0
85  // DEBUG
86  u[0] = &(vmf[Vars::xvel]);
87  u[1] = &(vmf[Vars::yvel]);
88  u[2] = &(vmf[Vars::zvel]);
89  computeDivergence(rhs[0], u, geom[0]);
90  Print() << "Max norm of divergence before solve at level 0 : " << rhs[0].norm0() << std::endl;
91 #endif
92 
93  for (int itp = 0; itp < solverChoice.ncorr; ++itp)
94  {
95  // Calculate u + dt*deltaf
96  for (int idim = 0; idim < 3; ++idim) {
97  MultiFab::Copy(u_plus_dtdf[0][idim], deltaf[0][idim], 0, 0, 1, 0);
98  u_plus_dtdf[0][0].mult(-l_dt,0,1,0);
99  }
100  MultiFab::Add(u_plus_dtdf[0][0], vmf[Vars::xvel], 0, 0, 1, 0);
101  MultiFab::Add(u_plus_dtdf[0][1], vmf[Vars::yvel], 0, 0, 1, 0);
102  MultiFab::Add(u_plus_dtdf[0][2], vmf[Vars::zvel], 0, 0, 1, 0);
103 
104  u[0] = &(u_plus_dtdf[0][0]);
105  u[1] = &(u_plus_dtdf[0][1]);
106  u[2] = &(u_plus_dtdf[0][2]);
107  computeDivergence(rhs[0], u, geom_tmp[0]);
108 
109 #if 0
110  // DEBUG
111  if (itp==0) {
112  for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
113  {
114  const Box& bx = mfi.tilebox();
115  const Array4<Real const>& divU = rhs[0].const_array(mfi);
116  const Array4<Real const>& uarr = vmf[Vars::xvel].const_array(mfi);
117  const Array4<Real const>& varr = vmf[Vars::yvel].const_array(mfi);
118  const Array4<Real const>& warr = vmf[Vars::zvel].const_array(mfi);
119  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
120  {
121  if ((i>=120) && (i<=139) && (j==0) && ((k>=127)&&(k<=128))) {
122  amrex::AllPrint() << "before project div"<<IntVect(i,j,k)<<" = "<< divU(i,j,k)
123  << " u: " << uarr(i,j,k) << " " << uarr(i+1,j,k)
124  << " v: " << varr(i,j,k) << " " << varr(i,j+1,k)
125  << " w: " << warr(i,j,k) << " " << warr(i,j,k+1)
126  << std::endl;
127  }
128  });
129  }
130  }
131 #endif
132 
133  // If all Neumann BCs, adjust RHS to make sure we can converge
134  if (need_adjust_rhs) {
135  Real offset = volWgtSumMF(lev, rhs[0], 0, false);
136  // amrex::Print() << "Poisson solvability offset = " << offset << std::endl;
137  rhs[0].plus(-offset, 0, 1);
138  }
139 
140  // Initialize phi to 0
141  phi[0].setVal(0.0);
142 
143  MLMG mlmg(*p_mlpoisson);
144  int max_iter = 100;
145  mlmg.setMaxIter(max_iter);
146 
147  mlmg.setVerbose(mg_verbose);
148  //mlmg.setBottomVerbose(mg_verbose);
149 
150  // solve for dt*p
151  mlmg.solve(GetVecOfPtrs(phi),
152  GetVecOfConstPtrs(rhs),
155 
156  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
157 
158  // Calculate new intermediate body force with updated gradp
159  if (thin_xforce[lev]) {
160  MultiFab::Copy( deltaf[0][0], fluxes[0][0], 0, 0, 1, 0);
161  ApplyInvertedMask(deltaf[0][0], *xflux_imask[0]);
162  }
163  if (thin_yforce[lev]) {
164  MultiFab::Copy( deltaf[0][1], fluxes[0][1], 0, 0, 1, 0);
165  ApplyInvertedMask(deltaf[0][1], *yflux_imask[0]);
166  }
167  if (thin_zforce[lev]) {
168  MultiFab::Copy( deltaf[0][2], fluxes[0][2], 0, 0, 1, 0);
169  ApplyInvertedMask(deltaf[0][2], *zflux_imask[0]);
170  }
171 
172  // DEBUG
173  // for (MFIter mfi(rhs[0], TilingIfNotGPU()); mfi.isValid(); ++mfi)
174  // {
175  // const Box& bx = mfi.tilebox();
176  // const Array4<Real const>& dfz_arr = deltaf[0][2].const_array(mfi);
177  // ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
178  // {
179  // if ((i>=120) && (i<=139) && (j==0) && (k==128)) {
180  // amrex::AllPrint()
181  // << " piter" << itp
182  // << " dfz"<<IntVect(i,j,k)<<" = "<< dfz_arr(i,j,k)
183  // << std::endl;
184  // }
185  // });
186  // }
187 
188  // Update pressure variable with phi -- note that phi is change in pressure, not the full pressure
189  MultiFab::Saxpy(pp_inc[lev], 1.0, phi[0],0,0,1,0);
190 
191  // Subtract grad(phi) from the velocity components
192  Real beta = 1.0;
193  MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
194  MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
195  MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
196  if (thin_xforce[lev]) {
197  ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
198  }
199  if (thin_yforce[lev]) {
200  ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
201  }
202  if (thin_zforce[lev]) {
203  ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
204  }
205  } // itp: pressure-force iterations
206 
207  // ****************************************************************************
208  // Define gradp from fluxes -- note that fluxes is dt * change in Gp
209  // ****************************************************************************
210  MultiFab::Saxpy(gradp[lev][GpVars::gpx],-1.0/l_dt,fluxes[0][0],0,0,1,0);
211  MultiFab::Saxpy(gradp[lev][GpVars::gpy],-1.0/l_dt,fluxes[0][1],0,0,1,0);
212  MultiFab::Saxpy(gradp[lev][GpVars::gpz],-1.0/l_dt,fluxes[0][2],0,0,1,0);
213 
214  gradp[lev][GpVars::gpx].FillBoundary(geom_tmp[0].periodicity());
215  gradp[lev][GpVars::gpy].FillBoundary(geom_tmp[0].periodicity());
216  gradp[lev][GpVars::gpz].FillBoundary(geom_tmp[0].periodicity());
217 
218  // Subtract grad(phi) from the velocity components
219 // Real beta = 1.0;
220 // for (int ilev = lev_min; ilev <= lev_max; ++ilev) {
221 // MultiFab::Saxpy(vmf[Vars::xvel], beta, fluxes[0][0], 0, 0, 1, 0);
222 // MultiFab::Saxpy(vmf[Vars::yvel], beta, fluxes[0][1], 0, 0, 1, 0);
223 // MultiFab::Saxpy(vmf[Vars::zvel], beta, fluxes[0][2], 0, 0, 1, 0);
224 // if (thin_xforce[lev]) {
225 // ApplyMask(vmf[Vars::xvel], *xflux_imask[0]);
226 // }
227 // if (thin_yforce[lev]) {
228 // ApplyMask(vmf[Vars::yvel], *yflux_imask[0]);
229 // }
230 // if (thin_zforce[lev]) {
231 // ApplyMask(vmf[Vars::zvel], *zflux_imask[0]);
232 // }
233 // }
234 
235 #if 0
236  // Confirm that the velocity is now divergence free
237  u[0] = &(vmf[Vars::xvel]);
238  u[1] = &(vmf[Vars::yvel]);
239  u[2] = &(vmf[Vars::zvel]);
240  computeDivergence(rhs[0], u, geom_tmp[0]);
241  Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl;
242 
243 #endif
244 }
AMREX_FORCE_INLINE IntVect offset(const int face_dir, const int normal)
Definition: ERF_ReadBndryPlanes.cpp:28
AMREX_GPU_HOST AMREX_FORCE_INLINE void ApplyInvertedMask(amrex::MultiFab &dst, const amrex::iMultiFab &imask, const int nghost=0)
Definition: ERF_Utils.H:421
amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > get_projection_bc(amrex::Orientation::Side side) const noexcept
Definition: ERF_SolveWithMLMG.cpp:17
bool projection_has_dirichlet(amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM > bcs) const
Definition: ERF_PoissonSolve_tb.cpp:8
amrex::Real volWgtSumMF(int lev, const amrex::MultiFab &mf, int comp, bool finemask)
Definition: ERF_WriteScalarProfiles.cpp:651
int ncorr
Definition: ERF_DataStruct.H:834
Here is the call graph for this function:

◆ projection_has_dirichlet()

bool ERF::projection_has_dirichlet ( amrex::Array< amrex::LinOpBCType, AMREX_SPACEDIM >  bcs) const
9 {
10  for (int dir = 0; dir < AMREX_SPACEDIM; ++dir) {
11  if (bcs[dir] == LinOpBCType::Dirichlet) return true;
12  }
13  return false;
14 }

◆ ReadCheckpointFile()

void ERF::ReadCheckpointFile ( )

ERF function for reading data from a checkpoint file during restart.

440 {
441  Print() << "Restart from native checkpoint " << restart_chkfile << "\n";
442 
443  // Header
444  std::string File(restart_chkfile + "/Header");
445 
446  VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
447 
448  Vector<char> fileCharPtr;
449  ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
450  std::string fileCharPtrString(fileCharPtr.dataPtr());
451  std::istringstream is(fileCharPtrString, std::istringstream::in);
452 
453  std::string line, word;
454 
455  int chk_ncomp_cons, chk_ncomp;
456 
457  // read in title line
458  std::getline(is, line);
459 
460  // read in finest_level
461  is >> finest_level;
462  GotoNextLine(is);
463 
464  // read the number of components
465  // for each variable we store
466 
467  // conservative, cell-centered vars
468  is >> chk_ncomp_cons;
469  GotoNextLine(is);
470 
471  // x-velocity on faces
472  is >> chk_ncomp;
473  GotoNextLine(is);
474  AMREX_ASSERT(chk_ncomp == 1);
475 
476  // y-velocity on faces
477  is >> chk_ncomp;
478  GotoNextLine(is);
479  AMREX_ASSERT(chk_ncomp == 1);
480 
481  // z-velocity on faces
482  is >> chk_ncomp;
483  GotoNextLine(is);
484  AMREX_ASSERT(chk_ncomp == 1);
485 
486  // read in array of istep
487  std::getline(is, line);
488  {
489  std::istringstream lis(line);
490  int i = 0;
491  while (lis >> word) {
492  istep[i++] = std::stoi(word);
493  }
494  }
495 
496  // read in array of dt
497  std::getline(is, line);
498  {
499  std::istringstream lis(line);
500  int i = 0;
501  while (lis >> word) {
502  dt[i++] = std::stod(word);
503  }
504  }
505 
506  // read in array of t_new
507  std::getline(is, line);
508  {
509  std::istringstream lis(line);
510  int i = 0;
511  while (lis >> word) {
512  t_new[i++] = std::stod(word);
513  }
514  }
515 
516  for (int lev = 0; lev <= finest_level; ++lev) {
517  // read in level 'lev' BoxArray from Header
518  BoxArray ba;
519  ba.readFrom(is);
520  GotoNextLine(is);
521 
522  // create a distribution mapping
523  DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
524 
525  MakeNewLevelFromScratch (lev, t_new[lev], ba, dm);
526  }
527 
528  // ncomp is only valid after we MakeNewLevelFromScratch (asks micro how many vars)
529  // NOTE: Data is written over ncomp, so check that we match the header file
530  int ncomp_cons = vars_new[0][Vars::cons].nComp();
531 
532  // NOTE: QKE was removed so this is for backward compatibility
533  AMREX_ASSERT((chk_ncomp_cons==ncomp_cons) || ((chk_ncomp_cons-1)==ncomp_cons));
534  //
535  // See if we have a written separate file that tells how many components and how many ghost cells
536  // we have of the base state
537  //
538  // If we can't find the file, then set the number of components to the original number = 3
539  //
540  int ncomp_base_to_read = 3;
541  IntVect ng_base = IntVect{1};
542  {
543  std::string BaseStateFile(restart_chkfile + "/num_base_state_comps");
544 
545  if (amrex::FileExists(BaseStateFile))
546  {
547  Vector<char> BaseStatefileCharPtr;
548  ParallelDescriptor::ReadAndBcastFile(BaseStateFile, BaseStatefileCharPtr);
549  std::string BaseStatefileCharPtrString(BaseStatefileCharPtr.dataPtr());
550 
551  // We set this to the default value of 3 but allow it be larger if th0 and qv0 were written
552  std::istringstream isb(BaseStatefileCharPtrString, std::istringstream::in);
553  isb >> ncomp_base_to_read;
554  isb >> ng_base;
555  }
556  }
557 
558  // read in the MultiFab data
559  for (int lev = 0; lev <= finest_level; ++lev)
560  {
561  // NOTE: For backward compatibility (chk file has QKE)
562  if ((chk_ncomp_cons-1)==ncomp_cons) {
563  MultiFab cons(grids[lev],dmap[lev],chk_ncomp_cons,0);
564  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
565 
566  // Copy up to RhoKE_comp
567  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,(RhoKE_comp+1),0);
568 
569  // Only if we have a PBL model do we need to copy QKE is src to KE in dst
570  if ( (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNN25) ||
571  (solverChoice.turbChoice[lev].pbl_type == PBLType::MYNNEDMF) ) {
572  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+1),RhoKE_comp,1,0);
573  vars_new[lev][Vars::cons].mult(0.5,RhoKE_comp,1,0);
574  }
575 
576  // Copy other components
577  int ncomp_remainder = ncomp_cons - (RhoKE_comp + 1);
578  MultiFab::Copy(vars_new[lev][Vars::cons],cons,(RhoKE_comp+2),(RhoKE_comp+1),ncomp_remainder,0);
579 
580  vars_new[lev][Vars::cons].setBndry(1.0e34);
581  } else {
582  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
583  VisMF::Read(cons, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
584  MultiFab::Copy(vars_new[lev][Vars::cons],cons,0,0,ncomp_cons,0);
585  vars_new[lev][Vars::cons].setBndry(1.0e34);
586  }
587 
588  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
589  VisMF::Read(xvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
590  MultiFab::Copy(vars_new[lev][Vars::xvel],xvel,0,0,1,0);
591  vars_new[lev][Vars::xvel].setBndry(1.0e34);
592 
593  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
594  VisMF::Read(yvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
595  MultiFab::Copy(vars_new[lev][Vars::yvel],yvel,0,0,1,0);
596  vars_new[lev][Vars::yvel].setBndry(1.0e34);
597 
598  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
599  VisMF::Read(zvel, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
600  MultiFab::Copy(vars_new[lev][Vars::zvel],zvel,0,0,1,0);
601  vars_new[lev][Vars::zvel].setBndry(1.0e34);
602 
603  if (solverChoice.anelastic[lev] == 1) {
604  MultiFab ppinc(grids[lev],dmap[lev],1,0);
605  VisMF::Read(ppinc, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PP_Inc"));
606  MultiFab::Copy(pp_inc[lev],ppinc,0,0,1,0);
607  pp_inc[lev].FillBoundary(geom[lev].periodicity());
608 
609  MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
610  VisMF::Read(gpx, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpx"));
611  MultiFab::Copy(gradp[lev][GpVars::gpx],gpx,0,0,1,0);
612  gradp[lev][GpVars::gpx].FillBoundary(geom[lev].periodicity());
613 
614  MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
615  VisMF::Read(gpy, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpy"));
616  MultiFab::Copy(gradp[lev][GpVars::gpy],gpy,0,0,1,0);
617  gradp[lev][GpVars::gpy].FillBoundary(geom[lev].periodicity());
618 
619  MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
620  VisMF::Read(gpz, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Gpz"));
621  MultiFab::Copy(gradp[lev][GpVars::gpz],gpz,0,0,1,0);
622  gradp[lev][GpVars::gpz].FillBoundary(geom[lev].periodicity());
623  }
624 
625  // Note that we read the ghost cells of the base state (unlike above)
626 
627  // The original base state only had 3 components and 1 ghost cell -- we read this
628  // here to be consistent with the old style
629  MultiFab base(grids[lev],dmap[lev],ncomp_base_to_read,ng_base);
630  VisMF::Read(base, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "BaseState"));
631 
632  MultiFab::Copy(base_state[lev],base,0,0,ncomp_base_to_read,ng_base);
633 
634  // Create theta0 from p0, rh0
635  if (ncomp_base_to_read < 4) {
636  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
637  {
638  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
639  const Box& bx = mfi.tilebox();
640  Array4<Real> const& fab = base_state[lev].array(mfi);
641  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
642  {
644  / fab(i,j,k,BaseState::r0_comp);
645  });
646  }
647  }
648  // Default theta0 to 0
649  if (ncomp_base_to_read < 5) {
650  for (MFIter mfi(base_state[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
651  {
652  // We only compute theta_0 on valid cells since we will impose domain BC's after restart
653  const Box& bx = mfi.tilebox();
654  Array4<Real> const& fab = base_state[lev].array(mfi);
655  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
656  {
657  fab(i,j,k,BaseState::qv0_comp) = 0.0;
658  });
659  }
660  }
661  base_state[lev].FillBoundary(geom[lev].periodicity());
662 
663  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
664  // Note that we also read the ghost cells of z_phys_nd
665  IntVect ng = z_phys_nd[lev]->nGrowVect();
666  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
667  VisMF::Read(z_height, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z_Phys_nd"));
668  MultiFab::Copy(*z_phys_nd[lev],z_height,0,0,1,ng);
670  }
671 
672  // Read in the moisture model restart variables
673  std::vector<int> qmoist_indices;
674  std::vector<std::string> qmoist_names;
675  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
676  int qmoist_nvar = qmoist_indices.size();
677  for (int var = 0; var < qmoist_nvar; var++) {
678  const int ncomp = 1;
679  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
680  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
681  VisMF::Read(moist_vars, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", qmoist_names[var]));
682  MultiFab::Copy(*(qmoist[lev][qmoist_indices[var]]),moist_vars,0,0,ncomp,ng_moist);
683  }
684 
685 #if defined(ERF_USE_WINDFARM)
686  if(solverChoice.windfarm_type == WindFarmType::Fitch or
687  solverChoice.windfarm_type == WindFarmType::EWP or
688  solverChoice.windfarm_type == WindFarmType::SimpleAD){
689  IntVect ng = Nturb[lev].nGrowVect();
690  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng);
691  VisMF::Read(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "NumTurb"));
692  MultiFab::Copy(Nturb[lev],mf_Nturb,0,0,1,ng);
693  }
694 #endif
695 
696  if (solverChoice.lsm_type != LandSurfaceType::None) {
697  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
698  BoxArray ba = lsm_data[lev][mvar]->boxArray();
699  DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
700  IntVect ng = lsm_data[lev][mvar]->nGrowVect();
701  int nvar = lsm_data[lev][mvar]->nComp();
702  MultiFab lsm_vars(ba,dm,nvar,ng);
703  VisMF::Read(lsm_vars, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LsmVars"));
704  MultiFab::Copy(*(lsm_data[lev][mvar]),lsm_vars,0,0,nvar,ng);
705  }
706  }
707 
708 
709  IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
710  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
711 
712  std::string MapFacMFileName(restart_chkfile + "/Level_0/MapFactor_mx_H");
713  if (amrex::FileExists(MapFacMFileName)) {
714  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_mx"));
715  } else {
716  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_m"));
717  }
718  MultiFab::Copy(*mapfac[lev][MapFacType::m_x],mf_m,0,0,1,ng);
719 
720 #if 0
722  VisMF::Read(mf_m, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_my"));
723  MultiFab::Copy(*mapfac[lev][MapFacType::m_y],mf_m,0,0,1,ng);
724  }
725 #endif
726 
727  ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
728  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
729 
730  std::string MapFacUFileName(restart_chkfile + "/Level_0/MapFactor_ux_H");
731  if (amrex::FileExists(MapFacUFileName)) {
732  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_ux"));
733  } else {
734  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_u"));
735  }
736  MultiFab::Copy(*mapfac[lev][MapFacType::u_x],mf_u,0,0,1,ng);
737 
738 #if 0
740  VisMF::Read(mf_u, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_uy"));
741  MultiFab::Copy(*mapfac[lev][MapFacType::u_y],mf_u,0,0,1,ng);
742  }
743 #endif
744 
745  ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
746  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
747 
748  std::string MapFacVFileName(restart_chkfile + "/Level_0/MapFactor_vx_H");
749  if (amrex::FileExists(MapFacVFileName)) {
750  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vx"));
751  } else {
752  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_v"));
753  }
754  MultiFab::Copy(*mapfac[lev][MapFacType::v_x],mf_v,0,0,1,ng);
755 
756 #if 0
758  VisMF::Read(mf_v, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MapFactor_vy"));
759  MultiFab::Copy(*mapfac[lev][MapFacType::v_y],mf_v,0,0,1,ng);
760  }
761 #endif
762 
763 
764  // NOTE: We read MOST data in ReadCheckpointFileMOST (see below)!
765 
766  // See if we wrote out SST data
767  std::string FirstSSTFileName(restart_chkfile + "/Level_0/SST_0_H");
768  if (amrex::FileExists(FirstSSTFileName))
769  {
770  amrex::Print() << "Reading SST data" << std::endl;
771  int ntimes = 1;
772  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
773  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
774  sst_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
775  for (int nt(0); nt<ntimes; ++nt) {
776  VisMF::Read(sst_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
777  "SST_" + std::to_string(nt)));
778  MultiFab::Copy(*sst_lev[lev][nt],sst_at_t,0,0,1,ng);
779  }
780  }
781 
782  // See if we wrote out TSK data
783  std::string FirstTSKFileName(restart_chkfile + "/Level_0/TSK_0_H");
784  if (amrex::FileExists(FirstTSKFileName))
785  {
786  amrex::Print() << "Reading TSK data" << std::endl;
787  int ntimes = 1;
788  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
789  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
790  tsk_lev[lev][0] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ng);
791  for (int nt(0); nt<ntimes; ++nt) {
792  VisMF::Read(tsk_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
793  "TSK_" + std::to_string(nt)));
794  MultiFab::Copy(*tsk_lev[lev][nt],tsk_at_t,0,0,1,ng);
795  }
796  }
797 
798  std::string LMaskFileName(restart_chkfile + "/Level_0/LMASK_0_H");
799  if (amrex::FileExists(LMaskFileName))
800  {
801  amrex::Print() << "Reading LMASK data" << std::endl;
802  int ntimes = 1;
803  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
804  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
805  for (int nt(0); nt<ntimes; ++nt) {
806  VisMF::Read(lmask_at_t, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_",
807  "LMASK_" + std::to_string(nt)));
808  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
809  const Box& bx = mfi.growntilebox();
810  Array4<int> const& dst_arr = lmask_lev[lev][nt]->array(mfi);
811  Array4<Real> const& src_arr = lmask_at_t.array(mfi);
812  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
813  {
814  dst_arr(i,j,k) = int(src_arr(i,j,k));
815  });
816  }
817  }
818  } else {
819  // Allow idealized cases over water, used to set lmask
820  ParmParse pp("erf");
821  int is_land;
822  if (pp.query("is_land", is_land, lev)) {
823  if (is_land == 1) {
824  amrex::Print() << "Level " << lev << " is land" << std::endl;
825  } else if (is_land == 0) {
826  amrex::Print() << "Level " << lev << " is water" << std::endl;
827  } else {
828  Error("is_land should be 0 or 1");
829  }
830  lmask_lev[lev][0]->setVal(is_land);
831  } else {
832  // Default to land everywhere if not specified
833  lmask_lev[lev][0]->setVal(1);
834  }
835  lmask_lev[lev][0]->FillBoundary(geom[lev].periodicity());
836  }
837 
838  IntVect ngv = ng; ngv[2] = 0;
839 
840  // Read lat/lon if it exists
842  amrex::Print() << "Reading Lat/Lon variables" << std::endl;
843  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
844  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
845  VisMF::Read(lat, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LAT"));
846  VisMF::Read(lon, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "LON"));
847  lat_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
848  lon_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
849  MultiFab::Copy(*lat_m[lev],lat,0,0,1,ngv);
850  MultiFab::Copy(*lon_m[lev],lon,0,0,1,ngv);
851  }
852 
853 #ifdef ERF_USE_NETCDF
854  // Read sinPhi and cosPhi if it exists
856  amrex::Print() << "Reading Coriolis factors" << std::endl;
857  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
858  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
859  VisMF::Read(sphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "SinPhi"));
860  VisMF::Read(cphi, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "CosPhi"));
861  sinPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
862  cosPhi_m[lev] = std::make_unique<MultiFab>(ba2d[lev],dmap[lev],1,ngv);
863  MultiFab::Copy(*sinPhi_m[lev],sphi,0,0,1,ngv);
864  MultiFab::Copy(*cosPhi_m[lev],cphi,0,0,1,ngv);
865  }
866 
867  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
868 
869  if (lev == 0) {
870  MultiFab tmp1d(ba1d[0],dmap[0],1,0);
871 
872  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C1H"));
873  MultiFab::Copy(*mf_C1H,tmp1d,0,0,1,0);
874 
875  VisMF::Read(tmp1d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "C2H"));
876  MultiFab::Copy(*mf_C2H,tmp1d,0,0,1,0);
877 
878  MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
879 
880  VisMF::Read(tmp2d, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "MUB"));
881  MultiFab::Copy(*mf_MUB,tmp2d,0,0,1,mf_MUB->nGrowVect());
882  }
883  }
884 #endif
885 
886  } // for lev
887 
888 #ifdef ERF_USE_PARTICLES
889  restartTracers((ParGDBBase*)GetParGDB(),restart_chkfile);
890  if (Microphysics::modelType(solverChoice.moisture_type) == MoistureModelType::Lagrangian) {
891  dynamic_cast<LagrangianMicrophysics&>(*micro).restartParticles((ParGDBBase*)GetParGDB(),restart_chkfile);
892  }
893 #endif
894 
895 #if 0
896 #ifdef ERF_USE_NETCDF
897  // Read bdy_data files
898  if ( ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
900  {
901  int ioproc = ParallelDescriptor::IOProcessorNumber(); // I/O rank
902  int num_time;
903  int num_var;
904  Vector<Box> bx_v;
905  if (ParallelDescriptor::IOProcessor()) {
906  // Open header file and read from it
907  std::ifstream bdy_h_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
908  bdy_h_file >> num_time;
909  bdy_h_file >> num_var;
910  bdy_h_file >> start_bdy_time;
911  bdy_h_file >> bdy_time_interval;
912  bdy_h_file >> real_width;
913  bx_v.resize(4*num_var);
914  for (int ivar(0); ivar<num_var; ++ivar) {
915  bdy_h_file >> bx_v[4*ivar ];
916  bdy_h_file >> bx_v[4*ivar+1];
917  bdy_h_file >> bx_v[4*ivar+2];
918  bdy_h_file >> bx_v[4*ivar+3];
919  }
920 
921  // IO size the FABs
922  bdy_data_xlo.resize(num_time);
923  bdy_data_xhi.resize(num_time);
924  bdy_data_ylo.resize(num_time);
925  bdy_data_yhi.resize(num_time);
926  for (int itime(0); itime<num_time; ++itime) {
927  bdy_data_xlo[itime].resize(num_var);
928  bdy_data_xhi[itime].resize(num_var);
929  bdy_data_ylo[itime].resize(num_var);
930  bdy_data_yhi[itime].resize(num_var);
931  for (int ivar(0); ivar<num_var; ++ivar) {
932  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
933  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
934  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
935  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
936  }
937  }
938 
939  // Open data file and read from it
940  std::ifstream bdy_d_file(MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
941  for (int itime(0); itime<num_time; ++itime) {
942  for (int ivar(0); ivar<num_var; ++ivar) {
943  bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
944  bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
945  bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
946  bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
947  }
948  }
949  } // IO
950 
951  // Broadcast the data
952  ParallelDescriptor::Barrier();
953  ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
954  ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
955  ParallelDescriptor::Bcast(&real_width,1,ioproc);
956  ParallelDescriptor::Bcast(&num_time,1,ioproc);
957  ParallelDescriptor::Bcast(&num_var,1,ioproc);
958 
959  // Everyone size their boxes
960  bx_v.resize(4*num_var);
961 
962  ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
963 
964  // Everyone but IO size their FABs
965  if (!ParallelDescriptor::IOProcessor()) {
966  bdy_data_xlo.resize(num_time);
967  bdy_data_xhi.resize(num_time);
968  bdy_data_ylo.resize(num_time);
969  bdy_data_yhi.resize(num_time);
970  for (int itime(0); itime<num_time; ++itime) {
971  bdy_data_xlo[itime].resize(num_var);
972  bdy_data_xhi[itime].resize(num_var);
973  bdy_data_ylo[itime].resize(num_var);
974  bdy_data_yhi[itime].resize(num_var);
975  for (int ivar(0); ivar<num_var; ++ivar) {
976  bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar ]);
977  bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
978  bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
979  bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
980  }
981  }
982  }
983 
984  for (int itime(0); itime<num_time; ++itime) {
985  for (int ivar(0); ivar<num_var; ++ivar) {
986  ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
987  ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
988  ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
989  ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
990  }
991  }
992  } // init_type == WRFInput or Metgrid
993 #endif
994 #endif
995 }
static void GotoNextLine(std::istream &is)
Definition: ERF_Checkpoint.cpp:16
void MakeNewLevelFromScratch(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:25
bool variable_coriolis
Definition: ERF_DataStruct.H:924
bool has_lat_lon
Definition: ERF_DataStruct.H:923
Here is the call graph for this function:

◆ ReadCheckpointFileSurfaceLayer()

void ERF::ReadCheckpointFileSurfaceLayer ( )

ERF function for reading additional data for MOST from a checkpoint file during restart.

This is called after the ABLMost object is instantiated.

1004 {
1005  for (int lev = 0; lev <= finest_level; ++lev)
1006  {
1007  amrex::Print() << "Reading MOST variables" << std::endl;
1008 
1009  IntVect ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
1010  MultiFab m_var(ba2d[lev],dmap[lev],1,ng);
1011  MultiFab* dst = nullptr;
1012 
1013  // U*
1014  std::string UstarFileName(restart_chkfile + "/Level_0/Ustar_H");
1015  if (amrex::FileExists(UstarFileName)) {
1016  dst = m_SurfaceLayer->get_u_star(lev);
1017  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Ustar"));
1018  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1019  }
1020 
1021  // W*
1022  std::string WstarFileName(restart_chkfile + "/Level_0/Wstar_H");
1023  if (amrex::FileExists(WstarFileName)) {
1024  dst = m_SurfaceLayer->get_w_star(lev);
1025  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Wstar"));
1026  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1027  }
1028 
1029  // T*
1030  std::string TstarFileName(restart_chkfile + "/Level_0/Tstar_H");
1031  if (amrex::FileExists(TstarFileName)) {
1032  dst = m_SurfaceLayer->get_t_star(lev);
1033  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Tstar"));
1034  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1035  }
1036 
1037  // Q*
1038  std::string QstarFileName(restart_chkfile + "/Level_0/Qstar_H");
1039  if (amrex::FileExists(QstarFileName)) {
1040  dst = m_SurfaceLayer->get_q_star(lev);
1041  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qstar"));
1042  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1043  }
1044 
1045  // Olen
1046  std::string OlenFileName(restart_chkfile + "/Level_0/Olen_H");
1047  if (amrex::FileExists(OlenFileName)) {
1048  dst = m_SurfaceLayer->get_olen(lev);
1049  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Olen"));
1050  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1051  }
1052 
1053  // Qsurf
1054  std::string QsurfFileName(restart_chkfile + "/Level_0/Qsurf_H");
1055  if (amrex::FileExists(QsurfFileName)) {
1056  dst = m_SurfaceLayer->get_q_surf(lev);
1057  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Qsurf"));
1058  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1059  }
1060 
1061  // PBLH
1062  std::string PBLHFileName(restart_chkfile + "/Level_0/PBLH_H");
1063  if (amrex::FileExists(PBLHFileName)) {
1064  dst = m_SurfaceLayer->get_pblh(lev);
1065  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "PBLH"));
1066  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1067  }
1068 
1069  // Z0
1070  std::string Z0FileName(restart_chkfile + "/Level_0/Z0_H");
1071  if (amrex::FileExists(Z0FileName)) {
1072  dst = m_SurfaceLayer->get_z0(lev);
1073  VisMF::Read(m_var, MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Z0"));
1074  MultiFab::Copy(*dst,m_var,0,0,1,ng);
1075  }
1076  }
1077 }

◆ ReadParameters()

void ERF::ReadParameters ( )
private
1870 {
1871  {
1872  ParmParse pp; // Traditionally, max_step and stop_time do not have prefix.
1873  pp.query("max_step", max_step);
1874  if (max_step < 0) {
1875  max_step = std::numeric_limits<int>::max();
1876  }
1877 
1878  std::string start_datetime, stop_datetime;
1879  if (pp.query("start_datetime", start_datetime)) {
1880  if (start_datetime.length() == 16) { // YYYY-MM-DD HH:MM
1881  start_datetime += ":00"; // add seconds
1882  }
1883  start_time = getEpochTime(start_datetime, datetime_format);
1884 
1885  if (pp.query("stop_datetime", stop_datetime)) {
1886  if (stop_datetime.length() == 16) { // YYYY-MM-DD HH:MM
1887  stop_datetime += ":00"; // add seconds
1888  }
1889  stop_time = getEpochTime(stop_datetime, datetime_format);
1890  }
1891 
1892  use_datetime = true;
1893 
1894  } else {
1895  pp.query("stop_time", stop_time);
1896  pp.query("start_time", start_time); // This is optional, it defaults to 0
1897  }
1898  }
1899 
1900  ParmParse pp(pp_prefix);
1901  ParmParse pp_amr("amr");
1902  {
1903  pp.query("regrid_level_0_on_restart", regrid_level_0_on_restart);
1904  pp.query("regrid_int", regrid_int);
1905  pp.query("check_file", check_file);
1906 
1907  // The regression tests use "amr.restart" and "amr.m_check_int" so we allow
1908  // for those or "erf.restart" / "erf.m_check_int" with the former taking
1909  // precedence if both are specified
1910  pp.query("check_int", m_check_int);
1911  pp.query("check_per", m_check_per);
1912  pp_amr.query("check_int", m_check_int);
1913  pp_amr.query("check_per", m_check_per);
1914 
1915  pp.query("restart", restart_chkfile);
1916  pp_amr.query("restart", restart_chkfile);
1917 
1918  // Verbosity
1919  pp.query("v", verbose);
1920  pp.query("mg_v", mg_verbose);
1921  pp.query("use_fft", use_fft);
1922 #ifndef ERF_USE_FFT
1923  if (use_fft) {
1924  amrex::Abort("You must build with USE_FFT in order to set use_fft = true in your inputs file");
1925  }
1926 #endif
1927 
1928  // Frequency of diagnostic output
1929  pp.query("sum_interval", sum_interval);
1930  pp.query("sum_period" , sum_per);
1931 
1932  pp.query("pert_interval", pert_interval);
1933 
1934  // Time step controls
1935  pp.query("cfl", cfl);
1936  pp.query("substepping_cfl", sub_cfl);
1937  pp.query("init_shrink", init_shrink);
1938  pp.query("change_max", change_max);
1939  pp.query("dt_max_initial", dt_max_initial);
1940  pp.query("dt_max", dt_max);
1941 
1942  fixed_dt.resize(max_level+1,-1.);
1943  fixed_fast_dt.resize(max_level+1,-1.);
1944 
1945  pp.query("fixed_dt", fixed_dt[0]);
1946  pp.query("fixed_fast_dt", fixed_fast_dt[0]);
1947 
1948  for (int lev = 1; lev <= max_level; lev++)
1949  {
1950  fixed_dt[lev] = fixed_dt[lev-1] / static_cast<Real>(MaxRefRatio(lev-1));
1951  fixed_fast_dt[lev] = fixed_fast_dt[lev-1] / static_cast<Real>(MaxRefRatio(lev-1));
1952  }
1953 
1954  pp.query("fixed_mri_dt_ratio", fixed_mri_dt_ratio);
1955 
1956  // We use this to keep track of how many boxes we read in from WRF initialization
1957  num_files_at_level.resize(max_level+1,0);
1958 
1959  // We use this to keep track of how many boxes are specified thru the refinement indicators
1960  num_boxes_at_level.resize(max_level+1,0);
1961  boxes_at_level.resize(max_level+1);
1962 
1963  // We always have exactly one file at level 0
1964  num_boxes_at_level[0] = 1;
1965  boxes_at_level[0].resize(1);
1966  boxes_at_level[0][0] = geom[0].Domain();
1967 
1968 #ifdef ERF_USE_NETCDF
1969  nc_init_file.resize(max_level+1);
1970 
1971  // NetCDF wrfinput initialization files -- possibly multiple files at each of multiple levels
1972  // but we always have exactly one file at level 0
1973  for (int lev = 0; lev <= max_level; lev++) {
1974  const std::string nc_file_names = Concatenate("nc_init_file_",lev,1);
1975  if (pp.contains(nc_file_names.c_str())) {
1976  int num_files = pp.countval(nc_file_names.c_str());
1977  num_files_at_level[lev] = num_files;
1978  nc_init_file[lev].resize(num_files);
1979  pp.queryarr(nc_file_names.c_str(), nc_init_file[lev],0,num_files);
1980  for (int j = 0; j < num_files; j++) {
1981  Print() << "Reading NC init file names at level " << lev << " and index " << j << " : " << nc_init_file[lev][j] << std::endl;
1982  } // j
1983  } // if pp.contains
1984  } // lev
1985 
1986  // NetCDF wrfbdy lateral boundary file
1987  if (pp.query("nc_bdy_file", nc_bdy_file)) {
1988  Print() << "Reading NC bdy file name " << nc_bdy_file << std::endl;
1989  }
1990 
1991  // NetCDF wrflow lateral boundary file
1992  if (pp.query("nc_low_file", nc_low_file)) {
1993  Print() << "Reading NC low file name " << nc_low_file << std::endl;
1994  }
1995 
1996 #endif
1997 
1998  // Options for vertical interpolation of met_em*.nc data.
1999  pp.query("metgrid_debug_quiescent", metgrid_debug_quiescent);
2000  pp.query("metgrid_debug_isothermal", metgrid_debug_isothermal);
2001  pp.query("metgrid_debug_dry", metgrid_debug_dry);
2002  pp.query("metgrid_debug_psfc", metgrid_debug_psfc);
2003  pp.query("metgrid_debug_msf", metgrid_debug_msf);
2004  pp.query("metgrid_interp_theta", metgrid_interp_theta);
2005  pp.query("metgrid_basic_linear", metgrid_basic_linear);
2006  pp.query("metgrid_use_below_sfc", metgrid_use_below_sfc);
2007  pp.query("metgrid_use_sfc", metgrid_use_sfc);
2008  pp.query("metgrid_retain_sfc", metgrid_retain_sfc);
2009  pp.query("metgrid_proximity", metgrid_proximity);
2010  pp.query("metgrid_order", metgrid_order);
2011  pp.query("metgrid_force_sfc_k", metgrid_force_sfc_k);
2012 
2013  // Set default to FullState for now ... later we will try Perturbation
2014  interpolation_type = StateInterpType::FullState;
2015  pp.query_enum_case_insensitive("interpolation_type" ,interpolation_type);
2016 
2017  PlotFileType plotfile3d_type_temp = PlotFileType::None;
2018  pp.query_enum_case_insensitive("plotfile_type" ,plotfile3d_type_temp);
2019  pp.query_enum_case_insensitive("plotfile_type_1",plotfile3d_type_1);
2020  pp.query_enum_case_insensitive("plotfile_type_2",plotfile3d_type_2);
2021 
2022  PlotFileType plotfile2d_type_temp = PlotFileType::None;
2023  pp.query_enum_case_insensitive("plotfile2d_type" ,plotfile2d_type_temp);
2024  pp.query_enum_case_insensitive("plotfile2d_type_1",plotfile2d_type_1);
2025  pp.query_enum_case_insensitive("plotfile2d_type_2",plotfile2d_type_2);
2026  //
2027  // This option is for backward consistency -- if only plotfile_type is set,
2028  // then it will be used for both 1 and 2 if and only if they are not set
2029  //
2030  // Default is native amrex if no type is specified
2031  //
2032  if (plotfile3d_type_temp == PlotFileType::None) {
2033  if (plotfile3d_type_1 == PlotFileType::None) {
2034  plotfile3d_type_1 = PlotFileType::Amrex;
2035  }
2036  if (plotfile3d_type_2 == PlotFileType::None) {
2037  plotfile3d_type_2 = PlotFileType::Amrex;
2038  }
2039  } else {
2040  if (plotfile3d_type_1 == PlotFileType::None) {
2041  plotfile3d_type_1 = plotfile3d_type_temp;
2042  } else {
2043  amrex::Abort("You must set either plotfile_type or plotfile_type_1, not both");
2044  }
2045  if (plotfile3d_type_2 == PlotFileType::None) {
2046  plotfile3d_type_2 = plotfile3d_type_temp;
2047  } else {
2048  amrex::Abort("You must set either plotfile_type or plotfile_type_2, not both");
2049  }
2050  }
2051  if (plotfile2d_type_temp == PlotFileType::None) {
2052  if (plotfile2d_type_1 == PlotFileType::None) {
2053  plotfile2d_type_1 = PlotFileType::Amrex;
2054  }
2055  if (plotfile2d_type_2 == PlotFileType::None) {
2056  plotfile2d_type_2 = PlotFileType::Amrex;
2057  }
2058  } else {
2059  if (plotfile2d_type_1 == PlotFileType::None) {
2060  plotfile2d_type_1 = plotfile2d_type_temp;
2061  } else {
2062  amrex::Abort("You must set either plotfile2d_type or plotfile2d_type_1, not both");
2063  }
2064  if (plotfile2d_type_2 == PlotFileType::None) {
2065  plotfile2d_type_2 = plotfile2d_type_temp;
2066  } else {
2067  amrex::Abort("You must set either plotfile2d_type or plotfile2d_type_2, not both");
2068  }
2069  }
2070 #ifndef ERF_USE_NETCDF
2071  if (plotfile3d_type_1 == PlotFileType::Netcdf ||
2072  plotfile3d_type_2 == PlotFileType::Netcdf ||
2073  plotfile2d_type_1 == PlotFileType::Netcdf ||
2074  plotfile2d_type_2 == PlotFileType::Netcdf) {
2075  amrex::Abort("Plotfile type = Netcdf is not allowed without USE_NETCDF = TRUE");
2076  }
2077 #endif
2078 
2079  pp.query("plot_file_1" , plot3d_file_1);
2080  pp.query("plot_file_2" , plot3d_file_2);
2081  pp.query("plot2d_file_1", plot2d_file_1);
2082  pp.query("plot2d_file_2", plot2d_file_2);
2083 
2084  pp.query("plot_int_1" , m_plot3d_int_1);
2085  pp.query("plot_int_2" , m_plot3d_int_2);
2086  pp.query("plot_per_1" , m_plot3d_per_1);
2087  pp.query("plot_per_2" , m_plot3d_per_2);
2088 
2089  pp.query("plot2d_int_1" , m_plot2d_int_1);
2090  pp.query("plot2d_int_2" , m_plot2d_int_2);
2091  pp.query("plot2d_per_1", m_plot2d_per_1);
2092  pp.query("plot2d_per_2", m_plot2d_per_2);
2093 
2094  pp.query("subvol_file", subvol_file);
2095  pp.query("subvol_int" , m_subvol_int);
2096  pp.query("subvol_per" , m_subvol_per);
2097 
2098  pp.query("expand_plotvars_to_unif_rr",m_expand_plotvars_to_unif_rr);
2099 
2100  pp.query("plot_face_vels",m_plot_face_vels);
2101 
2102  if ( (m_plot3d_int_1 > 0 && m_plot3d_per_1 > 0) ||
2103  (m_plot3d_int_2 > 0 && m_plot3d_per_2 > 0.) ) {
2104  Abort("Must choose only one of plot_int or plot_per");
2105  }
2106  if ( (m_plot2d_int_1 > 0 && m_plot2d_per_1 > 0) ||
2107  (m_plot2d_int_2 > 0 && m_plot2d_per_2 > 0.) ) {
2108  Abort("Must choose only one of plot_int or plot_per");
2109  }
2110 
2111  pp.query("profile_int", profile_int);
2112  pp.query("destag_profiles", destag_profiles);
2113 
2114  pp.query("plot_lsm", plot_lsm);
2115 #ifdef ERF_USE_RRTMGP
2116  pp.query("plot_rad", plot_rad);
2117 #endif
2118  pp.query("profile_rad_int", rad_datalog_int);
2119 
2120  pp.query("output_1d_column", output_1d_column);
2121  pp.query("column_per", column_per);
2122  pp.query("column_interval", column_interval);
2123  pp.query("column_loc_x", column_loc_x);
2124  pp.query("column_loc_y", column_loc_y);
2125  pp.query("column_file_name", column_file_name);
2126 
2127  // Sampler output frequency
2128  pp.query("sampler_per", sampler_per);
2129  pp.query("sampler_interval", sampler_interval);
2130 
2131  // Specify information about outputting planes of data
2132  pp.query("output_bndry_planes", output_bndry_planes);
2133  pp.query("bndry_output_planes_interval", bndry_output_planes_interval);
2134  pp.query("bndry_output_planes_per", bndry_output_planes_per);
2135  pp.query("bndry_output_start_time", bndry_output_planes_start_time);
2136 
2137  // Specify whether ingest boundary planes of data
2138  pp.query("input_bndry_planes", input_bndry_planes);
2139 
2140  // Query the set and total widths for wrfbdy interior ghost cells
2141  pp.query("real_width", real_width);
2142  pp.query("real_set_width", real_set_width);
2143 
2144  // If using real boundaries, do we extrapolate w (or set to 0)
2145  pp.query("real_extrap_w", real_extrap_w);
2146 
2147  // Query the set and total widths for crse-fine interior ghost cells
2148  pp.query("cf_width", cf_width);
2149  pp.query("cf_set_width", cf_set_width);
2150 
2151  // AmrMesh iterate on grids?
2152  bool iterate(true);
2153  pp_amr.query("iterate_grids",iterate);
2154  if (!iterate) SetIterateToFalse();
2155  }
2156 
2157 #ifdef ERF_USE_PARTICLES
2158  readTracersParams();
2159 #endif
2160 
2161  solverChoice.init_params(max_level,pp_prefix);
2162 
2163 #ifndef ERF_USE_NETCDF
2164  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(( (solverChoice.init_type != InitType::WRFInput) &&
2165  (solverChoice.init_type != InitType::Metgrid ) &&
2166  (solverChoice.init_type != InitType::NCFile ) ),
2167  "init_type cannot be 'WRFInput', 'MetGrid' or 'NCFile' if we don't build with netcdf!");
2168 #endif
2169 
2170  // Query the canopy model file name
2171  std::string forestfile;
2172  solverChoice.do_forest_drag = pp.query("forest_file", forestfile);
2174  for (int lev = 0; lev <= max_level; ++lev) {
2175  m_forest_drag[lev] = std::make_unique<ForestDrag>(forestfile);
2176  }
2177  }
2178 
2179  // If init from WRFInput or Metgrid make sure a valid file name is present
2180  if ((solverChoice.init_type == InitType::WRFInput) ||
2181  (solverChoice.init_type == InitType::Metgrid) ||
2182  (solverChoice.init_type == InitType::NCFile) ) {
2183  for (int lev = 0; lev <= max_level; lev++) {
2184  int num_files = nc_init_file[lev].size();
2185  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(num_files>0, "A file name must be present for init type WRFInput, Metgrid or NCFile.");
2186  for (int j = 0; j < num_files; j++) {
2187  AMREX_ALWAYS_ASSERT_WITH_MESSAGE(!nc_init_file[lev][j].empty(), "Valid file name must be present for init type WRFInput, Metgrid or NCFile.");
2188  } //j
2189  } // lev
2190  } // InitType
2191 
2192  // What type of land surface model to use
2193  // NOTE: Must be checked after init_params
2194  if (solverChoice.lsm_type == LandSurfaceType::SLM) {
2195  lsm.SetModel<SLM>();
2196  Print() << "SLM land surface model!\n";
2197  } else if (solverChoice.lsm_type == LandSurfaceType::MM5) {
2198  lsm.SetModel<MM5>();
2199  Print() << "MM5 land surface model!\n";
2200 #ifdef ERF_USE_NOAHMP
2201  } else if (solverChoice.lsm_type == LandSurfaceType::NOAHMP) {
2202  lsm.SetModel<NOAHMP>();
2203  Print() << "Noah-MP land surface model!\n";
2204 #endif
2205  } else if (solverChoice.lsm_type == LandSurfaceType::None) {
2206  lsm.SetModel<NullSurf>();
2207  Print() << "Null land surface model!\n";
2208  } else {
2209  Abort("Dont know this LandSurfaceType!") ;
2210  }
2211 
2212  if (verbose > 0) {
2213  solverChoice.display(max_level,pp_prefix);
2214  }
2215 
2217 }
AMREX_GPU_HOST AMREX_FORCE_INLINE std::time_t getEpochTime(const std::string &dateTime, const std::string &dateTimeFormat)
Definition: ERF_EpochTime.H:15
bool metgrid_basic_linear
Definition: ERF.H:1174
bool metgrid_debug_msf
Definition: ERF.H:1172
std::string plot2d_file_2
Definition: ERF.H:1024
std::string plot3d_file_1
Definition: ERF.H:1021
bool plot_rad
Definition: ERF.H:856
bool m_plot_face_vels
Definition: ERF.H:1037
std::string plot3d_file_2
Definition: ERF.H:1022
int regrid_int
Definition: ERF.H:1014
bool metgrid_retain_sfc
Definition: ERF.H:1177
bool metgrid_use_sfc
Definition: ERF.H:1176
amrex::Vector< int > num_files_at_level
Definition: ERF.H:766
bool metgrid_debug_quiescent
Definition: ERF.H:1168
bool metgrid_interp_theta
Definition: ERF.H:1173
bool regrid_level_0_on_restart
Definition: ERF.H:1018
int metgrid_force_sfc_k
Definition: ERF.H:1180
bool real_extrap_w
Definition: ERF.H:1162
bool metgrid_use_below_sfc
Definition: ERF.H:1175
std::string subvol_file
Definition: ERF.H:1025
amrex::Real metgrid_proximity
Definition: ERF.H:1178
std::string plot2d_file_1
Definition: ERF.H:1023
bool metgrid_debug_dry
Definition: ERF.H:1170
bool metgrid_debug_isothermal
Definition: ERF.H:1169
bool metgrid_debug_psfc
Definition: ERF.H:1171
static std::string nc_low_file
Definition: ERF.H:1165
void ParameterSanityChecks()
Definition: ERF.cpp:2221
bool m_expand_plotvars_to_unif_rr
Definition: ERF.H:1026
std::string check_file
Definition: ERF.H:1046
int metgrid_order
Definition: ERF.H:1179
bool plot_lsm
Definition: ERF.H:1039
void SetModel()
Definition: ERF_LandSurface.H:28
Definition: ERF_MM5.H:26
Definition: ERF_NOAHMP.H:49
Definition: ERF_NullSurf.H:8
Definition: ERF_SLM.H:26
void display(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:639
void init_params(int max_level, std::string pp_prefix)
Definition: ERF_DataStruct.H:125
Here is the call graph for this function:

◆ refinement_criteria_setup()

void ERF::refinement_criteria_setup ( )
private

Function to define the refinement criteria based on user input

224 {
225  if (max_level > 0)
226  {
227  ParmParse pp(pp_prefix);
228  Vector<std::string> refinement_indicators;
229  pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
230 
231  for (int i=0; i<refinement_indicators.size(); ++i)
232  {
233  std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
234 
235  ParmParse ppr(ref_prefix);
236  RealBox realbox;
237  int lev_for_box;
238 
239  int num_real_lo = ppr.countval("in_box_lo");
240  int num_indx_lo = ppr.countval("in_box_lo_indices");
241  int num_real_hi = ppr.countval("in_box_hi");
242  int num_indx_hi = ppr.countval("in_box_hi_indices");
243 
244  AMREX_ALWAYS_ASSERT(num_real_lo == num_real_hi);
245  AMREX_ALWAYS_ASSERT(num_indx_lo == num_indx_hi);
246 
247  if ( !((num_real_lo >= AMREX_SPACEDIM-1 && num_indx_lo == 0) ||
248  (num_indx_lo >= AMREX_SPACEDIM-1 && num_real_lo == 0) ||
249  (num_indx_lo == 0 && num_real_lo == 0)) )
250  {
251  amrex::Abort("Must only specify box for refinement using real OR index space");
252  }
253 
254  if (num_real_lo > 0) {
255  std::vector<Real> rbox_lo(3), rbox_hi(3);
256  ppr.get("max_level",lev_for_box);
257  if (lev_for_box <= max_level)
258  {
259  if (n_error_buf[0] != IntVect::TheZeroVector()) {
260  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
261  }
262 
263  const Real* plo = geom[lev_for_box].ProbLo();
264  const Real* phi = geom[lev_for_box].ProbHi();
265 
266  ppr.getarr("in_box_lo",rbox_lo,0,num_real_lo);
267  ppr.getarr("in_box_hi",rbox_hi,0,num_real_hi);
268 
269  if (rbox_lo[0] < plo[0]) rbox_lo[0] = plo[0];
270  if (rbox_lo[1] < plo[1]) rbox_lo[1] = plo[1];
271  if (rbox_hi[0] > phi[0]) rbox_hi[0] = phi[0];
272  if (rbox_hi[1] > phi[1]) rbox_hi[1] = phi[1];
273  if (num_real_lo < AMREX_SPACEDIM) {
274  rbox_lo[2] = plo[2];
275  rbox_hi[2] = phi[2];
276  }
277 
278  realbox = RealBox(&(rbox_lo[0]),&(rbox_hi[0]));
279 
280  Print() << "Realbox read in and intersected laterally with domain is " << realbox << std::endl;
281 
282  num_boxes_at_level[lev_for_box] += 1;
283 
284  int ilo, jlo, klo;
285  int ihi, jhi, khi;
286  const auto* dx = geom[lev_for_box].CellSize();
287  ilo = static_cast<int>((rbox_lo[0] - plo[0])/dx[0]);
288  jlo = static_cast<int>((rbox_lo[1] - plo[1])/dx[1]);
289  ihi = static_cast<int>((rbox_hi[0] - plo[0])/dx[0]-1);
290  jhi = static_cast<int>((rbox_hi[1] - plo[1])/dx[1]-1);
291  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
292  // Search for k indices corresponding to nominal grid
293  // AGL heights
294  const Box& domain = geom[lev_for_box].Domain();
295  klo = domain.smallEnd(2) - 1;
296  khi = domain.smallEnd(2) - 1;
297 
298  if (rbox_lo[2] <= zlevels_stag[lev_for_box][domain.smallEnd(2)])
299  {
300  klo = domain.smallEnd(2);
301  }
302  else
303  {
304  for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
305  if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
306  klo = k-1;
307  break;
308  }
309  }
310  }
311  AMREX_ASSERT(klo >= domain.smallEnd(2));
312 
313  if (rbox_hi[2] >= zlevels_stag[lev_for_box][domain.bigEnd(2)+1])
314  {
315  khi = domain.bigEnd(2);
316  }
317  else
318  {
319  for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
320  if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
321  khi = k-1;
322  break;
323  }
324  }
325  }
326  AMREX_ASSERT((khi <= domain.bigEnd(2)) && (khi > klo));
327 
328  // Need to update realbox because tagging is based on
329  // the initial _un_deformed grid
330  realbox = RealBox(plo[0]+ ilo *dx[0], plo[1]+ jlo *dx[1], plo[2]+ klo *dx[2],
331  plo[0]+(ihi+1)*dx[0], plo[1]+(jhi+1)*dx[1], plo[2]+(khi+1)*dx[2]);
332  } else {
333  klo = static_cast<int>((rbox_lo[2] - plo[2])/dx[2]);
334  khi = static_cast<int>((rbox_hi[2] - plo[2])/dx[2]-1);
335  }
336 
337  Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
338  if ( (ilo%ref_ratio[lev_for_box-1][0] != 0) || ((ihi+1)%ref_ratio[lev_for_box-1][0] != 0) ||
339  (jlo%ref_ratio[lev_for_box-1][1] != 0) || ((jhi+1)%ref_ratio[lev_for_box-1][1] != 0) ||
340  (klo%ref_ratio[lev_for_box-1][2] != 0) || ((khi+1)%ref_ratio[lev_for_box-1][2] != 0) )
341  {
342  amrex::Print() << "Box : " << bx << std::endl;
343  amrex::Print() << "RealBox : " << realbox << std::endl;
344  amrex::Print() << "ilo, ihi+1, jlo, jhi+1, klo, khi+1 by ref_ratio : "
345  << ilo%ref_ratio[lev_for_box-1][0] << " " << (ihi+1)%ref_ratio[lev_for_box-1][0] << " "
346  << jlo%ref_ratio[lev_for_box-1][1] << " " << (jhi+1)%ref_ratio[lev_for_box-1][1] << " "
347  << klo%ref_ratio[lev_for_box-1][2] << " " << (khi+1)%ref_ratio[lev_for_box-1][2] << std::endl;
348  amrex::Error("Fine box is not legit with this ref_ratio");
349  }
350  boxes_at_level[lev_for_box].push_back(bx);
351  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
352  } // lev
353 
354  if (solverChoice.init_type == InitType::WRFInput) {
355  if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
356  amrex::Error("Number of boxes doesn't match number of input files");
357 
358  }
359  }
360 
361  } else if (num_indx_lo > 0) {
362 
363  std::vector<int> box_lo(3), box_hi(3);
364  ppr.get("max_level",lev_for_box);
365  if (lev_for_box <= max_level)
366  {
367  if (n_error_buf[0] != IntVect::TheZeroVector()) {
368  amrex::Abort("Don't use n_error_buf > 0 when setting the box explicitly");
369  }
370 
371  ppr.getarr("in_box_lo_indices",box_lo,0,AMREX_SPACEDIM);
372  ppr.getarr("in_box_hi_indices",box_hi,0,AMREX_SPACEDIM);
373 
374  Box bx(IntVect(box_lo[0],box_lo[1],box_lo[2]),IntVect(box_hi[0],box_hi[1],box_hi[2]));
375  amrex::Print() << "BOX " << bx << std::endl;
376 
377  const auto* dx = geom[lev_for_box].CellSize();
378  const Real* plo = geom[lev_for_box].ProbLo();
379  realbox = RealBox(plo[0]+ box_lo[0] *dx[0], plo[1]+ box_lo[1] *dx[1], plo[2]+ box_lo[2] *dx[2],
380  plo[0]+(box_hi[0]+1)*dx[0], plo[1]+(box_hi[1]+1)*dx[1], plo[2]+(box_hi[2]+1)*dx[2]);
381 
382  Print() << "Reading " << bx << " at level " << lev_for_box << std::endl;
383  num_boxes_at_level[lev_for_box] += 1;
384 
385  if ( (box_lo[0]%ref_ratio[lev_for_box-1][0] != 0) || ((box_hi[0]+1)%ref_ratio[lev_for_box-1][0] != 0) ||
386  (box_lo[1]%ref_ratio[lev_for_box-1][1] != 0) || ((box_hi[1]+1)%ref_ratio[lev_for_box-1][1] != 0) ||
387  (box_lo[2]%ref_ratio[lev_for_box-1][2] != 0) || ((box_hi[2]+1)%ref_ratio[lev_for_box-1][2] != 0) )
388  amrex::Error("Fine box is not legit with this ref_ratio");
389  boxes_at_level[lev_for_box].push_back(bx);
390  Print() << "Saving in 'boxes at level' as " << bx << std::endl;
391  } // lev
392 
393  if (solverChoice.init_type == InitType::WRFInput) {
394  if (num_boxes_at_level[lev_for_box] != num_files_at_level[lev_for_box]) {
395  amrex::Error("Number of boxes doesn't match number of input files");
396 
397  }
398  }
399  }
400 
401  AMRErrorTagInfo info;
402 
403  if (realbox.ok()) {
404  info.SetRealBox(realbox);
405  }
406  if (ppr.countval("start_time") > 0) {
407  Real ref_min_time; ppr.get("start_time",ref_min_time);
408  info.SetMinTime(ref_min_time);
409  }
410  if (ppr.countval("end_time") > 0) {
411  Real ref_max_time; ppr.get("end_time",ref_max_time);
412  info.SetMaxTime(ref_max_time);
413  }
414  if (ppr.countval("max_level") > 0) {
415  int ref_max_level; ppr.get("max_level",ref_max_level);
416  info.SetMaxLevel(ref_max_level);
417  }
418 
419  if (ppr.countval("value_greater")) {
420  int num_val = ppr.countval("value_greater");
421  Vector<Real> value(num_val);
422  ppr.getarr("value_greater",value,0,num_val);
423  std::string field; ppr.get("field_name",field);
424  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
425  }
426  else if (ppr.countval("value_less")) {
427  int num_val = ppr.countval("value_less");
428  Vector<Real> value(num_val);
429  ppr.getarr("value_less",value,0,num_val);
430  std::string field; ppr.get("field_name",field);
431  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
432  }
433  else if (ppr.countval("adjacent_difference_greater")) {
434  int num_val = ppr.countval("adjacent_difference_greater");
435  Vector<Real> value(num_val);
436  ppr.getarr("adjacent_difference_greater",value,0,num_val);
437  std::string field; ppr.get("field_name",field);
438  ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
439  }
440  else if (realbox.ok())
441  {
442  ref_tags.push_back(AMRErrorTag(info));
443  } else {
444  Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
445  }
446  } // loop over criteria
447  } // if max_level > 0
448 }
Here is the call graph for this function:

◆ remake_zphys()

void ERF::remake_zphys ( int  lev,
amrex::Real  time,
std::unique_ptr< amrex::MultiFab > &  temp_zphys_nd 
)
607 {
608  if (lev > 0)
609  {
610  //
611  // First interpolate from coarser level
612  // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
613  // have been pre-filled - this includes ghost cells both inside and outside
614  // the domain
615  //
616  InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
617  IntVect(0,0,0), // do not fill ghost cells outside the domain
618  *z_phys_nd[lev-1], 0, 0, 1,
619  geom[lev-1], geom[lev],
620  refRatio(lev-1), &node_bilinear_interp,
622 
623  // This recomputes the fine values using the bottom terrain at the fine resolution,
624  // and also fills values of z_phys_nd outside the domain
625  make_terrain_fitted_coords(lev,geom[lev],*temp_zphys_nd,zlevels_stag[lev],phys_bc_type);
626 
627  std::swap(temp_zphys_nd, z_phys_nd[lev]);
628 
629  } // lev > 0
630 
631  if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
632  //
633  // This assumes we have already remade the EBGeometry
634  //
635  terrain_blanking[lev]->setVal(1.0);
636  MultiFab::Subtract(*terrain_blanking[lev], EBFactory(lev).getVolFrac(), 0, 0, 1, z_phys_nd[lev]->nGrowVect());
637  }
638 }
Here is the call graph for this function:

◆ RemakeLevel()

void ERF::RemakeLevel ( int  lev,
amrex::Real  time,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
override
455 {
456  if (verbose) {
457  amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
458  }
459 
460  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::MovingFittedMesh);
461 
462  BoxArray ba_old(vars_new[lev][Vars::cons].boxArray());
463  DistributionMapping dm_old(vars_new[lev][Vars::cons].DistributionMap());
464 
465  if (verbose) {
466  amrex::Print() <<" OLD BA AT LEVEL " << lev << " " << ba_old << std::endl;
467  }
468 
469  //
470  // Re-define subdomain at this level within the domain such that
471  // 1) all boxes in a given subdomain are "connected"
472  // 2) no boxes in a subdomain touch any boxes in any other subdomain
473  //
474  if (solverChoice.anelastic[lev] == 1) {
475  make_subdomains(ba.simplified_list(), subdomains[lev]);
476  }
477 
478  int ncomp_cons = vars_new[lev][Vars::cons].nComp();
479  IntVect ngrow_state = vars_new[lev][Vars::cons].nGrowVect();
480 
481  int ngrow_vels = ComputeGhostCells(solverChoice);
482 
483  Vector<MultiFab> temp_lev_new(Vars::NumTypes);
484  Vector<MultiFab> temp_lev_old(Vars::NumTypes);
485  MultiFab temp_base_state;
486 
487  std::unique_ptr<MultiFab> temp_zphys_nd;
488 
489  //********************************************************************************************
490  // This allocates all kinds of things, including but not limited to: solution arrays,
491  // terrain arrays and metrics, and base state.
492  // *******************************************************************************************
493  init_stuff(lev, ba, dm, temp_lev_new, temp_lev_old, temp_base_state, temp_zphys_nd);
494 
495  // ********************************************************************************************
496  // Build the data structures for terrain-related quantities
497  // ********************************************************************************************
498  if ( solverChoice.terrain_type == TerrainType::EB ||
499  solverChoice.terrain_type == TerrainType::ImmersedForcing)
500  {
501  const amrex::EB2::IndexSpace& ebis = amrex::EB2::IndexSpace::top();
502  const EB2::Level& eb_level = ebis.getLevel(geom[lev]);
503  if (solverChoice.terrain_type == TerrainType::EB) {
504  eb[lev]->make_all_factories(lev, geom[lev], ba, dm, eb_level);
505  } else if (solverChoice.terrain_type == TerrainType::ImmersedForcing) {
506  eb[lev]->make_cc_factory(lev, geom[lev], ba, dm, eb_level);
507  }
508  }
509  remake_zphys(lev, time, temp_zphys_nd);
511 
512  // ********************************************************************************************
513  // Make sure that detJ and z_phys_cc are the average of the data on a finer level if there is one
514  // ********************************************************************************************
515  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
516  for (int crse_lev = lev-1; crse_lev >= 0; crse_lev--) {
517  average_down( *detJ_cc[crse_lev+1], *detJ_cc[crse_lev], 0, 1, refRatio(crse_lev));
518  average_down(*z_phys_cc[crse_lev+1], *z_phys_cc[crse_lev], 0, 1, refRatio(crse_lev));
519  }
520  }
521 
522  // ********************************************************************************************
523  // Build the data structures for canopy model (depends upon z_phys)
524  // ********************************************************************************************
526  m_forest_drag[lev]->define_drag_field(ba, dm, geom[lev], z_phys_cc[lev].get(), z_phys_nd[lev].get());
527  }
528 
529  // *****************************************************************************************************
530  // Create the physbcs objects (after initializing the terrain but before calling FillCoarsePatch
531  // *****************************************************************************************************
532  make_physbcs(lev);
533 
534  // ********************************************************************************************
535  // Update the base state at this level by interpolation from coarser level AND copy
536  // from previous (pre-regrid) base_state array
537  // ********************************************************************************************
538  if (lev > 0) {
539  Interpolater* mapper = &cell_cons_interp;
540 
541  Vector<MultiFab*> fmf = {&base_state[lev ], &base_state[lev ]};
542  Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
543  Vector<Real> ftime = {time, time};
544  Vector<Real> ctime = {time, time};
545 
546  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
547  FillPatchTwoLevels(temp_base_state, temp_base_state.nGrowVect(), IntVect(0,0,0),
548  time, cmf, ctime, fmf, ftime,
549  0, 0, temp_base_state.nComp(), geom[lev-1], geom[lev],
550  refRatio(lev-1), mapper, domain_bcs_type,
552 
553  // Impose bc's outside the domain
554  (*physbcs_base[lev])(temp_base_state,0,temp_base_state.nComp(),base_state[lev].nGrowVect());
555 
556  // *************************************************************************************************
557  // This will fill the temporary MultiFabs with data from vars_new
558  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
559  // NOTE: we must create the new base state before calling FillPatch because we will
560  // interpolate perturbational quantities
561  // *************************************************************************************************
562  FillPatchFineLevel(lev, time, {&temp_lev_new[Vars::cons],&temp_lev_new[Vars::xvel],
563  &temp_lev_new[Vars::yvel],&temp_lev_new[Vars::zvel]},
564  {&temp_lev_new[Vars::cons],&rU_new[lev],&rV_new[lev],&rW_new[lev]},
565  base_state[lev], temp_base_state, false);
566  } else {
567  temp_base_state.ParallelCopy(base_state[lev],0,0,base_state[lev].nComp(),
568  base_state[lev].nGrowVect(),base_state[lev].nGrowVect());
569  temp_lev_new[Vars::cons].ParallelCopy(vars_new[lev][Vars::cons],0,0,ncomp_cons,ngrow_state,ngrow_state);
570  temp_lev_new[Vars::xvel].ParallelCopy(vars_new[lev][Vars::xvel],0,0, 1,ngrow_vels,ngrow_vels);
571  temp_lev_new[Vars::yvel].ParallelCopy(vars_new[lev][Vars::yvel],0,0, 1,ngrow_vels,ngrow_vels);
572 
573  temp_lev_new[Vars::zvel].setVal(0.);
574  temp_lev_new[Vars::zvel].ParallelCopy(vars_new[lev][Vars::zvel],0,0, 1,
575  IntVect(ngrow_vels,ngrow_vels,0),IntVect(ngrow_vels,ngrow_vels,0));
576  }
577 
578  // Now swap the pointers since we needed both old and new in the FillPatch
579  std::swap(temp_base_state, base_state[lev]);
580 
581  // ********************************************************************************************
582  // Copy from new into old just in case
583  // ********************************************************************************************
584  MultiFab::Copy(temp_lev_old[Vars::cons],temp_lev_new[Vars::cons],0,0,ncomp_cons,ngrow_state);
585  MultiFab::Copy(temp_lev_old[Vars::xvel],temp_lev_new[Vars::xvel],0,0, 1,ngrow_vels);
586  MultiFab::Copy(temp_lev_old[Vars::yvel],temp_lev_new[Vars::yvel],0,0, 1,ngrow_vels);
587  MultiFab::Copy(temp_lev_old[Vars::zvel],temp_lev_new[Vars::zvel],0,0, 1,IntVect(ngrow_vels,ngrow_vels,0));
588 
589  // ********************************************************************************************
590  // Now swap the pointers
591  // ********************************************************************************************
592  for (int var_idx = 0; var_idx < Vars::NumTypes; ++var_idx) {
593  std::swap(temp_lev_new[var_idx], vars_new[lev][var_idx]);
594  std::swap(temp_lev_old[var_idx], vars_old[lev][var_idx]);
595  }
596 
597  t_new[lev] = time;
598  t_old[lev] = time - 1.e200;
599 
600  // ********************************************************************************************
601  // Build the data structures for calculating diffusive/turbulent terms
602  // ********************************************************************************************
603  update_diffusive_arrays(lev, ba, dm);
604 
605  //********************************************************************************************
606  // Microphysics
607  // *******************************************************************************************
608  int q_size = micro->Get_Qmoist_Size(lev);
609  qmoist[lev].resize(q_size);
610  micro->Define(lev, solverChoice);
611  if (solverChoice.moisture_type != MoistureType::None)
612  {
613  micro->Init(lev, vars_new[lev][Vars::cons],
614  grids[lev], Geom(lev), 0.0,
615  z_phys_nd[lev], detJ_cc[lev]); // dummy dt value
616  }
617  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
618  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
619  }
620 
621  //********************************************************************************************
622  // Radiation
623  // *******************************************************************************************
624  if (solverChoice.rad_type != RadiationType::None)
625  {
626  rad[lev]->Init(geom[lev], ba, &vars_new[lev][Vars::cons]);
627  }
628 
629  // ********************************************************************************************
630  // Initialize the integrator class
631  // ********************************************************************************************
633 
634  // We need to re-define the FillPatcher if the grids have changed
635  if (lev > 0 && cf_width >= 0) {
636  bool ba_changed = (ba != ba_old);
637  bool dm_changed = (dm != dm_old);
638  if (ba_changed || dm_changed) {
640  }
641  }
642 
643  // ********************************************************************************************
644  // Update the SurfaceLayer arrays at this level
645  // ********************************************************************************************
646  if (phys_bc_type[Orientation(Direction::z,Orientation::low)] == ERF_BC::surface_layer) {
647  int nlevs = finest_level+1;
648  Vector<MultiFab*> mfv_old = {&vars_old[lev][Vars::cons], &vars_old[lev][Vars::xvel],
649  &vars_old[lev][Vars::yvel], &vars_old[lev][Vars::zvel]};
650  m_SurfaceLayer->make_SurfaceLayer_at_level(lev,nlevs,
651  mfv_old, Theta_prim[lev], Qv_prim[lev],
652  Qr_prim[lev], z_phys_nd[lev],
653  Hwave[lev].get(),Lwave[lev].get(),eddyDiffs_lev[lev].get(),
655  sst_lev[lev], tsk_lev[lev], lmask_lev[lev]);
656  }
657 
658  // These calls are done in AmrCore::regrid if this is a regrid at lev > 0
659  // For a level 0 regrid we must explicitly do them here
660  if (lev == 0) {
661  // Define grids[lev] to be ba
662  SetBoxArray(lev, ba);
663 
664  // Define dmap[lev] to be dm
665  SetDistributionMap(lev, dm);
666  }
667 
668 #ifdef ERF_USE_PARTICLES
669  particleData.Redistribute();
670 #endif
671 }
void remake_zphys(int lev, amrex::Real time, std::unique_ptr< amrex::MultiFab > &temp_zphys_nd)
Definition: ERF_MakeNewArrays.cpp:606

◆ restart()

void ERF::restart ( )
1686 {
1688 
1690  //
1691  // Coarsening before we split the grids ensures that each resulting
1692  // grid will have an even number of cells in each direction.
1693  //
1694  BoxArray new_ba(amrex::coarsen(Geom(0).Domain(),2));
1695  //
1696  // Now split up into list of grids within max_grid_size[0] limit.
1697  //
1698  new_ba.maxSize(max_grid_size[0]/2);
1699  //
1700  // Now refine these boxes back to level 0.
1701  //
1702  new_ba.refine(2);
1703 
1704  if (refine_grid_layout) {
1705  ChopGrids(0, new_ba, ParallelDescriptor::NProcs());
1706  }
1707 
1708  if (new_ba != grids[0]) {
1709  DistributionMapping new_dm(new_ba);
1710  RemakeLevel(0,t_new[0],new_ba,new_dm);
1711  }
1712  }
1713 
1714 #ifdef ERF_USE_PARTICLES
1715  // We call this here without knowing whether the particles have already been initialized or not
1716  initializeTracers((ParGDBBase*)GetParGDB(),z_phys_nd,t_new[0]);
1717 #endif
1718 
1719  Real cur_time = t_new[0];
1720  if (m_check_per > 0.) {last_check_file_time = cur_time;}
1721  if (m_plot2d_per_1 > 0.) {last_plot2d_file_time_1 = std::floor(cur_time/m_plot2d_per_1) * m_plot2d_per_1;}
1722  if (m_plot2d_per_2 > 0.) {last_plot2d_file_time_2 = std::floor(cur_time/m_plot2d_per_2) * m_plot2d_per_2;}
1723  if (m_plot3d_per_1 > 0.) {last_plot3d_file_time_1 = std::floor(cur_time/m_plot3d_per_1) * m_plot3d_per_1;}
1724  if (m_plot3d_per_2 > 0.) {last_plot3d_file_time_2 = std::floor(cur_time/m_plot3d_per_2) * m_plot3d_per_2;}
1725 
1726  if (m_check_int > 0.) {last_check_file_step = istep[0];}
1727  if (m_plot2d_int_1 > 0.) {last_plot2d_file_step_1 = istep[0];}
1728  if (m_plot2d_int_2 > 0.) {last_plot2d_file_step_2 = istep[0];}
1729  if (m_plot3d_int_1 > 0.) {last_plot3d_file_step_1 = istep[0];}
1730  if (m_plot3d_int_2 > 0.) {last_plot3d_file_step_2 = istep[0];}
1731 }
void RemakeLevel(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Definition: ERF_MakeNewLevel.cpp:454
void ReadCheckpointFile()
Definition: ERF_Checkpoint.cpp:439

◆ sample_lines()

void ERF::sample_lines ( int  lev,
amrex::Real  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling data along a line along the z-dimension at the (x,y) indices specified and writes it to an output file.

Parameters
levCurrent level
timeCurrent time
cellIntVect containing the x,y-dimension indices to sample along z
mfMultiFab from which we sample the data
564 {
565  int ifile = 0;
566 
567  const int ncomp = mf.nComp(); // cell-centered state vars
568 
569  MultiFab mf_vels(grids[lev], dmap[lev], AMREX_SPACEDIM, 0);
570  average_face_to_cellcenter(mf_vels, 0,
571  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
572 
573  //
574  // Sample the data at a line (in direction "dir") in space
575  // In this case we sample in the vertical direction so dir = 2
576  // The "k" value of "cell" is ignored
577  //
578  int dir = 2;
579  MultiFab my_line = get_line_data(mf, dir, cell);
580  MultiFab my_line_vels = get_line_data(mf_vels, dir, cell);
581  MultiFab my_line_tau11 = get_line_data(*Tau[lev][TauType::tau11], dir, cell);
582  MultiFab my_line_tau12 = get_line_data(*Tau[lev][TauType::tau12], dir, cell);
583  MultiFab my_line_tau13 = get_line_data(*Tau[lev][TauType::tau13], dir, cell);
584  MultiFab my_line_tau22 = get_line_data(*Tau[lev][TauType::tau22], dir, cell);
585  MultiFab my_line_tau23 = get_line_data(*Tau[lev][TauType::tau23], dir, cell);
586  MultiFab my_line_tau33 = get_line_data(*Tau[lev][TauType::tau33], dir, cell);
587 
588  for (MFIter mfi(my_line, false); mfi.isValid(); ++mfi)
589  {
590  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
591 
592  std::ostream& sample_log = SampleLineLog(ifile);
593  if (sample_log.good()) {
594  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << time;
595  const auto& my_line_arr = my_line[0].const_array();
596  const auto& my_line_vels_arr = my_line_vels[0].const_array();
597  const auto& my_line_tau11_arr = my_line_tau11[0].const_array();
598  const auto& my_line_tau12_arr = my_line_tau12[0].const_array();
599  const auto& my_line_tau13_arr = my_line_tau13[0].const_array();
600  const auto& my_line_tau22_arr = my_line_tau22[0].const_array();
601  const auto& my_line_tau23_arr = my_line_tau23[0].const_array();
602  const auto& my_line_tau33_arr = my_line_tau33[0].const_array();
603  const Box& my_box = my_line[0].box();
604  const int klo = my_box.smallEnd(2);
605  const int khi = my_box.bigEnd(2);
606  int i = cell[0];
607  int j = cell[1];
608  for (int n = 0; n < ncomp; n++) {
609  for (int k = klo; k <= khi; k++) {
610  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_arr(i,j,k,n);
611  }
612  }
613  for (int n = 0; n < AMREX_SPACEDIM; n++) {
614  for (int k = klo; k <= khi; k++) {
615  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_vels_arr(i,j,k,n);
616  }
617  }
618  for (int k = klo; k <= khi; k++) {
619  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau11_arr(i,j,k);
620  }
621  for (int k = klo; k <= khi; k++) {
622  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau12_arr(i,j,k);
623  }
624  for (int k = klo; k <= khi; k++) {
625  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau13_arr(i,j,k);
626  }
627  for (int k = klo; k <= khi; k++) {
628  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau22_arr(i,j,k);
629  }
630  for (int k = klo; k <= khi; k++) {
631  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau23_arr(i,j,k);
632  }
633  for (int k = klo; k <= khi; k++) {
634  sample_log << std::setw(datwidth) << std::setprecision(datprecision) << my_line_tau33_arr(i,j,k);
635  }
636  sample_log << std::endl;
637  } // if good
638  } // mfi
639 }
const int datwidth
Definition: ERF.H:978
AMREX_FORCE_INLINE std::ostream & SampleLineLog(int i)
Definition: ERF.H:1391
const int datprecision
Definition: ERF.H:979

◆ sample_points()

void ERF::sample_points ( int  lev,
amrex::Real  time,
amrex::IntVect  cell,
amrex::MultiFab &  mf 
)

Utility function for sampling MultiFab data at a specified cell index.

Parameters
levLevel for the associated MultiFab data
timeCurrent time
cellIntVect containing the indexes for the cell where we want to sample
mfMultiFab from which we wish to sample data
528 {
529  int ifile = 0;
530 
531  //
532  // Sample the data at a single point in space
533  //
534  int ncomp = mf.nComp();
535  Vector<Real> my_point = get_cell_data(mf, cell);
536 
537  if (!my_point.empty()) {
538 
539  // HERE DO WHATEVER YOU WANT TO THE DATA BEFORE WRITING
540 
541  std::ostream& sample_log = SamplePointLog(ifile);
542  if (sample_log.good()) {
543  sample_log << std::setw(datwidth) << time;
544  for (int i = 0; i < ncomp; ++i)
545  {
546  sample_log << std::setw(datwidth) << my_point[i];
547  }
548  sample_log << std::endl;
549  } // if good
550  } // only write from processor that holds the cell
551 }
AMREX_FORCE_INLINE std::ostream & SamplePointLog(int i)
Definition: ERF.H:1377

◆ SampleLine()

amrex::IntVect& ERF::SampleLine ( int  i)
inlineprivate
1418  {
1419  return sampleline[i];
1420  }

◆ SampleLineLog()

AMREX_FORCE_INLINE std::ostream& ERF::SampleLineLog ( int  i)
inlineprivate
1392  {
1393  return *samplelinelog[i];
1394  }

◆ SampleLineLogName()

std::string ERF::SampleLineLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith samplelinelog file.

1548 { return samplelinelogname[i]; }

◆ SamplePoint()

amrex::IntVect& ERF::SamplePoint ( int  i)
inlineprivate
1405  {
1406  return samplepoint[i];
1407  }

◆ SamplePointLog()

AMREX_FORCE_INLINE std::ostream& ERF::SamplePointLog ( int  i)
inlineprivate
1378  {
1379  return *sampleptlog[i];
1380  }

◆ SamplePointLogName()

std::string ERF::SamplePointLogName ( int  i) const
inlineprivatenoexcept

The filename of the ith sampleptlog file.

1545 { return sampleptlogname[i]; }

◆ setPlotVariables()

void ERF::setPlotVariables ( const std::string &  pp_plot_var_names,
amrex::Vector< std::string > &  plot_var_names 
)
private
24 {
25  ParmParse pp(pp_prefix);
26 
27  if (pp.contains(pp_plot_var_names.c_str()))
28  {
29  std::string nm;
30 
31  int nPltVars = pp.countval(pp_plot_var_names.c_str());
32 
33  for (int i = 0; i < nPltVars; i++)
34  {
35  pp.get(pp_plot_var_names.c_str(), nm, i);
36 
37  // Add the named variable to our list of plot variables
38  // if it is not already in the list
39  if (!containerHasElement(plot_var_names, nm)) {
40  plot_var_names.push_back(nm);
41  }
42  }
43  } else {
44  //
45  // The default is to add none of the variables to the list
46  //
47  plot_var_names.clear();
48  }
49 
50  // Get state variables in the same order as we define them,
51  // since they may be in any order in the input list
52  Vector<std::string> tmp_plot_names;
53 
54  for (int i = 0; i < cons_names.size(); ++i) {
55  if ( containerHasElement(plot_var_names, cons_names[i]) ) {
56  if (solverChoice.moisture_type == MoistureType::None) {
57  if (cons_names[i] != "rhoQ1" && cons_names[i] != "rhoQ2" && cons_names[i] != "rhoQ3" &&
58  cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
59  {
60  tmp_plot_names.push_back(cons_names[i]);
61  }
62  } else if (solverChoice.moisture_type == MoistureType::Kessler) { // allow rhoQ1, rhoQ2, rhoQ3
63  if (cons_names[i] != "rhoQ4" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
64  {
65  tmp_plot_names.push_back(cons_names[i]);
66  }
67  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
68  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
69  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow rhoQ1, rhoQ2
70  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ4" &&
71  cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
72  {
73  tmp_plot_names.push_back(cons_names[i]);
74  }
75  } else if ( (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
76  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow rhoQ1, rhoQ2, rhoQ4
77  if (cons_names[i] != "rhoQ3" && cons_names[i] != "rhoQ5" && cons_names[i] != "rhoQ6")
78  {
79  tmp_plot_names.push_back(cons_names[i]);
80  }
81  } else
82  {
83  // For moisture_type SAM and Morrison we have all six variables
84  tmp_plot_names.push_back(cons_names[i]);
85  }
86  }
87  }
88 
89  // check for velocity since it's not in cons_names
90  // if we are asked for any velocity component, we will need them all
91  if (containerHasElement(plot_var_names, "x_velocity") ||
92  containerHasElement(plot_var_names, "y_velocity") ||
93  containerHasElement(plot_var_names, "z_velocity")) {
94  tmp_plot_names.push_back("x_velocity");
95  tmp_plot_names.push_back("y_velocity");
96  tmp_plot_names.push_back("z_velocity");
97  }
98 
99  //
100  // If the model we are running doesn't have the variable listed in the inputs file,
101  // just ignore it rather than aborting
102  //
103  for (int i = 0; i < derived_names.size(); ++i) {
104  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
105  bool ok_to_add = ( (solverChoice.terrain_type == TerrainType::ImmersedForcing) ||
106  (derived_names[i] != "terrain_IB_mask") );
107  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
108  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
109  (derived_names[i] != "detJ") );
110  ok_to_add &= ( (SolverChoice::terrain_type == TerrainType::StaticFittedMesh) ||
111  (SolverChoice::terrain_type == TerrainType::MovingFittedMesh) ||
112  (derived_names[i] != "z_phys") );
113 #ifndef ERF_USE_WINDFARM
114  ok_to_add &= (derived_names[i] != "SMark0" && derived_names[i] != "SMark1");
115 #endif
116  if (ok_to_add)
117  {
118  if (solverChoice.moisture_type == MoistureType::None) { // no moist quantities allowed
119  if (derived_names[i] != "qv" && derived_names[i] != "qc" && derived_names[i] != "qrain" &&
120  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
121  derived_names[i] != "qt" && derived_names[i] != "qn" && derived_names[i] != "qp" &&
122  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
123  {
124  tmp_plot_names.push_back(derived_names[i]);
125  }
126  } else if ( (solverChoice.moisture_type == MoistureType::Kessler ) ||
127  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
128  (solverChoice.moisture_type == MoistureType::SAM_NoIce ) ) { // allow qv, qc, qrain
129  if (derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
130  derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
131  {
132  tmp_plot_names.push_back(derived_names[i]);
133  }
134  } else if ( (solverChoice.moisture_type == MoistureType::SatAdj) ||
135  (solverChoice.moisture_type == MoistureType::SAM_NoPrecip_NoIce) ||
136  (solverChoice.moisture_type == MoistureType::Kessler_NoRain) ) { // allow qv, qc
137  if (derived_names[i] != "qrain" &&
138  derived_names[i] != "qi" && derived_names[i] != "qsnow" && derived_names[i] != "qgraup" &&
139  derived_names[i] != "qp" &&
140  derived_names[i] != "rain_accum" && derived_names[i] != "snow_accum" && derived_names[i] != "graup_accum")
141  {
142  tmp_plot_names.push_back(derived_names[i]);
143  }
144  } else
145  {
146  // For moisture_type SAM and Morrison we have all moist quantities
147  tmp_plot_names.push_back(derived_names[i]);
148  }
149  } // use_terrain?
150  } // hasElement
151  }
152 
153 #ifdef ERF_USE_WINDFARM
154  for (int i = 0; i < derived_names.size(); ++i) {
155  if ( containerHasElement(plot_var_names, derived_names[i]) ) {
156  if(solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP) {
157  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0") {
158  tmp_plot_names.push_back(derived_names[i]);
159  }
160  }
161  if( solverChoice.windfarm_type == WindFarmType::SimpleAD or
162  solverChoice.windfarm_type == WindFarmType::GeneralAD ) {
163  if(derived_names[i] == "num_turb" or derived_names[i] == "SMark0" or derived_names[i] == "SMark1") {
164  tmp_plot_names.push_back(derived_names[i]);
165  }
166  }
167  }
168  }
169 #endif
170 
171 #ifdef ERF_USE_PARTICLES
172  const auto& particles_namelist( particleData.getNamesUnalloc() );
173  for (auto it = particles_namelist.cbegin(); it != particles_namelist.cend(); ++it) {
174  std::string tmp( (*it)+"_count" );
175  if (containerHasElement(plot_var_names, tmp) ) {
176  tmp_plot_names.push_back(tmp);
177  }
178  }
179 #endif
180  plot_var_names = tmp_plot_names;
181 }
const amrex::Vector< std::string > derived_names
Definition: ERF.H:1061
const amrex::Vector< std::string > cons_names
Definition: ERF.H:1054
Here is the call graph for this function:

◆ setRayleighRefFromSounding()

void ERF::setRayleighRefFromSounding ( bool  restarting)
private

Set Rayleigh mean profiles from input sounding.

Sets the Rayleigh Damping averaged quantities from an externally supplied input sounding data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
56 {
57  // If we are restarting then we haven't read the input_sounding file yet
58  // so we need to read it here
59  // TODO: should we store this information in the checkpoint file instead?
60  if (restarting) {
62  for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
64  }
65  }
66 
67  const Real* z_inp_sound = input_sounding_data.z_inp_sound[0].dataPtr();
68  const Real* U_inp_sound = input_sounding_data.U_inp_sound[0].dataPtr();
69  const Real* V_inp_sound = input_sounding_data.V_inp_sound[0].dataPtr();
70  const Real* theta_inp_sound = input_sounding_data.theta_inp_sound[0].dataPtr();
71  const int inp_sound_size = input_sounding_data.size(0);
72 
73  int refine_fac{1};
74  for (int lev = 0; lev <= finest_level; lev++)
75  {
76  const int klo = geom[lev].Domain().smallEnd(2);
77  const int khi = geom[lev].Domain().bigEnd(2);
78  const int Nz = khi - klo + 1;
79 
80  Vector<Real> zcc(Nz);
81  Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
82  zlevels_stag[0].begin()+khi/refine_fac+2);
83  expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
84 #if 0
85  amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
86  for (int k = 0; k < zlevels_sub.size(); k++) { amrex::AllPrint() << zlevels_sub[k] << " "; }
87  amrex::AllPrint() << " --> ";
88  for (int k = 0; k < Nz; k++) { amrex::AllPrint() << zcc[k] << " "; }
89  amrex::AllPrint() << std::endl;
90 #endif
91 
92  for (int k = 0; k < Nz; k++)
93  {
94  h_rayleigh_ptrs[lev][Rayleigh::ubar][k] = interpolate_1d(z_inp_sound, U_inp_sound, zcc[k], inp_sound_size);
95  h_rayleigh_ptrs[lev][Rayleigh::vbar][k] = interpolate_1d(z_inp_sound, V_inp_sound, zcc[k], inp_sound_size);
96  h_rayleigh_ptrs[lev][Rayleigh::wbar][k] = Real(0.0);
97  h_rayleigh_ptrs[lev][Rayleigh::thetabar][k] = interpolate_1d(z_inp_sound, theta_inp_sound, zcc[k], inp_sound_size);
98  }
99 
100  // Copy from host version to device version
101  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::ubar].begin(), h_rayleigh_ptrs[lev][Rayleigh::ubar].end(),
102  d_rayleigh_ptrs[lev][Rayleigh::ubar].begin());
103  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::vbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::vbar].end(),
104  d_rayleigh_ptrs[lev][Rayleigh::vbar].begin());
105  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::wbar].begin(), h_rayleigh_ptrs[lev][Rayleigh::wbar].end(),
106  d_rayleigh_ptrs[lev][Rayleigh::wbar].begin());
107  Gpu::copy(Gpu::hostToDevice, h_rayleigh_ptrs[lev][Rayleigh::thetabar].begin(), h_rayleigh_ptrs[lev][Rayleigh::thetabar].end(),
108  d_rayleigh_ptrs[lev][Rayleigh::thetabar].begin());
109 
110  refine_fac *= ref_ratio[lev][2];
111  }
112 }
AMREX_FORCE_INLINE void expand_and_interpolate_1d(amrex::Vector< amrex::Real > &znew, const amrex::Vector< amrex::Real > &zorig, int refine_fac, bool destag=false)
Definition: ERF_Interpolation_1D.H:85
amrex::Vector< amrex::Vector< amrex::Real > > theta_inp_sound
Definition: ERF_InputSoundingData.H:407
amrex::Vector< amrex::Vector< amrex::Real > > z_inp_sound
Definition: ERF_InputSoundingData.H:407
amrex::Vector< amrex::Vector< amrex::Real > > U_inp_sound
Definition: ERF_InputSoundingData.H:407
amrex::Vector< amrex::Vector< amrex::Real > > V_inp_sound
Definition: ERF_InputSoundingData.H:407
int size(int itime) const
Definition: ERF_InputSoundingData.H:382
Here is the call graph for this function:

◆ setRecordDataInfo()

void ERF::setRecordDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1448  {
1449  if (amrex::ParallelDescriptor::IOProcessor())
1450  {
1451  datalog[i] = std::make_unique<std::fstream>();
1452  datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1453  if (!datalog[i]->good()) {
1454  amrex::FileOpenFailed(filename);
1455  }
1456  }
1457  amrex::ParallelDescriptor::Barrier("ERF::setRecordDataInfo");
1458  }

◆ setRecordDerDataInfo()

void ERF::setRecordDerDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1461  {
1462  if (amrex::ParallelDescriptor::IOProcessor())
1463  {
1464  der_datalog[i] = std::make_unique<std::fstream>();
1465  der_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1466  if (!der_datalog[i]->good()) {
1467  amrex::FileOpenFailed(filename);
1468  }
1469  }
1470  amrex::ParallelDescriptor::Barrier("ERF::setRecordDerDataInfo");
1471  }

◆ setRecordEnergyDataInfo()

void ERF::setRecordEnergyDataInfo ( int  i,
const std::string &  filename 
)
inlineprivate
1474  {
1475  if (amrex::ParallelDescriptor::IOProcessor())
1476  {
1477  tot_e_datalog[i] = std::make_unique<std::fstream>();
1478  tot_e_datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1479  if (!tot_e_datalog[i]->good()) {
1480  amrex::FileOpenFailed(filename);
1481  }
1482  }
1483  amrex::ParallelDescriptor::Barrier("ERF::setRecordEnergyDataInfo");
1484  }

◆ setRecordSampleLineInfo()

void ERF::setRecordSampleLineInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1504  {
1505  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1506  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1507  {
1508  const amrex::Box& bx = mfi.validbox();
1509  if (bx.contains(cell)) {
1510  samplelinelog[i] = std::make_unique<std::fstream>();
1511  samplelinelog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1512  if (!samplelinelog[i]->good()) {
1513  amrex::FileOpenFailed(filename);
1514  }
1515  }
1516  }
1517  amrex::ParallelDescriptor::Barrier("ERF::setRecordSampleLineInfo");
1518  }

◆ setRecordSamplePointInfo()

void ERF::setRecordSamplePointInfo ( int  i,
int  lev,
amrex::IntVect &  cell,
const std::string &  filename 
)
inlineprivate
1487  {
1488  amrex::MultiFab dummy(grids[lev],dmap[lev],1,0);
1489  for (amrex::MFIter mfi(dummy); mfi.isValid(); ++mfi)
1490  {
1491  const amrex::Box& bx = mfi.validbox();
1492  if (bx.contains(cell)) {
1493  sampleptlog[i] = std::make_unique<std::fstream>();
1494  sampleptlog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
1495  if (!sampleptlog[i]->good()) {
1496  amrex::FileOpenFailed(filename);
1497  }
1498  }
1499  }
1500  amrex::ParallelDescriptor::Barrier("ERF::setRecordSamplePointInfo");
1501  }

◆ setSpongeRefFromSounding()

void ERF::setSpongeRefFromSounding ( bool  restarting)
private

Set sponge mean profiles from input sounding.

Sets the sponge damping averaged quantities from an externally supplied input sponge data file.

Parameters
[in]restartingBoolean parameter that indicates whether we are currently restarting from a checkpoint file.
66 {
67  // If we are restarting then we haven't read the input_sponge file yet
68  // so we need to read it here
69  // TODO: should we store this information in the checkpoint file instead?
70  if (restarting) {
72  }
73 
74  const Real* z_inp_sponge = input_sponge_data.z_inp_sponge.dataPtr();
75  const Real* U_inp_sponge = input_sponge_data.U_inp_sponge.dataPtr();
76  const Real* V_inp_sponge = input_sponge_data.V_inp_sponge.dataPtr();
77  const int inp_sponge_size = input_sponge_data.size();
78 
79  for (int lev = 0; lev <= finest_level; lev++)
80  {
81  const int khi = geom[lev].Domain().bigEnd()[2];
82  Vector<Real> zcc(khi+1);
83 
84  if (z_phys_cc[lev]) {
85  // use_terrain=1
86  // calculate the damping strength based on the max height at each k
88  } else {
89  const auto *const prob_lo = geom[lev].ProbLo();
90  const auto *const dx = geom[lev].CellSize();
91  for (int k = 0; k <= khi; k++)
92  {
93  zcc[k] = prob_lo[2] + (k+0.5) * dx[2];
94  }
95  }
96 
97  for (int k = 0; k <= khi; k++)
98  {
99  h_sponge_ptrs[lev][Sponge::ubar_sponge][k] = interpolate_1d(z_inp_sponge, U_inp_sponge, zcc[k], inp_sponge_size);
100  h_sponge_ptrs[lev][Sponge::vbar_sponge][k] = interpolate_1d(z_inp_sponge, V_inp_sponge, zcc[k], inp_sponge_size);
101  }
102 
103  // Copy from host version to device version
104  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::ubar_sponge].begin(), h_sponge_ptrs[lev][Sponge::ubar_sponge].end(),
105  d_sponge_ptrs[lev][Sponge::ubar_sponge].begin());
106  Gpu::copy(Gpu::hostToDevice, h_sponge_ptrs[lev][Sponge::vbar_sponge].begin(), h_sponge_ptrs[lev][Sponge::vbar_sponge].end(),
107  d_sponge_ptrs[lev][Sponge::vbar_sponge].begin());
108  }
109 }
AMREX_FORCE_INLINE void reduce_to_max_per_height(amrex::Vector< amrex::Real > &v, std::unique_ptr< amrex::MultiFab > &mf)
Definition: ERF_ParFunctions.H:8
amrex::Vector< amrex::Real > V_inp_sponge
Definition: ERF_InputSpongeData.H:114
amrex::Vector< amrex::Real > z_inp_sponge
Definition: ERF_InputSpongeData.H:114
amrex::Vector< amrex::Real > U_inp_sponge
Definition: ERF_InputSpongeData.H:114
int size() const
Definition: ERF_InputSpongeData.H:102
Here is the call graph for this function:

◆ solve_with_EB_mlmg()

void ERF::solve_with_EB_mlmg ( int  lev,
amrex::Vector< amrex::MultiFab > &  rhs,
amrex::Vector< amrex::MultiFab > &  p,
amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &  fluxes 
)

Solve the Poisson equation using EB_enabled MLMG Note that the level may or may not be level 0.

20 {
21  BL_PROFILE("ERF::solve_with_EB_mlmg()");
22 
23  auto const dom_lo = lbound(geom[lev].Domain());
24  auto const dom_hi = ubound(geom[lev].Domain());
25 
26  LPInfo info;
27  // Allow a hidden direction if the domain is one cell wide in any lateral direction
28  if (dom_lo.x == dom_hi.x) {
29  info.setHiddenDirection(0);
30  } else if (dom_lo.y == dom_hi.y) {
31  info.setHiddenDirection(1);
32  }
33 
34  // Make sure the solver only sees the levels over which we are solving
35  Vector<BoxArray> ba_tmp; ba_tmp.push_back(rhs[0].boxArray());
36  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(rhs[0].DistributionMap());
37  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
38 
39  auto bclo = get_projection_bc(Orientation::low);
40  auto bchi = get_projection_bc(Orientation::high);
41 
42  // amrex::Print() << "BCLO " << bclo[0] << " " << bclo[1] << " " << bclo[2] << std::endl;
43  // amrex::Print() << "BCHI " << bchi[0] << " " << bchi[1] << " " << bchi[2] << std::endl;
44 
47 
48  // ****************************************************************************
49  // Multigrid solve
50  // ****************************************************************************
51 
52  MLEBABecLap mleb (geom_tmp, ba_tmp, dm_tmp, info, {&EBFactory(lev)});
53 
54  mleb.setMaxOrder(2);
55  mleb.setDomainBC(bclo, bchi);
56  mleb.setLevelBC(0, nullptr);
57 
58  //
59  // This sets A = 0, B = 1 so that
60  // the operator A alpha - b del dot beta grad to b
61  // becomes - del dot beta grad
62  //
63  mleb.setScalars(0.0, 1.0);
64 
65  Array<MultiFab,AMREX_SPACEDIM> bcoef;
66  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
67  bcoef[idim].define(convert(ba_tmp[0],IntVect::TheDimensionVector(idim)),
68  dm_tmp[0], 1, 0, MFInfo(), EBFactory(lev));
69  bcoef[idim].setVal(-1.0);
70  }
71  mleb.setBCoeffs(0, amrex::GetArrOfConstPtrs(bcoef));
72 
73  MLMG mlmg(mleb);
74 
75  int max_iter = 100;
76  mlmg.setMaxIter(max_iter);
77  mlmg.setVerbose(mg_verbose);
78  mlmg.setBottomVerbose(0);
79 
80  mlmg.solve(GetVecOfPtrs(phi), GetVecOfConstPtrs(rhs), reltol, abstol);
81 
82  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
83 
84  ImposeBCsOnPhi(lev,phi[0], geom[lev].Domain());
85 
86  //
87  // This arises because we solve MINUS del dot beta grad phi = div (rho u)
88  //
89  fluxes[0][0].mult(-1.);
90  fluxes[0][1].mult(-1.);
91  fluxes[0][2].mult(-1.);
92 }
void ImposeBCsOnPhi(int lev, amrex::MultiFab &phi, const amrex::Box &subdomain)
Definition: ERF_ImposeBCsOnPhi.cpp:12

◆ solve_with_gmres()

void ERF::solve_with_gmres ( int  lev,
const amrex::Box &  subdomain,
amrex::MultiFab &  rhs,
amrex::MultiFab &  p,
amrex::Array< amrex::MultiFab, AMREX_SPACEDIM > &  fluxes,
amrex::MultiFab &  ax_sub,
amrex::MultiFab &  ay_sub,
amrex::MultiFab &  znd_sub 
)

Solve the Poisson equation using FFT-preconditioned GMRES

15 {
16 #ifdef ERF_USE_FFT
17  BL_PROFILE("ERF::solve_with_gmres()");
18 
21 
22  auto const dom_lo = lbound(Geom(lev).Domain());
23  auto const dom_hi = ubound(Geom(lev).Domain());
24 
25  auto const sub_lo = lbound(subdomain);
26  auto const sub_hi = ubound(subdomain);
27 
28  auto dx = Geom(lev).CellSizeArray();
29 
30  Geometry my_geom;
31 
32  Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
33  if (Geom(lev).isPeriodic(0) && sub_lo.x == dom_lo.x && sub_hi.x == dom_hi.x) { is_per[0] = 1;}
34  if (Geom(lev).isPeriodic(1) && sub_lo.y == dom_lo.y && sub_hi.y == dom_hi.y) { is_per[1] = 1;}
35 
36  int coord_sys = 0;
37 
38  // If subdomain == domain then we pass Geom(lev) to the FFT solver
39  if (subdomain == Geom(lev).Domain()) {
40  my_geom.define(Geom(lev).Domain(), Geom(lev).ProbDomain(), coord_sys, is_per);
41  } else {
42  // else we create a new geometry based only on the subdomain
43  // The information in my_geom used by the FFT routines is:
44  // 1) my_geom.Domain()
45  // 2) my_geom.CellSize()
46  // 3) my_geom.isAllPeriodic() / my_geom.periodicity()
47  RealBox rb( sub_lo.x *dx[0], sub_lo.y *dx[1], sub_lo.z *dx[2],
48  (sub_hi.x+1)*dx[0], (sub_hi.y+1)*dx[1], (sub_hi.z+1)*dx[2]);
49  my_geom.define(subdomain, rb, coord_sys, is_per);
50  }
51 
52  amrex::GMRES<MultiFab, TerrainPoisson> gmsolver;
53 
54  TerrainPoisson tp(my_geom, rhs.boxArray(), rhs.DistributionMap(), domain_bc_type,
55  stretched_dz_d[lev], ax_sub, ay_sub, &znd_sub);
56 
57  gmsolver.define(tp);
58 
59  gmsolver.setVerbose(mg_verbose);
60 
61  gmsolver.setRestartLength(50);
62 
63  tp.usePrecond(true);
64 
65  gmsolver.solve(phi, rhs, reltol, abstol);
66 
67  tp.getFluxes(phi, fluxes);
68 #else
69  amrex::ignore_unused(lev, rhs, phi, fluxes, ax_sub, ay_sub, znd_sub);
70 #endif
71 
72  // ****************************************************************************
73  // Impose bc's on pprime
74  // ****************************************************************************
75  ImposeBCsOnPhi(lev, phi, subdomain);
76 }

◆ solve_with_mlmg()

void ERF::solve_with_mlmg ( int  lev,
amrex::Vector< amrex::MultiFab > &  rhs,
amrex::Vector< amrex::MultiFab > &  p,
amrex::Vector< amrex::Array< amrex::MultiFab, AMREX_SPACEDIM >> &  fluxes 
)

Solve the Poisson equation using MLMG Note that the level may or may not be level 0.

41 {
42  BL_PROFILE("ERF::solve_with_mlmg()");
43 
44  auto const dom_lo = lbound(geom[lev].Domain());
45  auto const dom_hi = ubound(geom[lev].Domain());
46 
47  LPInfo info;
48  // Allow a hidden direction if the domain is one cell wide in any lateral direction
49  if (dom_lo.x == dom_hi.x) {
50  info.setHiddenDirection(0);
51  } else if (dom_lo.y == dom_hi.y) {
52  info.setHiddenDirection(1);
53  }
54 
55  // Make sure the solver only sees the levels over which we are solving
56  Vector<BoxArray> ba_tmp; ba_tmp.push_back(rhs[0].boxArray());
57  Vector<DistributionMapping> dm_tmp; dm_tmp.push_back(rhs[0].DistributionMap());
58  Vector<Geometry> geom_tmp; geom_tmp.push_back(geom[lev]);
59 
60  auto bclo = get_projection_bc(Orientation::low);
61  auto bchi = get_projection_bc(Orientation::high);
62 
63  // amrex::Print() << "BCLO " << bclo[0] << " " << bclo[1] << " " << bclo[2] << std::endl;
64  // amrex::Print() << "BCHI " << bchi[0] << " " << bchi[1] << " " << bchi[2] << std::endl;
65 
68 
69  // ****************************************************************************
70  // Multigrid solve
71  // ****************************************************************************
72 
73  MLPoisson mlpoisson(geom_tmp, ba_tmp, dm_tmp, info);
74  mlpoisson.setDomainBC(bclo, bchi);
75  if (lev > 0) {
76  mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann);
77  }
78  mlpoisson.setLevelBC(0, nullptr);
79 
80  // Use low order for outflow at physical boundaries
81  mlpoisson.setMaxOrder(2);
82 
83  MLMG mlmg(mlpoisson);
84  int max_iter = 100;
85  mlmg.setMaxIter(max_iter);
86 
87  mlmg.setVerbose(mg_verbose);
88  mlmg.setBottomVerbose(0);
89 
90  mlmg.solve(GetVecOfPtrs(phi),
91  GetVecOfConstPtrs(rhs),
92  reltol, abstol);
93  mlmg.getFluxes(GetVecOfArrOfPtrs(fluxes));
94 
95  // ****************************************************************************
96  // Impose bc's on pprime
97  // ****************************************************************************
98  ImposeBCsOnPhi(lev, phi[0], geom[lev].Domain());
99 }

◆ sum_derived_quantities()

void ERF::sum_derived_quantities ( amrex::Real  time)
188 {
189  if (verbose <= 0 || NumDerDataLogs() <= 0) return;
190 
191  int lev = 0;
192 
193  AMREX_ALWAYS_ASSERT(lev == 0);
194 
195  // ************************************************************************
196  // WARNING: we are not filling ghost cells other than periodic outside the domain
197  // ************************************************************************
198 
199  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
200  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
201 
202  // Average all three components of velocity (on faces) to the cell center
203  average_face_to_cellcenter(mf_cc_vel,0,
204  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
205  &vars_new[lev][Vars::yvel],
206  &vars_new[lev][Vars::zvel]});
207  mf_cc_vel.FillBoundary(geom[lev].periodicity());
208 
209  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
210  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
211  }
212 
213  MultiFab r_wted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
214  MultiFab unwted_magvelsq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
215  MultiFab enstrophysq(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
216  MultiFab theta_mf(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
217 
218 #ifdef _OPENMP
219 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
220 #endif
221  for (MFIter mfi(unwted_magvelsq, TilingIfNotGPU()); mfi.isValid(); ++mfi)
222  {
223  const Box& bx = mfi.tilebox();
224  auto& src_fab = mf_cc_vel[mfi];
225 
226  auto& dest1_fab = unwted_magvelsq[mfi];
227  derived::erf_dermagvelsq(bx, dest1_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
228 
229  auto& dest2_fab = enstrophysq[mfi];
230  derived::erf_derenstrophysq(bx, dest2_fab, 0, 1, src_fab, Geom(lev), t_new[0], nullptr, lev);
231  }
232 
233  // Copy the MF holding 1/2(u^2 + v^2 + w^2) into the MF that will hold 1/2 rho (u^2 + v^2 + w^2)d
234  MultiFab::Copy(r_wted_magvelsq, unwted_magvelsq, 0, 0, 1, 0);
235 
236  // Multiply the MF holding 1/2(u^2 + v^2 + w^2) by rho to get 1/2 rho (u^2 + v^2 + w^2)
237  MultiFab::Multiply(r_wted_magvelsq, vars_new[lev][Vars::cons], 0, 0, 1, 0);
238 
239  // Copy the MF holding (rho theta) into "theta_mf"
240  MultiFab::Copy(theta_mf, vars_new[lev][Vars::cons], RhoTheta_comp, 0, 1, 0);
241 
242  // Divide (rho theta) by rho to get theta in the MF "theta_mf"
243  MultiFab::Divide(theta_mf, vars_new[lev][Vars::cons], Rho_comp, 0, 1, 0);
244 
245  Real unwted_avg = volWgtSumMF(lev, unwted_magvelsq, 0, false);
246  Real r_wted_avg = volWgtSumMF(lev, r_wted_magvelsq, 0, false);
247  Real enstrsq_avg = volWgtSumMF(lev, enstrophysq, 0, false);
248  Real theta_avg = volWgtSumMF(lev, theta_mf, 0, false);
249 
250  // Get volume including terrain (consistent with volWgtSumMF routine)
251  MultiFab volume(grids[lev], dmap[lev], 1, 0);
252  auto const& dx = geom[lev].CellSizeArray();
253  Real cell_vol = dx[0]*dx[1]*dx[2];
254  volume.setVal(cell_vol);
255  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
256  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
257  }
258 #ifdef _OPENMP
259 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
260 #endif
261  for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
262  {
263  const Box& tbx = mfi.tilebox();
264  auto dst = volume.array(mfi);
265  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
266  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
267  ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
268  {
269  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
270  });
271  }
272  Real vol = volume.sum();
273 
274  unwted_avg /= vol;
275  r_wted_avg /= vol;
276  enstrsq_avg /= vol;
277  theta_avg /= vol;
278 
279  const int nfoo = 4;
280  Real foo[nfoo] = {unwted_avg,r_wted_avg,enstrsq_avg,theta_avg};
281 #ifdef AMREX_LAZY
282  Lazy::QueueReduction([=]() mutable {
283 #endif
284  ParallelDescriptor::ReduceRealSum(
285  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
286 
287  if (ParallelDescriptor::IOProcessor()) {
288  int i = 0;
289  unwted_avg = foo[i++];
290  r_wted_avg = foo[i++];
291  enstrsq_avg = foo[i++];
292  theta_avg = foo[i++];
293 
294  std::ostream& data_log_der = DerDataLog(0);
295 
296  if (time == 0.0) {
297  data_log_der << std::setw(datwidth) << " time";
298  data_log_der << std::setw(datwidth) << " ke_den";
299  data_log_der << std::setw(datwidth) << " velsq";
300  data_log_der << std::setw(datwidth) << " enstrophy";
301  data_log_der << std::setw(datwidth) << " int_energy";
302  data_log_der << std::endl;
303  }
304  data_log_der << std::setw(datwidth) << std::setprecision(timeprecision) << time;
305  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << unwted_avg;
306  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << r_wted_avg;
307  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << enstrsq_avg;
308  data_log_der << std::setw(datwidth) << std::setprecision(datprecision) << theta_avg;
309  data_log_der << std::endl;
310 
311  } // if IOProcessor
312 #ifdef AMREX_LAZY
313  }
314 #endif
315 }
AMREX_FORCE_INLINE std::ostream & DerDataLog(int i)
Definition: ERF.H:1355
AMREX_FORCE_INLINE int NumDerDataLogs() noexcept
Definition: ERF.H:1369
void erf_dermagvelsq(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:346
void erf_derenstrophysq(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:284
Here is the call graph for this function:

◆ sum_energy_quantities()

void ERF::sum_energy_quantities ( amrex::Real  time)
319 {
320  if ( (verbose <= 0) || (tot_e_datalog.size() < 1) ) { return; }
321 
322  int lev = 0;
323 
324  AMREX_ALWAYS_ASSERT(lev == 0);
325 
326  // ************************************************************************
327  // WARNING: we are not filling ghost cells other than periodic outside the domain
328  // ************************************************************************
329 
330  MultiFab mf_cc_vel(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
331  mf_cc_vel.setVal(0.); // We just do this to avoid uninitialized values
332 
333  // Average all three components of velocity (on faces) to the cell center
334  average_face_to_cellcenter(mf_cc_vel,0,
335  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
336  &vars_new[lev][Vars::yvel],
337  &vars_new[lev][Vars::zvel]});
338  mf_cc_vel.FillBoundary(geom[lev].periodicity());
339 
340  if (!geom[lev].isPeriodic(0) || !geom[lev].isPeriodic(1) || !geom[lev].isPeriodic(2)) {
341  amrex::Warning("Ghost cells outside non-periodic physical boundaries are not filled -- vel set to 0 there");
342  }
343 
344  MultiFab tot_mass (grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
345  MultiFab tot_energy(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(0,0,0));
346 
347  auto const& dx = geom[lev].CellSizeArray();
348  bool is_moist = (solverChoice.moisture_type != MoistureType::None);
349 
350 #ifdef _OPENMP
351 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
352 #endif
353  for (MFIter mfi(tot_mass, TilingIfNotGPU()); mfi.isValid(); ++mfi)
354  {
355  const Box& bx = mfi.tilebox();
356 
357  const Array4<Real>& cc_vel_arr = mf_cc_vel.array(mfi);
358  const Array4<Real>& tot_mass_arr = tot_mass.array(mfi);
359  const Array4<Real>& tot_energy_arr = tot_energy.array(mfi);
360  const Array4<const Real>& cons_arr = vars_new[lev][Vars::cons].const_array(mfi);
361  const Array4<const Real>& z_arr = (z_phys_nd[lev]) ? z_phys_nd[lev]->const_array(mfi) :
362  Array4<const Real>{};
363  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
364  {
365  Real Qv = (is_moist) ? cons_arr(i,j,k,RhoQ1_comp) : 0.0;
366  Real Qc = (is_moist) ? cons_arr(i,j,k,RhoQ2_comp) : 0.0;
367  Real Qt = Qv + Qc;
368  Real Rhod = cons_arr(i,j,k,Rho_comp);
369  Real Rhot = Rhod * (1.0 + Qt);
370  Real Temp = getTgivenRandRTh(Rhod, cons_arr(i,j,k,RhoTheta_comp), Qv);
371  Real TKE = 0.5 * ( cc_vel_arr(i,j,k,0)*cc_vel_arr(i,j,k,0)
372  + cc_vel_arr(i,j,k,1)*cc_vel_arr(i,j,k,1)
373  + cc_vel_arr(i,j,k,2)*cc_vel_arr(i,j,k,2) );
374  Real zval = (z_arr) ? z_arr(i,j,k) : Real(k)*dx[2];
375 
376  Real Cv = Cp_d - R_d;
377  Real Cvv = Cp_v - R_v;
378  Real Cpv = Cp_v;
379 
380  tot_mass_arr(i,j,k) = Rhot;
381  tot_energy_arr(i,j,k) = Rhod * ( (Cv + Cvv*Qv + Cpv*Qc)*Temp - L_v*Qc
382  + (1.0 + Qt)*TKE + (1.0 + Qt)*CONST_GRAV*zval );
383 
384  });
385 
386  }
387 
388  Real tot_mass_avg = volWgtSumMF(lev, tot_mass , 0, false);
389  Real tot_energy_avg = volWgtSumMF(lev, tot_energy, 0, false);
390 
391  // Get volume including terrain (consistent with volWgtSumMF routine)
392  MultiFab volume(grids[lev], dmap[lev], 1, 0);
393  Real cell_vol = dx[0]*dx[1]*dx[2];
394  volume.setVal(cell_vol);
395  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
396  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
397  }
398 #ifdef _OPENMP
399 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
400 #endif
401  for (MFIter mfi(volume, TilingIfNotGPU()); mfi.isValid(); ++mfi)
402  {
403  const Box& tbx = mfi.tilebox();
404  auto dst = volume.array(mfi);
405  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
406  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
407  ParallelFor(tbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
408  {
409  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
410  });
411  }
412  Real vol = volume.sum();
413 
414  // Divide by the volume
415  tot_mass_avg /= vol;
416  tot_energy_avg /= vol;
417 
418  const int nfoo = 2;
419  Real foo[nfoo] = {tot_mass_avg,tot_energy_avg};
420 #ifdef AMREX_LAZY
421  Lazy::QueueReduction([=]() mutable {
422 #endif
423  ParallelDescriptor::ReduceRealSum(
424  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
425 
426  if (ParallelDescriptor::IOProcessor()) {
427  int i = 0;
428  tot_mass_avg = foo[i++];
429  tot_energy_avg = foo[i++];
430 
431  std::ostream& data_log_energy = *tot_e_datalog[0];
432 
433  if (time == 0.0) {
434  data_log_energy << std::setw(datwidth) << " time";
435  data_log_energy << std::setw(datwidth) << " tot_mass";
436  data_log_energy << std::setw(datwidth) << " tot_energy";
437  data_log_energy << std::endl;
438  }
439  data_log_energy << std::setw(datwidth) << std::setprecision(timeprecision) << time;
440  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_mass_avg;
441  data_log_energy << std::setw(datwidth) << std::setprecision(datprecision) << tot_energy_avg;
442  data_log_energy << std::endl;
443 
444  } // if IOProcessor
445 #ifdef AMREX_LAZY
446  }
447 #endif
448 }
constexpr amrex::Real R_v
Definition: ERF_Constants.H:11
constexpr amrex::Real Cp_d
Definition: ERF_Constants.H:12
constexpr amrex::Real CONST_GRAV
Definition: ERF_Constants.H:21
constexpr amrex::Real Cp_v
Definition: ERF_Constants.H:13
constexpr amrex::Real R_d
Definition: ERF_Constants.H:10
constexpr amrex::Real L_v
Definition: ERF_Constants.H:16
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real getTgivenRandRTh(const amrex::Real rho, const amrex::Real rhotheta, const amrex::Real qv=0.0)
Definition: ERF_EOS.H:46
Here is the call graph for this function:

◆ sum_integrated_quantities()

void ERF::sum_integrated_quantities ( amrex::Real  time)

Computes the integrated quantities on the grid such as the total scalar and total mass quantities. Prints and writes to output file.

Parameters
timeCurrent time
16 {
17  BL_PROFILE("ERF::sum_integrated_quantities()");
18 
19  if (verbose <= 0)
20  return;
21 
22  // Single level sum
23  Real mass_sl;
24 
25  // Multilevel sums
26  Real mass_ml = 0.0;
27  Real rhth_ml = 0.0;
28  Real scal_ml = 0.0;
29  Real mois_ml = 0.0;
30 
31 #if 1
32  mass_sl = volWgtSumMF(0,vars_new[0][Vars::cons],Rho_comp,false);
33  for (int lev = 0; lev <= finest_level; lev++) {
34  mass_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],Rho_comp,true);
35  }
36 #else
37  for (int lev = 0; lev <= finest_level; lev++) {
38  MultiFab pert_dens(vars_new[lev][Vars::cons].boxArray(),
39  vars_new[lev][Vars::cons].DistributionMap(),
40  1,0);
41  MultiFab r_hse (base_state[lev], make_alias, BaseState::r0_comp, 1);
42  for ( MFIter mfi(pert_dens,TilingIfNotGPU()); mfi.isValid(); ++mfi)
43  {
44  const Box& bx = mfi.tilebox();
45  const Array4<Real >& pert_dens_arr = pert_dens.array(mfi);
46  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
47  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
48  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
49  pert_dens_arr(i, j, k, 0) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
50  });
51  }
52  if (lev == 0) {
53  mass_sl = volWgtSumMF(0,pert_dens,0,false);
54  }
55  mass_ml += volWgtSumMF(lev,pert_dens,0,true);
56  } // lev
57 #endif
58 
59  Real rhth_sl = volWgtSumMF(0,vars_new[0][Vars::cons], RhoTheta_comp,false);
60  Real scal_sl = volWgtSumMF(0,vars_new[0][Vars::cons],RhoScalar_comp,false);
61  Real mois_sl = 0.0;
62  if (solverChoice.moisture_type != MoistureType::None) {
63  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
64  for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
65  mois_sl += volWgtSumMF(0,vars_new[0][Vars::cons],RhoQ1_comp+qoff,false);
66  }
67  }
68 
69  for (int lev = 0; lev <= finest_level; lev++) {
70  rhth_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons], RhoTheta_comp,true);
71  scal_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoScalar_comp,true);
72  if (solverChoice.moisture_type != MoistureType::None) {
73  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
74  for (int qoff(0); qoff<n_qstate_moist; ++qoff) {
75  mois_ml += volWgtSumMF(lev,vars_new[lev][Vars::cons],RhoQ1_comp+qoff,false);
76  }
77  }
78  }
79 
80  Gpu::HostVector<Real> h_avg_ustar; h_avg_ustar.resize(1);
81  Gpu::HostVector<Real> h_avg_tstar; h_avg_tstar.resize(1);
82  Gpu::HostVector<Real> h_avg_olen; h_avg_olen.resize(1);
83  if ((m_SurfaceLayer != nullptr) && (NumDataLogs() > 0)) {
84  Box domain = geom[0].Domain();
85  int zdir = 2;
86  h_avg_ustar = sumToLine(*m_SurfaceLayer->get_u_star(0),0,1,domain,zdir);
87  h_avg_tstar = sumToLine(*m_SurfaceLayer->get_t_star(0),0,1,domain,zdir);
88  h_avg_olen = sumToLine(*m_SurfaceLayer->get_olen(0) ,0,1,domain,zdir);
89 
90  // Divide by the total number of cells we are averaging over
91  Real area_z = static_cast<Real>(domain.length(0)*domain.length(1));
92  h_avg_ustar[0] /= area_z;
93  h_avg_tstar[0] /= area_z;
94  h_avg_olen[0] /= area_z;
95 
96  } else {
97  h_avg_ustar[0] = 0.;
98  h_avg_tstar[0] = 0.;
99  h_avg_olen[0] = 0.;
100  }
101 
102  const int nfoo = 8;
103  Real foo[nfoo] = {mass_sl,rhth_sl,scal_sl,mois_sl,mass_ml,rhth_ml,scal_ml,mois_ml};
104 #ifdef AMREX_LAZY
105  Lazy::QueueReduction([=]() mutable {
106 #endif
107  ParallelDescriptor::ReduceRealSum(
108  foo, nfoo, ParallelDescriptor::IOProcessorNumber());
109 
110  if (ParallelDescriptor::IOProcessor()) {
111  int i = 0;
112  mass_sl = foo[i++];
113  rhth_sl = foo[i++];
114  scal_sl = foo[i++];
115  mois_sl = foo[i++];
116  mass_ml = foo[i++];
117  rhth_ml = foo[i++];
118  scal_ml = foo[i++];
119  mois_ml = foo[i++];
120 
121  Print() << '\n';
122  Print() << "TIME= " << std::setw(datwidth) << std::setprecision(timeprecision) << std::left << time << '\n';
123  if (finest_level == 0) {
124 #if 1
125  Print() << " MASS = " << mass_sl << '\n';
126 #else
127  Print() << " PERT MASS = " << mass_sl << '\n';
128 #endif
129  Print() << " RHO THETA = " << rhth_sl << '\n';
130  Print() << " RHO SCALAR = " << scal_sl << '\n';
131  Print() << " RHO QTOTAL = " << mois_sl << '\n';
132  } else {
133 #if 1
134  Print() << " MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
135 #else
136  Print() << " PERT MASS SL/ML = " << mass_sl << " " << mass_ml << '\n';
137 #endif
138  Print() << " RHO THETA SL/ML = " << rhth_sl << " " << rhth_ml << '\n';
139  Print() << " RHO SCALAR SL/ML = " << scal_sl << " " << scal_ml << '\n';
140  Print() << " RHO QTOTAL SL/ML = " << mois_sl << " " << mois_ml << '\n';
141  }
142 
143  // The first data log only holds scalars
144  if (NumDataLogs() > 0)
145  {
146  int n_d = 0;
147  std::ostream& data_log1 = DataLog(n_d);
148  if (data_log1.good()) {
149  if (time == 0.0) {
150  data_log1 << std::setw(datwidth) << " time";
151  data_log1 << std::setw(datwidth) << " u_star";
152  data_log1 << std::setw(datwidth) << " t_star";
153  data_log1 << std::setw(datwidth) << " olen";
154  data_log1 << std::endl;
155  } // time = 0
156 
157  // Write the quantities at this time
158  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time;
159  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_ustar[0];
160  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_tstar[0];
161  data_log1 << std::setw(datwidth) << std::setprecision(datprecision) << h_avg_olen[0];
162  data_log1 << std::endl;
163  } // if good
164  } // loop over i
165  } // if IOProcessor
166 #ifdef AMREX_LAZY
167  });
168 #endif
169 
170  // This is just an alias for convenience
171  int lev = 0;
172  if (NumSamplePointLogs() > 0 && NumSamplePoints() > 0) {
173  for (int i = 0; i < NumSamplePoints(); ++i)
174  {
175  sample_points(lev, time, SamplePoint(i), vars_new[lev][Vars::cons]);
176  }
177  }
178  if (NumSampleLineLogs() > 0 && NumSampleLines() > 0) {
179  for (int i = 0; i < NumSampleLines(); ++i)
180  {
181  sample_lines(lev, time, SampleLine(i), vars_new[lev][Vars::cons]);
182  }
183  }
184 }
AMREX_FORCE_INLINE int NumSampleLineLogs() noexcept
Definition: ERF.H:1398
AMREX_FORCE_INLINE int NumSamplePointLogs() noexcept
Definition: ERF.H:1384
amrex::IntVect & SampleLine(int i)
Definition: ERF.H:1417
AMREX_FORCE_INLINE int NumSamplePoints() noexcept
Definition: ERF.H:1411
AMREX_FORCE_INLINE int NumSampleLines() noexcept
Definition: ERF.H:1424
amrex::IntVect & SamplePoint(int i)
Definition: ERF.H:1404
void sample_points(int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:527
AMREX_FORCE_INLINE std::ostream & DataLog(int i)
Definition: ERF.H:1348
AMREX_FORCE_INLINE int NumDataLogs() noexcept
Definition: ERF.H:1362
void sample_lines(int lev, amrex::Real time, amrex::IntVect cell, amrex::MultiFab &mf)
Definition: ERF_WriteScalarProfiles.cpp:563

◆ timeStep()

void ERF::timeStep ( int  lev,
amrex::Real  time,
int  iteration 
)
private

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters
[in]levlevel of refinement (coarsest level is 0)
[in]timestart time for time advance
[in]iterationtime step counter
18 {
19  //
20  // We need to FillPatch the coarse level before assessing whether to regrid
21  // We have not done the swap yet so we fill the "new" which will become the "old"
22  //
23  MultiFab& S_new = vars_new[lev][Vars::cons];
24  MultiFab& U_new = vars_new[lev][Vars::xvel];
25  MultiFab& V_new = vars_new[lev][Vars::yvel];
26  MultiFab& W_new = vars_new[lev][Vars::zvel];
27 
28 #ifdef ERF_USE_NETCDF
29  //
30  // Since we now only read in a subset of the time slices in wrfbdy we need to check
31  // whether it's time to read in more
32  //
33  if (solverChoice.use_real_bcs && (lev==0)) {
34  Real dT = bdy_time_interval;
35 
36  int n_time_old = static_cast<int>( (time ) / dT);
37  int n_time_new = static_cast<int>( (time+dt[lev]) / dT);
38 
39  int ntimes = bdy_data_xlo.size();
40  for (int itime = 0; itime < ntimes; itime++)
41  {
42  //if (bdy_data_xlo[itime].size() > 0) {
43  // amrex::Print() << "HAVE DATA AT TIME " << itime << std::endl;
44  //} else {
45  // amrex::Print() << " NO DATA AT TIME " << itime << std::endl;
46  //}
47 
48  bool clear_itime = (itime < n_time_old);
49 
50  if (clear_itime && bdy_data_xlo[itime].size() > 0) {
51  bdy_data_xlo[itime].clear();
52  //amrex::Print() << "CLEAR DATA AT TIME " << itime << std::endl;
53  }
54 
55  bool need_itime = (itime >= n_time_old && itime <= n_time_new+1);
56  //if (need_itime) amrex::Print() << "NEED DATA AT TIME " << itime << std::endl;
57 
58  if (bdy_data_xlo[itime].size() == 0 && need_itime) {
59  read_from_wrfbdy(itime,nc_bdy_file,geom[0].Domain(),
60  bdy_data_xlo,bdy_data_xhi,bdy_data_ylo,bdy_data_yhi,
61  real_width);
62 
63  bool use_moist = (solverChoice.moisture_type != MoistureType::None);
64  convert_all_wrfbdy_data(itime, geom[0].Domain(), bdy_data_xlo, bdy_data_xhi, bdy_data_ylo, bdy_data_yhi,
65  *mf_MUB, *mf_C1H, *mf_C2H,
67  geom[lev], use_moist);
68  }
69  } // itime
70  } // use_real_bcs && lev == 0
71 #endif
72 
73  //
74  // NOTE: the momenta here are not fillpatched (they are only used as scratch space)
75  //
76  if (lev == 0) {
77  FillPatchCrseLevel(lev, time, {&S_new, &U_new, &V_new, &W_new});
78  } else if (lev < finest_level) {
79  FillPatchFineLevel(lev, time, {&S_new, &U_new, &V_new, &W_new},
80  {&S_new, &rU_new[lev], &rV_new[lev], &rW_new[lev]},
81  base_state[lev], base_state[lev]);
82  }
83 
84  if (regrid_int > 0) // We may need to regrid
85  {
86  // help keep track of whether a level was already regridded
87  // from a coarser level call to regrid
88  static Vector<int> last_regrid_step(max_level+1, 0);
89 
90  // regrid changes level "lev+1" so we don't regrid on max_level
91  // also make sure we don't regrid fine levels again if
92  // it was taken care of during a coarser regrid
93  if (lev < max_level)
94  {
95  if ( (istep[lev] % regrid_int == 0) && (istep[lev] > last_regrid_step[lev]) )
96  {
97  // regrid could add newly refine levels (if finest_level < max_level)
98  // so we save the previous finest level index
99  int old_finest = finest_level;
100 
101  regrid(lev, time);
102 
103 #ifdef ERF_USE_PARTICLES
104  if (finest_level != old_finest) {
105  particleData.Redistribute();
106  }
107 #endif
108 
109  // mark that we have regridded this level already
110  for (int k = lev; k <= finest_level; ++k) {
111  last_regrid_step[k] = istep[k];
112  }
113 
114  // if there are newly created levels, set the time step
115  for (int k = old_finest+1; k <= finest_level; ++k) {
116  dt[k] = dt[k-1] / MaxRefRatio(k-1);
117  }
118  } // if
119  } // lev
120  }
121 
122  // Update what we call "old" and "new" time
123  t_old[lev] = t_new[lev];
124  t_new[lev] += dt[lev];
125 
126  if (Verbose()) {
127  amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
128  amrex::Print() << std::setprecision(timeprecision)
129  << "ADVANCE from elapsed time = " << t_old[lev] << " to " << t_new[lev]
130  << " with dt = " << dt[lev] << std::endl;
131  }
132 
133 #ifdef ERF_USE_WW3_COUPLING
134  amrex::Print() << " About to call send_to_ww3 from ERF_Timestep" << std::endl;
135  send_to_ww3(lev);
136  amrex::Print() << " About to call read_waves from ERF_Timestep" << std::endl;
137  read_waves(lev);
138  //send_to_ww3(lev);
139  //read_waves(lev);
140  //send_to_ww3(lev);
141 #endif
142 
143  // Advance a single level for a single time step
144  Advance(lev, time, dt[lev], istep[lev], nsubsteps[lev]);
145 
146  ++istep[lev];
147 
148  if (Verbose()) {
149  amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
150  amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
151  }
152 
153  if (lev < finest_level)
154  {
155  // recursive call for next-finer level
156  for (int i = 1; i <= nsubsteps[lev+1]; ++i)
157  {
158  Real strt_time_for_fine = time + (i-1)*dt[lev+1];
159  timeStep(lev+1, strt_time_for_fine, i);
160  }
161  }
162 
163  if (verbose && lev == 0 && solverChoice.moisture_type != MoistureType::None) {
164  amrex::Print() << "Cloud fraction " << time << " " << cloud_fraction(time) << std::endl;
165  }
166 }
amrex::Real cloud_fraction(amrex::Real time)
Definition: ERF_WriteScalarProfiles.cpp:451
void Advance(int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
Definition: ERF_Advance.cpp:23

◆ turbPert_amplitude()

void ERF::turbPert_amplitude ( const int  lev)
private
33 {
34  // Accessing data
35  auto& lev_new = vars_new[lev];
36 
37  // Creating local data
38  int ncons = lev_new[Vars::cons].nComp();
39  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
40 
41  // Defining BoxArray type
42  auto m_ixtype = cons_data.boxArray().ixType();
43 
44 #ifdef _OPENMP
45 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
46 #endif
47  for (MFIter mfi(lev_new[Vars::cons], TileNoZ()); mfi.isValid(); ++mfi) {
48  const Box &bx = mfi.validbox();
49  const auto &cons_pert_arr = cons_data.array(mfi); // Address of perturbation array
50  const amrex::Array4<const amrex::Real> &pert_cell = turbPert.pb_cell[lev].array(mfi); // per-cell perturbation stored in structure
51 
52  turbPert.apply_tpi(lev, bx, RhoTheta_comp, m_ixtype, cons_pert_arr, pert_cell);
53  } // mfi
54 }
Here is the call graph for this function:

◆ turbPert_update()

void ERF::turbPert_update ( const int  lev,
const amrex::Real  dt 
)
private
13 {
14  // Accessing data
15  auto& lev_new = vars_new[lev];
16 
17  // Create aliases to state data to pass to calc_tpi_update
18  int ncons = lev_new[Vars::cons].nComp();
19  MultiFab cons_data(lev_new[Vars::cons], make_alias, 0, ncons);
20  MultiFab xvel_data(lev_new[Vars::xvel], make_alias, 0, 1);
21  MultiFab yvel_data(lev_new[Vars::yvel], make_alias, 0, 1);
22 
23  // Computing perturbation update time
24  turbPert.calc_tpi_update(lev, local_dt, xvel_data, yvel_data, cons_data);
25 
26  Print() << "Successfully initialized turbulent perturbation update time and amplitude with type: "<< turbPert.pt_type <<"\n";
27 }
int pt_type
Definition: ERF_TurbPertStruct.H:631

◆ update_diffusive_arrays()

void ERF::update_diffusive_arrays ( int  lev,
const amrex::BoxArray &  ba,
const amrex::DistributionMapping &  dm 
)
private
446 {
447  // ********************************************************************************************
448  // Diffusive terms
449  // ********************************************************************************************
450  bool l_use_terrain = (SolverChoice::terrain_type != TerrainType::None);
451  bool l_use_kturb = solverChoice.turbChoice[lev].use_kturb;
452  bool l_use_diff = ( (solverChoice.diffChoice.molec_diff_type != MolecDiffType::None) ||
453  l_use_kturb );
454  bool l_need_SmnSmn = solverChoice.turbChoice[lev].use_keqn;
455  bool l_use_moist = ( solverChoice.moisture_type != MoistureType::None );
456  bool l_rotate = ( solverChoice.use_rotate_surface_flux );
457 
458  BoxArray ba12 = convert(ba, IntVect(1,1,0));
459  BoxArray ba13 = convert(ba, IntVect(1,0,1));
460  BoxArray ba23 = convert(ba, IntVect(0,1,1));
461 
462  Tau[lev].resize(9);
463 
464  if (l_use_diff) {
465  //
466  // NOTE: We require ghost cells in the vertical when allowing grids that don't
467  // cover the entire vertical extent of the domain at this level
468  //
469  for (int i = 0; i < 3; i++) {
470  Tau[lev][i] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) );
471  }
472  Tau[lev][TauType::tau12] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
473  Tau[lev][TauType::tau13] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
474  Tau[lev][TauType::tau23] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
475  if (l_use_terrain) {
476  Tau[lev][TauType::tau21] = std::make_unique<MultiFab>( ba12, dm, 1, IntVect(1,1,1) );
477  Tau[lev][TauType::tau31] = std::make_unique<MultiFab>( ba13, dm, 1, IntVect(1,1,1) );
478  Tau[lev][TauType::tau32] = std::make_unique<MultiFab>( ba23, dm, 1, IntVect(1,1,1) );
479  } else {
480  Tau[lev][TauType::tau21] = nullptr;
481  Tau[lev][TauType::tau31] = nullptr;
482  Tau[lev][TauType::tau32] = nullptr;
483  }
484  SFS_hfx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
485  SFS_hfx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
486  SFS_hfx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
487  SFS_diss_lev[lev] = std::make_unique<MultiFab>( ba , dm, 1, IntVect(1,1,1) );
488  SFS_hfx1_lev[lev]->setVal(0.);
489  SFS_hfx2_lev[lev]->setVal(0.);
490  SFS_hfx3_lev[lev]->setVal(0.);
491  SFS_diss_lev[lev]->setVal(0.);
492  if (l_use_moist) {
493  SFS_q1fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
494  SFS_q2fx3_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,0,1)), dm, 1, IntVect(1,1,1) );
495  SFS_q1fx3_lev[lev]->setVal(0.0);
496  SFS_q2fx3_lev[lev]->setVal(0.0);
497  if (l_rotate) {
498  SFS_q1fx1_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(1,0,0)), dm, 1, IntVect(1,1,1) );
499  SFS_q1fx2_lev[lev] = std::make_unique<MultiFab>( convert(ba,IntVect(0,1,0)), dm, 1, IntVect(1,1,1) );
500  SFS_q1fx1_lev[lev]->setVal(0.0);
501  SFS_q1fx2_lev[lev]->setVal(0.0);
502  } else {
503  SFS_q1fx1_lev[lev] = nullptr;
504  SFS_q1fx2_lev[lev] = nullptr;
505  }
506  } else {
507  SFS_q1fx1_lev[lev] = nullptr;
508  SFS_q1fx2_lev[lev] = nullptr;
509  SFS_q1fx3_lev[lev] = nullptr;
510  SFS_q2fx3_lev[lev] = nullptr;
511  }
512  } else {
513  for (int i = 0; i < 9; i++) {
514  Tau[lev][i] = nullptr;
515  }
516  SFS_hfx1_lev[lev] = nullptr; SFS_hfx2_lev[lev] = nullptr; SFS_hfx3_lev[lev] = nullptr;
517  SFS_diss_lev[lev] = nullptr;
518  }
519 
520  if (l_use_kturb) {
521  eddyDiffs_lev[lev] = std::make_unique<MultiFab>(ba, dm, EddyDiff::NumDiffs, 2);
522  eddyDiffs_lev[lev]->setVal(0.0);
523  if(l_need_SmnSmn) {
524  SmnSmn_lev[lev] = std::make_unique<MultiFab>( ba, dm, 1, 0 );
525  } else {
526  SmnSmn_lev[lev] = nullptr;
527  }
528  } else {
529  eddyDiffs_lev[lev] = nullptr;
530  SmnSmn_lev[lev] = nullptr;
531  }
532 }
@ NumDiffs
Definition: ERF_IndexDefines.H:181

◆ update_terrain_arrays()

void ERF::update_terrain_arrays ( int  lev)
641 {
642  if (SolverChoice::mesh_type == MeshType::StretchedDz ||
643  SolverChoice::mesh_type == MeshType::VariableDz) {
644  make_J(geom[lev],*z_phys_nd[lev],*detJ_cc[lev]);
645  make_areas(geom[lev],*z_phys_nd[lev],*ax[lev],*ay[lev],*az[lev]);
646  make_zcc(geom[lev],*z_phys_nd[lev],*z_phys_cc[lev]);
647  } else { // MeshType::ConstantDz
648  if (SolverChoice::terrain_type == TerrainType::EB) {
649  const auto& ebfact = *eb[lev]->get_const_factory();
650  const MultiFab& volfrac = ebfact.getVolFrac();
651  detJ_cc[lev] = std::make_unique<MultiFab>(volfrac, amrex::make_alias, 0, volfrac.nComp());
652 
653  // Array<const MultiCutFab*, AMREX_SPACEDIM> areafrac = ebfact.getAreaFrac();
654  // ax[lev] = std::make_unique<MultiFab>(*(areafrac[0]), amrex::make_alias, 0, areafrac[0]->nComp());
655  // ay[lev] = std::make_unique<MultiFab>(*(areafrac[1]), amrex::make_alias, 0, areafrac[1]->nComp());
656  // az[lev] = std::make_unique<MultiFab>(*(areafrac[2]), amrex::make_alias, 0, areafrac[2]->nComp());
657  }
658  }
659 }
void make_areas(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &ax, MultiFab &ay, MultiFab &az)
Definition: ERF_TerrainMetrics.cpp:558
void make_J(const Geometry &geom, MultiFab &z_phys_nd, MultiFab &detJ_cc)
Definition: ERF_TerrainMetrics.cpp:520
Here is the call graph for this function:

◆ volWgtSumMF()

Real ERF::volWgtSumMF ( int  lev,
const amrex::MultiFab &  mf,
int  comp,
bool  finemask 
)

Utility function for computing a volume weighted sum of MultiFab data for a single component

Parameters
levCurrent level
mfMultiFab on which we do the volume weighted sum
compIndex of the component we want to sum
localBoolean sets whether or not to reduce the sum over the domain (false) or compute sums local to each MPI rank (true)
finemaskIf a finer level is available, determines whether we mask fine data
655 {
656  BL_PROFILE("ERF::volWgtSumMF()");
657 
658  Real sum = 0.0;
659  MultiFab tmp(grids[lev], dmap[lev], 1, 0);
660  MultiFab::Copy(tmp, mf, comp, 0, 1, 0);
661 
662  // The quantity that is conserved is not (rho S), but rather (rho S / m^2) where
663  // m is the map scale factor at cell centers
664 #ifdef _OPENMP
665 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
666 #endif
667  for (MFIter mfi(tmp, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
668  const Box& bx = mfi.tilebox();
669  const auto dst = tmp.array(mfi);
670  const auto& mfx = mapfac[lev][MapFacType::m_x]->const_array(mfi);
671  const auto& mfy = mapfac[lev][MapFacType::m_y]->const_array(mfi);
672  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
673  {
674  dst(i,j,k) /= (mfx(i,j,0)*mfy(i,j,0));
675  });
676  } // mfi
677 
678  if (lev < finest_level && finemask) {
679  const MultiFab& mask = build_fine_mask(lev+1);
680  MultiFab::Multiply(tmp, mask, 0, 0, 1, 0);
681  }
682 
683  // Get volume including terrain (consistent with volWgtSumMF routine)
684  MultiFab volume(grids[lev], dmap[lev], 1, 0);
685  auto const& dx = geom[lev].CellSizeArray();
686  Real cell_vol = dx[0]*dx[1]*dx[2];
687  volume.setVal(cell_vol);
688  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
689  MultiFab::Multiply(volume, *detJ_cc[lev], 0, 0, 1, 0);
690  }
691 
692  //
693  // Note that when we send in local = true, NO ParallelAllReduce::Sum
694  // is called inside the Dot product -- we will do that before we print
695  //
696  bool local = true;
697  sum = MultiFab::Dot(tmp, 0, volume, 0, 1, 0, local);
698 
699  return sum;
700 }
amrex::MultiFab & build_fine_mask(int lev)
Definition: ERF_WriteScalarProfiles.cpp:710

◆ WeatherDataInterpolation()

void ERF::WeatherDataInterpolation ( const amrex::Real  time)
498 {
499 
500  static Real next_read_forecast_time = -1.0;
501 
502  if (next_read_forecast_time < 0.0) {
503  int next_multiple = static_cast<int>(time / 10800.0);
504  next_read_forecast_time = next_multiple * 10800.0;
505  }
506  if (time >= next_read_forecast_time) {
507 
508  std::string folder = solverChoice.hindcast_boundary_data_dir;
509 
510  // Check if folder exists and is a directory
511  if (!fs::exists(folder) || !fs::is_directory(folder)) {
512  throw std::runtime_error("Error: Folder '" + folder + "' does not exist or is not a directory.");
513  }
514 
515  std::vector<std::string> bin_files;
516 
517  for (const auto& entry : fs::directory_iterator(folder)) {
518  if (!entry.is_regular_file()) continue;
519 
520  std::string fname = entry.path().filename().string();
521  if (fname.size() >= 4 && fname.substr(fname.size() - 4) == ".bin") {
522  bin_files.push_back(entry.path().string());
523  }
524  }
525  std::sort(bin_files.begin(), bin_files.end());
526 
527  // Check if no .bin files were found
528  if (bin_files.empty()) {
529  throw std::runtime_error("Error: No .bin files found in folder '" + folder + "'.");
530  }
531 
532  std::string filename1, filename2;
533 
534  int idx1 = static_cast<int>(time / 10800.0);
535  int idx2 = static_cast<int>(time / 10800.0)+1;
536  std::cout << "Reading weather data " << time << " " << idx1 << " " << idx2 <<" " << bin_files.size() << std::endl;
537 
538  if (idx2 >= static_cast<int>(bin_files.size())) {
539  throw std::runtime_error("Error: Not enough .bin files to cover time " + std::to_string(time));
540  }
541 
542  filename1 = bin_files[idx1];
543  filename2 = bin_files[idx2];
544 
545  BoxArray nba;
546  DistributionMapping dm;
547  Geometry geom_weather;
548 
549  //Read in weather_forecast_1
551  geom_weather,
552  nba,
553  dm);
554 
555  FillWeatherDataMultiFab(filename1,
556  geom_weather,
557  nba,
558  dm,
560 
563 
564  FillWeatherDataMultiFab(filename2,
565  geom_weather,
566  nba,
567  dm,
571 
573 
574  next_read_forecast_time += 10800.0;
575  }
576  Real alpha1 = 1.0 - (time - next_read_forecast_time)/10800.0;
577  Real alpha2 = 1.0 - alpha1;
578 
579  MultiFab& erf_mf_cons = forecast_state_interp[0][Vars::cons];
580  MultiFab& erf_mf_xvel = forecast_state_interp[0][Vars::xvel];
581  MultiFab& erf_mf_yvel = forecast_state_interp[0][Vars::yvel];
582  //MultiFab& erf_mf_zvel = forecast_state_interp[0][Vars::zvel];
583  MultiFab& erf_mf_latlon = forecast_state_interp[0][4];
584 
585  MultiFab::LinComb(forecast_state_interp[0][Vars::cons],
586  alpha1, forecast_state_1[0][Vars::cons], 0,
587  alpha2, forecast_state_2[0][Vars::cons], 0,
588  0, erf_mf_cons.nComp(), forecast_state_interp[0][Vars::cons].nGrow());
589  MultiFab::LinComb(forecast_state_interp[0][Vars::xvel],
590  alpha1, forecast_state_1[0][Vars::xvel], 0,
591  alpha2, forecast_state_2[0][Vars::xvel], 0,
592  0, erf_mf_xvel.nComp(), forecast_state_interp[0][Vars::xvel].nGrow());
593  MultiFab::LinComb(forecast_state_interp[0][Vars::yvel],
594  alpha1, forecast_state_1[0][Vars::yvel], 0,
595  alpha2, forecast_state_2[0][Vars::yvel], 0,
596  0, erf_mf_yvel.nComp(), forecast_state_interp[0][Vars::yvel].nGrow());
597  MultiFab::LinComb(forecast_state_interp[0][4],
598  alpha1, forecast_state_1[0][4], 0,
599  alpha2, forecast_state_2[0][4], 0,
600  0, erf_mf_latlon.nComp(), forecast_state_interp[0][4].nGrow());
601 
602 
603  /*Vector<std::string> varnames_plot_mf = {
604  "rho", "rhotheta", "rhoqv", "rhoqc", "rhoqr", "xvel", "yvel", "zvel", "latitude", "longitude"
605  }; // Customize variable names
606 
607  std::string pltname = "plt_interp";
608 
609  MultiFab plot_mf(erf_mf_cons.boxArray(), erf_mf_cons.DistributionMap(),
610  10, 0);
611 
612  plot_mf.setVal(0.0);
613 
614  for (MFIter mfi(plot_mf); mfi.isValid(); ++mfi) {
615  const Array4<Real> &plot_mf_arr = plot_mf.array(mfi);
616  const Array4<Real> &erf_mf_cons_arr = erf_mf_cons.array(mfi);
617  const Array4<Real> &erf_mf_xvel_arr = erf_mf_xvel.array(mfi);
618  const Array4<Real> &erf_mf_yvel_arr = erf_mf_yvel.array(mfi);
619  const Array4<Real> &erf_mf_zvel_arr = erf_mf_zvel.array(mfi);
620  const Array4<Real> &erf_mf_latlon_arr = erf_mf_latlon.array(mfi);
621 
622  const Box& bx = mfi.validbox();
623 
624  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) {
625  plot_mf_arr(i,j,k,0) = erf_mf_cons_arr(i,j,k,Rho_comp);
626  plot_mf_arr(i,j,k,1) = erf_mf_cons_arr(i,j,k,RhoTheta_comp);
627  plot_mf_arr(i,j,k,2) = erf_mf_cons_arr(i,j,k,RhoQ1_comp);
628  plot_mf_arr(i,j,k,3) = erf_mf_cons_arr(i,j,k,RhoQ2_comp);
629  plot_mf_arr(i,j,k,4) = erf_mf_cons_arr(i,j,k,RhoQ3_comp);
630 
631  plot_mf_arr(i,j,k,5) = (erf_mf_xvel_arr(i,j,k,0) + erf_mf_xvel_arr(i+1,j,k,0))/2.0;
632  plot_mf_arr(i,j,k,6) = (erf_mf_yvel_arr(i,j,k,0) + erf_mf_yvel_arr(i,j+1,k,0))/2.0;
633  plot_mf_arr(i,j,k,7) = (erf_mf_zvel_arr(i,j,k,0) + erf_mf_zvel_arr(i,j,k+1,0))/2.0;
634 
635  plot_mf_arr(i,j,k,8) = erf_mf_latlon_arr(i,j,k,0);
636  plot_mf_arr(i,j,k,9) = erf_mf_latlon_arr(i,j,k,1);
637  });
638  }
639 
640 
641  WriteSingleLevelPlotfile(
642  pltname,
643  plot_mf,
644  varnames_plot_mf,
645  geom[0],
646  time,
647  0 // level
648  );*/
649 
650 
651 
652 }
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_interp
Definition: ERF.H:157
void CreateWeatherDataGeomBoxArrayDistMap(const std::string &filename, amrex::Geometry &geom_weather, amrex::BoxArray &nba, amrex::DistributionMapping &dm)
Definition: ERF_WeatherDataInterpolation.cpp:155
void CreateForecastStateMultiFabs(amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
Definition: ERF_WeatherDataInterpolation.cpp:249
void FillWeatherDataMultiFab(const std::string &filename, const amrex::Geometry &geom_weather, const amrex::BoxArray &nba, const amrex::DistributionMapping &dm, amrex::Vector< amrex::MultiFab > &weather_forecast_data)
Definition: ERF_WeatherDataInterpolation.cpp:438
amrex::Vector< amrex::MultiFab > weather_forecast_data_2
Definition: ERF.H:154
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_2
Definition: ERF.H:156
amrex::Vector< amrex::Vector< amrex::MultiFab > > forecast_state_1
Definition: ERF.H:155
amrex::Vector< amrex::MultiFab > weather_forecast_data_1
Definition: ERF.H:154
void InterpWeatherDataOntoMesh(const amrex::Geometry &geom_weather, amrex::MultiFab &weather_forecast_interp, amrex::Vector< amrex::Vector< amrex::MultiFab >> &forecast_state)
Definition: ERF_WeatherDataInterpolation.cpp:268
std::string hindcast_boundary_data_dir
Definition: ERF_DataStruct.H:953

◆ Write2DPlotFile()

void ERF::Write2DPlotFile ( int  which,
PlotFileType  plotfile_type,
amrex::Vector< std::string >  plot_var_names 
)
1860 {
1861  const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
1862  const int ncomp_mf = varnames.size();
1863 
1864  if (ncomp_mf == 0) return;
1865 
1866  // Vector of MultiFabs for cell-centered data
1867  Vector<MultiFab> mf(finest_level+1);
1868  for (int lev = 0; lev <= finest_level; ++lev) {
1869  mf[lev].define(ba2d[lev], dmap[lev], ncomp_mf, 0);
1870  }
1871 
1872 
1873  // **********************************************************************************************
1874  // (Effectively) 2D arrays
1875  // **********************************************************************************************
1876  for (int lev = 0; lev <= finest_level; ++lev)
1877  {
1878  int mf_comp = 0;
1879 
1880  // Set all components to zero in case they aren't defined below
1881  mf[lev].setVal(0.0);
1882 
1883  if (containerHasElement(plot_var_names, "mapfac")) {
1884 #ifdef _OPENMP
1885 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1886 #endif
1887  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1888  {
1889  const Box& bx = mfi.tilebox();
1890  const Array4<Real>& derdat = mf[lev].array(mfi);
1891  const Array4<Real>& mf_m = mapfac[lev][MapFacType::m_x]->array(mfi);
1892  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1893  derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
1894  });
1895  }
1896  mf_comp++;
1897  }
1898 
1899  if (containerHasElement(plot_var_names, "lat_m")) {
1900  if (lat_m[lev]) {
1901 #ifdef _OPENMP
1902 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1903 #endif
1904  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1905  {
1906  const Box& bx = mfi.tilebox();
1907  const Array4<Real>& derdat = mf[lev].array(mfi);
1908  const Array4<Real>& data = lat_m[lev]->array(mfi);
1909  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1910  derdat(i, j, k, mf_comp) = data(i,j,0);
1911  });
1912  }
1913  }
1914  mf_comp++;
1915  } // lat_m
1916 
1917  if (containerHasElement(plot_var_names, "lon_m")) {
1918  if (lon_m[lev]) {
1919 #ifdef _OPENMP
1920 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1921 #endif
1922  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1923  {
1924  const Box& bx = mfi.tilebox();
1925  const Array4<Real>& derdat = mf[lev].array(mfi);
1926  const Array4<Real>& data = lon_m[lev]->array(mfi);
1927  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1928  derdat(i, j, k, mf_comp) = data(i,j,0);
1929  });
1930  }
1931  } else {
1932  mf[lev].setVal(0.0,mf_comp,1,0);
1933  }
1934 
1935  mf_comp++;
1936 
1937  } // lon_m
1938  }
1939 
1940  // mf[0].setVal(1.0);
1941 
1942  std::string plotfilename;
1943  if (which == 1) {
1944  plotfilename = Concatenate(plot2d_file_1, istep[0], 5);
1945  } else if (which == 2) {
1946  plotfilename = Concatenate(plot2d_file_2, istep[0], 5);
1947  }
1948 
1949  Vector<Geometry> my_geom(finest_level+1);
1950 
1951  Array<int,AMREX_SPACEDIM> is_per; is_per[0] = 0; is_per[1] = 0; is_per[2] = 0;
1952  if (geom[0].isPeriodic(0)) { is_per[0] = 1;}
1953  if (geom[0].isPeriodic(1)) { is_per[1] = 1;}
1954 
1955  int coord_sys = 0;
1956 
1957  for (int lev = 0; lev <= finest_level; lev++)
1958  {
1959  Box slab = makeSlab(geom[lev].Domain(),2,0);
1960  auto const slab_lo = lbound(slab);
1961  auto const slab_hi = ubound(slab);
1962 
1963  // Create a new geometry based only on the 2D slab
1964  // We need
1965  // 1) my_geom.Domain()
1966  // 2) my_geom.CellSize()
1967  // 3) my_geom.periodicity()
1968  const auto dx = geom[lev].CellSize();
1969  RealBox rb( slab_lo.x *dx[0], slab_lo.y *dx[1], slab_lo.z *dx[2],
1970  (slab_hi.x+1)*dx[0], (slab_hi.y+1)*dx[1], (slab_hi.z+1)*dx[2]);
1971  my_geom[lev].define(slab, rb, coord_sys, is_per);
1972  }
1973 
1974  if (plotfile_type == PlotFileType::Amrex)
1975  {
1976  Print() << "Writing 2D native plotfile " << plotfilename << "\n";
1977  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1978  GetVecOfConstPtrs(mf),
1979  varnames, my_geom, t_new[0], istep, refRatio());
1980  writeJobInfo(plotfilename);
1981 
1982 #ifdef ERF_USE_NETCDF
1983  } else if (plotfile_type == PlotFileType::Netcdf) {
1984  int lev = 0;
1985  int l_which = 0;
1986  const Real* p_lo = my_geom[lev].ProbLo();
1987  const Real* p_hi = my_geom[lev].ProbHi();
1988  const auto dx = my_geom[lev].CellSize();
1989  writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1990  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],dx[2]}, {dx[0],dx[1],dx[2]},
1991  my_geom[lev].Domain(), t_new[0], start_bdy_time);
1992 #endif
1993  } else {
1994  // Here we assume the plotfile_type is PlotFileType::None
1995  Print() << "Writing no 2D plotfile since plotfile_type is none" << std::endl;
1996  }
1997 }
void writeNCPlotFile(int lev, int which_subdomain, const std::string &dir, const Vector< const MultiFab * > &plotMF, const Vector< std::string > &plot_var_names, const Vector< int > &, Array< Real, AMREX_SPACEDIM > prob_lo, Array< Real, AMREX_SPACEDIM > prob_hi, Array< Real, AMREX_SPACEDIM > dx_in, const Box &subdomain, const Real time, const Real start_bdy_time)
Definition: ERF_NCPlotFile.cpp:14
static amrex::Vector< std::string > PlotFileVarNames(amrex::Vector< std::string > plot_var_names)
Definition: ERF_Plotfile.cpp:251
void writeJobInfo(const std::string &dir) const
Definition: ERF_WriteJobInfo.cpp:9
Here is the call graph for this function:

◆ Write3DPlotFile()

void ERF::Write3DPlotFile ( int  which,
PlotFileType  plotfile_type,
amrex::Vector< std::string >  plot_var_names 
)
264 {
265  const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
266  const int ncomp_mf = varnames.size();
267 
268  int ncomp_cons = vars_new[0][Vars::cons].nComp();
269 
270  if (ncomp_mf == 0) return;
271 
272  // We Fillpatch here because some of the derived quantities require derivatives
273  // which require ghost cells to be filled. We do not need to call FillPatcher
274  // because we don't need to set interior fine points.
275  // NOTE: the momenta here are only used as scratch space, the momenta themselves are not fillpatched
276 
277  // Level 0 FilLPatch
279  &vars_new[0][Vars::yvel], &vars_new[0][Vars::zvel]});
280 
281  for (int lev = 1; lev <= finest_level; ++lev) {
282  bool fillset = false;
283  FillPatchFineLevel(lev, t_new[lev], {&vars_new[lev][Vars::cons], &vars_new[lev][Vars::xvel],
284  &vars_new[lev][Vars::yvel], &vars_new[lev][Vars::zvel]},
285  {&vars_new[lev][Vars::cons], &rU_new[lev], &rV_new[lev], &rW_new[lev]},
286  base_state[lev], base_state[lev], fillset);
287  }
288 
289  // Get qmoist pointers if using moisture
290  bool use_moisture = (solverChoice.moisture_type != MoistureType::None);
291  for (int lev = 0; lev <= finest_level; ++lev) {
292  for (int mvar(0); mvar<qmoist[lev].size(); ++mvar) {
293  qmoist[lev][mvar] = micro->Get_Qmoist_Ptr(lev,mvar);
294  }
295  }
296 
297  // Vector of MultiFabs for cell-centered data
298  Vector<MultiFab> mf(finest_level+1);
299  for (int lev = 0; lev <= finest_level; ++lev) {
300  mf[lev].define(grids[lev], dmap[lev], ncomp_mf, 0);
301  }
302 
303  // Vector of MultiFabs for nodal data
304  Vector<MultiFab> mf_nd(finest_level+1);
305  if ( SolverChoice::mesh_type != MeshType::ConstantDz) {
306  for (int lev = 0; lev <= finest_level; ++lev) {
307  BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
308  mf_nd[lev].define(nodal_grids, dmap[lev], 3, 0);
309  mf_nd[lev].setVal(0.);
310  }
311  }
312 
313  // Vector of MultiFabs for face-centered velocity
314  Vector<MultiFab> mf_u(finest_level+1);
315  Vector<MultiFab> mf_v(finest_level+1);
316  Vector<MultiFab> mf_w(finest_level+1);
317  if (m_plot_face_vels) {
318  for (int lev = 0; lev <= finest_level; ++lev) {
319  BoxArray grid_stag_u(grids[lev]); grid_stag_u.surroundingNodes(0);
320  BoxArray grid_stag_v(grids[lev]); grid_stag_v.surroundingNodes(1);
321  BoxArray grid_stag_w(grids[lev]); grid_stag_w.surroundingNodes(2);
322  mf_u[lev].define(grid_stag_u, dmap[lev], 1, 0);
323  mf_v[lev].define(grid_stag_v, dmap[lev], 1, 0);
324  mf_w[lev].define(grid_stag_w, dmap[lev], 1, 0);
325  MultiFab::Copy(mf_u[lev],vars_new[lev][Vars::xvel],0,0,1,0);
326  MultiFab::Copy(mf_v[lev],vars_new[lev][Vars::yvel],0,0,1,0);
327  MultiFab::Copy(mf_w[lev],vars_new[lev][Vars::zvel],0,0,1,0);
328  }
329  }
330 
331  // Array of MultiFabs for cell-centered velocity
332  Vector<MultiFab> mf_cc_vel(finest_level+1);
333 
334  if (containerHasElement(plot_var_names, "x_velocity" ) ||
335  containerHasElement(plot_var_names, "y_velocity" ) ||
336  containerHasElement(plot_var_names, "z_velocity" ) ||
337  containerHasElement(plot_var_names, "magvel" ) ||
338  containerHasElement(plot_var_names, "vorticity_x") ||
339  containerHasElement(plot_var_names, "vorticity_y") ||
340  containerHasElement(plot_var_names, "vorticity_z") ) {
341 
342  for (int lev = 0; lev <= finest_level; ++lev) {
343  mf_cc_vel[lev].define(grids[lev], dmap[lev], AMREX_SPACEDIM, IntVect(1,1,1));
344  mf_cc_vel[lev].setVal(-1.e20);
345  average_face_to_cellcenter(mf_cc_vel[lev],0,
346  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],
347  &vars_new[lev][Vars::yvel],
348  &vars_new[lev][Vars::zvel]});
349  } // lev
350  } // if (vel or vort)
351 
352  // We need ghost cells if computing vorticity
353  if ( containerHasElement(plot_var_names, "vorticity_x")||
354  containerHasElement(plot_var_names, "vorticity_y") ||
355  containerHasElement(plot_var_names, "vorticity_z") )
356  {
357  amrex::Interpolater* mapper = &cell_cons_interp;
358  for (int lev = 1; lev <= finest_level; ++lev)
359  {
360  Vector<MultiFab*> fmf = {&(mf_cc_vel[lev]), &(mf_cc_vel[lev])};
361  Vector<Real> ftime = {t_new[lev], t_new[lev]};
362  Vector<MultiFab*> cmf = {&mf_cc_vel[lev-1], &mf_cc_vel[lev-1]};
363  Vector<Real> ctime = {t_new[lev], t_new[lev]};
364 
365  FillBdyCCVels(mf_cc_vel,lev-1);
366 
367  // Call FillPatch which ASSUMES that all ghost cells at lev-1 have already been filled
368  FillPatchTwoLevels(mf_cc_vel[lev], mf_cc_vel[lev].nGrowVect(), IntVect(0,0,0),
369  t_new[lev], cmf, ctime, fmf, ftime,
370  0, 0, mf_cc_vel[lev].nComp(), geom[lev-1], geom[lev],
371  refRatio(lev-1), mapper, domain_bcs_type,
373  } // lev
374  FillBdyCCVels(mf_cc_vel);
375  } // if (vort)
376 
377 
378  for (int lev = 0; lev <= finest_level; ++lev)
379  {
380  int mf_comp = 0;
381 
382  BoxArray ba(vars_new[lev][Vars::cons].boxArray());
383  DistributionMapping dm = vars_new[lev][Vars::cons].DistributionMap();
384 
385  // First, copy any of the conserved state variables into the output plotfile
386  for (int i = 0; i < cons_names.size(); ++i) {
387  if (containerHasElement(plot_var_names, cons_names[i])) {
388  MultiFab::Copy(mf[lev],vars_new[lev][Vars::cons],i,mf_comp,1,0);
389  mf_comp++;
390  }
391  }
392 
393  // Next, check for velocities
394  if (containerHasElement(plot_var_names, "x_velocity")) {
395  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 0, mf_comp, 1, 0);
396  mf_comp += 1;
397  }
398  if (containerHasElement(plot_var_names, "y_velocity")) {
399  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 1, mf_comp, 1, 0);
400  mf_comp += 1;
401  }
402  if (containerHasElement(plot_var_names, "z_velocity")) {
403  MultiFab::Copy(mf[lev], mf_cc_vel[lev], 2, mf_comp, 1, 0);
404  mf_comp += 1;
405  }
406 
407  // Create multifabs for HSE and pressure fields used to derive other quantities
408  MultiFab r_hse(base_state[lev], make_alias, BaseState::r0_comp , 1);
409  MultiFab p_hse(base_state[lev], make_alias, BaseState::p0_comp , 1);
410  MultiFab th_hse(base_state[lev], make_alias, BaseState::th0_comp, 1);
411 
412  MultiFab pressure;
413 
414  if (solverChoice.anelastic[lev] == 0) {
415  if (containerHasElement(plot_var_names, "pressure") ||
416  containerHasElement(plot_var_names, "pert_pres") ||
417  containerHasElement(plot_var_names, "dpdx") ||
418  containerHasElement(plot_var_names, "dpdy") ||
419  containerHasElement(plot_var_names, "dpdz") ||
420  containerHasElement(plot_var_names, "eq_pot_temp") ||
421  containerHasElement(plot_var_names, "qsat"))
422  {
423  int ng = (containerHasElement(plot_var_names, "dpdx") || containerHasElement(plot_var_names, "dpdy") ||
424  containerHasElement(plot_var_names, "dpdz")) ? 1 : 0;
425 
426  // Allocate space for pressure
427  pressure.define(ba,dm,1,ng);
428 
429  if (ng > 0) {
430  // Default to p_hse as a way of filling ghost cells at domain boundaries
431  MultiFab::Copy(pressure,p_hse,0,0,1,1);
432  }
433 #ifdef _OPENMP
434 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
435 #endif
436  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
437  {
438  const Box& gbx = mfi.growntilebox(IntVect(ng,ng,0));
439 
440  const Array4<Real >& p_arr = pressure.array(mfi);
441  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
442  const int ncomp = vars_new[lev][Vars::cons].nComp();
443 
444  ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
445  {
446  Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0;
447  const Real rhotheta = S_arr(i,j,k,RhoTheta_comp);
448  p_arr(i, j, k) = getPgivenRTh(rhotheta,qv_for_p);
449  });
450  } // mfi
451  pressure.FillBoundary(geom[lev].periodicity());
452  } // compute compressible pressure
453  } // not anelastic
454  else {
455  if (containerHasElement(plot_var_names, "dpdx") ||
456  containerHasElement(plot_var_names, "dpdy") ||
457  containerHasElement(plot_var_names, "dpdz") ||
458  containerHasElement(plot_var_names, "eq_pot_temp") ||
459  containerHasElement(plot_var_names, "qsat"))
460  {
461  // Copy p_hse into pressure if using anelastic
462  pressure.define(ba,dm,1,0);
463  MultiFab::Copy(pressure,p_hse,0,0,1,0);
464  }
465  }
466 
467  // Finally, check for any derived quantities and compute them, inserting
468  // them into our output multifab
469  auto calculate_derived = [&](const std::string& der_name,
470  MultiFab& src_mf,
471  decltype(derived::erf_dernull)& der_function)
472  {
473  if (containerHasElement(plot_var_names, der_name)) {
474  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
475 #ifdef _OPENMP
476 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
477 #endif
478  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
479  {
480  const Box& bx = mfi.tilebox();
481  auto& dfab = dmf[mfi];
482  auto& sfab = src_mf[mfi];
483  der_function(bx, dfab, 0, 1, sfab, Geom(lev), t_new[0], nullptr, lev);
484  }
485 
486  mf_comp++;
487  }
488  };
489 
490  // *****************************************************************************************
491  // NOTE: All derived variables computed below **MUST MATCH THE ORDER** of "derived_names"
492  // defined in ERF.H
493  // *****************************************************************************************
494 
495  calculate_derived("soundspeed", vars_new[lev][Vars::cons], derived::erf_dersoundspeed);
496  if (use_moisture) {
497  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dermoisttemp);
498  } else {
499  calculate_derived("temp", vars_new[lev][Vars::cons], derived::erf_dertemp);
500  }
501  calculate_derived("theta", vars_new[lev][Vars::cons], derived::erf_dertheta);
502  calculate_derived("KE", vars_new[lev][Vars::cons], derived::erf_derKE);
503  calculate_derived("scalar", vars_new[lev][Vars::cons], derived::erf_derscalar);
504  calculate_derived("vorticity_x", mf_cc_vel[lev] , derived::erf_dervortx);
505  calculate_derived("vorticity_y", mf_cc_vel[lev] , derived::erf_dervorty);
506  calculate_derived("vorticity_z", mf_cc_vel[lev] , derived::erf_dervortz);
507  calculate_derived("magvel" , mf_cc_vel[lev] , derived::erf_dermagvel);
508 
509  if (containerHasElement(plot_var_names, "divU"))
510  {
511  // TODO TODO TODO -- we need to convert w to omega here!!
512  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
513  Array<MultiFab const*, AMREX_SPACEDIM> u;
514  u[0] = &(vars_new[lev][Vars::xvel]);
515  u[1] = &(vars_new[lev][Vars::yvel]);
516  u[2] = &(vars_new[lev][Vars::zvel]);
517  compute_divergence (lev, dmf, u, geom[lev]);
518  mf_comp += 1;
519  }
520 
521  if (containerHasElement(plot_var_names, "pres_hse"))
522  {
523  MultiFab::Copy(mf[lev],p_hse,0,mf_comp,1,0);
524  mf_comp += 1;
525  }
526  if (containerHasElement(plot_var_names, "dens_hse"))
527  {
528  MultiFab::Copy(mf[lev],r_hse,0,mf_comp,1,0);
529  mf_comp += 1;
530  }
531  if (containerHasElement(plot_var_names, "theta_hse"))
532  {
533  MultiFab::Copy(mf[lev],th_hse,0,mf_comp,1,0);
534  mf_comp += 1;
535  }
536 
537  if (containerHasElement(plot_var_names, "pressure"))
538  {
539  if (solverChoice.anelastic[lev] == 1) {
540  MultiFab::Copy(mf[lev], p_hse, 0, mf_comp, 1, 0);
541  } else {
542  MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
543  }
544 
545  mf_comp += 1;
546  }
547 
548  if (containerHasElement(plot_var_names, "pert_pres"))
549  {
550  if (solverChoice.anelastic[lev] == 1) {
551  MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0);
552  } else {
553  MultiFab::Copy(mf[lev], pressure, 0, mf_comp, 1, 0);
554  MultiFab::Subtract(mf[lev],p_hse,0,mf_comp,1,IntVect{0});
555  }
556  mf_comp += 1;
557  }
558 
559  if (containerHasElement(plot_var_names, "pert_dens"))
560  {
561 #ifdef _OPENMP
562 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
563 #endif
564  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
565  {
566  const Box& bx = mfi.tilebox();
567  const Array4<Real>& derdat = mf[lev].array(mfi);
568  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
569  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
570  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
571  derdat(i, j, k, mf_comp) = S_arr(i,j,k,Rho_comp) - r0_arr(i,j,k);
572  });
573  }
574  mf_comp ++;
575  }
576 
577  if (containerHasElement(plot_var_names, "eq_pot_temp"))
578  {
579 #ifdef _OPENMP
580 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
581 #endif
582  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
583  {
584  const Box& bx = mfi.tilebox();
585  const Array4<Real>& derdat = mf[lev].array(mfi);
586  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
587  const Array4<Real const>& p_arr = pressure.const_array(mfi);
588  const int ncomp = vars_new[lev][Vars::cons].nComp();
589  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
590  Real qv = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
591  Real qc = (use_moisture && (ncomp > RhoQ2_comp)) ? S_arr(i,j,k,RhoQ2_comp)/S_arr(i,j,k,Rho_comp) : 0.0;
592  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
593  Real fac = Cp_d + Cp_l*(qv + qc);
594  Real pv = erf_esatw(T)*100.0;
595 
596  derdat(i, j, k, mf_comp) = T*std::pow((p_arr(i,j,k) - pv)/p_0, -R_d/fac)*std::exp(L_v*qv/(fac*T)) ;
597  });
598  }
599  mf_comp ++;
600  }
601 
602 #ifdef ERF_USE_WINDFARM
603  if ( containerHasElement(plot_var_names, "num_turb") and
604  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
605  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
606  {
607  MultiFab::Copy(mf[lev],Nturb[lev],0,mf_comp,1,0);
608  mf_comp ++;
609  }
610 
611  if ( containerHasElement(plot_var_names, "SMark0") and
612  (solverChoice.windfarm_type == WindFarmType::Fitch or solverChoice.windfarm_type == WindFarmType::EWP or
613  solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD) )
614  {
615  MultiFab::Copy(mf[lev],SMark[lev],0,mf_comp,1,0);
616  mf_comp ++;
617  }
618 
619  if (containerHasElement(plot_var_names, "SMark1") and
620  (solverChoice.windfarm_type == WindFarmType::SimpleAD or solverChoice.windfarm_type == WindFarmType::GeneralAD))
621  {
622  MultiFab::Copy(mf[lev],SMark[lev],1,mf_comp,1,0);
623  mf_comp ++;
624  }
625 #endif
626 
627  // **********************************************************************************************
628  // Allocate space if we are computing any pressure gradients
629  // **********************************************************************************************
630 
631  Vector<MultiFab> gradp_temp; gradp_temp.resize(AMREX_SPACEDIM);
632  if (containerHasElement(plot_var_names, "dpdx") ||
633  containerHasElement(plot_var_names, "dpdy") ||
634  containerHasElement(plot_var_names, "dpdz") ||
635  containerHasElement(plot_var_names, "pres_hse_x") ||
636  containerHasElement(plot_var_names, "pres_hse_y"))
637  {
638  gradp_temp[GpVars::gpx].define(convert(ba, IntVect(1,0,0)), dm, 1, 1); gradp_temp[GpVars::gpx].setVal(0.);
639  gradp_temp[GpVars::gpy].define(convert(ba, IntVect(0,1,0)), dm, 1, 1); gradp_temp[GpVars::gpy].setVal(0.);
640  gradp_temp[GpVars::gpz].define(convert(ba, IntVect(0,0,1)), dm, 1, 1); gradp_temp[GpVars::gpz].setVal(0.);
641  }
642 
643  // **********************************************************************************************
644  // These are based on computing gradient of full pressure
645  // **********************************************************************************************
646 
647  if (solverChoice.anelastic[lev] == 0) {
648  if ( (containerHasElement(plot_var_names, "dpdx")) ||
649  (containerHasElement(plot_var_names, "dpdy")) ||
650  (containerHasElement(plot_var_names, "dpdz")) ) {
651  compute_gradp(pressure, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), get_eb(lev), gradp_temp, solverChoice);
652  }
653  }
654 
655  if (containerHasElement(plot_var_names, "dpdx"))
656  {
657  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
658  {
659  const Box& bx = mfi.tilebox();
660  const Array4<Real >& derdat = mf[lev].array(mfi);
661  const Array4<Real const>& gpx_arr = (solverChoice.anelastic[lev] == 1) ?
662  gradp[lev][GpVars::gpx].array(mfi) : gradp_temp[GpVars::gpx].array(mfi);
663  const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
664  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
665  derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
666  });
667  }
668  mf_comp ++;
669  } // dpdx
670  if (containerHasElement(plot_var_names, "dpdy"))
671  {
672  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
673  {
674  const Box& bx = mfi.tilebox();
675  const Array4<Real >& derdat = mf[lev].array(mfi);
676  const Array4<Real const>& gpy_arr = (solverChoice.anelastic[lev] == 1) ?
677  gradp[lev][GpVars::gpy].array(mfi) : gradp_temp[GpVars::gpy].array(mfi);
678  const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
679  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
680  derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
681  });
682  }
683  mf_comp ++;
684  } // dpdy
685  if (containerHasElement(plot_var_names, "dpdz"))
686  {
687  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
688  {
689  const Box& bx = mfi.tilebox();
690  const Array4<Real >& derdat = mf[lev].array(mfi);
691  const Array4<Real const>& gpz_arr = (solverChoice.anelastic[lev] == 1) ?
692  gradp[lev][GpVars::gpz].array(mfi) : gradp_temp[GpVars::gpz].array(mfi);
693  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
694  derdat(i ,j ,k, mf_comp) = 0.5 * (gpz_arr(i,j,k+1) + gpz_arr(i,j,k));
695  });
696  }
697  mf_comp ++;
698  } // dpdz
699 
700  // **********************************************************************************************
701  // These are based on computing gradient of basestate pressure
702  // **********************************************************************************************
703 
704  if ( (containerHasElement(plot_var_names, "pres_hse_x")) ||
705  (containerHasElement(plot_var_names, "pres_hse_y")) ) {
706  compute_gradp(p_hse, geom[lev], *z_phys_nd[lev].get(), *z_phys_cc[lev].get(), get_eb(lev), gradp_temp, solverChoice);
707  }
708 
709  if (containerHasElement(plot_var_names, "pres_hse_x"))
710  {
711  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
712  {
713  const Box& bx = mfi.tilebox();
714  const Array4<Real >& derdat = mf[lev].array(mfi);
715  const Array4<Real const>& gpx_arr = gradp_temp[0].array(mfi);
716  const Array4<Real const>& mf_mx_arr = mapfac[lev][MapFacType::m_x]->const_array(mfi);
717  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
718  derdat(i ,j ,k, mf_comp) = 0.5 * (gpx_arr(i+1,j,k) + gpx_arr(i,j,k)) * mf_mx_arr(i,j,0);
719  });
720  }
721  mf_comp += 1;
722  } // pres_hse_x
723 
724  if (containerHasElement(plot_var_names, "pres_hse_y"))
725  {
726  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
727  {
728  const Box& bx = mfi.tilebox();
729  const Array4<Real >& derdat = mf[lev].array(mfi);
730  const Array4<Real const>& gpy_arr = gradp_temp[1].array(mfi);
731  const Array4<Real const>& mf_my_arr = mapfac[lev][MapFacType::m_y]->const_array(mfi);
732  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
733  derdat(i ,j ,k, mf_comp) = 0.5 * (gpy_arr(i,j+1,k) + gpy_arr(i,j,k)) * mf_my_arr(i,j,0);
734  });
735  }
736  mf_comp += 1;
737  } // pres_hse_y
738 
739  // **********************************************************************************************
740  // Metric terms
741  // **********************************************************************************************
742 
743  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
744  if (containerHasElement(plot_var_names, "z_phys"))
745  {
746  MultiFab::Copy(mf[lev],*z_phys_cc[lev],0,mf_comp,1,0);
747  mf_comp ++;
748  }
749 
750  if (containerHasElement(plot_var_names, "detJ"))
751  {
752  MultiFab::Copy(mf[lev],*detJ_cc[lev],0,mf_comp,1,0);
753  mf_comp ++;
754  }
755  } // use_terrain
756 
757  if (containerHasElement(plot_var_names, "mapfac")) {
758  amrex::Print() << "You are plotting a 3D version of mapfac; we suggest using the 2D plotfile instead" << std::endl;
759 #ifdef _OPENMP
760 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
761 #endif
762  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
763  {
764  const Box& bx = mfi.tilebox();
765  const Array4<Real>& derdat = mf[lev].array(mfi);
766  const Array4<Real>& mf_m = mapfac[lev][MapFacType::m_x]->array(mfi);
767  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
768  derdat(i ,j ,k, mf_comp) = mf_m(i,j,0);
769  });
770  }
771  mf_comp ++;
772  }
773 
774  if (containerHasElement(plot_var_names, "lat_m")) {
775  amrex::Print() << "You are plotting a 3D version of lat_m; we suggest using the 2D plotfile instead" << std::endl;
776  if (lat_m[lev]) {
777 #ifdef _OPENMP
778 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
779 #endif
780  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
781  {
782  const Box& bx = mfi.tilebox();
783  const Array4<Real>& derdat = mf[lev].array(mfi);
784  const Array4<Real>& data = lat_m[lev]->array(mfi);
785  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
786  derdat(i, j, k, mf_comp) = data(i,j,0);
787  });
788  }
789  } else {
790  mf[lev].setVal(0.0,mf_comp,1,0);
791  }
792  mf_comp++;
793  } // lat_m
794 
795  if (containerHasElement(plot_var_names, "lon_m")) {
796  amrex::Print() << "You are plotting a 3D version of lon_m; we suggest using the 2D plotfile instead" << std::endl;
797  if (lon_m[lev]) {
798 #ifdef _OPENMP
799 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
800 #endif
801  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
802  {
803  const Box& bx = mfi.tilebox();
804  const Array4<Real>& derdat = mf[lev].array(mfi);
805  const Array4<Real>& data = lon_m[lev]->array(mfi);
806  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
807  derdat(i, j, k, mf_comp) = data(i,j,0);
808  });
809  }
810  } else {
811  mf[lev].setVal(0.0,mf_comp,1,0);
812  }
813  mf_comp++;
814  } // lon_m
815 
817  if (containerHasElement(plot_var_names, "u_t_avg")) {
818 #ifdef _OPENMP
819 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
820 #endif
821  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
822  {
823  const Box& bx = mfi.tilebox();
824  const Array4<Real>& derdat = mf[lev].array(mfi);
825  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
826  const Real norm = t_avg_cnt[lev];
827  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
828  {
829  derdat(i ,j ,k, mf_comp) = data(i,j,k,0) / norm;
830  });
831  }
832  mf_comp ++;
833  }
834 
835  if (containerHasElement(plot_var_names, "v_t_avg")) {
836 #ifdef _OPENMP
837 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
838 #endif
839  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
840  {
841  const Box& bx = mfi.tilebox();
842  const Array4<Real>& derdat = mf[lev].array(mfi);
843  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
844  const Real norm = t_avg_cnt[lev];
845  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
846  {
847  derdat(i ,j ,k, mf_comp) = data(i,j,k,1) / norm;
848  });
849  }
850  mf_comp ++;
851  }
852 
853  if (containerHasElement(plot_var_names, "w_t_avg")) {
854 #ifdef _OPENMP
855 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
856 #endif
857  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
858  {
859  const Box& bx = mfi.tilebox();
860  const Array4<Real>& derdat = mf[lev].array(mfi);
861  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
862  const Real norm = t_avg_cnt[lev];
863  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
864  {
865  derdat(i ,j ,k, mf_comp) = data(i,j,k,2) / norm;
866  });
867  }
868  mf_comp ++;
869  }
870 
871  if (containerHasElement(plot_var_names, "umag_t_avg")) {
872 #ifdef _OPENMP
873 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
874 #endif
875  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
876  {
877  const Box& bx = mfi.tilebox();
878  const Array4<Real>& derdat = mf[lev].array(mfi);
879  const Array4<Real>& data = vel_t_avg[lev]->array(mfi);
880  const Real norm = t_avg_cnt[lev];
881  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
882  {
883  derdat(i ,j ,k, mf_comp) = data(i,j,k,3) / norm;
884  });
885  }
886  mf_comp ++;
887  }
888  }
889 
890  if (containerHasElement(plot_var_names, "nut")) {
891  MultiFab dmf(mf[lev], make_alias, mf_comp, 1);
892  MultiFab cmf(vars_new[lev][Vars::cons], make_alias, 0, 1); // to provide rho only
893 #ifdef _OPENMP
894 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
895 #endif
896  for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi)
897  {
898  const Box& bx = mfi.tilebox();
899  auto prim = dmf[mfi].array();
900  auto const cons = cmf[mfi].const_array();
901  auto const diff = (*eddyDiffs_lev[lev])[mfi].const_array();
902  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
903  {
904  const Real rho = cons(i, j, k, Rho_comp);
905  const Real Kmv = diff(i, j, k, EddyDiff::Mom_v);
906  prim(i,j,k) = Kmv / rho;
907  });
908  }
909 
910  mf_comp++;
911  }
912 
913  if (containerHasElement(plot_var_names, "Kmv")) {
914  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_v,mf_comp,1,0);
915  mf_comp ++;
916  }
917  if (containerHasElement(plot_var_names, "Kmh")) {
918  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Mom_h,mf_comp,1,0);
919  mf_comp ++;
920  }
921  if (containerHasElement(plot_var_names, "Khv")) {
922  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_v,mf_comp,1,0);
923  mf_comp ++;
924  }
925  if (containerHasElement(plot_var_names, "Khh")) {
926  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Theta_h,mf_comp,1,0);
927  mf_comp ++;
928  }
929  if (containerHasElement(plot_var_names, "Lturb")) {
930  MultiFab::Copy(mf[lev],*eddyDiffs_lev[lev],EddyDiff::Turb_lengthscale,mf_comp,1,0);
931  mf_comp ++;
932  }
933  if (containerHasElement(plot_var_names, "walldist")) {
934  MultiFab::Copy(mf[lev],*walldist[lev],0,mf_comp,1,0);
935  mf_comp ++;
936  }
937  if (containerHasElement(plot_var_names, "diss")) {
938  MultiFab::Copy(mf[lev],*SFS_diss_lev[lev],0,mf_comp,1,0);
939  mf_comp ++;
940  }
941 
942  // TODO: The size of the q variables can vary with different
943  // moisture models. Therefore, certain components may
944  // reside at different indices. For example, Kessler is
945  // warm but precipitating. This puts qp at index 3.
946  // However, SAM is cold and precipitating so qp is index 4.
947  // Need to built an external enum struct or a better pathway.
948 
949  // NOTE: Protect against accessing non-existent data
950  if (use_moisture) {
951  int n_qstate_moist = micro->Get_Qstate_Moist_Size();
952 
953  // Moist density
954  if(containerHasElement(plot_var_names, "moist_density"))
955  {
956  int n_start = RhoQ1_comp; // qv
957  int n_end = RhoQ2_comp; // qc
958  if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
959  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
960  for (int n_comp(n_start); n_comp <= n_end; ++n_comp) {
961  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
962  }
963  mf_comp += 1;
964  }
965 
966  if(containerHasElement(plot_var_names, "qv") && (n_qstate_moist >= 1))
967  {
968  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ1_comp, mf_comp, 1, 0);
969  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
970  mf_comp += 1;
971  }
972 
973  if(containerHasElement(plot_var_names, "qc") && (n_qstate_moist >= 2))
974  {
975  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ2_comp, mf_comp, 1, 0);
976  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
977  mf_comp += 1;
978  }
979 
980  if(containerHasElement(plot_var_names, "qi") && (n_qstate_moist >= 4))
981  {
982  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ3_comp, mf_comp, 1, 0);
983  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
984  mf_comp += 1;
985  }
986 
987  if(containerHasElement(plot_var_names, "qrain") && (n_qstate_moist >= 3))
988  {
989  int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
990  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start , mf_comp, 1, 0);
991  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
992  mf_comp += 1;
993  }
994 
995  if(containerHasElement(plot_var_names, "qsnow") && (n_qstate_moist >= 5))
996  {
997  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ5_comp, mf_comp, 1, 0);
998  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
999  mf_comp += 1;
1000  }
1001 
1002  if(containerHasElement(plot_var_names, "qgraup") && (n_qstate_moist >= 6))
1003  {
1004  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], RhoQ6_comp, mf_comp, 1, 0);
1005  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp, mf_comp, 1, 0);
1006  mf_comp += 1;
1007  }
1008 
1009  // Precipitating + non-precipitating components
1010  //--------------------------------------------------------------------------
1011  if(containerHasElement(plot_var_names, "qt"))
1012  {
1013  int n_start = RhoQ1_comp; // qv
1014  int n_end = n_start + n_qstate_moist;
1015  MultiFab::Copy(mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1016  for (int n_comp(n_start+1); n_comp < n_end; ++n_comp) {
1017  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1018  }
1019  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1020  mf_comp += 1;
1021  }
1022 
1023  // Non-precipitating components
1024  //--------------------------------------------------------------------------
1025  if (containerHasElement(plot_var_names, "qn"))
1026  {
1027  int n_start = RhoQ1_comp; // qv
1028  int n_end = RhoQ2_comp; // qc
1029  if (n_qstate_moist > 3) n_end = RhoQ3_comp; // qi
1030  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1031  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1032  MultiFab::Add(mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1033  }
1034  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1035  mf_comp += 1;
1036  }
1037 
1038  // Precipitating components
1039  //--------------------------------------------------------------------------
1040  if(containerHasElement(plot_var_names, "qp") && (n_qstate_moist >= 3))
1041  {
1042  int n_start = (n_qstate_moist > 3) ? RhoQ4_comp : RhoQ3_comp;
1043  int n_end = ncomp_cons - 1;
1044  MultiFab::Copy( mf[lev], vars_new[lev][Vars::cons], n_start, mf_comp, 1, 0);
1045  for (int n_comp(n_start+1); n_comp <= n_end; ++n_comp) {
1046  MultiFab::Add( mf[lev], vars_new[lev][Vars::cons], n_comp, mf_comp, 1, 0);
1047  }
1048  MultiFab::Divide(mf[lev], vars_new[lev][Vars::cons], Rho_comp , mf_comp, 1, 0);
1049  mf_comp += 1;
1050  }
1051 
1052  if (containerHasElement(plot_var_names, "qsat"))
1053  {
1054 #ifdef _OPENMP
1055 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1056 #endif
1057  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1058  {
1059  const Box& bx = mfi.tilebox();
1060  const Array4<Real>& derdat = mf[lev].array(mfi);
1061  const Array4<Real const>& p_arr = pressure.array(mfi);
1062  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1063  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1064  {
1065  Real qv = S_arr(i,j,k,RhoQ1_comp) / S_arr(i,j,k,Rho_comp);
1066  Real T = getTgivenRandRTh(S_arr(i,j,k,Rho_comp), S_arr(i,j,k,RhoTheta_comp), qv);
1067  Real p = p_arr(i,j,k) * Real(0.01);
1068  erf_qsatw(T, p, derdat(i,j,k,mf_comp));
1069  });
1070  }
1071  mf_comp ++;
1072  }
1073 
1074  if ( (solverChoice.moisture_type == MoistureType::Kessler) ||
1075  (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ||
1076  (solverChoice.moisture_type == MoistureType::SAM_NoIce) )
1077  {
1078  int offset = (solverChoice.moisture_type == MoistureType::Morrison_NoIce) ? 5 : 0;
1079  if (containerHasElement(plot_var_names, "rain_accum"))
1080  {
1081  MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
1082  mf_comp += 1;
1083  }
1084  }
1085  else if ( (solverChoice.moisture_type == MoistureType::SAM) ||
1086  (solverChoice.moisture_type == MoistureType::Morrison) )
1087  {
1088  int offset = (solverChoice.moisture_type == MoistureType::Morrison) ? 5 : 0;
1089  if (containerHasElement(plot_var_names, "rain_accum"))
1090  {
1091  MultiFab::Copy(mf[lev],*(qmoist[lev][offset]),0,mf_comp,1,0);
1092  mf_comp += 1;
1093  }
1094  if (containerHasElement(plot_var_names, "snow_accum"))
1095  {
1096  MultiFab::Copy(mf[lev],*(qmoist[lev][offset+1]),0,mf_comp,1,0);
1097  mf_comp += 1;
1098  }
1099  if (containerHasElement(plot_var_names, "graup_accum"))
1100  {
1101  MultiFab::Copy(mf[lev],*(qmoist[lev][offset+2]),0,mf_comp,1,0);
1102  mf_comp += 1;
1103  }
1104  }
1105 
1106  if (containerHasElement(plot_var_names, "reflectivity")) {
1107  if (solverChoice.moisture_type == MoistureType::Morrison) {
1108 
1109  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1110  const Box& bx = mfi.tilebox();
1111  const Array4<Real>& derdat = mf[lev].array(mfi);
1112  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1113  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
1114 
1115  Real rho = S_arr(i,j,k,Rho_comp);
1116  Real qv = std::max(0.0,S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp));
1117  Real qpr = std::max(0.0,S_arr(i,j,k,RhoQ4_comp)/S_arr(i,j,k,Rho_comp));
1118  Real qps = std::max(0.0,S_arr(i,j,k,RhoQ5_comp)/S_arr(i,j,k,Rho_comp));
1119  Real qpg = std::max(0.0,S_arr(i,j,k,RhoQ6_comp)/S_arr(i,j,k,Rho_comp));
1120 
1121  Real temp = getTgivenRandRTh(S_arr(i,j,k,Rho_comp),
1122  S_arr(i,j,k,RhoTheta_comp),
1123  qv);
1124  derdat(i, j, k, mf_comp) = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
1125  1, 1, 1, 1) ;
1126  });
1127  }
1128  mf_comp ++;
1129  }
1130  }
1131 
1132  if (solverChoice.moisture_type == MoistureType::Morrison) {
1133  if (containerHasElement(plot_var_names, "max_reflectivity")) {
1134  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1135  const Box& bx = mfi.tilebox();
1136 
1137  const Array4<Real>& derdat = mf[lev].array(mfi);
1138  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1139 
1140  // collapse to i,j box (ignore vertical for now)
1141  Box b2d = bx;
1142  b2d.setSmall(2,0);
1143  b2d.setBig(2,0);
1144 
1145  ParallelFor(b2d, [=] AMREX_GPU_DEVICE(int i, int j, int) noexcept {
1146 
1147  Real max_dbz = -1.0e30;
1148 
1149  // find max reflectivity over k
1150  for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
1151  Real rho = S_arr(i,j,k,Rho_comp);
1152  Real qv = std::max(0.0, S_arr(i,j,k,RhoQ1_comp)/rho);
1153  Real qpr = std::max(0.0, S_arr(i,j,k,RhoQ4_comp)/rho);
1154  Real qps = std::max(0.0, S_arr(i,j,k,RhoQ5_comp)/rho);
1155  Real qpg = std::max(0.0, S_arr(i,j,k,RhoQ6_comp)/rho);
1156 
1157  Real temp = getTgivenRandRTh(rho, S_arr(i,j,k,RhoTheta_comp), qv);
1158 
1159  Real dbz = compute_max_reflectivity_dbz(rho, temp, qpr, qps, qpg,
1160  1, 1, 1, 1);
1161  max_dbz = amrex::max(max_dbz, dbz);
1162  }
1163 
1164  // store max_dbz into *all* levels for this (i,j)
1165  for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
1166  derdat(i, j, k, mf_comp) = max_dbz;
1167  }
1168  });
1169  }
1170  mf_comp++;
1171  }
1172  }
1173  } // use_moisture
1174 
1175  if (containerHasElement(plot_var_names, "terrain_IB_mask"))
1176  {
1177  MultiFab* terrain_blank = terrain_blanking[lev].get();
1178  MultiFab::Copy(mf[lev],*terrain_blank,0,mf_comp,1,0);
1179  mf_comp ++;
1180  }
1181 
1182  if (containerHasElement(plot_var_names, "volfrac")) {
1183  if ( solverChoice.terrain_type == TerrainType::EB ||
1184  solverChoice.terrain_type == TerrainType::ImmersedForcing)
1185  {
1186  MultiFab::Copy(mf[lev], EBFactory(lev).getVolFrac(), 0, mf_comp, 1, 0);
1187  } else {
1188  mf[lev].setVal(1.0, mf_comp, 1, 0);
1189  }
1190  mf_comp += 1;
1191  }
1192 
1193 #ifdef ERF_COMPUTE_ERROR
1194  // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some
1195  if (containerHasElement(plot_var_names, "xvel_err") ||
1196  containerHasElement(plot_var_names, "yvel_err") ||
1197  containerHasElement(plot_var_names, "zvel_err"))
1198  {
1199  //
1200  // Moving terrain ANALYTICAL
1201  //
1202  Real H = geom[lev].ProbHi()[2];
1203  Real Ampl = 0.16;
1204  Real wavelength = 100.;
1205  Real kp = 2. * PI / wavelength;
1206  Real g = CONST_GRAV;
1207  Real omega = std::sqrt(g * kp);
1208  Real omega_t = omega * t_new[lev];
1209 
1210  const auto dx = geom[lev].CellSizeArray();
1211 
1212 #ifdef _OPENMP
1213 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1214 #endif
1215  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1216  {
1217  const Box& bx = mfi.validbox();
1218  Box xbx(bx); xbx.surroundingNodes(0);
1219  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1220  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1221 
1222  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1223 
1224  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1225  {
1226  Real x = i * dx[0];
1227  Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1228 
1229  Real z_base = Ampl * std::sin(kp * x - omega_t);
1230  z -= z_base;
1231 
1232  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1233 
1234  xvel_arr(i,j,k) -= -Ampl * omega * fac * std::sin(kp * x - omega_t);
1235  });
1236 
1237  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1238  {
1239  Real x = (i + 0.5) * dx[0];
1240  Real z = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1241 
1242  Real z_base = Ampl * std::sin(kp * x - omega_t);
1243  z -= z_base;
1244 
1245  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1246 
1247  zvel_arr(i,j,k) -= Ampl * omega * fac * std::cos(kp * x - omega_t);
1248  });
1249  }
1250 
1251  MultiFab temp_mf(mf[lev].boxArray(), mf[lev].DistributionMap(), AMREX_SPACEDIM, 0);
1252  average_face_to_cellcenter(temp_mf,0,
1253  Array<const MultiFab*,3>{&vars_new[lev][Vars::xvel],&vars_new[lev][Vars::yvel],&vars_new[lev][Vars::zvel]});
1254 
1255  if (containerHasElement(plot_var_names, "xvel_err")) {
1256  MultiFab::Copy(mf[lev],temp_mf,0,mf_comp,1,0);
1257  mf_comp += 1;
1258  }
1259  if (containerHasElement(plot_var_names, "yvel_err")) {
1260  MultiFab::Copy(mf[lev],temp_mf,1,mf_comp,1,0);
1261  mf_comp += 1;
1262  }
1263  if (containerHasElement(plot_var_names, "zvel_err")) {
1264  MultiFab::Copy(mf[lev],temp_mf,2,mf_comp,1,0);
1265  mf_comp += 1;
1266  }
1267 
1268  // Now restore the velocities to what they were
1269 #ifdef _OPENMP
1270 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1271 #endif
1272  for (MFIter mfi(mf[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
1273  {
1274  const Box& bx = mfi.validbox();
1275  Box xbx(bx); xbx.surroundingNodes(0);
1276 
1277  const Array4<Real> xvel_arr = vars_new[lev][Vars::xvel].array(mfi);
1278  const Array4<Real> zvel_arr = vars_new[lev][Vars::zvel].array(mfi);
1279 
1280  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1281 
1282  ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1283  {
1284  Real x = i * dx[0];
1285  Real z = 0.25 * (z_nd(i,j,k) + z_nd(i,j+1,k) + z_nd(i,j,k+1) + z_nd(i,j+1,k+1));
1286  Real z_base = Ampl * std::sin(kp * x - omega_t);
1287 
1288  z -= z_base;
1289 
1290  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1291  xvel_arr(i,j,k) += -Ampl * omega * fac * std::sin(kp * x - omega_t);
1292  });
1293  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1294  {
1295  Real x = (i + 0.5) * dx[0];
1296  Real z = 0.25 * ( z_nd(i,j,k) + z_nd(i+1,j,k) + z_nd(i,j+1,k) + z_nd(i+1,j+1,k));
1297  Real z_base = Ampl * std::sin(kp * x - omega_t);
1298 
1299  z -= z_base;
1300  Real fac = std::sinh( kp * (z - H) ) / std::sinh(kp * H);
1301 
1302  zvel_arr(i,j,k) += Ampl * omega * fac * std::cos(kp * x - omega_t);
1303  });
1304  }
1305  } // end xvel_err, yvel_err, zvel_err
1306 
1307  if (containerHasElement(plot_var_names, "pp_err"))
1308  {
1309  // Moving terrain ANALYTICAL
1310 #ifdef _OPENMP
1311 #pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
1312 #endif
1313  for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi)
1314  {
1315  const Box& bx = mfi.tilebox();
1316  const Array4<Real>& derdat = mf[lev].array(mfi);
1317  const Array4<Real const>& p0_arr = p_hse.const_array(mfi);
1318  const Array4<Real const>& S_arr = vars_new[lev][Vars::cons].const_array(mfi);
1319 
1320  const auto dx = geom[lev].CellSizeArray();
1321  const Array4<Real const>& z_nd = z_phys_nd[lev]->const_array(mfi);
1322  const Array4<Real const>& p_arr = pressure.const_array(mfi);
1323  const Array4<Real const>& r0_arr = r_hse.const_array(mfi);
1324 
1325  Real H = geom[lev].ProbHi()[2];
1326  Real Ampl = 0.16;
1327  Real wavelength = 100.;
1328  Real kp = 2. * PI / wavelength;
1329  Real g = CONST_GRAV;
1330  Real omega = std::sqrt(g * kp);
1331  Real omega_t = omega * t_new[lev];
1332 
1333  ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept
1334  {
1335  derdat(i, j, k, mf_comp) = p_arr(i,j,k) - p0_arr(i,j,k);
1336 
1337  Real rho_hse = r0_arr(i,j,k);
1338 
1339  Real x = (i + 0.5) * dx[0];
1340  Real z = 0.125 * ( z_nd(i,j,k ) + z_nd(i+1,j,k ) + z_nd(i,j+1,k ) + z_nd(i+1,j+1,k )
1341  +z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) );
1342  Real z_base = Ampl * std::sin(kp * x - omega_t);
1343 
1344  z -= z_base;
1345  Real fac = std::cosh( kp * (z - H) ) / std::sinh(kp * H);
1346  Real pprime_exact = -(Ampl * omega * omega / kp) * fac *
1347  std::sin(kp * x - omega_t) * r0_arr(i,j,k);
1348 
1349  derdat(i,j,k,mf_comp) -= pprime_exact;
1350  });
1351  }
1352  mf_comp += 1;
1353  }
1354 #endif
1355 
1356  if (solverChoice.rad_type != RadiationType::None) {
1357  if (containerHasElement(plot_var_names, "qsrc_sw")) {
1358  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 0, mf_comp, 1, 0);
1359  mf_comp += 1;
1360  }
1361  if (containerHasElement(plot_var_names, "qsrc_lw")) {
1362  MultiFab::Copy(mf[lev], *(qheating_rates[lev]), 1, mf_comp, 1, 0);
1363  mf_comp += 1;
1364  }
1365  }
1366 
1367  // *****************************************************************************************
1368  // End of derived variables corresponding to "derived_names" in ERF.H
1369  //
1370  // Particles and microphysics can provide additional outputs, which are handled below.
1371  // *****************************************************************************************
1372 
1373 #ifdef ERF_USE_PARTICLES
1374  const auto& particles_namelist( particleData.getNames() );
1375 
1376  if (containerHasElement(plot_var_names, "tracer_particles_count")) {
1377  if (particles_namelist.size() == 0) {
1378  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
1379  temp_dat.setVal(0);
1380  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1381  mf_comp += 1;
1382  } else {
1383  for (ParticlesNamesVector::size_type i = 0; i < particles_namelist.size(); i++) {
1384  if (containerHasElement(plot_var_names, std::string(particles_namelist[i]+"_count"))) {
1385  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
1386  temp_dat.setVal(0);
1387  if (particleData.HasSpecies(particles_namelist[i])) {
1388  particleData[particles_namelist[i]]->Increment(temp_dat, lev);
1389  }
1390  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1391  mf_comp += 1;
1392  }
1393  }
1394  }
1395  }
1396 
1397  Vector<std::string> particle_mesh_plot_names(0);
1398  particleData.GetMeshPlotVarNames( particle_mesh_plot_names );
1399 
1400  for (int i = 0; i < particle_mesh_plot_names.size(); i++) {
1401  std::string plot_var_name(particle_mesh_plot_names[i]);
1402  if (containerHasElement(plot_var_names, plot_var_name) ) {
1403  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
1404  temp_dat.setVal(0);
1405  particleData.GetMeshPlotVar(plot_var_name, temp_dat, lev);
1406  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1407  mf_comp += 1;
1408  }
1409  }
1410 #endif
1411 
1412  {
1413  Vector<std::string> microphysics_plot_names;
1414  micro->GetPlotVarNames(microphysics_plot_names);
1415  for (auto& plot_name : microphysics_plot_names) {
1416  if (containerHasElement(plot_var_names, plot_name)) {
1417  MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 1);
1418  temp_dat.setVal(0);
1419  micro->GetPlotVar(plot_name, temp_dat, lev);
1420  MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
1421  mf_comp += 1;
1422  }
1423  }
1424  }
1425 
1426 
1427  }
1428 
1429  if (solverChoice.terrain_type == TerrainType::EB)
1430  {
1431  for (int lev = 0; lev <= finest_level; ++lev) {
1432  EB_set_covered(mf[lev], 0.0);
1433  }
1434  }
1435 
1436  // Fill terrain distortion MF (nu_nd)
1437  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1438  for (int lev(0); lev <= finest_level; ++lev) {
1439  MultiFab::Copy(mf_nd[lev],*z_phys_nd[lev],0,2,1,0);
1440  Real dz = Geom()[lev].CellSizeArray()[2];
1441  for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
1442  const Box& bx = mfi.tilebox();
1443  Array4<Real> mf_arr = mf_nd[lev].array(mfi);
1444  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
1445  {
1446  mf_arr(i,j,k,2) -= k * dz;
1447  });
1448  }
1449  }
1450  }
1451 
1452  std::string plotfilename;
1453  std::string plotfilenameU;
1454  std::string plotfilenameV;
1455  std::string plotfilenameW;
1456  if (which == 1) {
1457  plotfilename = Concatenate(plot3d_file_1, istep[0], 5);
1458  plotfilenameU = Concatenate(plot3d_file_1+"U", istep[0], 5);
1459  plotfilenameV = Concatenate(plot3d_file_1+"V", istep[0], 5);
1460  plotfilenameW = Concatenate(plot3d_file_1+"W", istep[0], 5);
1461  } else if (which == 2) {
1462  plotfilename = Concatenate(plot3d_file_2, istep[0], 5);
1463  plotfilenameU = Concatenate(plot3d_file_2+"U", istep[0], 5);
1464  plotfilenameV = Concatenate(plot3d_file_2+"V", istep[0], 5);
1465  plotfilenameW = Concatenate(plot3d_file_2+"W", istep[0], 5);
1466  }
1467 
1468  // LSM writes it's own data
1469  if (which==1 && plot_lsm) {
1470  lsm.Plot_Lsm_Data(t_new[0], istep, refRatio());
1471  }
1472 
1473 #ifdef ERF_USE_RRTMGP
1474  /*
1475  // write additional RRTMGP data
1476  // TODO: currently single level only
1477  if (which==1 && plot_rad) {
1478  rad[0]->writePlotfile(plot_file_1, t_new[0], istep[0]);
1479  }
1480  */
1481 #endif
1482 
1483  // Single level
1484  if (finest_level == 0)
1485  {
1486  if (plotfile_type == PlotFileType::Amrex)
1487  {
1488  Print() << "Writing native 3D plotfile " << plotfilename << "\n";
1489  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1490  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1491  GetVecOfConstPtrs(mf),
1492  GetVecOfConstPtrs(mf_nd),
1493  varnames,
1494  Geom(), t_new[0], istep, refRatio());
1495  } else {
1496  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1497  GetVecOfConstPtrs(mf),
1498  varnames,
1499  Geom(), t_new[0], istep, refRatio());
1500  }
1501  writeJobInfo(plotfilename);
1502 
1503  if (m_plot_face_vels) {
1504  Print() << "Writing face velocities" << std::endl;
1505  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1506  GetVecOfConstPtrs(mf_u),
1507  {"x_velocity_stag"},
1508  Geom(), t_new[0], istep, refRatio());
1509  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1510  GetVecOfConstPtrs(mf_v),
1511  {"y_velocity_stag"},
1512  Geom(), t_new[0], istep, refRatio());
1513  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1514  GetVecOfConstPtrs(mf_w),
1515  {"z_velocity_stag"},
1516  Geom(), t_new[0], istep, refRatio());
1517  }
1518 
1519 #ifdef ERF_USE_PARTICLES
1520  particleData.writePlotFile(plotfilename);
1521 #endif
1522 #ifdef ERF_USE_NETCDF
1523  } else if (plotfile_type == PlotFileType::Netcdf) {
1524  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
1525  int lev = 0;
1526  int l_which = 0;
1527  const Real* p_lo = geom[lev].ProbLo();
1528  const Real* p_hi = geom[lev].ProbHi();
1529  const auto dx = geom[lev].CellSize();
1530  writeNCPlotFile(lev, l_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1531  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
1532  geom[lev].Domain(), t_new[0], start_bdy_time);
1533 #endif
1534  } else {
1535  // Here we assume the plotfile_type is PlotFileType::None
1536  Print() << "Writing no 3D plotfile since plotfile_type is none" << std::endl;
1537  }
1538 
1539  } else { // Multilevel
1540 
1541  if (plotfile_type == PlotFileType::Amrex) {
1542 
1543  int lev0 = 0;
1544  int desired_ratio = std::max(std::max(ref_ratio[lev0][0],ref_ratio[lev0][1]),ref_ratio[lev0][2]);
1545  bool any_ratio_one = ( ( (ref_ratio[lev0][0] == 1) || (ref_ratio[lev0][1] == 1) ) ||
1546  (ref_ratio[lev0][2] == 1) );
1547  for (int lev = 1; lev < finest_level; lev++) {
1548  any_ratio_one = any_ratio_one ||
1549  ( ( (ref_ratio[lev][0] == 1) || (ref_ratio[lev][1] == 1) ) ||
1550  (ref_ratio[lev][2] == 1) );
1551  }
1552 
1553  if (any_ratio_one && m_expand_plotvars_to_unif_rr)
1554  {
1555  Vector<IntVect> r2(finest_level);
1556  Vector<Geometry> g2(finest_level+1);
1557  Vector<MultiFab> mf2(finest_level+1);
1558 
1559  mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
1560 
1561  // Copy level 0 as is
1562  MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
1563 
1564  // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
1565  Array<int,AMREX_SPACEDIM> periodicity =
1566  {Geom()[lev0].isPeriodic(0),Geom()[lev0].isPeriodic(1),Geom()[lev0].isPeriodic(2)};
1567  g2[lev0].define(Geom()[lev0].Domain(),&(Geom()[lev0].ProbDomain()),0,periodicity.data());
1568 
1569  r2[0] = IntVect(desired_ratio/ref_ratio[lev0][0],
1570  desired_ratio/ref_ratio[lev0][1],
1571  desired_ratio/ref_ratio[lev0][2]);
1572 
1573  for (int lev = 1; lev <= finest_level; ++lev) {
1574  if (lev > 1) {
1575  r2[lev-1][0] = r2[lev-2][0] * desired_ratio / ref_ratio[lev-1][0];
1576  r2[lev-1][1] = r2[lev-2][1] * desired_ratio / ref_ratio[lev-1][1];
1577  r2[lev-1][2] = r2[lev-2][2] * desired_ratio / ref_ratio[lev-1][2];
1578  }
1579 
1580  mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
1581 
1582  // Set the new problem domain
1583  Box d2(Geom()[lev].Domain());
1584  d2.refine(r2[lev-1]);
1585 
1586  g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
1587  }
1588 
1589  //
1590  // We need to make a temporary that is the size of ncomp_mf
1591  // in order to not get an out of bounds error
1592  // even though the values will not be used
1593  //
1594  Vector<BCRec> temp_domain_bcs_type;
1595  temp_domain_bcs_type.resize(ncomp_mf);
1596 
1597  //
1598  // Do piecewise constant interpolation of mf into mf2
1599  //
1600  for (int lev = 1; lev <= finest_level; ++lev) {
1601  Interpolater* mapper_c = &pc_interp;
1602  InterpFromCoarseLevel(mf2[lev], t_new[lev], mf[lev],
1603  0, 0, ncomp_mf,
1604  geom[lev], g2[lev],
1606  r2[lev-1], mapper_c, temp_domain_bcs_type, 0);
1607  }
1608 
1609  // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
1610  Vector<IntVect> rr(finest_level);
1611  for (int lev = 0; lev < finest_level; ++lev) {
1612  rr[lev] = IntVect(desired_ratio);
1613  }
1614 
1615  Print() << "Writing 3D plotfile " << plotfilename << "\n";
1616  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1617  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1618  GetVecOfConstPtrs(mf2),
1619  GetVecOfConstPtrs(mf_nd),
1620  varnames,
1621  g2, t_new[0], istep, rr);
1622  } else {
1623  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1624  GetVecOfConstPtrs(mf2), varnames,
1625  g2, t_new[0], istep, rr);
1626  }
1627 
1628  } else {
1629  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
1630  WriteMultiLevelPlotfileWithTerrain(plotfilename, finest_level+1,
1631  GetVecOfConstPtrs(mf),
1632  GetVecOfConstPtrs(mf_nd),
1633  varnames,
1634  geom, t_new[0], istep, ref_ratio);
1635  } else {
1636  WriteMultiLevelPlotfile(plotfilename, finest_level+1,
1637  GetVecOfConstPtrs(mf), varnames,
1638  geom, t_new[0], istep, ref_ratio);
1639  }
1640  if (m_plot_face_vels) {
1641  Print() << "Writing face velocities" << std::endl;
1642  WriteMultiLevelPlotfile(plotfilenameU, finest_level+1,
1643  GetVecOfConstPtrs(mf_u),
1644  {"x_velocity_stag"},
1645  geom, t_new[0], istep, ref_ratio);
1646  WriteMultiLevelPlotfile(plotfilenameV, finest_level+1,
1647  GetVecOfConstPtrs(mf_v),
1648  {"y_velocity_stag"},
1649  geom, t_new[0], istep, ref_ratio);
1650  WriteMultiLevelPlotfile(plotfilenameW, finest_level+1,
1651  GetVecOfConstPtrs(mf_w),
1652  {"z_velocity_stag"},
1653  geom, t_new[0], istep, ref_ratio);
1654  }
1655  } // ref_ratio test
1656 
1657  writeJobInfo(plotfilename);
1658 
1659 #ifdef ERF_USE_PARTICLES
1660  particleData.writePlotFile(plotfilename);
1661 #endif
1662 
1663 #ifdef ERF_USE_NETCDF
1664  } else if (plotfile_type == PlotFileType::Netcdf) {
1665  AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::StaticFittedMesh);
1666  for (int lev = 0; lev <= finest_level; ++lev) {
1667  for (int which_box = 0; which_box < num_boxes_at_level[lev]; which_box++) {
1668  Box bounding_region = (lev == 0) ? geom[lev].Domain() : boxes_at_level[lev][which_box];
1669  const Real* p_lo = geom[lev].ProbLo();
1670  const Real* p_hi = geom[lev].ProbHi();
1671  const auto dx = geom[lev].CellSizeArray();
1672  writeNCPlotFile(lev, which_box, plotfilename, GetVecOfConstPtrs(mf), varnames, istep,
1673  {p_lo[0],p_lo[1],p_lo[2]},{p_hi[0],p_hi[1],p_hi[2]}, {dx[0],dx[1],dx[2]},
1674  bounding_region, t_new[0], start_bdy_time);
1675  }
1676  }
1677 #endif
1678  }
1679  } // end multi-level
1680 }
constexpr amrex::Real PI
Definition: ERF_Constants.H:6
constexpr amrex::Real Cp_l
Definition: ERF_Constants.H:14
#define RhoQ4_comp
Definition: ERF_IndexDefines.H:45
void compute_gradp(const MultiFab &p, const Geometry &geom, MultiFab &z_phys_nd, MultiFab &z_phys_cc, const eb_ &ebfact, Vector< MultiFab > &gradp, const SolverChoice &solverChoice)
Definition: ERF_MakeGradP.cpp:74
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real erf_esatw(amrex::Real t)
Definition: ERF_MicrophysicsUtils.H:65
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void erf_qsatw(amrex::Real t, amrex::Real p, amrex::Real &qsatw)
Definition: ERF_MicrophysicsUtils.H:163
PhysBCFunctNoOp null_bc_for_fill
Definition: ERF_Plotfile.cpp:7
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real compute_max_reflectivity_dbz(amrex::Real rho_air, amrex::Real tmk, amrex::Real qra, amrex::Real qsn, amrex::Real qgr, int in0r, int in0s, int in0g, int iliqskin)
Definition: ERF_StormDiagnostics.H:13
void WriteMultiLevelPlotfileWithTerrain(const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
Definition: ERF_Plotfile.cpp:1683
void Plot_Lsm_Data(amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &ref_ratio)
Definition: ERF_LandSurface.H:110
@ Turb_lengthscale
Definition: ERF_IndexDefines.H:180
@ Mom_h
Definition: ERF_IndexDefines.H:170
@ Theta_h
Definition: ERF_IndexDefines.H:171
@ qpg
Definition: ERF_Morrison.H:41
@ qps
Definition: ERF_Morrison.H:40
@ qpr
Definition: ERF_Morrison.H:39
void erf_dervortx(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:200
void erf_dervorty(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &geomdata, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:228
void erf_dermagvel(const amrex::Box &bx, amrex::FArrayBox &derfab, int dcomp, int ncomp, const amrex::FArrayBox &datfab, const amrex::Geometry &, amrex::Real, const int *, const int)
Definition: ERF_Derive.cpp:319
void erf_dernull(const Box &, FArrayBox &, int, int, const FArrayBox &, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:39
void erf_dertemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:91
void erf_derKE(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:186
void erf_dermoisttemp(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:113
void erf_dersoundspeed(const Box &bx, FArrayBox &derfab, int, int, const FArrayBox &datfab, const Geometry &, Real, const int *, const int)
Definition: ERF_Derive.cpp:58
real(c_double), parameter g
Definition: ERF_module_model_constants.F90:19
Here is the call graph for this function:

◆ write_1D_profiles()

void ERF::write_1D_profiles ( amrex::Real  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Parameters
timeCurrent time
18 {
19  BL_PROFILE("ERF::write_1D_profiles()");
20 
21  if (NumDataLogs() > 1)
22  {
23  // Define the 1d arrays we will need
24  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
25  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
26  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
27  Gpu::HostVector<Real> h_avg_wthv;
28  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
29  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
30  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
31  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
32  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
33  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
34 
35  if (NumDataLogs() > 1) {
37  h_avg_u, h_avg_v, h_avg_w,
38  h_avg_rho, h_avg_th, h_avg_ksgs,
39  h_avg_Kmv, h_avg_Khv,
40  h_avg_qv, h_avg_qc, h_avg_qr,
41  h_avg_wqv, h_avg_wqc, h_avg_wqr,
42  h_avg_qi, h_avg_qs, h_avg_qg,
43  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
44  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
45  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
46  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
47  h_avg_wthv);
48  }
49 
50  if (NumDataLogs() > 3 && time > 0.) {
51  derive_stress_profiles(h_avg_tau11, h_avg_tau12, h_avg_tau13,
52  h_avg_tau22, h_avg_tau23, h_avg_tau33,
53  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
54  h_avg_sgsdiss);
55  }
56 
57  int hu_size = h_avg_u.size();
58 
59  auto const& dx = geom[0].CellSizeArray();
60  if (ParallelDescriptor::IOProcessor()) {
61  if (NumDataLogs() > 1) {
62  std::ostream& data_log1 = DataLog(1);
63  if (data_log1.good()) {
64  // Write the quantities at this time
65  for (int k = 0; k < hu_size; k++) {
66  Real z;
67  if (zlevels_stag[0].size() > 1) {
68  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
69  } else {
70  z = (k + 0.5)* dx[2];
71  }
72  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
73  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
74  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
75  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
76  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
77  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
78  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
79  << std::endl;
80  } // loop over z
81  } // if good
82  } // NumDataLogs
83 
84  if (NumDataLogs() > 2) {
85  std::ostream& data_log2 = DataLog(2);
86  if (data_log2.good()) {
87  // Write the perturbational quantities at this time
88  for (int k = 0; k < hu_size; k++) {
89  Real z;
90  if (zlevels_stag[0].size() > 1) {
91  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
92  } else {
93  z = (k + 0.5)* dx[2];
94  }
95  Real thv = h_avg_th[k] * (1 + 0.61*h_avg_qv[k] - h_avg_qc[k] - h_avg_qr[k]);
96  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
97  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
98  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " "
99  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " "
100  << h_avg_uw[k] - h_avg_u[k]*h_avg_w[k] << " "
101  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " "
102  << h_avg_vw[k] - h_avg_v[k]*h_avg_w[k] << " "
103  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " "
104  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " "
105  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " "
106  << h_avg_wth[k] - h_avg_w[k]*h_avg_th[k] << " "
107  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " "
108  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
109  // - <u_i u_i> * <u_j>
110  // - 2*<u_i> * <u_i u_j>
111  // + 2*<u_i>*<u_i> * <u_j>
112  << h_avg_uiuiu[k]
113  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_u[k]
114  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + h_avg_w[k]*h_avg_uw[k])
115  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_u[k]
116  << " " // (u'_i u'_i)u'
117  << h_avg_uiuiv[k]
118  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_v[k]
119  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + h_avg_w[k]*h_avg_vw[k])
120  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_v[k]
121  << " " // (u'_i u'_i)v'
122  << h_avg_uiuiw[k]
123  - (h_avg_uu[k] + h_avg_vv[k] + h_avg_ww[k])*h_avg_w[k]
124  - 2*(h_avg_u[k]*h_avg_uw[k] + h_avg_v[k]*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
125  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
126  << " " // (u'_i u'_i)w'
127  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " "
128  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " "
129  << h_avg_pw[k] - h_avg_p[k]*h_avg_w[k] << " "
130  << h_avg_wqv[k] - h_avg_qv[k]*h_avg_w[k] << " "
131  << h_avg_wqc[k] - h_avg_qc[k]*h_avg_w[k] << " "
132  << h_avg_wqr[k] - h_avg_qr[k]*h_avg_w[k] << " "
133  << h_avg_wthv[k] - h_avg_w[k]*thv
134  << std::endl;
135  } // loop over z
136  } // if good
137  } // NumDataLogs
138 
139  if (NumDataLogs() > 3 && time > 0.) {
140  std::ostream& data_log3 = DataLog(3);
141  if (data_log3.good()) {
142  // Write the average stresses
143  for (int k = 0; k < hu_size; k++) {
144  Real z;
145  if (zlevels_stag[0].size() > 1) {
146  z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
147  } else {
148  z = (k + 0.5)* dx[2];
149  }
150  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
151  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
152  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
153  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
154  << h_avg_sgshfx[k] << " "
155  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
156  << h_avg_sgsdiss[k]
157  << std::endl;
158  } // loop over z
159  } // if good
160  } // if (NumDataLogs() > 3)
161  } // if IOProcessor
162  } // if (NumDataLogs() > 1)
163 }
void derive_diag_profiles(amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles.cpp:190
void derive_stress_profiles(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles.cpp:475

◆ write_1D_profiles_stag()

void ERF::write_1D_profiles_stag ( amrex::Real  time)

Writes 1-dimensional averaged quantities as profiles to output log files at the given time.

Quantities are output at their native grid locations. Therefore, w and associated flux quantities <(•)'w'>, tau13, and tau23 (where '•' includes u, v, p, theta, ...) will be output at staggered heights (i.e., coincident with z faces) rather than cell-center heights to avoid performing additional averaging. Unstaggered (i.e., cell-centered) quantities are output alongside staggered quantities at the lower cell faces in the log file; these quantities will have a zero value at the big end, corresponding to k=Nz+1.

The structure of file should follow ERF_Write1DProfiles.cpp

Parameters
timeCurrent time
26 {
27  BL_PROFILE("ERF::write_1D_profiles()");
28 
29  if (NumDataLogs() > 1)
30  {
31  // Define the 1d arrays we will need
32  Gpu::HostVector<Real> h_avg_u, h_avg_v, h_avg_w;
33  Gpu::HostVector<Real> h_avg_rho, h_avg_th, h_avg_ksgs, h_avg_Kmv, h_avg_Khv;
34  Gpu::HostVector<Real> h_avg_qv, h_avg_qc, h_avg_qr, h_avg_wqv, h_avg_wqc, h_avg_wqr, h_avg_qi, h_avg_qs, h_avg_qg;
35  Gpu::HostVector<Real> h_avg_wthv;
36  Gpu::HostVector<Real> h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth;
37  Gpu::HostVector<Real> h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww;
38  Gpu::HostVector<Real> h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw;
39  Gpu::HostVector<Real> h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw;
40  Gpu::HostVector<Real> h_avg_tau11, h_avg_tau12, h_avg_tau13, h_avg_tau22, h_avg_tau23, h_avg_tau33;
41  Gpu::HostVector<Real> h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx, h_avg_sgsdiss; // only output tau_{theta,w} and epsilon for now
42 
43  if (NumDataLogs() > 1) {
45  h_avg_u, h_avg_v, h_avg_w,
46  h_avg_rho, h_avg_th, h_avg_ksgs,
47  h_avg_Kmv, h_avg_Khv,
48  h_avg_qv, h_avg_qc, h_avg_qr,
49  h_avg_wqv, h_avg_wqc, h_avg_wqr,
50  h_avg_qi, h_avg_qs, h_avg_qg,
51  h_avg_uu, h_avg_uv, h_avg_uw, h_avg_vv, h_avg_vw, h_avg_ww,
52  h_avg_uth, h_avg_vth, h_avg_wth, h_avg_thth,
53  h_avg_uiuiu, h_avg_uiuiv, h_avg_uiuiw,
54  h_avg_p, h_avg_pu, h_avg_pv, h_avg_pw,
55  h_avg_wthv);
56  }
57 
58  if (NumDataLogs() > 3 && time > 0.) {
59  derive_stress_profiles_stag(h_avg_tau11, h_avg_tau12, h_avg_tau13,
60  h_avg_tau22, h_avg_tau23, h_avg_tau33,
61  h_avg_sgshfx, h_avg_sgsq1fx, h_avg_sgsq2fx,
62  h_avg_sgsdiss);
63  }
64 
65  int unstag_size = h_avg_w.size() - 1; // _un_staggered heights
66 
67  auto const& dx = geom[0].CellSizeArray();
68  if (ParallelDescriptor::IOProcessor()) {
69  if (NumDataLogs() > 1) {
70  std::ostream& data_log1 = DataLog(1);
71  if (data_log1.good()) {
72  // Write the quantities at this time
73  for (int k = 0; k < unstag_size; k++) {
74  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
75  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
76  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
77  << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
78  << h_avg_rho[k] << " " << h_avg_th[k] << " " << h_avg_ksgs[k] << " "
79  << h_avg_Kmv[k] << " " << h_avg_Khv[k] << " "
80  << h_avg_qv[k] << " " << h_avg_qc[k] << " " << h_avg_qr[k] << " "
81  << h_avg_qi[k] << " " << h_avg_qs[k] << " " << h_avg_qg[k]
82  << std::endl;
83  } // loop over z
84  // Write top face values
85  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
86  data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
87  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
88  << 0 << " " << 0 << " " << h_avg_w[unstag_size] << " "
89  << 0 << " " << 0 << " " << 0 << " " // rho, theta, ksgs
90  << 0 << " " << 0 << " " // Kmv, Khv
91  << 0 << " " << 0 << " " << 0 << " " // qv, qc, qr
92  << 0 << " " << 0 << " " << 0 // qi, qs, qg
93  << std::endl;
94  } // if good
95  } // NumDataLogs
96 
97  if (NumDataLogs() > 2) {
98  std::ostream& data_log2 = DataLog(2);
99  if (data_log2.good()) {
100  // Write the perturbational quantities at this time
101  // For surface values (k=0), assume w = uw = vw = ww = 0
102  Real w_cc = h_avg_w[1] / 2; // w at first cell center
103  Real uw_cc = h_avg_uw[1] / 2; // u*w at first cell center
104  Real vw_cc = h_avg_vw[1] / 2; // v*w at first cell center
105  Real ww_cc = h_avg_ww[1] / 2; // w*w at first cell center
106  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
107  << std::setw(datwidth) << std::setprecision(datprecision) << 0 << " "
108  << h_avg_uu[0] - h_avg_u[0]*h_avg_u[0] << " " // u'u'
109  << h_avg_uv[0] - h_avg_u[0]*h_avg_v[0] << " " // u'v'
110  << 0 << " " // u'w'
111  << h_avg_vv[0] - h_avg_v[0]*h_avg_v[0] << " " // v'v'
112  << 0 << " " // v'w'
113  << 0 << " " // w'w'
114  << h_avg_uth[0] - h_avg_u[0]*h_avg_th[0] << " " // u'th'
115  << h_avg_vth[0] - h_avg_v[0]*h_avg_th[0] << " " // v'th'
116  << 0 << " " // w'th'
117  << h_avg_thth[0] - h_avg_th[0]*h_avg_th[0] << " " // th'th'
118  << h_avg_uiuiu[0]
119  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_u[0]
120  - 2*(h_avg_u[0]*h_avg_uu[0] + h_avg_v[0]*h_avg_uv[0] + w_cc*uw_cc)
121  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_u[0]
122  << " " // (u'_i u'_i)u'
123  << h_avg_uiuiv[0]
124  - (h_avg_uu[0] + h_avg_vv[0] + ww_cc)*h_avg_v[0]
125  - 2*(h_avg_u[0]*h_avg_uv[0] + h_avg_v[0]*h_avg_vv[0] + w_cc*vw_cc)
126  + 2*(h_avg_u[0]*h_avg_u[0] + h_avg_v[0]*h_avg_v[0] + w_cc*w_cc)*h_avg_v[0]
127  << " " // (u'_i u'_i)v'
128  << 0 << " " // (u'_i u'_i)w'
129  << h_avg_pu[0] - h_avg_p[0]*h_avg_u[0] << " " // p'u'
130  << h_avg_pv[0] - h_avg_p[0]*h_avg_v[0] << " " // p'v'
131  << 0 << " " // p'w'
132  << 0 << " " // qv'w'
133  << 0 << " " // qc'w'
134  << 0 << " " // qr'w'
135  << 0 // thv'w'
136  << std::endl;
137 
138  // For internal values, interpolate scalar quantities to faces
139  for (int k = 1; k < unstag_size; k++) {
140  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
141  Real uface = 0.5*(h_avg_u[k] + h_avg_u[k-1]);
142  Real vface = 0.5*(h_avg_v[k] + h_avg_v[k-1]);
143  Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]);
144  Real pface = 0.5*(h_avg_p[k] + h_avg_p[k-1]);
145  Real qvface = 0.5*(h_avg_qv[k] + h_avg_qv[k-1]);
146  Real qcface = 0.5*(h_avg_qc[k] + h_avg_qc[k-1]);
147  Real qrface = 0.5*(h_avg_qr[k] + h_avg_qr[k-1]);
148  Real uuface = 0.5*(h_avg_uu[k] + h_avg_uu[k-1]);
149  Real vvface = 0.5*(h_avg_vv[k] + h_avg_vv[k-1]);
150  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
151  w_cc = 0.5*(h_avg_w[k-1] + h_avg_w[k]);
152  uw_cc = 0.5*(h_avg_uw[k-1] + h_avg_uw[k]);
153  vw_cc = 0.5*(h_avg_vw[k-1] + h_avg_vw[k]);
154  ww_cc = 0.5*(h_avg_ww[k-1] + h_avg_ww[k]);
155  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
156  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
157  << h_avg_uu[k] - h_avg_u[k]*h_avg_u[k] << " " // u'u'
158  << h_avg_uv[k] - h_avg_u[k]*h_avg_v[k] << " " // u'v'
159  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
160  << h_avg_vv[k] - h_avg_v[k]*h_avg_v[k] << " " // v'v'
161  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
162  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
163  << h_avg_uth[k] - h_avg_u[k]*h_avg_th[k] << " " // u'th'
164  << h_avg_vth[k] - h_avg_v[k]*h_avg_th[k] << " " // v'th'
165  << h_avg_wth[k] - h_avg_w[k]*thface << " " // w'th'
166  << h_avg_thth[k] - h_avg_th[k]*h_avg_th[k] << " " // th'th'
167  // Note: <u'_i u'_i u'_j> = <u_i u_i u_j>
168  // - <u_i u_i> * <u_j>
169  // - 2*<u_i> * <u_i u_j>
170  // + 2*<u_i>*<u_i> * <u_j>
171  << h_avg_uiuiu[k]
172  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_u[k]
173  - 2*(h_avg_u[k]*h_avg_uu[k] + h_avg_v[k]*h_avg_uv[k] + w_cc*uw_cc)
174  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_u[k]
175  << " " // cell-centered (u'_i u'_i)u'
176  << h_avg_uiuiv[k]
177  - (h_avg_uu[k] + h_avg_vv[k] + ww_cc)*h_avg_v[k]
178  - 2*(h_avg_u[k]*h_avg_uv[k] + h_avg_v[k]*h_avg_vv[k] + w_cc*vw_cc)
179  + 2*(h_avg_u[k]*h_avg_u[k] + h_avg_v[k]*h_avg_v[k] + w_cc*w_cc)*h_avg_v[k]
180  << " " // cell-centered (u'_i u'_i)v'
181  << h_avg_uiuiw[k]
182  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
183  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
184  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
185  << " " // face-centered (u'_i u'_i)w'
186  << h_avg_pu[k] - h_avg_p[k]*h_avg_u[k] << " " // cell-centered p'u'
187  << h_avg_pv[k] - h_avg_p[k]*h_avg_v[k] << " " // cell-centered p'v'
188  << h_avg_pw[k] - pface*h_avg_w[k] << " " // face-centered p'w'
189  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
190  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
191  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
192  << h_avg_wthv[k] - thvface*h_avg_w[k]
193  << std::endl;
194  } // loop over z
195 
196  // Write top face values, extrapolating scalar quantities
197  const int k = unstag_size;
198  Real uface = 1.5*h_avg_u[k-1] - 0.5*h_avg_u[k-2];
199  Real vface = 1.5*h_avg_v[k-1] - 0.5*h_avg_v[k-2];
200  Real thface = 1.5*h_avg_th[k-1] - 0.5*h_avg_th[k-2];
201  Real pface = 1.5*h_avg_p[k-1] - 0.5*h_avg_p[k-2];
202  Real qvface = 1.5*h_avg_qv[k-1] - 0.5*h_avg_qv[k-2];
203  Real qcface = 1.5*h_avg_qc[k-1] - 0.5*h_avg_qc[k-2];
204  Real qrface = 1.5*h_avg_qr[k-1] - 0.5*h_avg_qr[k-2];
205  Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2];
206  Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2];
207  Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
208  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
209  data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
210  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
211  << 0 << " " // u'u'
212  << 0 << " " // u'v'
213  << h_avg_uw[k] - uface*h_avg_w[k] << " " // u'w'
214  << 0 << " " // v'v'
215  << h_avg_vw[k] - vface*h_avg_w[k] << " " // v'w'
216  << h_avg_ww[k] - h_avg_w[k]*h_avg_w[k] << " " // w'w'
217  << 0 << " " // u'th'
218  << 0 << " " // v'th'
219  << h_avg_wth[k] - thface*h_avg_w[k] << " " // w'th'
220  << 0 << " " // th'th'
221  << 0 << " " // (u'_i u'_i)u'
222  << 0 << " " // (u'_i u'_i)v'
223  << h_avg_uiuiw[k]
224  - (uuface + vvface + h_avg_ww[k])*h_avg_w[k]
225  - 2*(uface*h_avg_uw[k] + vface*h_avg_vw[k] + h_avg_w[k]*h_avg_ww[k])
226  + 2*(uface*uface + vface*vface + h_avg_w[k]*h_avg_w[k])*h_avg_w[k]
227  << " " // (u'_i u'_i)w'
228  << 0 << " " // pu'
229  << 0 << " " // pv'
230  << h_avg_pw[k] - pface*h_avg_w[k] << " " // pw'
231  << h_avg_wqv[k] - qvface*h_avg_w[k] << " "
232  << h_avg_wqc[k] - qcface*h_avg_w[k] << " "
233  << h_avg_wqr[k] - qrface*h_avg_w[k] << " "
234  << h_avg_wthv[k] - thvface*h_avg_w[k]
235  << std::endl;
236  } // if good
237  } // NumDataLogs
238 
239  if (NumDataLogs() > 3 && time > 0.) {
240  std::ostream& data_log3 = DataLog(3);
241  if (data_log3.good()) {
242  // Write the average stresses
243  for (int k = 0; k < unstag_size; k++) {
244  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
245  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
246  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
247  << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
248  << h_avg_tau22[k] << " " << h_avg_tau23[k] << " " << h_avg_tau33[k] << " "
249  << h_avg_sgshfx[k] << " "
250  << h_avg_sgsq1fx[k] << " " << h_avg_sgsq2fx[k] << " "
251  << h_avg_sgsdiss[k]
252  << std::endl;
253  } // loop over z
254  // Write top face values
255  Real NANval = 0.0;
256  Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
257  data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
258  << std::setw(datwidth) << std::setprecision(datprecision) << z << " "
259  << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
260  << NANval << " " << h_avg_tau23[unstag_size] << " " << NANval << " "
261  << h_avg_sgshfx[unstag_size] << " "
262  << h_avg_sgsq1fx[unstag_size] << " " << h_avg_sgsq2fx[unstag_size] << " "
263  << NANval
264  << std::endl;
265  } // if good
266  } // if (NumDataLogs() > 3)
267  } // if IOProcessor
268  } // if (NumDataLogs() > 1)
269 }
void derive_diag_profiles_stag(amrex::Real time, amrex::Gpu::HostVector< amrex::Real > &h_avg_u, amrex::Gpu::HostVector< amrex::Real > &h_avg_v, amrex::Gpu::HostVector< amrex::Real > &h_avg_w, amrex::Gpu::HostVector< amrex::Real > &h_avg_rho, amrex::Gpu::HostVector< amrex::Real > &h_avg_th, amrex::Gpu::HostVector< amrex::Real > &h_avg_ksgs, amrex::Gpu::HostVector< amrex::Real > &h_avg_Kmv, amrex::Gpu::HostVector< amrex::Real > &h_avg_Khv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qv, amrex::Gpu::HostVector< amrex::Real > &h_avg_qc, amrex::Gpu::HostVector< amrex::Real > &h_avg_qr, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqv, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqc, amrex::Gpu::HostVector< amrex::Real > &h_avg_wqr, amrex::Gpu::HostVector< amrex::Real > &h_avg_qi, amrex::Gpu::HostVector< amrex::Real > &h_avg_qs, amrex::Gpu::HostVector< amrex::Real > &h_avg_qg, amrex::Gpu::HostVector< amrex::Real > &h_avg_uu, amrex::Gpu::HostVector< amrex::Real > &h_avg_uv, amrex::Gpu::HostVector< amrex::Real > &h_avg_uw, amrex::Gpu::HostVector< amrex::Real > &h_avg_vv, amrex::Gpu::HostVector< amrex::Real > &h_avg_vw, amrex::Gpu::HostVector< amrex::Real > &h_avg_ww, amrex::Gpu::HostVector< amrex::Real > &h_avg_uth, amrex::Gpu::HostVector< amrex::Real > &h_avg_vth, amrex::Gpu::HostVector< amrex::Real > &h_avg_wth, amrex::Gpu::HostVector< amrex::Real > &h_avg_thth, amrex::Gpu::HostVector< amrex::Real > &h_avg_ku, amrex::Gpu::HostVector< amrex::Real > &h_avg_kv, amrex::Gpu::HostVector< amrex::Real > &h_avg_kw, amrex::Gpu::HostVector< amrex::Real > &h_avg_p, amrex::Gpu::HostVector< amrex::Real > &h_avg_pu, amrex::Gpu::HostVector< amrex::Real > &h_avg_pv, amrex::Gpu::HostVector< amrex::Real > &h_avg_pw, amrex::Gpu::HostVector< amrex::Real > &h_avg_wthv)
Definition: ERF_Write1DProfiles_stag.cpp:296
void derive_stress_profiles_stag(amrex::Gpu::HostVector< amrex::Real > &h_avg_tau11, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau12, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau13, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau22, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau23, amrex::Gpu::HostVector< amrex::Real > &h_avg_tau33, amrex::Gpu::HostVector< amrex::Real > &h_avg_hfx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q1fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_q2fx3, amrex::Gpu::HostVector< amrex::Real > &h_avg_diss)
Definition: ERF_Write1DProfiles_stag.cpp:605

◆ writeBuildInfo()

void ERF::writeBuildInfo ( std::ostream &  os)
static
138 {
139  std::string PrettyLine = std::string(78, '=') + "\n";
140  std::string OtherLine = std::string(78, '-') + "\n";
141  std::string SkipSpace = std::string(8, ' ');
142 
143  // build information
144  os << PrettyLine;
145  os << " ERF Build Information\n";
146  os << PrettyLine;
147 
148  os << "build date: " << buildInfoGetBuildDate() << "\n";
149  os << "build machine: " << buildInfoGetBuildMachine() << "\n";
150  os << "build dir: " << buildInfoGetBuildDir() << "\n";
151  os << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
152 
153  os << "\n";
154 
155  os << "COMP: " << buildInfoGetComp() << "\n";
156  os << "COMP version: " << buildInfoGetCompVersion() << "\n";
157 
158  os << "C++ compiler: " << buildInfoGetCXXName() << "\n";
159  os << "C++ flags: " << buildInfoGetCXXFlags() << "\n";
160 
161  os << "\n";
162 
163  os << "Link flags: " << buildInfoGetLinkFlags() << "\n";
164  os << "Libraries: " << buildInfoGetLibraries() << "\n";
165 
166  os << "\n";
167 
168  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
169  os << buildInfoGetModuleName(n) << ": "
170  << buildInfoGetModuleVal(n) << "\n";
171  }
172 
173  os << "\n";
174  const char* githash1 = buildInfoGetGitHash(1);
175  const char* githash2 = buildInfoGetGitHash(2);
176  if (strlen(githash1) > 0) {
177  os << "ERF git hash: " << githash1 << "\n";
178  }
179  if (strlen(githash2) > 0) {
180  os << "AMReX git hash: " << githash2 << "\n";
181  }
182 
183  const char* buildgithash = buildInfoGetBuildGitHash();
184  const char* buildgitname = buildInfoGetBuildGitName();
185  if (strlen(buildgithash) > 0) {
186  os << buildgitname << " git hash: " << buildgithash << "\n";
187  }
188 
189  os << "\n";
190  os << " ERF Compile time variables: \n";
191 
192  os << "\n";
193  os << " ERF Defines: \n";
194 #ifdef _OPENMP
195  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
196  << std::endl;
197 #else
198  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
199  << std::endl;
200 #endif
201 
202 #ifdef MPI_VERSION
203  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
204  << MPI_VERSION << std::endl;
205 #else
206  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
207  << "UNDEFINED" << std::endl;
208 #endif
209 
210 #ifdef MPI_SUBVERSION
211  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
212  << MPI_SUBVERSION << std::endl;
213 #else
214  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
215  << "UNDEFINED" << std::endl;
216 #endif
217 
218  os << "\n\n";
219 }

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ WriteCheckpointFile()

void ERF::WriteCheckpointFile ( ) const

ERF function for writing a checkpoint file.

27 {
28  // chk00010 write a checkpoint file with this root directory
29  // chk00010/Header this contains information you need to save (e.g., finest_level, t_new, etc.) and also
30  // the BoxArrays at each level
31  // chk00010/Level_0/
32  // chk00010/Level_1/
33  // etc. these subdirectories will hold the MultiFab data at each level of refinement
34 
35  // checkpoint file name, e.g., chk00010
36  const std::string& checkpointname = Concatenate(check_file,istep[0],5);
37 
38  Print() << "Writing native checkpoint " << checkpointname << "\n";
39 
40  const int nlevels = finest_level+1;
41 
42  // ---- prebuild a hierarchy of directories
43  // ---- dirName is built first. if dirName exists, it is renamed. then build
44  // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
45  // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
46  // ---- after all directories are built
47  // ---- ParallelDescriptor::IOProcessor() creates the directories
48  PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
49 
50  int ncomp_cons = vars_new[0][Vars::cons].nComp();
51 
52  // write Header file
53  if (ParallelDescriptor::IOProcessor()) {
54 
55  std::string HeaderFileName(checkpointname + "/Header");
56  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
57  std::ofstream HeaderFile;
58  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
59  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
60  std::ofstream::trunc |
61  std::ofstream::binary);
62  if(! HeaderFile.good()) {
63  FileOpenFailed(HeaderFileName);
64  }
65 
66  HeaderFile.precision(17);
67 
68  // write out title line
69  HeaderFile << "Checkpoint file for ERF\n";
70 
71  // write out finest_level
72  HeaderFile << finest_level << "\n";
73 
74  // write the number of components
75  // for each variable we store
76 
77  // conservative, cell-centered vars
78  HeaderFile << ncomp_cons << "\n";
79 
80  // x-velocity on faces
81  HeaderFile << 1 << "\n";
82 
83  // y-velocity on faces
84  HeaderFile << 1 << "\n";
85 
86  // z-velocity on faces
87  HeaderFile << 1 << "\n";
88 
89  // write out array of istep
90  for (int i = 0; i < istep.size(); ++i) {
91  HeaderFile << istep[i] << " ";
92  }
93  HeaderFile << "\n";
94 
95  // write out array of dt
96  for (int i = 0; i < dt.size(); ++i) {
97  HeaderFile << dt[i] << " ";
98  }
99  HeaderFile << "\n";
100 
101  // write out array of t_new
102  for (int i = 0; i < t_new.size(); ++i) {
103  HeaderFile << t_new[i] << " ";
104  }
105  HeaderFile << "\n";
106 
107  // write the BoxArray at each level
108  for (int lev = 0; lev <= finest_level; ++lev) {
109  boxArray(lev).writeOn(HeaderFile);
110  HeaderFile << '\n';
111  }
112 
113  // Write separate file that tells how many components we have of the base state
114  std::string BaseStateFileName(checkpointname + "/num_base_state_comps");
115  std::ofstream BaseStateFile;
116  BaseStateFile.open(BaseStateFileName.c_str(), std::ofstream::out |
117  std::ofstream::trunc |
118  std::ofstream::binary);
119  if(! BaseStateFile.good()) {
120  FileOpenFailed(BaseStateFileName);
121  } else {
122  // write out number of components in base state
123  BaseStateFile << BaseState::num_comps << "\n";
124  BaseStateFile << base_state[0].nGrowVect() << "\n";
125  }
126  }
127 
128  // write the MultiFab data to, e.g., chk00010/Level_0/
129  // Here we make copies of the MultiFab with no ghost cells
130  for (int lev = 0; lev <= finest_level; ++lev)
131  {
132  MultiFab cons(grids[lev],dmap[lev],ncomp_cons,0);
133  MultiFab::Copy(cons,vars_new[lev][Vars::cons],0,0,ncomp_cons,0);
134  VisMF::Write(cons, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
135 
136  MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
137  MultiFab::Copy(xvel,vars_new[lev][Vars::xvel],0,0,1,0);
138  VisMF::Write(xvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
139 
140  MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
141  MultiFab::Copy(yvel,vars_new[lev][Vars::yvel],0,0,1,0);
142  VisMF::Write(yvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
143 
144  MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
145  MultiFab::Copy(zvel,vars_new[lev][Vars::zvel],0,0,1,0);
146  VisMF::Write(zvel, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
147 
148  if (solverChoice.anelastic[lev] == 1) {
149  MultiFab ppinc(grids[lev],dmap[lev],1,0);
150  MultiFab::Copy(ppinc,pp_inc[lev],0,0,1,0);
151  VisMF::Write(ppinc, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PP_Inc"));
152 
153  MultiFab gpx(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
154  MultiFab::Copy(gpx,gradp[lev][GpVars::gpx],0,0,1,0);
155  VisMF::Write(gpx, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpx"));
156 
157  MultiFab gpy(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
158  MultiFab::Copy(gpy,gradp[lev][GpVars::gpy],0,0,1,0);
159  VisMF::Write(gpy, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpy"));
160 
161  MultiFab gpz(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
162  MultiFab::Copy(gpz,gradp[lev][GpVars::gpz],0,0,1,0);
163  VisMF::Write(gpz, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Gpz"));
164  }
165 
166  // Note that we write the ghost cells of the base state (unlike above)
167  IntVect ng_base = base_state[lev].nGrowVect();
168  int ncomp_base = base_state[lev].nComp();
169  MultiFab base(grids[lev],dmap[lev],ncomp_base,ng_base);
170  MultiFab::Copy(base,base_state[lev],0,0,ncomp_base,ng_base);
171  VisMF::Write(base, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "BaseState"));
172 
173  if (SolverChoice::mesh_type != MeshType::ConstantDz) {
174  // Note that we also write the ghost cells of z_phys_nd
175  IntVect ng = z_phys_nd[lev]->nGrowVect();
176  MultiFab z_height(convert(grids[lev],IntVect(1,1,1)),dmap[lev],1,ng);
177  MultiFab::Copy(z_height,*z_phys_nd[lev],0,0,1,ng);
178  VisMF::Write(z_height, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z_Phys_nd"));
179  }
180 
181  // We must read and write qmoist with ghost cells because we don't directly impose BCs on these vars
182  // Write the moisture model restart variables
183  std::vector<int> qmoist_indices;
184  std::vector<std::string> qmoist_names;
185  micro->Get_Qmoist_Restart_Vars(lev, solverChoice, qmoist_indices, qmoist_names);
186  int qmoist_nvar = qmoist_indices.size();
187  for (int var = 0; var < qmoist_nvar; var++) {
188  const int ncomp = 1;
189  IntVect ng_moist = qmoist[lev][qmoist_indices[var]]->nGrowVect();
190  MultiFab moist_vars(grids[lev],dmap[lev],ncomp,ng_moist);
191  MultiFab::Copy(moist_vars,*(qmoist[lev][qmoist_indices[var]]),0,0,ncomp,ng_moist);
192  VisMF::Write(moist_vars, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", qmoist_names[var]));
193  }
194 
195 #if defined(ERF_USE_WINDFARM)
196  if(solverChoice.windfarm_type == WindFarmType::Fitch or
197  solverChoice.windfarm_type == WindFarmType::EWP or
198  solverChoice.windfarm_type == WindFarmType::SimpleAD){
199  IntVect ng_turb = Nturb[lev].nGrowVect();
200  MultiFab mf_Nturb(grids[lev],dmap[lev],1,ng_turb);
201  MultiFab::Copy(mf_Nturb,Nturb[lev],0,0,1,ng_turb);
202  VisMF::Write(mf_Nturb, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "NumTurb"));
203  }
204 #endif
205 
206  if (solverChoice.lsm_type != LandSurfaceType::None) {
207  for (int mvar(0); mvar<lsm_data[lev].size(); ++mvar) {
208  BoxArray ba = lsm_data[lev][mvar]->boxArray();
209  DistributionMapping dm = lsm_data[lev][mvar]->DistributionMap();
210  IntVect ng = lsm_data[lev][mvar]->nGrowVect();
211  int nvar = lsm_data[lev][mvar]->nComp();
212  MultiFab lsm_vars(ba,dm,nvar,ng);
213  MultiFab::Copy(lsm_vars,*(lsm_data[lev][mvar]),0,0,nvar,ng);
214  VisMF::Write(lsm_vars, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LsmVars"));
215  }
216  }
217 
218  IntVect ng = mapfac[lev][MapFacType::m_x]->nGrowVect();
219  MultiFab mf_m(ba2d[lev],dmap[lev],1,ng);
220  MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_x],0,0,1,ng);
221  VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_mx"));
222 
223 #if 0
225  MultiFab::Copy(mf_m,*mapfac[lev][MapFacType::m_y],0,0,1,ng);
226  VisMF::Write(mf_m, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_my"));
227  }
228 #endif
229 
230  ng = mapfac[lev][MapFacType::u_x]->nGrowVect();
231  MultiFab mf_u(convert(ba2d[lev],IntVect(1,0,0)),dmap[lev],1,ng);
232  MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_x],0,0,1,ng);
233  VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_ux"));
234 
235 #if 0
237  MultiFab::Copy(mf_u,*mapfac[lev][MapFacType::u_y],0,0,1,ng);
238  VisMF::Write(mf_u, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_uy"));
239  }
240 #endif
241 
242  ng = mapfac[lev][MapFacType::v_x]->nGrowVect();
243  MultiFab mf_v(convert(ba2d[lev],IntVect(0,1,0)),dmap[lev],1,ng);
244  MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_x],0,0,1,ng);
245  VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vx"));
246 
247 #if 0
249  MultiFab::Copy(mf_v,*mapfac[lev][MapFacType::v_y],0,0,1,ng);
250  VisMF::Write(mf_v, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MapFactor_vy"));
251  }
252 #endif
253 
254  if (m_SurfaceLayer) {
255  amrex::Print() << "Writing SurfaceLayer variables at level " << lev << std::endl;
256  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
257  MultiFab m_var(ba2d[lev],dmap[lev],1,ng);
258  MultiFab* src = nullptr;
259 
260  // U*
261  src = m_SurfaceLayer->get_u_star(lev);
262  MultiFab::Copy(m_var,*src,0,0,1,ng);
263  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Ustar"));
264 
265  // W*
266  src = m_SurfaceLayer->get_w_star(lev);
267  MultiFab::Copy(m_var,*src,0,0,1,ng);
268  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Wstar"));
269 
270  // T*
271  src = m_SurfaceLayer->get_t_star(lev);
272  MultiFab::Copy(m_var,*src,0,0,1,ng);
273  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Tstar"));
274 
275  // Q*
276  src = m_SurfaceLayer->get_q_star(lev);
277  MultiFab::Copy(m_var,*src,0,0,1,ng);
278  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qstar"));
279 
280  // Olen
281  src = m_SurfaceLayer->get_olen(lev);
282  MultiFab::Copy(m_var,*src,0,0,1,ng);
283  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Olen"));
284 
285  // Qsurf
286  src = m_SurfaceLayer->get_q_surf(lev);
287  MultiFab::Copy(m_var,*src,0,0,1,ng);
288  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Qsurf"));
289 
290  // PBLH
291  src = m_SurfaceLayer->get_pblh(lev);
292  MultiFab::Copy(m_var,*src,0,0,1,ng);
293  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "PBLH"));
294 
295  // Z0
296  src = m_SurfaceLayer->get_z0(lev);
297  MultiFab::Copy(m_var,*src,0,0,1,ng);
298  VisMF::Write(m_var, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Z0"));
299  }
300 
301  if (sst_lev[lev][0]) {
302  int ntimes = 1;
303  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
304  MultiFab sst_at_t(ba2d[lev],dmap[lev],1,ng);
305  for (int nt(0); nt<ntimes; ++nt) {
306  MultiFab::Copy(sst_at_t,*sst_lev[lev][nt],0,0,1,ng);
307  VisMF::Write(sst_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
308  "SST_" + std::to_string(nt)));
309  }
310  }
311 
312  if (tsk_lev[lev][0]) {
313  int ntimes = 1;
314  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
315  MultiFab tsk_at_t(ba2d[lev],dmap[lev],1,ng);
316  for (int nt(0); nt<ntimes; ++nt) {
317  MultiFab::Copy(tsk_at_t,*tsk_lev[lev][nt],0,0,1,ng);
318  VisMF::Write(tsk_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
319  "TSK_" + std::to_string(nt)));
320  }
321  }
322 
323  {
324  int ntimes = 1;
325  ng = vars_new[lev][Vars::cons].nGrowVect(); ng[2]=0;
326  MultiFab lmask_at_t(ba2d[lev],dmap[lev],1,ng);
327  for (int nt(0); nt<ntimes; ++nt) {
328  for (MFIter mfi(lmask_at_t); mfi.isValid(); ++mfi) {
329  const Box& bx = mfi.growntilebox();
330  Array4<int> const& src_arr = lmask_lev[lev][nt]->array(mfi);
331  Array4<Real> const& dst_arr = lmask_at_t.array(mfi);
332  ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
333  {
334  dst_arr(i,j,k) = Real(src_arr(i,j,k));
335  });
336  }
337  VisMF::Write(lmask_at_t, MultiFabFileFullPrefix(lev, checkpointname, "Level_",
338  "LMASK_" + std::to_string(nt)));
339  }
340  }
341 
342  IntVect ngv = ng; ngv[2] = 0;
343 
344  // Write lat/lon if it exists
345  if (lat_m[lev] && lon_m[lev] && solverChoice.has_lat_lon) {
346  amrex::Print() << "Writing Lat/Lon variables at level " << lev << std::endl;
347  MultiFab lat(ba2d[lev],dmap[lev],1,ngv);
348  MultiFab lon(ba2d[lev],dmap[lev],1,ngv);
349  MultiFab::Copy(lat,*lat_m[lev],0,0,1,ngv);
350  MultiFab::Copy(lon,*lon_m[lev],0,0,1,ngv);
351  VisMF::Write(lat, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LAT"));
352  VisMF::Write(lon, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "LON"));
353  }
354 
355 
356 #ifdef ERF_USE_NETCDF
357  // Write sinPhi and cosPhi if it exists
358  if (cosPhi_m[lev] && sinPhi_m[lev] && solverChoice.variable_coriolis) {
359  amrex::Print() << "Writing Coriolis factors at level " << lev << std::endl;
360  MultiFab sphi(ba2d[lev],dmap[lev],1,ngv);
361  MultiFab cphi(ba2d[lev],dmap[lev],1,ngv);
362  MultiFab::Copy(sphi,*sinPhi_m[lev],0,0,1,ngv);
363  MultiFab::Copy(cphi,*cosPhi_m[lev],0,0,1,ngv);
364  VisMF::Write(sphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "SinPhi"));
365  VisMF::Write(cphi, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "CosPhi"));
366  }
367 
368  if (solverChoice.use_real_bcs && solverChoice.init_type == InitType::WRFInput) {
369  amrex::Print() << "Writing C1H/C2H/MUB variables at level " << lev << std::endl;
370  MultiFab tmp1d(ba1d[0],dmap[0],1,0);
371 
372  MultiFab::Copy(tmp1d,*mf_C1H,0,0,1,0);
373  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C1H"));
374 
375  MultiFab::Copy(tmp1d,*mf_C2H,0,0,1,0);
376  VisMF::Write(tmp1d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "C2H"));
377 
378  MultiFab tmp2d(ba2d[0],dmap[0],1,mf_MUB->nGrowVect());
379 
380  MultiFab::Copy(tmp2d,*mf_MUB,0,0,1,mf_MUB->nGrowVect());
381  VisMF::Write(tmp2d, MultiFabFileFullPrefix(lev, checkpointname, "Level_", "MUB"));
382  }
383 #endif
384 
385  } // for lev
386 
387 #ifdef ERF_USE_PARTICLES
388  particleData.Checkpoint(checkpointname);
389 #endif
390 
391 #if 0
392 #ifdef ERF_USE_NETCDF
393  // Write bdy_data files
394  if ( ParallelDescriptor::IOProcessor() &&
395  ((solverChoice.init_type==InitType::WRFInput) || (solverChoice.init_type==InitType::Metgrid)) &&
397  {
398  // Vector dimensions
399  int num_time = bdy_data_xlo.size();
400  int num_var = bdy_data_xlo[0].size();
401 
402  // Open header file and write to it
403  std::ofstream bdy_h_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
404  bdy_h_file << std::setprecision(1) << std::fixed;
405  bdy_h_file << num_time << "\n";
406  bdy_h_file << num_var << "\n";
407  bdy_h_file << start_bdy_time << "\n";
408  bdy_h_file << bdy_time_interval << "\n";
409  bdy_h_file << real_width << "\n";
410  for (int ivar(0); ivar<num_var; ++ivar) {
411  bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
412  bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
413  bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
414  bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
415  }
416 
417  // Open data file and write to it
418  std::ofstream bdy_d_file(MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
419  for (int itime(0); itime<num_time; ++itime) {
420  if (bdy_data_xlo[itime].size() > 0) {
421  for (int ivar(0); ivar<num_var; ++ivar) {
422  bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
423  bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
424  bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
425  bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
426  }
427  }
428  }
429  }
430 #endif
431 #endif
432 
433 }

◆ WriteGenericPlotfileHeaderWithTerrain()

void ERF::WriteGenericPlotfileHeaderWithTerrain ( std::ostream &  HeaderFile,
int  nlevels,
const amrex::Vector< amrex::BoxArray > &  bArray,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
amrex::Real  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName,
const std::string &  levelPrefix,
const std::string &  mfPrefix 
) const
1781 {
1782  AMREX_ALWAYS_ASSERT(nlevels <= bArray.size());
1783  AMREX_ALWAYS_ASSERT(nlevels <= my_ref_ratio.size()+1);
1784  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
1785 
1786  HeaderFile.precision(17);
1787 
1788  // ---- this is the generic plot file type name
1789  HeaderFile << versionName << '\n';
1790 
1791  HeaderFile << varnames.size() << '\n';
1792 
1793  for (int ivar = 0; ivar < varnames.size(); ++ivar) {
1794  HeaderFile << varnames[ivar] << "\n";
1795  }
1796  HeaderFile << AMREX_SPACEDIM << '\n';
1797  HeaderFile << my_time << '\n';
1798  HeaderFile << finest_level << '\n';
1799  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1800  HeaderFile << my_geom[0].ProbLo(i) << ' ';
1801  }
1802  HeaderFile << '\n';
1803  for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1804  HeaderFile << my_geom[0].ProbHi(i) << ' ';
1805  }
1806  HeaderFile << '\n';
1807  for (int i = 0; i < finest_level; ++i) {
1808  HeaderFile << my_ref_ratio[i][0] << ' ';
1809  }
1810  HeaderFile << '\n';
1811  for (int i = 0; i <= finest_level; ++i) {
1812  HeaderFile << my_geom[i].Domain() << ' ';
1813  }
1814  HeaderFile << '\n';
1815  for (int i = 0; i <= finest_level; ++i) {
1816  HeaderFile << level_steps[i] << ' ';
1817  }
1818  HeaderFile << '\n';
1819  for (int i = 0; i <= finest_level; ++i) {
1820  for (int k = 0; k < AMREX_SPACEDIM; ++k) {
1821  HeaderFile << my_geom[i].CellSize()[k] << ' ';
1822  }
1823  HeaderFile << '\n';
1824  }
1825  HeaderFile << (int) my_geom[0].Coord() << '\n';
1826  HeaderFile << "0\n";
1827 
1828  for (int level = 0; level <= finest_level; ++level) {
1829  HeaderFile << level << ' ' << bArray[level].size() << ' ' << my_time << '\n';
1830  HeaderFile << level_steps[level] << '\n';
1831 
1832  const IntVect& domain_lo = my_geom[level].Domain().smallEnd();
1833  for (int i = 0; i < bArray[level].size(); ++i)
1834  {
1835  // Need to shift because the RealBox ctor we call takes the
1836  // physical location of index (0,0,0). This does not affect
1837  // the usual cases where the domain index starts with 0.
1838  const Box& b = shift(bArray[level][i], -domain_lo);
1839  RealBox loc = RealBox(b, my_geom[level].CellSize(), my_geom[level].ProbLo());
1840  for (int n = 0; n < AMREX_SPACEDIM; ++n) {
1841  HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
1842  }
1843  }
1844 
1845  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
1846  }
1847  HeaderFile << "1" << "\n";
1848  HeaderFile << "3" << "\n";
1849  HeaderFile << "amrexvec_nu_x" << "\n";
1850  HeaderFile << "amrexvec_nu_y" << "\n";
1851  HeaderFile << "amrexvec_nu_z" << "\n";
1852  std::string mf_nodal_prefix = "Nu_nd";
1853  for (int level = 0; level <= finest_level; ++level) {
1854  HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
1855  }
1856 }
Coord
Definition: ERF_DataStruct.H:85

◆ writeJobInfo()

void ERF::writeJobInfo ( const std::string &  dir) const
10 {
11  // job_info file with details about the run
12  std::ofstream jobInfoFile;
13  std::string FullPathJobInfoFile = dir;
14  FullPathJobInfoFile += "/job_info";
15  jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out);
16 
17  std::string PrettyLine = "==================================================="
18  "============================\n";
19  std::string OtherLine = "----------------------------------------------------"
20  "----------------------------\n";
21  std::string SkipSpace = " ";
22 
23  // job information
24  jobInfoFile << PrettyLine;
25  jobInfoFile << " ERF Job Information\n";
26  jobInfoFile << PrettyLine;
27 
28  jobInfoFile << "inputs file: " << inputs_name << "\n\n";
29 
30  jobInfoFile << "number of MPI processes: "
31  << ParallelDescriptor::NProcs() << "\n";
32 #ifdef _OPENMP
33  jobInfoFile << "number of threads: " << omp_get_max_threads() << "\n";
34 #endif
35 
36  jobInfoFile << "\n";
37  jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
38  << getCPUTime() / 3600.0;
39 
40  jobInfoFile << "\n\n";
41 
42  // plotfile information
43  jobInfoFile << PrettyLine;
44  jobInfoFile << " Plotfile Information\n";
45  jobInfoFile << PrettyLine;
46 
47  time_t now = time(nullptr);
48 
49  // Convert now to tm struct for local timezone
50  tm* localtm = localtime(&now);
51  jobInfoFile << "output data / time: " << asctime(localtm);
52 
53  std::string currentDir = FileSystem::CurrentPath();
54  jobInfoFile << "output dir: " << currentDir << "\n";
55 
56  jobInfoFile << "\n\n";
57 
58  // build information
59  jobInfoFile << PrettyLine;
60  jobInfoFile << " Build Information\n";
61  jobInfoFile << PrettyLine;
62 
63  jobInfoFile << "build date: " << buildInfoGetBuildDate() << "\n";
64  jobInfoFile << "build machine: " << buildInfoGetBuildMachine() << "\n";
65  jobInfoFile << "build dir: " << buildInfoGetBuildDir() << "\n";
66  jobInfoFile << "AMReX dir: " << buildInfoGetAMReXDir() << "\n";
67 
68  jobInfoFile << "\n";
69 
70  jobInfoFile << "COMP: " << buildInfoGetComp() << "\n";
71  jobInfoFile << "COMP version: " << buildInfoGetCompVersion() << "\n";
72 
73  jobInfoFile << "\n";
74 
75  for (int n = 1; n <= buildInfoGetNumModules(); n++) {
76  jobInfoFile << buildInfoGetModuleName(n) << ": "
77  << buildInfoGetModuleVal(n) << "\n";
78  }
79 
80  jobInfoFile << "\n";
81 
82  const char* githash1 = buildInfoGetGitHash(1);
83  const char* githash2 = buildInfoGetGitHash(2);
84  if (strlen(githash1) > 0) {
85  jobInfoFile << "ERF git hash: " << githash1 << "\n";
86  }
87  if (strlen(githash2) > 0) {
88  jobInfoFile << "AMReX git hash: " << githash2 << "\n";
89  }
90 
91  const char* buildgithash = buildInfoGetBuildGitHash();
92  const char* buildgitname = buildInfoGetBuildGitName();
93  if (strlen(buildgithash) > 0) {
94  jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
95  }
96 
97  jobInfoFile << "\n\n";
98 
99  // grid information
100  jobInfoFile << PrettyLine;
101  jobInfoFile << " Grid Information\n";
102  jobInfoFile << PrettyLine;
103 
104  int f_lev = finest_level;
105 
106  for (int i = 0; i <= f_lev; i++) {
107  jobInfoFile << " level: " << i << "\n";
108  jobInfoFile << " number of boxes = " << grids[i].size() << "\n";
109  jobInfoFile << " maximum zones = ";
110  for (int n = 0; n < AMREX_SPACEDIM; n++) {
111  jobInfoFile << geom[i].Domain().length(n) << " ";
112  }
113  jobInfoFile << "\n\n";
114  }
115 
116  jobInfoFile << " Boundary conditions\n";
117 
118  jobInfoFile << " -x: " << domain_bc_type[0] << "\n";
119  jobInfoFile << " +x: " << domain_bc_type[3] << "\n";
120  jobInfoFile << " -y: " << domain_bc_type[1] << "\n";
121  jobInfoFile << " +y: " << domain_bc_type[4] << "\n";
122  jobInfoFile << " -z: " << domain_bc_type[2] << "\n";
123  jobInfoFile << " +z: " << domain_bc_type[5] << "\n";
124 
125  jobInfoFile << "\n\n";
126 
127  // runtime parameters
128  jobInfoFile << PrettyLine;
129  jobInfoFile << " Inputs File Parameters\n";
130  jobInfoFile << PrettyLine;
131 
132  ParmParse::dumpTable(jobInfoFile, true);
133  jobInfoFile.close();
134 }
std::string inputs_name
Definition: main.cpp:14
static amrex::Real getCPUTime()
Definition: ERF.H:1433
Here is the call graph for this function:

◆ WriteMultiLevelPlotfileWithTerrain()

void ERF::WriteMultiLevelPlotfileWithTerrain ( const std::string &  plotfilename,
int  nlevels,
const amrex::Vector< const amrex::MultiFab * > &  mf,
const amrex::Vector< const amrex::MultiFab * > &  mf_nd,
const amrex::Vector< std::string > &  varnames,
const amrex::Vector< amrex::Geometry > &  my_geom,
amrex::Real  time,
const amrex::Vector< int > &  level_steps,
const amrex::Vector< amrex::IntVect > &  my_ref_ratio,
const std::string &  versionName = "HyperCLaw-V1.1",
const std::string &  levelPrefix = "Level_",
const std::string &  mfPrefix = "Cell",
const amrex::Vector< std::string > &  extra_dirs = amrex::Vector<std::string>() 
) const
1695 {
1696  BL_PROFILE("WriteMultiLevelPlotfileWithTerrain()");
1697 
1698  AMREX_ALWAYS_ASSERT(nlevels <= mf.size());
1699  AMREX_ALWAYS_ASSERT(nlevels <= rr.size()+1);
1700  AMREX_ALWAYS_ASSERT(nlevels <= level_steps.size());
1701  AMREX_ALWAYS_ASSERT(mf[0]->nComp() == varnames.size());
1702 
1703  bool callBarrier(false);
1704  PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
1705  if (!extra_dirs.empty()) {
1706  for (const auto& d : extra_dirs) {
1707  const std::string ed = plotfilename+"/"+d;
1708  PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
1709  }
1710  }
1711  ParallelDescriptor::Barrier();
1712 
1713  if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
1714  Vector<BoxArray> boxArrays(nlevels);
1715  for(int level(0); level < boxArrays.size(); ++level) {
1716  boxArrays[level] = mf[level]->boxArray();
1717  }
1718 
1719  auto f = [=]() {
1720  VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
1721  std::string HeaderFileName(plotfilename + "/Header");
1722  std::ofstream HeaderFile;
1723  HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
1724  HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
1725  std::ofstream::trunc |
1726  std::ofstream::binary);
1727  if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
1728  WriteGenericPlotfileHeaderWithTerrain(HeaderFile, nlevels, boxArrays, varnames,
1729  my_geom, time, level_steps, rr, versionName,
1730  levelPrefix, mfPrefix);
1731  };
1732 
1733  if (AsyncOut::UseAsyncOut()) {
1734  AsyncOut::Submit(std::move(f));
1735  } else {
1736  f();
1737  }
1738  }
1739 
1740  std::string mf_nodal_prefix = "Nu_nd";
1741  for (int level = 0; level <= finest_level; ++level)
1742  {
1743  if (AsyncOut::UseAsyncOut()) {
1744  VisMF::AsyncWrite(*mf[level],
1745  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
1746  true);
1747  VisMF::AsyncWrite(*mf_nd[level],
1748  MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
1749  true);
1750  } else {
1751  const MultiFab* data;
1752  std::unique_ptr<MultiFab> mf_tmp;
1753  if (mf[level]->nGrowVect() != 0) {
1754  mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
1755  mf[level]->DistributionMap(),
1756  mf[level]->nComp(), 0, MFInfo(),
1757  mf[level]->Factory());
1758  MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
1759  data = mf_tmp.get();
1760  } else {
1761  data = mf[level];
1762  }
1763  VisMF::Write(*data , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
1764  VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
1765  }
1766  }
1767 }
void WriteGenericPlotfileHeaderWithTerrain(std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, const amrex::Vector< amrex::Geometry > &my_geom, amrex::Real time, const amrex::Vector< int > &level_steps, const amrex::Vector< amrex::IntVect > &my_ref_ratio, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
Definition: ERF_Plotfile.cpp:1770

◆ WriteMyEBSurface()

void ERF::WriteMyEBSurface ( )
6 {
7  using namespace amrex;
8 
9  amrex::Print() << "Writing the geometry to a vtp file.\n" << std::endl;
10 
11  // Only write at the finest level!
12  int lev = finest_level;
13 
14  BoxArray & ba = grids[lev];
15  DistributionMapping & dm = dmap[lev];
16 
17  const EBFArrayBoxFactory* ebfact = &EBFactory(lev);
18 
19  WriteEBSurface(ba,dm,Geom(lev),ebfact);
20 }
Definition: ERF_ConsoleIO.cpp:12
Here is the call graph for this function:

◆ writeNow()

bool ERF::writeNow ( const amrex::Real  cur_time,
const int  nstep,
const int  plot_int,
const amrex::Real  plot_per,
const amrex::Real  dt_0,
amrex::Real last_file_time 
)
2519 {
2520  bool write_now = false;
2521 
2522  if ( plot_int > 0) {
2523 
2524  write_now = (nstep % plot_int == 0);
2525 
2526  } else if (plot_per > 0.0) {
2527 
2528  amrex::Print() << "CUR NEXT PER " << cur_time << " " << next_file_time << " " << plot_per << std::endl;
2529 
2530  // Only write now if nstep newly matches the number of elapsed periods
2531  write_now = (cur_time > (next_file_time - Real(0.1)*dt_0));
2532  }
2533 
2534  return write_now;
2535 }

◆ WriteSubvolume()

void ERF::WriteSubvolume ( )
10 {
11  ParmParse pp("erf.subvol");
12 
13  Vector<Real> origin;
14  Vector< int> ncell;
15  Vector<Real> delta;
16 
17  // **************************************************************
18  // Read in the origin, number of cells in each dir, and resolution
19  // **************************************************************
20  pp.getarr("origin",origin,0,AMREX_SPACEDIM);
21  pp.getarr("nxnynz", ncell,0,AMREX_SPACEDIM);
22  pp.getarr("dxdydz", delta,0,AMREX_SPACEDIM);
23 
24  int lev_for_sub = 0;
25 
26  bool found = false;
27  for (int i = 0; i <= finest_level; i++) {
28  if (!found) {
29  if (almostEqual(delta[0],geom[i].CellSize(0)) &&
30  almostEqual(delta[1],geom[i].CellSize(1)) &&
31  almostEqual(delta[2],geom[i].CellSize(2)) ) {
32 
33  // amrex::Print() << "XDIR " << delta[0] << " " << geom[i].CellSize(0) << std::endl;
34  // amrex::Print() << "YDIR " << delta[1] << " " << geom[i].CellSize(1) << std::endl;
35  // amrex::Print() << "ZDIR " << delta[2] << " " << geom[i].CellSize(2) << std::endl;
36  amrex::Print() << "Resolution specified matches that of level " << i << std::endl;
37  found = true;
38  lev_for_sub = i;
39  }
40  }
41  }
42 
43  if (!found) {
44  amrex::Abort("Resolution specified for subvol does not match the resolution of any of the levels.");
45  }
46 
47  // **************************************************************
48  // Now that we know which level we're at, we can figure out which (i,j,k) the origin corresponds to
49  // Note we use 1.0001 as a fudge factor since the division of two reals --> integer will do a floor
50  // **************************************************************
51  int i0 = static_cast<int>((origin[0] - geom[lev_for_sub].ProbLo(0)) * 1.0001 / delta[0]);
52  int j0 = static_cast<int>((origin[1] - geom[lev_for_sub].ProbLo(1)) * 1.0001 / delta[1]);
53  int k0 = static_cast<int>((origin[2] - geom[lev_for_sub].ProbLo(2)) * 1.0001 / delta[2]);
54 
55  found = false;
56  if (almostEqual(geom[lev_for_sub].ProbLo(0)+i0*delta[0],origin[0]) &&
57  almostEqual(geom[lev_for_sub].ProbLo(1)+j0*delta[1],origin[1]) &&
58  almostEqual(geom[lev_for_sub].ProbLo(2)+k0*delta[2],origin[2]) )
59  {
60  amrex::Print() << "Specified origin is the lower left corner of cell " << IntVect(i0,j0,k0) << std::endl;
61  found = true;
62  }
63 
64  if (!found) {
65  amrex::Abort("Origin specified does not correspond to a node at this level.");
66  }
67 
68  Box domain(geom[lev_for_sub].Domain());
69 
70  Box bx(IntVect(i0,j0,k0),IntVect(i0+ncell[0]-1,j0+ncell[1]-1,k0+ncell[2]-1));
71  amrex::Print() << "Box requested is " << bx << std::endl;
72 
73  if (!domain.contains(bx))
74  {
75  amrex::Abort("Box requested is larger than the existing domain");
76  }
77 
78  Vector<int> cs(3);
79  int count = pp.countval("chunk_size");
80  if (count > 0) {
81  pp.queryarr("chunk_size",cs,0,AMREX_SPACEDIM);
82  } else {
83  cs[0] = max_grid_size[0][0];
84  cs[1] = max_grid_size[0][1];
85  cs[2] = max_grid_size[0][2];
86  }
87  IntVect chunk_size(cs[0],cs[1],cs[2]);
88 
89  BoxArray ba(bx);
90  ba.maxSize(chunk_size);
91 
92  amrex::Print() << "BoxArray is " << ba << std::endl;
93 
94  int ncomp_mf = AMREX_SPACEDIM;
95 
96  DistributionMapping dm(ba);
97 
98  MultiFab mf(ba, dm, ncomp_mf, 0);
99 
100  MultiFab mf_cc_vel(grids[lev_for_sub], dmap[lev_for_sub], ncomp_mf, 0);
101  average_face_to_cellcenter(mf_cc_vel,0,
102  Array<const MultiFab*,3>{&vars_new[lev_for_sub][Vars::xvel],
103  &vars_new[lev_for_sub][Vars::yvel],
104  &vars_new[lev_for_sub][Vars::zvel]});
105 
106  mf.ParallelCopy(mf_cc_vel,0,0,AMREX_SPACEDIM,0,0);
107 
108  std::string subvol_filename = Concatenate(subvol_file, istep[0], 5);
109 
110  Vector<std::string> varnames;
111  varnames.push_back("x_velocity");
112  varnames.push_back("y_velocity");
113  varnames.push_back("z_velocity");
114 
115  Real time = t_new[lev_for_sub];
116 
117  amrex::Print() <<"Writing subvolume into " << subvol_filename << std::endl;
118  WriteSingleLevelPlotfile(subvol_filename,mf,varnames,geom[lev_for_sub],time,istep[0]);
119 }
real(c_double), private cs
Definition: ERF_module_mp_morr_two_moment.F90:202
Here is the call graph for this function:

◆ WriteVTKPolyline()

void ERF::WriteVTKPolyline ( const std::string &  filename,
amrex::Vector< std::array< amrex::Real, 2 >> &  points_xy 
)
595 {
596  std::ofstream vtkfile(filename);
597  if (!vtkfile.is_open()) {
598  std::cerr << "Error: Cannot open file " << filename << std::endl;
599  return;
600  }
601 
602  int num_points = points_xy.size();
603  if (num_points == 0) {
604  vtkfile << "# vtk DataFile Version 3.0\n";
605  vtkfile << "Hurricane Track\n";
606  vtkfile << "ASCII\n";
607  vtkfile << "DATASET POLYDATA\n";
608  vtkfile << "POINTS " << num_points << " float\n";
609  vtkfile.close();
610  return;
611  }
612  if (num_points < 2) {
613  points_xy.push_back(points_xy[0]);
614  }
615  num_points = points_xy.size();
616 
617  vtkfile << "# vtk DataFile Version 3.0\n";
618  vtkfile << "Hurricane Track\n";
619  vtkfile << "ASCII\n";
620  vtkfile << "DATASET POLYDATA\n";
621 
622  // Write points (Z=0 assumed)
623  vtkfile << "POINTS " << num_points << " float\n";
624  for (const auto& pt : points_xy) {
625  vtkfile << pt[0] << " " << pt[1] << " 10000.0\n";
626  }
627 
628  // Write polyline connectivity
629  vtkfile << "LINES 1 " << num_points + 1 << "\n";
630  vtkfile << num_points << " ";
631  for (int i = 0; i < num_points; ++i) {
632  vtkfile << i << " ";
633  }
634  vtkfile << "\n";
635 
636  vtkfile.close();
637 }

Member Data Documentation

◆ advflux_reg

amrex::Vector<amrex::YAFluxRegister*> ERF::advflux_reg
private

Referenced by getAdvFluxReg().

◆ ax

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax
private

◆ ax_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ax_src
private

◆ ay

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay
private

◆ ay_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::ay_src
private

◆ az

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az
private

◆ az_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::az_src
private

◆ ba1d

amrex::Vector<amrex::BoxArray> ERF::ba1d
private

◆ ba2d

amrex::Vector<amrex::BoxArray> ERF::ba2d
private

◆ base_state

amrex::Vector<amrex::MultiFab> ERF::base_state
private

◆ base_state_new

amrex::Vector<amrex::MultiFab> ERF::base_state_new
private

◆ bndry_output_planes_interval

int ERF::bndry_output_planes_interval = -1
staticprivate

◆ bndry_output_planes_per

Real ERF::bndry_output_planes_per = -1.0
staticprivate

◆ bndry_output_planes_start_time

Real ERF::bndry_output_planes_start_time = 0.0
staticprivate

◆ boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > ERF::boxes_at_level
private

◆ cf_set_width

int ERF::cf_set_width {0}
private

◆ cf_width

int ERF::cf_width {0}
private

◆ cfl

Real ERF::cfl = 0.8
staticprivate

◆ change_max

Real ERF::change_max = 1.1
staticprivate

◆ check_file

std::string ERF::check_file {"chk"}
private

◆ column_file_name

std::string ERF::column_file_name = "column_data.nc"
staticprivate

◆ column_interval

int ERF::column_interval = -1
staticprivate

◆ column_loc_x

Real ERF::column_loc_x = 0.0
staticprivate

◆ column_loc_y

Real ERF::column_loc_y = 0.0
staticprivate

◆ column_per

Real ERF::column_per = -1.0
staticprivate

◆ cons_names

const amrex::Vector<std::string> ERF::cons_names
private
Initial value:
{"density", "rhotheta", "rhoKE", "rhoadv_0",
"rhoQ1", "rhoQ2", "rhoQ3",
"rhoQ4", "rhoQ5", "rhoQ6"}

◆ cosPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::cosPhi_m
private

◆ d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_density
private

◆ d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_pressure
private

◆ d_havg_qc

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qc
private

◆ d_havg_qv

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_qv
private

◆ d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> ERF::d_havg_temperature
private

◆ d_rayleigh_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_rayleigh_ptrs
private

◆ d_rhoqt_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhoqt_src
private

◆ d_rhotheta_src

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_rhotheta_src
private

◆ d_sponge_ptrs

amrex::Vector<amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > > ERF::d_sponge_ptrs
private

◆ d_u_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_u_geos
private

◆ d_v_geos

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_v_geos
private

◆ d_w_subsid

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::d_w_subsid
private

◆ data_sampler

std::unique_ptr<SampleData> ERF::data_sampler = nullptr
private

◆ datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::datalog
private

◆ datalogname

amrex::Vector<std::string> ERF::datalogname
private

Referenced by DataLogName().

◆ datetime_format

const std::string ERF::datetime_format = "%Y-%m-%d %H:%M:%S"
private

◆ datprecision

const int ERF::datprecision = 6
private

◆ datwidth

const int ERF::datwidth = 14
private

◆ der_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::der_datalog
private

◆ der_datalogname

amrex::Vector<std::string> ERF::der_datalogname
private

Referenced by DerDataLogName().

◆ derived_names

const amrex::Vector<std::string> ERF::derived_names
private

◆ destag_profiles

bool ERF::destag_profiles = true
private

◆ detJ_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc
private

◆ detJ_cc_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_new
private

◆ detJ_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::detJ_cc_src
private

◆ domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> ERF::domain_bc_type
private

◆ domain_bcs_type

amrex::Vector<amrex::BCRec> ERF::domain_bcs_type
private

◆ domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> ERF::domain_bcs_type_d
private

◆ dt

amrex::Vector<amrex::Real> ERF::dt
private

◆ dt_max

Real ERF::dt_max = 1.0e9
staticprivate

◆ dt_max_initial

Real ERF::dt_max_initial = 2.0e100
staticprivate

◆ dt_mri_ratio

amrex::Vector<long> ERF::dt_mri_ratio
private

◆ dz_min

amrex::Vector<amrex::Real> ERF::dz_min
private

◆ eb

amrex::Vector<std::unique_ptr<eb_> > ERF::eb
private

Referenced by EBFactory(), and get_eb().

◆ eddyDiffs_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::eddyDiffs_lev
private

◆ fine_mask

amrex::MultiFab ERF::fine_mask
private

◆ finished_wave

bool ERF::finished_wave = false
private

◆ fixed_dt

amrex::Vector<amrex::Real> ERF::fixed_dt
private

◆ fixed_fast_dt

amrex::Vector<amrex::Real> ERF::fixed_fast_dt
private

◆ fixed_mri_dt_ratio

int ERF::fixed_mri_dt_ratio = 0
staticprivate

◆ forecast_state_1

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_1

◆ forecast_state_2

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_2

◆ forecast_state_interp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::forecast_state_interp

◆ FPr_c

amrex::Vector<ERFFillPatcher> ERF::FPr_c
private

◆ FPr_u

amrex::Vector<ERFFillPatcher> ERF::FPr_u
private

◆ FPr_v

amrex::Vector<ERFFillPatcher> ERF::FPr_v
private

◆ FPr_w

amrex::Vector<ERFFillPatcher> ERF::FPr_w
private

◆ gradp

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::gradp
private

◆ h_havg_density

amrex::Vector<amrex::Real> ERF::h_havg_density
private

◆ h_havg_pressure

amrex::Vector<amrex::Real> ERF::h_havg_pressure
private

◆ h_havg_qc

amrex::Vector<amrex::Real> ERF::h_havg_qc
private

◆ h_havg_qv

amrex::Vector<amrex::Real> ERF::h_havg_qv
private

◆ h_havg_temperature

amrex::Vector<amrex::Real> ERF::h_havg_temperature
private

◆ h_rayleigh_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_rayleigh_ptrs
private

◆ h_rhoqt_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhoqt_src
private

◆ h_rhotheta_src

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_rhotheta_src
private

◆ h_sponge_ptrs

amrex::Vector<amrex::Vector< amrex::Vector<amrex::Real> > > ERF::h_sponge_ptrs
private

◆ h_u_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_u_geos
private

◆ h_v_geos

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_v_geos
private

◆ h_w_subsid

amrex::Vector< amrex::Vector<amrex::Real> > ERF::h_w_subsid
private

◆ hurricane_track_xy

amrex::Vector<std::array<amrex::Real, 2> > ERF::hurricane_track_xy

◆ Hwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave
private

◆ Hwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Hwave_onegrid
private

◆ init_shrink

Real ERF::init_shrink = 1.0
staticprivate

◆ input_bndry_planes

int ERF::input_bndry_planes = 0
staticprivate

◆ input_sounding_data

InputSoundingData ERF::input_sounding_data
private

◆ input_sponge_data

InputSpongeData ERF::input_sponge_data
private

◆ interpolation_type

StateInterpType ERF::interpolation_type
staticprivate

◆ istep

amrex::Vector<int> ERF::istep
private

◆ last_check_file_step

int ERF::last_check_file_step = -1
staticprivate

◆ last_check_file_time

Real ERF::last_check_file_time = 0.0
staticprivate

◆ last_plot2d_file_step_1

int ERF::last_plot2d_file_step_1 = -1
staticprivate

◆ last_plot2d_file_step_2

int ERF::last_plot2d_file_step_2 = -1
staticprivate

◆ last_plot2d_file_time_1

Real ERF::last_plot2d_file_time_1 = 0.0
staticprivate

◆ last_plot2d_file_time_2

Real ERF::last_plot2d_file_time_2 = 0.0
staticprivate

◆ last_plot3d_file_step_1

int ERF::last_plot3d_file_step_1 = -1
staticprivate

◆ last_plot3d_file_step_2

int ERF::last_plot3d_file_step_2 = -1
staticprivate

◆ last_plot3d_file_time_1

Real ERF::last_plot3d_file_time_1 = 0.0
staticprivate

◆ last_plot3d_file_time_2

Real ERF::last_plot3d_file_time_2 = 0.0
staticprivate

◆ last_subvol_step

int ERF::last_subvol_step = -1
staticprivate

◆ last_subvol_time

Real ERF::last_subvol_time = 0.0
staticprivate

◆ lat_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lat_m
private

◆ lmask_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab> > > ERF::lmask_lev
private

◆ lon_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::lon_m
private

◆ lsm

LandSurface ERF::lsm
private

◆ lsm_data

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_data
private

◆ lsm_data_name

amrex::Vector<std::string> ERF::lsm_data_name
private

◆ lsm_flux

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::lsm_flux
private

◆ lsm_flux_name

amrex::Vector<std::string> ERF::lsm_flux_name
private

◆ Lwave

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave
private

◆ Lwave_onegrid

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Lwave_onegrid
private

◆ m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_extdir_vals
private

◆ m_bc_neumann_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>, AMREX_SPACEDIM+NBCVAR_max> ERF::m_bc_neumann_vals
private

◆ m_check_int

int ERF::m_check_int = -1
private

◆ m_check_per

amrex::Real ERF::m_check_per = -1.0
private

◆ m_expand_plotvars_to_unif_rr

bool ERF::m_expand_plotvars_to_unif_rr = false
private

◆ m_forest_drag

amrex::Vector<std::unique_ptr<ForestDrag> > ERF::m_forest_drag
private

◆ m_plot2d_int_1

int ERF::m_plot2d_int_1 = -1
private

◆ m_plot2d_int_2

int ERF::m_plot2d_int_2 = -1
private

◆ m_plot2d_per_1

amrex::Real ERF::m_plot2d_per_1 = -1.0
private

◆ m_plot2d_per_2

amrex::Real ERF::m_plot2d_per_2 = -1.0
private

◆ m_plot3d_int_1

int ERF::m_plot3d_int_1 = -1
private

◆ m_plot3d_int_2

int ERF::m_plot3d_int_2 = -1
private

◆ m_plot3d_per_1

amrex::Real ERF::m_plot3d_per_1 = -1.0
private

◆ m_plot3d_per_2

amrex::Real ERF::m_plot3d_per_2 = -1.0
private

◆ m_plot_face_vels

bool ERF::m_plot_face_vels = false
private

◆ m_r2d

std::unique_ptr<ReadBndryPlanes> ERF::m_r2d = nullptr
private

◆ m_subvol_int

int ERF::m_subvol_int = -1
private

◆ m_subvol_per

amrex::Real ERF::m_subvol_per = -1.0
private

◆ m_SurfaceLayer

std::unique_ptr<SurfaceLayer> ERF::m_SurfaceLayer = nullptr
private

◆ m_w2d

std::unique_ptr<WriteBndryPlanes> ERF::m_w2d = nullptr
private

◆ mapfac

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::mapfac
private

◆ max_step

int ERF::max_step = -1
private

◆ metgrid_basic_linear

bool ERF::metgrid_basic_linear {false}
private

◆ metgrid_debug_dry

bool ERF::metgrid_debug_dry {false}
private

◆ metgrid_debug_isothermal

bool ERF::metgrid_debug_isothermal {false}
private

◆ metgrid_debug_msf

bool ERF::metgrid_debug_msf {false}
private

◆ metgrid_debug_psfc

bool ERF::metgrid_debug_psfc {false}
private

◆ metgrid_debug_quiescent

bool ERF::metgrid_debug_quiescent {false}
private

◆ metgrid_force_sfc_k

int ERF::metgrid_force_sfc_k {6}
private

◆ metgrid_interp_theta

bool ERF::metgrid_interp_theta {false}
private

◆ metgrid_order

int ERF::metgrid_order {2}
private

◆ metgrid_proximity

amrex::Real ERF::metgrid_proximity {500.0}
private

◆ metgrid_retain_sfc

bool ERF::metgrid_retain_sfc {false}
private

◆ metgrid_use_below_sfc

bool ERF::metgrid_use_below_sfc {true}
private

◆ metgrid_use_sfc

bool ERF::metgrid_use_sfc {true}
private

◆ mf_C1H

std::unique_ptr<amrex::MultiFab> ERF::mf_C1H
private

◆ mf_C2H

std::unique_ptr<amrex::MultiFab> ERF::mf_C2H
private

◆ mf_MUB

std::unique_ptr<amrex::MultiFab> ERF::mf_MUB
private

◆ mf_PSFC

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::mf_PSFC
private

◆ mg_verbose

int ERF::mg_verbose = 0
staticprivate

◆ micro

std::unique_ptr<Microphysics> ERF::micro
private

◆ mri_integrator_mem

amrex::Vector<std::unique_ptr<MRISplitIntegrator<amrex::Vector<amrex::MultiFab> > > > ERF::mri_integrator_mem
private

◆ nc_bdy_file

std::string ERF::nc_bdy_file
staticprivate

◆ nc_init_file

Vector< Vector< std::string > > ERF::nc_init_file = {{""}}
staticprivate

◆ nc_low_file

std::string ERF::nc_low_file
staticprivate

◆ ng_dens_hse

int ERF::ng_dens_hse
staticprivate

◆ ng_pres_hse

int ERF::ng_pres_hse
staticprivate

◆ nsubsteps

amrex::Vector<int> ERF::nsubsteps
private

◆ num_boxes_at_level

amrex::Vector<int> ERF::num_boxes_at_level
private

◆ num_files_at_level

amrex::Vector<int> ERF::num_files_at_level
private

◆ output_1d_column

int ERF::output_1d_column = 0
staticprivate

◆ output_bndry_planes

int ERF::output_bndry_planes = 0
staticprivate

◆ pert_interval

int ERF::pert_interval = -1
staticprivate

◆ phys_bc_type

amrex::GpuArray<ERF_BC, AMREX_SPACEDIM*2> ERF::phys_bc_type
private

◆ physbcs_base

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_base> > ERF::physbcs_base
private

◆ physbcs_cons

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_cons> > ERF::physbcs_cons
private

◆ physbcs_u

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_u> > ERF::physbcs_u
private

◆ physbcs_v

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_v> > ERF::physbcs_v
private

◆ physbcs_w

amrex::Vector<std::unique_ptr<ERFPhysBCFunct_w> > ERF::physbcs_w
private

◆ plot2d_file_1

std::string ERF::plot2d_file_1 {"plt2d_1_"}
private

◆ plot2d_file_2

std::string ERF::plot2d_file_2 {"plt2d_2_"}
private

◆ plot2d_var_names_1

amrex::Vector<std::string> ERF::plot2d_var_names_1
private

◆ plot2d_var_names_2

amrex::Vector<std::string> ERF::plot2d_var_names_2
private

◆ plot3d_file_1

std::string ERF::plot3d_file_1 {"plt_1_"}
private

◆ plot3d_file_2

std::string ERF::plot3d_file_2 {"plt_2_"}
private

◆ plot3d_var_names_1

amrex::Vector<std::string> ERF::plot3d_var_names_1
private

◆ plot3d_var_names_2

amrex::Vector<std::string> ERF::plot3d_var_names_2
private

◆ plot_file_on_restart

bool ERF::plot_file_on_restart = true
staticprivate

◆ plot_lsm

bool ERF::plot_lsm = false
private

◆ plot_rad

bool ERF::plot_rad = false
private

◆ plotfile2d_type_1

PlotFileType ERF::plotfile2d_type_1 = PlotFileType::None
staticprivate

◆ plotfile2d_type_2

PlotFileType ERF::plotfile2d_type_2 = PlotFileType::None
staticprivate

◆ plotfile3d_type_1

PlotFileType ERF::plotfile3d_type_1 = PlotFileType::None
staticprivate

◆ plotfile3d_type_2

PlotFileType ERF::plotfile3d_type_2 = PlotFileType::None
staticprivate

◆ pp_inc

amrex::Vector<amrex::MultiFab> ERF::pp_inc
private

◆ pp_prefix

std::string ERF::pp_prefix {"erf"}

◆ previousCPUTimeUsed

Real ERF::previousCPUTimeUsed = 0.0
staticprivate

Referenced by getCPUTime().

◆ prob

std::unique_ptr<ProblemBase> ERF::prob = nullptr
private

◆ profile_int

int ERF::profile_int = -1
private

◆ qheating_rates

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::qheating_rates
private

◆ qmoist

amrex::Vector<amrex::Vector<amrex::MultiFab*> > ERF::qmoist
private

◆ Qr_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qr_prim
private

◆ Qv_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Qv_prim
private

◆ rad

amrex::Vector<std::unique_ptr<IRadiation> > ERF::rad
private

◆ rad_datalog_int

int ERF::rad_datalog_int = -1
private

◆ real_extrap_w

bool ERF::real_extrap_w {true}
private

◆ real_set_width

int ERF::real_set_width {0}
private

◆ real_width

int ERF::real_width {0}
private

◆ ref_tags

Vector< AMRErrorTag > ERF::ref_tags
staticprivate

◆ regrid_int

int ERF::regrid_int = -1
private

◆ regrid_level_0_on_restart

bool ERF::regrid_level_0_on_restart = false
private

◆ restart_chkfile

std::string ERF::restart_chkfile = ""
private

◆ rU_new

amrex::Vector<amrex::MultiFab> ERF::rU_new
private

◆ rU_old

amrex::Vector<amrex::MultiFab> ERF::rU_old
private

◆ rV_new

amrex::Vector<amrex::MultiFab> ERF::rV_new
private

◆ rV_old

amrex::Vector<amrex::MultiFab> ERF::rV_old
private

◆ rW_new

amrex::Vector<amrex::MultiFab> ERF::rW_new
private

◆ rW_old

amrex::Vector<amrex::MultiFab> ERF::rW_old
private

◆ sampleline

amrex::Vector<amrex::IntVect> ERF::sampleline
private

Referenced by NumSampleLines(), and SampleLine().

◆ samplelinelog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::samplelinelog
private

◆ samplelinelogname

amrex::Vector<std::string> ERF::samplelinelogname
private

Referenced by SampleLineLogName().

◆ samplepoint

amrex::Vector<amrex::IntVect> ERF::samplepoint
private

Referenced by NumSamplePoints(), and SamplePoint().

◆ sampleptlog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::sampleptlog
private

◆ sampleptlogname

amrex::Vector<std::string> ERF::sampleptlogname
private

Referenced by SamplePointLogName().

◆ sampler_interval

int ERF::sampler_interval = -1
private

◆ sampler_per

amrex::Real ERF::sampler_per = -1.0
private

◆ SFS_diss_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_diss_lev
private

◆ SFS_hfx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx1_lev
private

◆ SFS_hfx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx2_lev
private

◆ SFS_hfx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_hfx3_lev
private

◆ SFS_q1fx1_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx1_lev
private

◆ SFS_q1fx2_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx2_lev
private

◆ SFS_q1fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q1fx3_lev
private

◆ SFS_q2fx3_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SFS_q2fx3_lev
private

◆ sinPhi_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sinPhi_m
private

◆ SmnSmn_lev

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::SmnSmn_lev
private

◆ solar_zenith

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::solar_zenith
private

◆ solverChoice

SolverChoice ERF::solverChoice
staticprivate

◆ sponge_type

std::string ERF::sponge_type
staticprivate

◆ sst_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::sst_lev
private

◆ start_time

Real ERF::start_time = 0.0
staticprivate

◆ startCPUTime

Real ERF::startCPUTime = 0.0
staticprivate

Referenced by getCPUTime().

◆ stop_time

Real ERF::stop_time = std::numeric_limits<amrex::Real>::max()
staticprivate

◆ stretched_dz_d

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::stretched_dz_d
private

◆ stretched_dz_h

amrex::Vector<amrex::Vector<amrex::Real> > ERF::stretched_dz_h
private

◆ sub_cfl

Real ERF::sub_cfl = 1.0
staticprivate

◆ subdomains

amrex::Vector<amrex::Vector<amrex::BoxArray> > ERF::subdomains
private

◆ subvol_file

std::string ERF::subvol_file {"subvol"}
private

◆ sum_interval

int ERF::sum_interval = -1
staticprivate

◆ sum_per

Real ERF::sum_per = -1.0
staticprivate

◆ sw_lw_fluxes

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::sw_lw_fluxes
private

◆ t_avg_cnt

amrex::Vector<amrex::Real> ERF::t_avg_cnt
private

◆ t_new

amrex::Vector<amrex::Real> ERF::t_new
private

◆ t_old

amrex::Vector<amrex::Real> ERF::t_old
private

◆ Tau

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::Tau
private

◆ terrain_blanking

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::terrain_blanking
private

◆ th_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::th_bc_data
private

◆ Theta_prim

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::Theta_prim
private

◆ thin_xforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_xforce
private

◆ thin_yforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_yforce
private

◆ thin_zforce

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::thin_zforce
private

◆ timeprecision

const int ERF::timeprecision = 13
private

◆ tot_e_datalog

amrex::Vector<std::unique_ptr<std::fstream> > ERF::tot_e_datalog
private

Referenced by setRecordEnergyDataInfo().

◆ tot_e_datalogname

amrex::Vector<std::string> ERF::tot_e_datalogname
private

◆ tsk_lev

amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab> > > ERF::tsk_lev
private

◆ turbPert

TurbulentPerturbation ERF::turbPert
private

◆ use_datetime

bool ERF::use_datetime = false
private

◆ use_fft

bool ERF::use_fft = false
staticprivate

◆ vars_new

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_new
private

◆ vars_old

amrex::Vector<amrex::Vector<amrex::MultiFab> > ERF::vars_old
private

◆ vel_t_avg

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::vel_t_avg
private

◆ verbose

int ERF::verbose = 0
staticprivate

◆ walldist

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::walldist
private

◆ weather_forecast_data_1

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_1

◆ weather_forecast_data_2

amrex::Vector<amrex::MultiFab> ERF::weather_forecast_data_2

◆ xflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::xflux_imask
private

◆ xvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::xvel_bc_data
private

◆ yflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::yflux_imask
private

◆ yvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::yvel_bc_data
private

◆ z_phys_cc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc
private

◆ z_phys_cc_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_cc_src
private

◆ z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd
private

◆ z_phys_nd_new

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_new
private

◆ z_phys_nd_src

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_phys_nd_src
private

◆ z_t_rk

amrex::Vector<std::unique_ptr<amrex::MultiFab> > ERF::z_t_rk
private

◆ zflux_imask

amrex::Vector<std::unique_ptr<amrex::iMultiFab> > ERF::zflux_imask
private

◆ zlevels_stag

amrex::Vector<amrex::Vector<amrex::Real> > ERF::zlevels_stag
private

◆ zmom_crse_rhs

amrex::Vector<amrex::MultiFab> ERF::zmom_crse_rhs
private

◆ zvel_bc_data

amrex::Vector<amrex::Gpu::DeviceVector<amrex::Real> > ERF::zvel_bc_data
private

The documentation for this class was generated from the following files: