main.cpp File Reference

#include <iostream>
#include <AMReX.H>
#include <AMReX_BLProfiler.H>
#include <AMReX_ParallelDescriptor.H>
#include "ERF.H"
#include "ERF_InputsName.H"

Include dependency graph for main.cpp:

Functions
void	add_par ()
int	main (int argc, char *argv[])

Variables
std::string	inputs_name

Function Documentation

add_par()

void add_par

(

)

Function to set the refine_grid_layout flags to (1,1,0) by default since the ERF default is different from the amrex default (1,1,1) Also set max_grid_size to very large since the only reason for chopping grids is if Nprocs > Ngrids, and set no_box_split_dir to 2 so that grids are never decomposed in the vertical direction

                {
   ParmParse pp("amr");
   ParmParse pp_erf("erf");
 
   // Set the refine_grid_layout flags to (1,1,0) by default
   pp.add("refine_grid_layout_x",1);
   pp.add("refine_grid_layout_y",1);
   pp.add("refine_grid_layout_z",0);
 
   // no_box_split_dir tells amrex that the grids must never be split in that
   // direction; max_grid_size and refine_grid_layout are then ignored there.
   // ERF sets this to 2 (the z-direction) by default so that no grid is ever
   // decomposed in the vertical -- no two grids then share a face normal to z, so
   // no column is chopped between them, which is required, e.g., by the implicit
   // vertical diffusion solves and by the implicit acoustic substepping.  (Several
   // grids may still sit over the same column, as long as they do not touch; what
   // those solves cannot do is split one contiguous column at a box seam.)  Setting
   // amr.no_box_split_dir = -1 in the inputs file restores the amrex behavior of
   // allowing grids to be chopped in every direction; ERF::ReadParameters rejects
   // that if any level uses erf.substepping_type = Implicit.
   int no_box_split_dir = 2;
   pp.queryAdd("no_box_split_dir",no_box_split_dir);
 
   // n_proper is the minimum number of coarse cells between coarse-fine boundaries
   // between levels (ell and ell+1) and levels (ell-1 and ell).   We want this to be
   // greater than or equal to the stencil width (a function of spatial order) divided by
   // ref_ratio (which can be 2,3 or 4).  This ensures that fillpatch at level (ell)
   // does not need to reach beyond level (ell-1). Here to be conservative we set this to 2
   // (rather than the amrex default of 1).
   pp.add("n_proper",2);
 
   int max_grid_size = 2048;
   pp.queryAdd("max_grid_size",max_grid_size);
 
   // This will set the default value of blocking_factor to be 1, but will allow
   //     the user to override it in the inputs file or on command line
   int blocking_factor = 1;
   pp.queryAdd("blocking_factor",blocking_factor);
 
   int n_error_buf = 0;
   pp.queryAdd("n_error_buf",n_error_buf);
 
   amrex::Vector<int> legacy_n_cell;
   amrex::Vector<int> erf_n_cell;
   const bool has_legacy_n_cell = pp.queryarr("n_cell", legacy_n_cell);
   const bool has_erf_n_cell = pp_erf.queryarr("n_cell", erf_n_cell);
   if (has_erf_n_cell && has_legacy_n_cell) {
      amrex::Abort("erf.n_cell and amr.n_cell are both specified. Please use only one!");
   }
   if (has_erf_n_cell) {
      pp.addarr("n_cell", erf_n_cell);
   }
 
   ParmParse pp_geometry("geometry");
   amrex::Vector<amrex::Real> legacy_prob_lo;
   amrex::Vector<amrex::Real> legacy_prob_hi;
   amrex::Vector<amrex::Real> legacy_prob_extent;
   amrex::Vector<int> legacy_is_periodic;
   amrex::Vector<amrex::Real> erf_prob_lo;
   amrex::Vector<amrex::Real> erf_prob_hi;
   amrex::Vector<amrex::Real> erf_prob_extent;
   amrex::Vector<int> erf_is_periodic;
 
   const bool has_legacy_prob_lo = pp_geometry.queryarr("prob_lo", legacy_prob_lo);
   const bool has_legacy_prob_hi = pp_geometry.queryarr("prob_hi", legacy_prob_hi);
   const bool has_legacy_prob_extent = pp_geometry.queryarr("prob_extent", legacy_prob_extent);
   const bool has_legacy_is_periodic = pp_geometry.queryarr("is_periodic", legacy_is_periodic);
 
   const bool has_erf_prob_lo = pp_erf.queryarr("prob_lo", erf_prob_lo);
   const bool has_erf_prob_hi = pp_erf.queryarr("prob_hi", erf_prob_hi);
   const bool has_erf_prob_extent = pp_erf.queryarr("prob_extent", erf_prob_extent);
   const bool has_erf_is_periodic = pp_erf.queryarr("is_periodic", erf_is_periodic);
 
   if (has_erf_prob_lo && has_legacy_prob_lo) {
      amrex::Abort("erf.prob_lo and geometry.prob_lo are both specified. Please use only one!");
   }
   if ((has_erf_prob_hi || has_erf_prob_extent) && !has_erf_prob_lo) {
      amrex::Abort("ERF prefixed geometry requires erf.prob_lo together with erf.prob_hi or erf.prob_extent.");
   }
   if (has_erf_is_periodic && has_legacy_is_periodic) {
      amrex::Abort("erf.is_periodic and geometry.is_periodic are both specified. Please use only one!");
   }
   if (has_erf_prob_hi && has_legacy_prob_hi) {
      amrex::Abort("erf.prob_hi and geometry.prob_hi are both specified. Please use only one!");
   }
   if (has_erf_prob_extent && has_legacy_prob_extent) {
      amrex::Abort("erf.prob_extent and geometry.prob_extent are both specified. Please use only one!");
   }
   if (has_erf_prob_hi || has_erf_prob_extent) {
      if (has_legacy_prob_hi || has_legacy_prob_extent) {
         amrex::Abort("ERF prefixed geometry and legacy geometry.prob_hi/prob_extent are both specified. Please use only one form.");
      }
   }
   if (has_erf_prob_hi && has_erf_prob_extent) {
      amrex::Vector<amrex::Real> derived_prob_hi(erf_prob_extent.size(), 0.0);
      for (int i = 0; i < static_cast<int>(erf_prob_extent.size()); ++i) {
         derived_prob_hi[i] = (i < static_cast<int>(erf_prob_lo.size()) ? erf_prob_lo[i] : 0.0) + erf_prob_extent[i];
      }
      AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
         derived_prob_hi.size() == erf_prob_hi.size(),
         "erf.prob_hi and erf.prob_extent have inconsistent lengths.");
      for (int i = 0; i < static_cast<int>(erf_prob_hi.size()); ++i) {
         if (derived_prob_hi[i] != erf_prob_hi[i]) {
            amrex::Abort("erf.prob_hi and erf.prob_extent are both specified but inconsistent.");
         }
      }
   }
 
   if (has_erf_prob_lo) {
      pp_geometry.addarr("prob_lo", erf_prob_lo);
   }
   if (has_erf_prob_hi) {
      pp_geometry.addarr("prob_hi", erf_prob_hi);
   } else if (has_erf_prob_extent) {
      amrex::Vector<amrex::Real> derived_prob_hi(erf_prob_extent.size(), 0.0);
      for (int i = 0; i < static_cast<int>(erf_prob_extent.size()); ++i) {
         derived_prob_hi[i] = (i < static_cast<int>(erf_prob_lo.size()) ? erf_prob_lo[i] : 0.0) + erf_prob_extent[i];
      }
      pp_geometry.addarr("prob_hi", derived_prob_hi);
   }
   if (has_erf_is_periodic) {
      pp_geometry.addarr("is_periodic", erf_is_periodic);
   }
}

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

main()

int main	(	int	argc,
		char *	argv[]
	)

Main driver – creates the ERF object, calls ERF.InitData() and ERF.Evolve()

{
 
auto finalize_mpi_and_return = [](int code) {
#ifdef AMREX_USE_MPI
#ifdef ERF_USE_WW3_COUPLING
    amrex::MPMD::Finalize();
#else
    MPI_Finalize();
#endif
#endif
return code;
};
 
#if defined(AMREX_MPI_THREAD_MULTIPLE)
    int requested = MPI_THREAD_MULTIPLE;
    int provided = -1;
    MPI_Init_thread(&argc, &argv, requested, &provided);
#elif defined(AMREX_USE_MPI)
    MPI_Init(&argc, &argv);
#endif
 
    if (argc < 2) {
        // Print usage and exit with error code if no input file was provided.
        ERF::print_usage(MPI_COMM_WORLD, std::cout);
        ERF::print_error(
            MPI_COMM_WORLD, "No input file provided. Exiting!!");
        return 1;
    }
 
    // Look for "-h" or "--help" flag and print usage
    for (auto i = 1; i < argc; i++) {
        const std::string param(argv[i]);
        if ((param == "--help") || (param == "-h") || (param == "--usage")) {
            ERF::print_banner(MPI_COMM_WORLD, std::cout);
            ERF::print_usage(MPI_COMM_WORLD, std::cout);
            return 0;
        }
    }
 
    if (argc >= 2) {
        for (auto i = 1; i < argc; i++) {
            if (std::string(argv[i]) == "--describe") {
                ERF::writeBuildInfo(std::cout);
                return finalize_mpi_and_return(0);
            }
        }
    }
 
    if (!strchr(argv[1], '=') && !amrex::FileSystem::Exists(std::string(argv[1])))
    {
        // Print usage and exit with error code if we cannot find the input file
        ERF::print_usage(MPI_COMM_WORLD, std::cout);
        ERF::print_error(
            MPI_COMM_WORLD, "Input file does not exist = " +
                                std::string(argv[1]) + ". Exiting!!");
        return finalize_mpi_and_return(1);
    }
 
  //  print_banner(MPI_COMM_WORLD, std::cout);
    // Check to see if the command line contains --describe
    if (argc >= 2) {
        for (auto i = 1; i < argc; i++) {
            if (std::string(argv[i]) == "--describe") {
                ERF::writeBuildInfo(std::cout);
                return 0;
            }
        }
    }
#ifdef ERF_USE_WW3_COUPLING
    MPI_Comm comm = amrex::MPMD::Initialize(argc, argv);
    amrex::Initialize(argc,argv,true,comm,add_par);
#else
    amrex::Initialize(argc,argv,true,MPI_COMM_WORLD,add_par);
#endif
 
    // Only when argv[1] is actually an inputs file (same test the surrounding code uses).
    if (argc > 1 && std::string(argv[1]).find('=') == std::string::npos) {
        CheckForDuplicateInputs(argv[1]);
    }
 
#ifdef ERF_USE_KOKKOS
    // Initialize kokkos
    if (!Kokkos::is_initialized()) {
        Kokkos::initialize(Kokkos::InitializationSettings()
                           .set_device_id(amrex::Gpu::Device::deviceId()));
    }
#endif
 
    // Save the inputs file name for later.
    if (!strchr(argv[1], '=')) {
      inputs_name = argv[1];
    }
 
    // timer for profiling
    BL_PROFILE_VAR("main()", pmain);
 
    // wallclock time
    const double strt_total = amrex::second();
 
    {
        // constructor - reads in parameters from inputs file
        //             - sizes multilevel arrays and data structures
        ERF erf;
 
        // initialize AMR data
        erf.InitData();
 
        // advance solution to final time
        erf.Evolve();
 
        // wallclock time
        double end_total = amrex::second() - strt_total;
 
        // print wallclock time
        ParallelDescriptor::ReduceRealMax(end_total ,ParallelDescriptor::IOProcessorNumber());
        if (erf.Verbose()) {
            amrex::Print() << "\nTotal Time: " << end_total << '\n';
        }
    }
 
    // destroy timer for profiling
    BL_PROFILE_VAR_STOP(pmain);
#ifdef ERF_USE_WW3_COUPLING
    MPI_Barrier(MPI_COMM_WORLD);
#endif
 
#ifdef ERF_USE_KOKKOS
    Kokkos::finalize();
#endif
 
    amrex::Finalize();
#ifdef AMREX_USE_MPI
#ifdef ERF_USE_WW3_COUPLING
    amrex::MPMD::Finalize();
#else
    MPI_Finalize();
#endif
#endif
}

Here is the call graph for this function:

Variable Documentation

inputs_name

std::string inputs_name

Referenced by main(), and ERF::writeJobInfo().