main.cpp File Reference

#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <iostream>
#include <limits>
#include <AMReX.H>
#include <AMReX_BLProfiler.H>
#include <AMReX_ParallelDescriptor.H>
#include "ERF.H"
#include "ERF_InputsName.H"

Include dependency graph for main.cpp:

Functions
void	add_par ()
static void	erf_flush_fortran_units ()
void	install_exit_trap ()
int	main (int argc, char *argv[])

Variables
std::string	inputs_name

Function Documentation

add_par()

void add_par

(

)

Function to set the refine_grid_layout flags to (1,1,0) by default since the ERF default is different from the amrex default (1,1,1) Also set max_grid_size to very large since the only reason for chopping grids is if Nprocs > Ngrids, and set no_box_split_dir to 2 so that grids are never decomposed in the vertical direction

                {
   ParmParse pp("amr");
   ParmParse pp_erf("erf");
 
   // Set the refine_grid_layout flags to (1,1,0) by default
   pp.add("refine_grid_layout_x",1);
   pp.add("refine_grid_layout_y",1);
   pp.add("refine_grid_layout_z",0);
 
   // no_box_split_dir tells amrex that the grids must never be split in that
   // direction; max_grid_size and refine_grid_layout are then ignored there.
   // ERF sets this to 2 (the z-direction) by default so that no grid is ever
   // decomposed in the vertical -- no two grids then share a face normal to z, so
   // no column is chopped between them, which is required, e.g., by the implicit
   // vertical diffusion solves and by the implicit acoustic substepping.  (Several
   // grids may still sit over the same column, as long as they do not touch; what
   // those solves cannot do is split one contiguous column at a box seam.)  Setting
   // amr.no_box_split_dir = -1 in the inputs file restores the amrex behavior of
   // allowing grids to be chopped in every direction; ERF::ReadParameters rejects
   // that if any level uses erf.substepping_type = Implicit.
   int no_box_split_dir = 2;
   pp.queryAdd("no_box_split_dir",no_box_split_dir);
 
   // n_proper is the minimum number of coarse cells between coarse-fine boundaries
   // between levels (ell and ell+1) and levels (ell-1 and ell).   We want this to be
   // greater than or equal to the stencil width (a function of spatial order) divided by
   // ref_ratio (which can be 2,3 or 4).  This ensures that fillpatch at level (ell)
   // does not need to reach beyond level (ell-1). Here to be conservative we set this to 2
   // (rather than the amrex default of 1).
   pp.add("n_proper",2);
 
   int max_grid_size = 2048;
   pp.queryAdd("max_grid_size",max_grid_size);
 
   // This will set the default value of blocking_factor to be 1, but will allow
   //     the user to override it in the inputs file or on command line
   int blocking_factor = 1;
   pp.queryAdd("blocking_factor",blocking_factor);
 
   int n_error_buf = 0;
   pp.queryAdd("n_error_buf",n_error_buf);
 
   amrex::Vector<int> legacy_n_cell;
   amrex::Vector<int> erf_n_cell;
   const bool has_legacy_n_cell = pp.queryarr("n_cell", legacy_n_cell);
   const bool has_erf_n_cell = pp_erf.queryarr("n_cell", erf_n_cell);
   if (has_erf_n_cell && has_legacy_n_cell) {
      amrex::Abort("erf.n_cell and amr.n_cell are both specified. Please use only one!");
   }
   if (has_erf_n_cell) {
      pp.addarr("n_cell", erf_n_cell);
   }
 
   ParmParse pp_geometry("geometry");
   amrex::Vector<amrex::Real> legacy_prob_lo;
   amrex::Vector<amrex::Real> legacy_prob_hi;
   amrex::Vector<amrex::Real> legacy_prob_extent;
   amrex::Vector<int> legacy_is_periodic;
   amrex::Vector<amrex::Real> erf_prob_lo;
   amrex::Vector<amrex::Real> erf_prob_hi;
   amrex::Vector<amrex::Real> erf_prob_extent;
   amrex::Vector<int> erf_is_periodic;
 
   const bool has_legacy_prob_lo = pp_geometry.queryarr("prob_lo", legacy_prob_lo);
   const bool has_legacy_prob_hi = pp_geometry.queryarr("prob_hi", legacy_prob_hi);
   const bool has_legacy_prob_extent = pp_geometry.queryarr("prob_extent", legacy_prob_extent);
   const bool has_legacy_is_periodic = pp_geometry.queryarr("is_periodic", legacy_is_periodic);
 
   const bool has_erf_prob_lo = pp_erf.queryarr("prob_lo", erf_prob_lo);
   const bool has_erf_prob_hi = pp_erf.queryarr("prob_hi", erf_prob_hi);
   const bool has_erf_prob_extent = pp_erf.queryarr("prob_extent", erf_prob_extent);
   const bool has_erf_is_periodic = pp_erf.queryarr("is_periodic", erf_is_periodic);
 
   if (has_erf_prob_lo && has_legacy_prob_lo) {
      amrex::Abort("erf.prob_lo and geometry.prob_lo are both specified. Please use only one!");
   }
   if ((has_erf_prob_hi || has_erf_prob_extent) && !has_erf_prob_lo) {
      amrex::Abort("ERF prefixed geometry requires erf.prob_lo together with erf.prob_hi or erf.prob_extent.");
   }
   if (has_erf_is_periodic && has_legacy_is_periodic) {
      amrex::Abort("erf.is_periodic and geometry.is_periodic are both specified. Please use only one!");
   }
   if (has_erf_prob_hi && has_legacy_prob_hi) {
      amrex::Abort("erf.prob_hi and geometry.prob_hi are both specified. Please use only one!");
   }
   if (has_erf_prob_extent && has_legacy_prob_extent) {
      amrex::Abort("erf.prob_extent and geometry.prob_extent are both specified. Please use only one!");
   }
   if (has_erf_prob_hi || has_erf_prob_extent) {
      if (has_legacy_prob_hi || has_legacy_prob_extent) {
         amrex::Abort("ERF prefixed geometry and legacy geometry.prob_hi/prob_extent are both specified. Please use only one form.");
      }
   }
   if (has_erf_prob_hi && has_erf_prob_extent) {
      amrex::Vector<amrex::Real> derived_prob_hi(erf_prob_extent.size(), 0.0);
      for (int i = 0; i < static_cast<int>(erf_prob_extent.size()); ++i) {
         derived_prob_hi[i] = (i < static_cast<int>(erf_prob_lo.size()) ? erf_prob_lo[i] : 0.0) + erf_prob_extent[i];
      }
      AMREX_ALWAYS_ASSERT_WITH_MESSAGE(
         derived_prob_hi.size() == erf_prob_hi.size(),
         "erf.prob_hi and erf.prob_extent have inconsistent lengths.");
      // Allow for rounding in prob_lo + prob_extent, relative to the inputs' magnitude
      for (int i = 0; i < static_cast<int>(erf_prob_hi.size()); ++i) {
         const amrex::Real lo = (i < static_cast<int>(erf_prob_lo.size()) ? erf_prob_lo[i] : 0.0);
         const amrex::Real scale = std::max({std::abs(lo), std::abs(erf_prob_extent[i]), std::abs(erf_prob_hi[i])});
         const amrex::Real tol = amrex::Real(100.0) * std::numeric_limits<amrex::Real>::epsilon() * scale;
         if (std::abs(derived_prob_hi[i] - erf_prob_hi[i]) > tol) {
            amrex::Abort("erf.prob_hi and erf.prob_extent are both specified but inconsistent.");
         }
      }
   }
 
   if (has_erf_prob_lo) {
      pp_geometry.addarr("prob_lo", erf_prob_lo);
   }
   if (has_erf_prob_hi) {
      pp_geometry.addarr("prob_hi", erf_prob_hi);
   } else if (has_erf_prob_extent) {
      amrex::Vector<amrex::Real> derived_prob_hi(erf_prob_extent.size(), 0.0);
      for (int i = 0; i < static_cast<int>(erf_prob_extent.size()); ++i) {
         derived_prob_hi[i] = (i < static_cast<int>(erf_prob_lo.size()) ? erf_prob_lo[i] : 0.0) + erf_prob_extent[i];
      }
      pp_geometry.addarr("prob_hi", derived_prob_hi);
   }
   if (has_erf_is_periodic) {
      pp_geometry.addarr("is_periodic", erf_is_periodic);
   }
}

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

erf_flush_fortran_units()

static void erf_flush_fortran_units ( )

static

{}

Referenced by install_exit_trap().

Here is the caller graph for this function:

install_exit_trap()

void install_exit_trap

(

)

Make a run that leaves through exit() before amrex::Finalize report failure.

A normal run calls amrex::Finalize before main returns, so a process that reaches exit() while AMReX is still initialised has stopped part-way – with no final plotfile or checkpoint – whatever status it asked for. The ways in are a Fortran STOP in a coupled library (Noah-MP has 27, all exit status 0) and the exit() calls left in ERF itself, some of which also pass 0. Run under Open MPI's launcher such a run is still flagged, because the rank never calls MPI_Finalize; run directly, or under a launcher that does not enforce that, it reports success. Every return from main after amrex::Initialize goes through amrex::Finalize, so this never fires on a normal exit.

amrex::Abort ends in MPI_Abort, or std::_Exit in a serial build. Open MPI's MPI_Abort kills the job without exit(), but MPICH's (and Cray MPICH's) ends in exit(errorcode), so there the trap runs after an abort as well: the run is still reported as a failure, with status 1 in place of the abort's code, and the message below allows for it.

A Fortran STOP or an exit() fires on the one rank that hit the problem; ending only that rank would leave the others blocked in their next collective until the launcher notices (Open MPI does; srun without –kill-on-bad-exit and some PMI setups may not). So the trap calls MPI_Abort to tear the whole job down, guarded so that an MPI whose own MPI_Abort ends in exit() (MPICH, Cray MPICH) cannot re-enter it, and skipped when MPI is not initialised or already finalised.

std::_Exit is the only way to change the status from here, and it skips the rest of the exit sequence, which is what would flush the output buffers. Standard output is fully buffered when it is redirected to a file, so flush everything first – C++ streams, C stdio and the Fortran units – or the lines that explain the stop are lost.

{
    std::atexit([]() {
        if (amrex::Initialized()) {
            std::cout.flush();
            std::clog.flush();
            std::cerr.flush();
            erf_flush_fortran_units();
            std::fflush(nullptr);
            std::fputs("ERF: exit() was called while AMReX was still initialised, so the run "
                       "stopped before finishing; the reason should be printed above (a "
                       "Fortran STOP in a coupled library such as Noah-MP, an error exit in "
                       "ERF, or an amrex::Abort under an MPI whose MPI_Abort calls exit()). "
                       "Reporting failure whatever exit status was requested.\n",
                       stderr);
            std::fflush(stderr);
#ifdef AMREX_USE_MPI
            // atexit handlers run single-threaded, so a plain flag is enough of a
            // reentrancy guard for the MPI_Abort-calls-exit() implementations.
            static bool aborting = false;
            int mpi_up = 0, mpi_done = 0;
            MPI_Initialized(&mpi_up);
            MPI_Finalized(&mpi_done);
            if (mpi_up && !mpi_done && !aborting) {
                aborting = true;
                MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
            }
#endif
            std::_Exit(EXIT_FAILURE);
        }
    });
}

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

main()

int main	(	int	argc,
		char *	argv[]
	)

Main driver – creates the ERF object, calls ERF.InitData() and ERF.Evolve()

{
 
// For the early exits below, which all come before amrex::Initialize (and, with
// WW3 coupling, before amrex::MPMD::Initialize): MPI is ours to finalize.
auto finalize_mpi_and_return = [](int code) {
#ifdef AMREX_USE_MPI
    MPI_Finalize();
#endif
    return code;
};
 
#if defined(AMREX_MPI_THREAD_MULTIPLE)
    int requested = MPI_THREAD_MULTIPLE;
    int provided = -1;
    MPI_Init_thread(&argc, &argv, requested, &provided);
#elif defined(AMREX_USE_MPI)
    MPI_Init(&argc, &argv);
#endif
 
    if (argc < 2) {
        // Print usage and exit with error code if no input file was provided.
        ERF::print_usage(MPI_COMM_WORLD, std::cout);
        ERF::print_error(
            MPI_COMM_WORLD, "No input file provided. Exiting!!");
        return finalize_mpi_and_return(1);
    }
 
    // Look for "-h" or "--help" flag and print usage
    for (auto i = 1; i < argc; i++) {
        const std::string param(argv[i]);
        if ((param == "--help") || (param == "-h") || (param == "--usage")) {
            ERF::print_banner(MPI_COMM_WORLD, std::cout);
            ERF::print_usage(MPI_COMM_WORLD, std::cout);
            return finalize_mpi_and_return(0);
        }
    }
 
    if (argc >= 2) {
        for (auto i = 1; i < argc; i++) {
            if (std::string(argv[i]) == "--describe") {
                ERF::writeBuildInfo(std::cout);
                return finalize_mpi_and_return(0);
            }
        }
    }
 
    if (!strchr(argv[1], '=') && !amrex::FileSystem::Exists(std::string(argv[1])))
    {
        // Print usage and exit with error code if we cannot find the input file
        ERF::print_usage(MPI_COMM_WORLD, std::cout);
        ERF::print_error(
            MPI_COMM_WORLD, "Input file does not exist = " +
                                std::string(argv[1]) + ". Exiting!!");
        return finalize_mpi_and_return(1);
    }
 
#ifdef ERF_USE_WW3_COUPLING
    MPI_Comm comm = amrex::MPMD::Initialize(argc, argv);
    amrex::Initialize(argc,argv,true,comm,add_par);
#else
    amrex::Initialize(argc,argv,true,MPI_COMM_WORLD,add_par);
#endif
 
    install_exit_trap();
 
    // Only when argv[1] is actually an inputs file (same test the surrounding code uses).
    if (argc > 1 && std::string(argv[1]).find('=') == std::string::npos) {
        CheckForDuplicateInputs(argv[1]);
    }
 
#ifdef ERF_USE_KOKKOS
    // Initialize kokkos
    if (!Kokkos::is_initialized()) {
        Kokkos::initialize(Kokkos::InitializationSettings()
                           .set_device_id(amrex::Gpu::Device::deviceId()));
    }
#endif
 
    // Save the inputs file name for later.
    if (!strchr(argv[1], '=')) {
      inputs_name = argv[1];
    }
 
    // timer for profiling
    BL_PROFILE_VAR("main()", pmain);
 
    // wallclock time
    const double strt_total = amrex::second();
 
    {
        // constructor - reads in parameters from inputs file
        //             - sizes multilevel arrays and data structures
        ERF erf;
 
        // initialize AMR data
        erf.InitData();
 
        // advance solution to final time
        erf.Evolve();
 
        // wallclock time
        double end_total = amrex::second() - strt_total;
 
        // print wallclock time
        ParallelDescriptor::ReduceRealMax(end_total ,ParallelDescriptor::IOProcessorNumber());
        if (erf.Verbose()) {
            amrex::Print() << "\nTotal Time: " << end_total << '\n';
        }
    }
 
    // destroy timer for profiling
    BL_PROFILE_VAR_STOP(pmain);
#ifdef ERF_USE_WW3_COUPLING
    MPI_Barrier(MPI_COMM_WORLD);
#endif
 
#ifdef ERF_USE_KOKKOS
    Kokkos::finalize();
#endif
 
    amrex::Finalize();
#ifdef AMREX_USE_MPI
#ifdef ERF_USE_WW3_COUPLING
    // MPI was initialized above, before amrex::MPMD::Initialize, so MPMD::Finalize
    // frees its communicator but leaves MPI_Finalize to us.
    amrex::MPMD::Finalize();
#endif
    MPI_Finalize();
#endif
}

Here is the call graph for this function:

Variable Documentation

inputs_name

std::string inputs_name

Referenced by main(), and ERF::writeJobInfo().