ERF
Energy Research and Forecasting: An Atmospheric Modeling Code
Functions
ERF_NCPlotFile.cpp File Reference
#include <iomanip>
#include <iostream>
#include <string>
#include <ctime>
#include <AMReX_Utility.H>
#include <AMReX_MultiFab.H>
#include "ERF_Constants.H"
#include "ERF_NCInterface.H"
Include dependency graph for ERF_NCPlotFile.cpp:

Functions

void writeNCPlotFile (int lev, int which_subdomain, const std::string &dir, const Vector< const MultiFab * > &plotMF, const Vector< std::string > &plot_var_names, const Vector< int > &, Array< Real, AMREX_SPACEDIM > prob_lo, Array< Real, AMREX_SPACEDIM > prob_hi, Array< Real, AMREX_SPACEDIM > dx_in, const Box &subdomain, const Real time, const Real start_bdy_time)
 

Function Documentation

◆ writeNCPlotFile()

void writeNCPlotFile ( int  lev,
int  which_subdomain,
const std::string &  dir,
const Vector< const MultiFab * > &  plotMF,
const Vector< std::string > &  plot_var_names,
const Vector< int > &  ,
Array< Real, AMREX_SPACEDIM >  prob_lo,
Array< Real, AMREX_SPACEDIM >  prob_hi,
Array< Real, AMREX_SPACEDIM >  dx_in,
const Box &  subdomain,
const Real  time,
const Real  start_bdy_time 
)
24 {
25  //
26  // Set the full IO path for NetCDF output
27  //
28  std::string FullPath = dir;
29  if (lev == 0) {
30  const std::string& extension = amrex::Concatenate("_d",lev+1,2);
31  FullPath += extension + ".nc";
32  } else {
33  const std::string& extension = amrex::Concatenate("_d",lev+1+which_subdomain,2);
34  FullPath += extension + ".nc";
35  }
36 
37  Print() << "Writing level " << lev << " NetCDF plot file " << FullPath << std::endl;
38 
39  //
40  // Open netcdf file to write data
41  //
42  auto ncf = ncutils::NCFile::create_par(FullPath, NC_NETCDF4 | NC_MPIIO,
43  amrex::ParallelContext::CommunicatorSub(), MPI_INFO_NULL);
44 
45  auto ba = plotMF[lev]->boxArray();
46  auto dm = plotMF[lev]->DistributionMap();
47 
48  int nblocks = ba.size();
49 
50  int nx = subdomain.length(0);
51  int ny = subdomain.length(1);
52  int nz = subdomain.length(2);
53 
54  int num_pts = nx*ny*nz;
55 
56  int n_data_items = plotMF[lev]->nComp();
57 
58  //
59  // Start Define stuff
60  //
61  ncf.enter_def_mode();
62 
63  const std::string nt_name = "num_time_steps";
64  const std::string nb_name = "num_blocks";
65  const std::string np_name = "num_pts";
66  const std::string nx_name = "nx";
67  const std::string ny_name = "ny";
68  const std::string nz_name = "nz";
69 
70  const std::string ndim_name = "num_geo_dims";
71 
72  ncf.put_attr("title", "ERF NetCDF Plot data output");
73 
74  ncf.def_dim(ndim_name, AMREX_SPACEDIM);
75  ncf.def_dim(np_name , num_pts);
76  ncf.def_dim(nb_name , nblocks);
77 
78  ncf.def_dim(nt_name, NC_UNLIMITED);
79  ncf.def_dim(nx_name, nx);
80  ncf.def_dim(ny_name, ny);
81  ncf.def_dim(nz_name, nz);
82 
83  ncf.def_var("probLo" , NC_FLOAT, {ndim_name});
84  ncf.def_var("probHi" , NC_FLOAT, {ndim_name});
85 
86  ncf.def_var("Geom.smallend", NC_INT, {ndim_name});
87  ncf.def_var("Geom.bigend" , NC_INT, {ndim_name});
88  ncf.def_var("CellSize" , NC_FLOAT, {ndim_name});
89 
90  ncf.def_var("x_grid", NC_DOUBLE, {np_name});
91  ncf.def_var("y_grid", NC_DOUBLE, {np_name});
92  ncf.def_var("z_grid", NC_DOUBLE, {np_name});
93 
94  for (int i = 0; i < plot_var_names.size(); i++) {
95  ncf.def_var(plot_var_names[i], NC_DOUBLE, {nz_name, ny_name, nx_name});
96  }
97 
98  ncf.exit_def_mode();
99  //
100  // End Define stuff
101  //
102 
103  // We are doing single-level writes but it doesn't have to be level 0
104  //
105  // Write out the netcdf plotfile head information.
106  //
107  if (n_data_items == 0) {
108  amrex::Error("Must specify at least one valid data item to plot");
109  }
110 
111  ncf.put_attr("number_variables", std::vector<int>{n_data_items});
112  ncf.put_attr("space_dimension", std::vector<int>{AMREX_SPACEDIM});
113  ncf.put_attr("current_time", std::vector<double>{time});
114  ncf.put_attr("start_time", std::vector<double>{start_bdy_time});
115  ncf.put_attr("CurrentLevel", std::vector<int>{lev});
116 
117  Real dx[AMREX_SPACEDIM];
118  for (int i = 0; i < AMREX_SPACEDIM; i++) {
119  dx[i] = dx_in[i];
120  }
121 
122  amrex::Vector<Real> probLo;
123  amrex::Vector<Real> probHi;
124  for (int i = 0; i < AMREX_SPACEDIM; i++) {
125  probLo.push_back(prob_lo[i]);
126  probHi.push_back(prob_hi[i]);
127  }
128 
129  auto nc_probLo = ncf.var("probLo");
130  nc_probLo.par_access(NC_COLLECTIVE);
131  nc_probLo.put(probLo.data(), {0}, {AMREX_SPACEDIM});
132 
133  auto nc_probHi = ncf.var("probHi");
134  nc_probHi.par_access(NC_COLLECTIVE);
135  nc_probHi.put(probHi.data(), {0}, {AMREX_SPACEDIM});
136 
137  amrex::Vector<int> smallend;
138  amrex::Vector<int> bigend;
139  smallend.clear(); bigend.clear();
140  for (int j = 0; j < AMREX_SPACEDIM; j++) {
141  smallend.push_back(subdomain.smallEnd(j));
142  bigend.push_back(subdomain.bigEnd(j));
143  }
144 
145  auto nc_Geom_smallend = ncf.var("Geom.smallend");
146  nc_Geom_smallend.par_access(NC_COLLECTIVE);
147  nc_Geom_smallend.put(smallend.data(), {0}, {AMREX_SPACEDIM});
148 
149  auto nc_Geom_bigend = ncf.var("Geom.bigend");
150  nc_Geom_bigend.par_access(NC_COLLECTIVE);
151  nc_Geom_bigend.put(bigend.data(), {0}, {AMREX_SPACEDIM});
152 
153  amrex::Vector<Real> CellSize;
154  CellSize.clear();
155  for (Real& j : dx) {
156  CellSize.push_back(j);
157  }
158  auto nc_CellSize = ncf.var("CellSize");
159  nc_CellSize.par_access(NC_COLLECTIVE);
160  nc_CellSize.put(CellSize.data(), {0}, {AMREX_SPACEDIM});
161 
162  ncf.put_attr("DefaultGeometry", std::vector<int>{amrex::DefaultGeometry().Coord()});
163 
164  std::vector<Real> x_grid;
165  std::vector<Real> y_grid;
166  std::vector<Real> z_grid;
167  long unsigned goffset = 0;
168  long unsigned glen = 0;
169 
170  // *******************************************************************************
171  // NOTE: the (x,y,z) output here are for a mesh withOUT terrain-fitted coordinates
172  // *******************************************************************************
173  for (int i = 0; i < ba.size(); ++i) {
174  auto bx = ba[i];
175  if (subdomain.contains(bx)) {
176  x_grid.clear(); y_grid.clear(); z_grid.clear();
177  for (auto k3 = 0; k3 < bx.length(2); ++k3) {
178  for (auto k2 = 0; k2 < bx.length(1); ++k2) {
179  for (auto k1 = 0; k1 < bx.length(0); ++k1) {
180  x_grid.push_back(prob_lo[0]+dx[0]*(static_cast<Real>(k1)+myhalf));
181  y_grid.push_back(prob_lo[1]+dx[1]*(static_cast<Real>(k2)+myhalf));
182  z_grid.push_back(prob_lo[2]+dx[2]*(static_cast<Real>(k3)+myhalf));
183  }
184  }
185  }
186 
187  goffset += glen;
188  glen = bx.numPts();
189 
190  auto nc_x_grid = ncf.var("x_grid");
191  auto nc_y_grid = ncf.var("y_grid");
192  auto nc_z_grid = ncf.var("z_grid");
193 
194  nc_x_grid.par_access(NC_COLLECTIVE);
195  nc_y_grid.par_access(NC_COLLECTIVE);
196  nc_z_grid.par_access(NC_COLLECTIVE);
197 
198  nc_x_grid.put(x_grid.data(), {goffset}, {glen});
199  nc_y_grid.put(y_grid.data(), {goffset}, {glen});
200  nc_z_grid.put(z_grid.data(), {goffset}, {glen});
201  }
202  }
203 
204  const int ncomp = plotMF[lev]->nComp();
205 
206  for (MFIter mfi(*plotMF[lev]); mfi.isValid(); ++mfi)
207  {
208  auto bx = mfi.validbox();
209 
210  if (subdomain.contains(bx))
211 
212  {
213  //
214  // These are the dimensions of the data we write for only this box
215  //
216  long unsigned local_nx = bx.length()[0];
217  long unsigned local_ny = bx.length()[1];
218  long unsigned local_nz = bx.length()[2];
219 
220  long unsigned local_start_x = static_cast<long unsigned>(bx.smallEnd()[0]-subdomain.smallEnd()[0]);
221  long unsigned local_start_y = static_cast<long unsigned>(bx.smallEnd()[1]-subdomain.smallEnd()[1]);
222  long unsigned local_start_z = static_cast<long unsigned>(bx.smallEnd()[2]-subdomain.smallEnd()[2]);
223 
224  for (int k(0); k < ncomp; ++k) {
225  FArrayBox tmp;
226  tmp.resize(bx, 1, amrex::The_Pinned_Arena());
227  tmp.template copy<RunOn::Device>((*plotMF[lev])[mfi.index()], k, 0, 1);
228  Gpu::streamSynchronize();
229 
230  auto nc_plot_var = ncf.var(plot_var_names[k]);
231  nc_plot_var.par_access(NC_COLLECTIVE);
232  nc_plot_var.put(tmp.dataPtr(), {local_start_z,local_start_y,local_start_x},
233  {local_nz, local_ny, local_nx});
234  }
235  }
236  }
237  ncf.close();
238 }
myhalf
constexpr amrex::Real myhalf
Definition: ERF_Constants.H:11
dx
const Real dx
Definition: ERF_InitCustomPert_ABL.H:23
prob_lo
const amrex::Real * prob_lo
Definition: ERF_InitCustomPert_DataAssimilation_ISV.H:16
prob_hi
const amrex::Real * prob_hi
Definition: ERF_InitCustomPert_DataAssimilation_ISV.H:17
Real
amrex::Real Real
Definition: ERF_ShocInterface.H:19
ncutils::NCFile::create_par
static NCFile create_par(const std::string &name, const int cmode=NC_CLOBBER|NC_NETCDF4|NC_MPIIO, MPI_Comm comm=MPI_COMM_WORLD, MPI_Info info=MPI_INFO_NULL)
Definition: ERF_NCInterface.cpp:714
module_mp_morr_two_moment::k1
real(c_double), private k1
Definition: ERF_module_mp_morr_two_moment.F90:213

Referenced by ERF::Write2DPlotFile(), and ERF::Write3DPlotFile().

Here is the call graph for this function:
Here is the caller graph for this function: