ERF
Energy Research and Forecasting: An Atmospheric Modeling Code
Functions
ERF_NCPlotFile.cpp File Reference
#include <iomanip>
#include <iostream>
#include <string>
#include <ctime>
#include <AMReX_Utility.H>
#include <AMReX_MultiFab.H>
#include "ERF_Constants.H"
#include "ERF_DataStruct.H"
#include "ERF_NCPlotFile.H"
#include "ERF_NCInterface.H"
Include dependency graph for ERF_NCPlotFile.cpp:

Functions

void writeNCPlotFile (int lev, int which_subdomain, const std::string &dir, const Vector< const MultiFab * > &plotMF, const Vector< std::string > &plot_var_names, const Vector< int > &, Array< Real, AMREX_SPACEDIM > prob_lo, Array< Real, AMREX_SPACEDIM > prob_hi, Array< Real, AMREX_SPACEDIM > dx_in, const Box &subdomain, const Real time, const Real start_bdy_time, const SolverChoice &solverChoice, const Vector< Real > &zlevels_stag)
 

Function Documentation

◆ writeNCPlotFile()

void writeNCPlotFile ( int  lev,
int  which_subdomain,
const std::string &  dir,
const Vector< const MultiFab * > &  plotMF,
const Vector< std::string > &  plot_var_names,
const Vector< int > &  ,
Array< Real, AMREX_SPACEDIM >  prob_lo,
Array< Real, AMREX_SPACEDIM >  prob_hi,
Array< Real, AMREX_SPACEDIM >  dx_in,
const Box &  subdomain,
const Real  time,
const Real  start_bdy_time,
const SolverChoice &  solverChoice,
const Vector< Real > &  zlevels_stag 
)
29 {
30  //
31  // Set the full IO path for NetCDF output
32  //
33  std::string FullPath = dir;
34  if (lev == 0) {
35  const std::string& extension = amrex::Concatenate("_d",lev+1,2);
36  FullPath += extension + ".nc";
37  } else {
38  const std::string& extension = amrex::Concatenate("_d",lev+1+which_subdomain,2);
39  FullPath += extension + ".nc";
40  }
41 
42  Print() << "Writing level " << lev << " NetCDF plot file " << FullPath << std::endl;
43 
44  //
45  // Open netcdf file to write data
46  //
47  auto ncf = ncutils::NCFile::create_par(FullPath, NC_NETCDF4 | NC_MPIIO,
48  amrex::ParallelContext::CommunicatorSub(), MPI_INFO_NULL);
49 
50  auto ba = plotMF[lev]->boxArray();
51  auto dm = plotMF[lev]->DistributionMap();
52 
53  int nblocks = ba.size();
54 
55  int nx = subdomain.length(0);
56  int ny = subdomain.length(1);
57  int nz = subdomain.length(2);
58 
59  int num_pts = nx*ny*nz;
60 
61  int n_data_items = plotMF[lev]->nComp();
62 
63  //
64  // Start Define stuff
65  //
66  ncf.enter_def_mode();
67 
68  const std::string nt_name = "num_time_steps";
69  const std::string nb_name = "num_blocks";
70  const std::string np_name = "num_pts";
71  const std::string nx_name = "nx";
72  const std::string ny_name = "ny";
73  const std::string nz_name = "nz";
74 
75  const std::string ndim_name = "num_geo_dims";
76 
77  ncf.put_attr("title", "ERF NetCDF Plot data output");
78 
79  ncf.def_dim(ndim_name, AMREX_SPACEDIM);
80  ncf.def_dim(np_name , num_pts);
81  ncf.def_dim(nb_name , nblocks);
82 
83  ncf.def_dim(nt_name, NC_UNLIMITED);
84  ncf.def_dim(nx_name, nx);
85  ncf.def_dim(ny_name, ny);
86  ncf.def_dim(nz_name, nz);
87 
88  ncf.def_var("probLo" , NC_FLOAT, {ndim_name});
89  ncf.def_var("probHi" , NC_FLOAT, {ndim_name});
90 
91  ncf.def_var("Geom.smallend", NC_INT, {ndim_name});
92  ncf.def_var("Geom.bigend" , NC_INT, {ndim_name});
93  ncf.def_var("CellSize" , NC_FLOAT, {ndim_name});
94 
95  ncf.def_var("x_grid", NC_DOUBLE, {np_name});
96  ncf.def_var("y_grid", NC_DOUBLE, {np_name});
97  ncf.def_var("z_grid", NC_DOUBLE, {np_name});
98 
99  for (int i = 0; i < plot_var_names.size(); i++) {
100  ncf.def_var(plot_var_names[i], NC_DOUBLE, {nz_name, ny_name, nx_name});
101  }
102 
103  ncf.exit_def_mode();
104  //
105  // End Define stuff
106  //
107 
108  // We are doing single-level writes but it doesn't have to be level 0
109  //
110  // Write out the netcdf plotfile head information.
111  //
112  if (n_data_items == 0) {
113  amrex::Error("Must specify at least one valid data item to plot");
114  }
115 
116  ncf.put_attr("number_variables", std::vector<int>{n_data_items});
117  ncf.put_attr("space_dimension", std::vector<int>{AMREX_SPACEDIM});
118  ncf.put_attr("current_time", std::vector<double>{time});
119  ncf.put_attr("start_time", std::vector<double>{start_bdy_time});
120  ncf.put_attr("CurrentLevel", std::vector<int>{lev});
121 
122  Real dx[AMREX_SPACEDIM];
123  for (int i = 0; i < AMREX_SPACEDIM; i++) {
124  dx[i] = dx_in[i];
125  }
126 
127  amrex::Vector<Real> probLo;
128  amrex::Vector<Real> probHi;
129  for (int i = 0; i < AMREX_SPACEDIM; i++) {
130  probLo.push_back(prob_lo[i]);
131  probHi.push_back(prob_hi[i]);
132  }
133 
134  auto nc_probLo = ncf.var("probLo");
135  nc_probLo.par_access(NC_COLLECTIVE);
136  nc_probLo.put(probLo.data(), {0}, {AMREX_SPACEDIM});
137 
138  auto nc_probHi = ncf.var("probHi");
139  nc_probHi.par_access(NC_COLLECTIVE);
140  nc_probHi.put(probHi.data(), {0}, {AMREX_SPACEDIM});
141 
142  amrex::Vector<int> smallend;
143  amrex::Vector<int> bigend;
144  smallend.clear(); bigend.clear();
145  for (int j = 0; j < AMREX_SPACEDIM; j++) {
146  smallend.push_back(subdomain.smallEnd(j));
147  bigend.push_back(subdomain.bigEnd(j));
148  }
149 
150  auto nc_Geom_smallend = ncf.var("Geom.smallend");
151  nc_Geom_smallend.par_access(NC_COLLECTIVE);
152  nc_Geom_smallend.put(smallend.data(), {0}, {AMREX_SPACEDIM});
153 
154  auto nc_Geom_bigend = ncf.var("Geom.bigend");
155  nc_Geom_bigend.par_access(NC_COLLECTIVE);
156  nc_Geom_bigend.put(bigend.data(), {0}, {AMREX_SPACEDIM});
157 
158  amrex::Vector<Real> CellSize;
159  CellSize.clear();
160  for (Real& j : dx) {
161  CellSize.push_back(j);
162  }
163  auto nc_CellSize = ncf.var("CellSize");
164  nc_CellSize.par_access(NC_COLLECTIVE);
165  nc_CellSize.put(CellSize.data(), {0}, {AMREX_SPACEDIM});
166 
167  ncf.put_attr("DefaultGeometry", std::vector<int>{amrex::DefaultGeometry().Coord()});
168 
169  std::vector<Real> x_grid;
170  std::vector<Real> y_grid;
171  std::vector<Real> z_grid;
172  long unsigned goffset = 0;
173  long unsigned glen = 0;
174 
175  // *******************************************************************************
176  // NOTE: the (x,y,z) output here are for a mesh withOUT terrain-fitted coordinates
177  // *******************************************************************************
178  if (solverChoice.mesh_type == MeshType::ConstantDz) {
179  for (int i = 0; i < ba.size(); ++i) {
180  auto bx = ba[i];
181  if (subdomain.contains(bx)) {
182  x_grid.clear(); y_grid.clear(); z_grid.clear();
183  for (auto k3 = 0; k3 < bx.length(2); ++k3) {
184  for (auto k2 = 0; k2 < bx.length(1); ++k2) {
185  for (auto k1 = 0; k1 < bx.length(0); ++k1) {
186  x_grid.push_back(prob_lo[0]+dx[0]*(static_cast<Real>(k1)+myhalf));
187  y_grid.push_back(prob_lo[1]+dx[1]*(static_cast<Real>(k2)+myhalf));
188  z_grid.push_back(prob_lo[2]+dx[2]*(static_cast<Real>(k3)+myhalf));
189  }
190  }
191  }
192 
193  goffset += glen;
194  glen = bx.numPts();
195 
196  auto nc_x_grid = ncf.var("x_grid");
197  auto nc_y_grid = ncf.var("y_grid");
198  auto nc_z_grid = ncf.var("z_grid");
199 
200  nc_x_grid.par_access(NC_COLLECTIVE);
201  nc_y_grid.par_access(NC_COLLECTIVE);
202  nc_z_grid.par_access(NC_COLLECTIVE);
203 
204  nc_x_grid.put(x_grid.data(), {goffset}, {glen});
205  nc_y_grid.put(y_grid.data(), {goffset}, {glen});
206  nc_z_grid.put(z_grid.data(), {goffset}, {glen});
207  }
208  }
209  }
210  else if (solverChoice.mesh_type == MeshType::StretchedDz)
211  {
212  for (int i = 0; i < ba.size(); ++i) {
213  auto bx = ba[i];
214  if (subdomain.contains(bx)) {
215  x_grid.clear(); y_grid.clear(); z_grid.clear();
216  for (auto k3 = 0; k3 < bx.length(2); ++k3) {
217  for (auto k2 = 0; k2 < bx.length(1); ++k2) {
218  for (auto k1 = 0; k1 < bx.length(0); ++k1) {
219  x_grid.push_back(prob_lo[0]+dx[0]*(static_cast<Real>(k1)+myhalf));
220  y_grid.push_back(prob_lo[1]+dx[1]*(static_cast<Real>(k2)+myhalf));
221  z_grid.push_back(myhalf * (zlevels_stag[k3] + zlevels_stag[k3+1]));
222  }
223  }
224  }
225 
226  goffset += glen;
227  glen = bx.numPts();
228 
229  auto nc_x_grid = ncf.var("x_grid");
230  auto nc_y_grid = ncf.var("y_grid");
231  auto nc_z_grid = ncf.var("z_grid");
232 
233  nc_x_grid.par_access(NC_COLLECTIVE);
234  nc_y_grid.par_access(NC_COLLECTIVE);
235  nc_z_grid.par_access(NC_COLLECTIVE);
236 
237  nc_x_grid.put(x_grid.data(), {goffset}, {glen});
238  nc_y_grid.put(y_grid.data(), {goffset}, {glen});
239  nc_z_grid.put(z_grid.data(), {goffset}, {glen});
240  }
241  }
242  }
243 
244  const int ncomp = plotMF[lev]->nComp();
245 
246  for (MFIter mfi(*plotMF[lev]); mfi.isValid(); ++mfi)
247  {
248  auto bx = mfi.validbox();
249 
250  if (subdomain.contains(bx))
251 
252  {
253  //
254  // These are the dimensions of the data we write for only this box
255  //
256  long unsigned local_nx = bx.length()[0];
257  long unsigned local_ny = bx.length()[1];
258  long unsigned local_nz = bx.length()[2];
259 
260  long unsigned local_start_x = static_cast<long unsigned>(bx.smallEnd()[0]-subdomain.smallEnd()[0]);
261  long unsigned local_start_y = static_cast<long unsigned>(bx.smallEnd()[1]-subdomain.smallEnd()[1]);
262  long unsigned local_start_z = static_cast<long unsigned>(bx.smallEnd()[2]-subdomain.smallEnd()[2]);
263 
264  for (int k(0); k < ncomp; ++k) {
265  FArrayBox tmp;
266  tmp.resize(bx, 1, amrex::The_Pinned_Arena());
267  tmp.template copy<RunOn::Device>((*plotMF[lev])[mfi.index()], k, 0, 1);
268  Gpu::streamSynchronize();
269 
270  auto nc_plot_var = ncf.var(plot_var_names[k]);
271  nc_plot_var.par_access(NC_COLLECTIVE);
272  nc_plot_var.put(tmp.dataPtr(), {local_start_z,local_start_y,local_start_x},
273  {local_nz, local_ny, local_nx});
274  }
275  }
276  }
277  ncf.close();
278 }
myhalf
constexpr amrex::Real myhalf
Definition: ERF_Constants.H:13
dx
const Real dx
Definition: ERF_InitCustomPert_ABL.H:23
prob_lo
const amrex::Real * prob_lo
Definition: ERF_InitCustomPert_DataAssimilation_ISV.H:16
prob_hi
const amrex::Real * prob_hi
Definition: ERF_InitCustomPert_DataAssimilation_ISV.H:17
Real
amrex::Real Real
Definition: ERF_ShocInterface.H:19
ncutils::NCFile::create_par
static NCFile create_par(const std::string &name, const int cmode=NC_CLOBBER|NC_NETCDF4|NC_MPIIO, MPI_Comm comm=MPI_COMM_WORLD, MPI_Info info=MPI_INFO_NULL)
Definition: ERF_NCInterface.cpp:714
WSM6SedCellScratch::tmp
@ tmp
Definition: ERF_AdvanceWSM6.cpp:114
module_mp_morr_two_moment::k1
real(c_double), private k1
Definition: ERF_module_mp_morr_two_moment.F90:213
SolverChoice::mesh_type
static MeshType mesh_type
Definition: ERF_DataStruct.H:1211

Referenced by ERF::Write2DPlotFile(), and ERF::Write3DPlotFile().

Here is the call graph for this function:
Here is the caller graph for this function: