ERF
Energy Research and Forecasting: An Atmospheric Modeling Code
•All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Functions
ERF_NCPlotFile.cpp File Reference
#include <iomanip>
#include <iostream>
#include <string>
#include <ctime>
#include <AMReX_Utility.H>
#include <AMReX_MultiFab.H>
#include "ERF_NCInterface.H"
Include dependency graph for ERF_NCPlotFile.cpp:

Functions

void writeNCPlotFile (int lev, int which_subdomain, const std::string &dir, const Vector< const MultiFab * > &plotMF, const Vector< std::string > &plot_var_names, const Vector< int > &, Array< Real, AMREX_SPACEDIM > prob_lo, Array< Real, AMREX_SPACEDIM > prob_hi, Array< Real, AMREX_SPACEDIM > dx_in, const Box &subdomain, const Real time, const Real start_bdy_time)
 

Function Documentation

◆ writeNCPlotFile()

void writeNCPlotFile ( int  lev,
int  which_subdomain,
const std::string &  dir,
const Vector< const MultiFab * > &  plotMF,
const Vector< std::string > &  plot_var_names,
const Vector< int > &  ,
Array< Real, AMREX_SPACEDIM >  prob_lo,
Array< Real, AMREX_SPACEDIM >  prob_hi,
Array< Real, AMREX_SPACEDIM >  dx_in,
const Box &  subdomain,
const Real  time,
const Real  start_bdy_time 
)
23 {
24  //
25  // Set the full IO path for NetCDF output
26  //
27  std::string FullPath = dir;
28  if (lev == 0) {
29  const std::string& extension = amrex::Concatenate("_d",lev+1,2);
30  FullPath += extension + ".nc";
31  } else {
32  const std::string& extension = amrex::Concatenate("_d",lev+1+which_subdomain,2);
33  FullPath += extension + ".nc";
34  }
35 
36  Print() << "Writing level " << lev << " NetCDF plot file " << FullPath << std::endl;
37 
38  //
39  // Open netcdf file to write data
40  //
41  auto ncf = ncutils::NCFile::create_par(FullPath, NC_NETCDF4 | NC_MPIIO,
42  amrex::ParallelContext::CommunicatorSub(), MPI_INFO_NULL);
43 
44  auto ba = plotMF[lev]->boxArray();
45  auto dm = plotMF[lev]->DistributionMap();
46 
47  int nblocks = ba.size();
48 
49  int nx = subdomain.length(0);
50  int ny = subdomain.length(1);
51  int nz = subdomain.length(2);
52 
53  int num_pts = nx*ny*nz;
54 
55  int n_data_items = plotMF[lev]->nComp();
56 
57  //
58  // Start Define stuff
59  //
60  ncf.enter_def_mode();
61 
62  const std::string nt_name = "num_time_steps";
63  const std::string nb_name = "num_blocks";
64  const std::string np_name = "num_pts";
65  const std::string nx_name = "nx";
66  const std::string ny_name = "ny";
67  const std::string nz_name = "nz";
68 
69  const std::string ndim_name = "num_geo_dims";
70 
71  ncf.put_attr("title", "ERF NetCDF Plot data output");
72 
73  ncf.def_dim(ndim_name, AMREX_SPACEDIM);
74  ncf.def_dim(np_name , num_pts);
75  ncf.def_dim(nb_name , nblocks);
76 
77  ncf.def_dim(nt_name, NC_UNLIMITED);
78  ncf.def_dim(nx_name, nx);
79  ncf.def_dim(ny_name, ny);
80  ncf.def_dim(nz_name, nz);
81 
82  ncf.def_var("probLo" , NC_FLOAT, {ndim_name});
83  ncf.def_var("probHi" , NC_FLOAT, {ndim_name});
84 
85  ncf.def_var("Geom.smallend", NC_INT, {ndim_name});
86  ncf.def_var("Geom.bigend" , NC_INT, {ndim_name});
87  ncf.def_var("CellSize" , NC_FLOAT, {ndim_name});
88 
89  ncf.def_var("x_grid", NC_DOUBLE, {np_name});
90  ncf.def_var("y_grid", NC_DOUBLE, {np_name});
91  ncf.def_var("z_grid", NC_DOUBLE, {np_name});
92 
93  for (int i = 0; i < plot_var_names.size(); i++) {
94  ncf.def_var(plot_var_names[i], NC_DOUBLE, {nz_name, ny_name, nx_name});
95  }
96 
97  ncf.exit_def_mode();
98  //
99  // End Define stuff
100  //
101 
102  // We are doing single-level writes but it doesn't have to be level 0
103  //
104  // Write out the netcdf plotfile head information.
105  //
106  if (n_data_items == 0) {
107  amrex::Error("Must specify at least one valid data item to plot");
108  }
109 
110  ncf.put_attr("number_variables", std::vector<int>{n_data_items});
111  ncf.put_attr("space_dimension", std::vector<int>{AMREX_SPACEDIM});
112  ncf.put_attr("current_time", std::vector<double>{time});
113  ncf.put_attr("start_time", std::vector<double>{start_bdy_time});
114  ncf.put_attr("CurrentLevel", std::vector<int>{lev});
115 
116  Real dx[AMREX_SPACEDIM];
117  for (int i = 0; i < AMREX_SPACEDIM; i++) {
118  dx[i] = dx_in[i];
119  }
120 
121  amrex::Vector<Real> probLo;
122  amrex::Vector<Real> probHi;
123  for (int i = 0; i < AMREX_SPACEDIM; i++) {
124  probLo.push_back(prob_lo[i]);
125  probHi.push_back(prob_hi[i]);
126  }
127 
128  auto nc_probLo = ncf.var("probLo");
129  nc_probLo.par_access(NC_COLLECTIVE);
130  nc_probLo.put(probLo.data(), {0}, {AMREX_SPACEDIM});
131 
132  auto nc_probHi = ncf.var("probHi");
133  nc_probHi.par_access(NC_COLLECTIVE);
134  nc_probHi.put(probHi.data(), {0}, {AMREX_SPACEDIM});
135 
136  amrex::Vector<int> smallend;
137  amrex::Vector<int> bigend;
138  smallend.clear(); bigend.clear();
139  for (int j = 0; j < AMREX_SPACEDIM; j++) {
140  smallend.push_back(subdomain.smallEnd(j));
141  bigend.push_back(subdomain.bigEnd(j));
142  }
143 
144  auto nc_Geom_smallend = ncf.var("Geom.smallend");
145  nc_Geom_smallend.par_access(NC_COLLECTIVE);
146  nc_Geom_smallend.put(smallend.data(), {0}, {AMREX_SPACEDIM});
147 
148  auto nc_Geom_bigend = ncf.var("Geom.bigend");
149  nc_Geom_bigend.par_access(NC_COLLECTIVE);
150  nc_Geom_bigend.put(bigend.data(), {0}, {AMREX_SPACEDIM});
151 
152  amrex::Vector<Real> CellSize;
153  CellSize.clear();
154  for (double & j : dx) {
155  CellSize.push_back(j);
156  }
157  auto nc_CellSize = ncf.var("CellSize");
158  nc_CellSize.par_access(NC_COLLECTIVE);
159  nc_CellSize.put(CellSize.data(), {0}, {AMREX_SPACEDIM});
160 
161  ncf.put_attr("DefaultGeometry", std::vector<int>{amrex::DefaultGeometry().Coord()});
162 
163  std::vector<Real> x_grid;
164  std::vector<Real> y_grid;
165  std::vector<Real> z_grid;
166  long unsigned goffset = 0;
167  long unsigned glen = 0;
168 
169  // *******************************************************************************
170  // NOTE: the (x,y,z) output here are for a mesh withOUT terrain-fitted coordinates
171  // *******************************************************************************
172  for (int i = 0; i < ba.size(); ++i) {
173  auto bx = ba[i];
174  if (subdomain.contains(bx)) {
175  x_grid.clear(); y_grid.clear(); z_grid.clear();
176  for (auto k3 = 0; k3 < bx.length(2); ++k3) {
177  for (auto k2 = 0; k2 < bx.length(1); ++k2) {
178  for (auto k1 = 0; k1 < bx.length(0); ++k1) {
179  x_grid.push_back(prob_lo[0]+dx[0]*(static_cast<Real>(k1)+0.5));
180  y_grid.push_back(prob_lo[1]+dx[1]*(static_cast<Real>(k2)+0.5));
181  z_grid.push_back(prob_lo[2]+dx[2]*(static_cast<Real>(k3)+0.5));
182  }
183  }
184  }
185 
186  goffset += glen;
187  glen = ba.numPts();
188 
189  auto nc_x_grid = ncf.var("x_grid");
190  auto nc_y_grid = ncf.var("y_grid");
191  auto nc_z_grid = ncf.var("z_grid");
192 
193  nc_x_grid.par_access(NC_COLLECTIVE);
194  nc_y_grid.par_access(NC_COLLECTIVE);
195  nc_z_grid.par_access(NC_COLLECTIVE);
196 
197  nc_x_grid.put(x_grid.data(), {goffset}, {glen});
198  nc_y_grid.put(y_grid.data(), {goffset}, {glen});
199  nc_z_grid.put(z_grid.data(), {goffset}, {glen});
200  }
201  }
202 
203  const int ncomp = plotMF[lev]->nComp();
204 
205  for (MFIter mfi(*plotMF[lev]); mfi.isValid(); ++mfi)
206  {
207  auto bx = mfi.validbox();
208 
209  if (subdomain.contains(bx))
210 
211  {
212  //
213  // These are the dimensions of the data we write for only this box
214  //
215  long unsigned local_nx = bx.length()[0];
216  long unsigned local_ny = bx.length()[1];
217  long unsigned local_nz = bx.length()[2];
218 
219  long unsigned local_start_x = static_cast<long unsigned>(bx.smallEnd()[0]);
220  long unsigned local_start_y = static_cast<long unsigned>(bx.smallEnd()[1]);
221  long unsigned local_start_z = static_cast<long unsigned>(bx.smallEnd()[2]);
222 
223  for (int k(0); k < ncomp; ++k) {
224  FArrayBox tmp;
225  tmp.resize(bx, 1, amrex::The_Pinned_Arena());
226  tmp.template copy<RunOn::Device>((*plotMF[lev])[mfi.index()], k, 0, 1);
227  Gpu::streamSynchronize();
228 
229  auto nc_plot_var = ncf.var(plot_var_names[k]);
230  nc_plot_var.par_access(NC_COLLECTIVE);
231  nc_plot_var.put(tmp.dataPtr(), {local_start_z,local_start_y,local_start_x},
232  {local_nz, local_ny, local_nx});
233  }
234  }
235  }
236  ncf.close();
237 }
ncutils::NCFile::create_par
static NCFile create_par(const std::string &name, const int cmode=NC_CLOBBER|NC_NETCDF4|NC_MPIIO, MPI_Comm comm=MPI_COMM_WORLD, MPI_Info info=MPI_INFO_NULL)
Definition: ERF_NCInterface.cpp:714
module_mp_morr_two_moment::k1
real(c_double), private k1
Definition: ERF_module_mp_morr_two_moment.F90:212

Referenced by ERF::WritePlotFile().

Here is the call graph for this function:
Here is the caller graph for this function: