VAPoR
3.0.0
|
Implements the VDC abstract class, providing storage of VDC data in NetCDF files. More...
#include <VDCNetCDF.h>
Public Member Functions | |
VDCNetCDF (int numthreads=0, size_t master_theshold=10 *1024 *1024, size_t variable_threshold=100 *1024 *1024) | |
virtual | ~VDCNetCDF () |
virtual int | GetPath (string varname, size_t ts, string &path, size_t &file_ts, size_t &max_ts) const |
virtual int | GetDimLensAtLevel (string varname, int level, std::vector< size_t > &dims_at_level, vector< size_t > &bs_at_level) const |
virtual int | Initialize (const vector< string > &paths, AccessMode mode, size_t chunksizehint=0) |
virtual int | Initialize (string path, AccessMode mode, size_t chunksizehint=0) |
size_t | GetMasterThreshold () const |
size_t | GetVariableThreshold () const |
int | OpenVariableRead (size_t ts, string varname, int level=0, int lod=-1) |
int | CloseVariable () |
int | OpenVariableWrite (size_t ts, string varname, int lod=-1) |
int | Write (const float *region) |
int | WriteSlice (const float *slice) |
int | Read (float *region) |
int | ReadSlice (float *slice) |
int | ReadRegion (const std::vector< size_t > &min, const std::vector< size_t > &max, float *region) |
int | ReadRegionBlock (const vector< size_t > &min, const vector< size_t > &max, float *region) |
int | PutVar (string varname, int lod, const float *data) |
int | PutVar (size_t ts, string varname, int lod, const float *data) |
int | GetVar (string varname, int level, int lod, float *data) |
int | GetVar (size_t ts, string varname, int level, int lod, float *data) |
bool | CompressionInfo (std::vector< size_t > bs, string wname, size_t &nlevels, size_t &maxcratio) const |
virtual bool | VariableExists (size_t ts, string varname, int reflevel=0, int lod=0) const |
Public Member Functions inherited from VAPoR::VDC | |
VDC () | |
virtual | ~VDC () |
virtual int | Initialize (const std::vector< string > &paths, AccessMode mode) |
virtual int | Initialize (const std::vector< string > &paths) |
int | SetCompressionBlock (std::vector< size_t > bs, string wname, std::vector< size_t > cratios) |
void | GetCompressionBlock (std::vector< size_t > &bs, string &wname, std::vector< size_t > &cratios) const |
void | SetPeriodicBoundary (std::vector< bool > periodic) |
std::vector< bool > | GetPeriodicBoundary () const |
int | DefineDimension (string dimname, size_t length, int axis) |
bool | GetDimension (string dimname, DC::Dimension &dimension) const |
std::vector< string > | GetDimensionNames () const |
int | DefineCoordVar (string varname, std::vector< string > dimnames, string units, int axis, XType type, bool compressed) |
int | DefineCoordVarUniform (string varname, std::vector< string > dimname, string units, int axis, XType type, bool compressed) |
bool | GetCoordVarInfo (string varname, std::vector< string > &dimnames, string &units, int &axis, XType &type, bool &compressed, bool &uniform) const |
bool | GetCoordVarInfo (string varname, DC::CoordVar &cvar) const |
int | DefineDataVar (string varname, std::vector< string > dimnames, std::vector< string > coordvars, string units, XType type, bool compressed) |
int | DefineDataVar (string varname, std::vector< string > dimnames, std::vector< string > coordvars, string units, XType type, double missing_value, string maskvar) |
bool | GetDataVarInfo (string varname, std::vector< string > &dimnames, std::vector< string > &coordvars, string &units, XType &type, bool &compressed, string &maskvar) const |
bool | GetDataVarInfo (string varname, DC::DataVar &datavar) const |
bool | GetBaseVarInfo (string varname, DC::BaseVar &var) const |
virtual std::vector< string > | GetDataVarNames () const |
virtual std::vector< string > | GetCoordVarNames () const |
int | GetNumRefLevels (string varname) const |
int | PutAtt (string varname, string attname, XType type, const vector< double > &values) |
int | PutAtt (string varname, string attname, XType type, const vector< long > &values) |
int | PutAtt (string varname, string attname, XType type, const string &values) |
int | GetAtt (string varname, string attname, vector< double > &values) const |
int | GetAtt (string varname, string attname, vector< long > &values) const |
int | GetAtt (string varname, string attname, string &values) const |
std::vector< string > | GetAttNames (string varname) const |
XType | GetAttType (string varname, string attname) const |
virtual int | GetMapProjection (string lonname, string latname, string &projstring) const |
virtual int | SetMapProjection (string lonname, string latname, string projstring) |
int | EndDefine () |
virtual int | GetDimLensAtLevel (string varname, int level, std::vector< size_t > &dims_at_level, std::vector< size_t > &bs_at_level) const =0 |
virtual int | ReadRegion (const vector< size_t > &min, const vector< size_t > &max, float *region)=0 |
virtual bool | CompressionInfo (vector< size_t > bs, string wname, size_t &nlevels, size_t &maxcratio) const =0 |
Public Member Functions inherited from VAPoR::DC | |
DC () | |
virtual | ~DC () |
virtual int | Initialize (const vector< string > &paths)=0 |
virtual bool | GetDimension (string dimname, size_t &length, int &axis) const |
virtual std::vector< string > | GetDataVarNames (int ndim, bool spatial) const |
virtual std::vector< string > | GetCoordVarNames (int ndim, bool spatial) const |
virtual bool | IsTimeVarying (string varname) const |
virtual bool | IsCompressed (string varname) const |
virtual int | GetNumTimeSteps (string varname) const |
virtual int | GetCRatios (string varname, vector< size_t > &cratios) const |
virtual bool | IsDataVar (string varname) const |
virtual bool | IsCoordVar (string varname) const |
virtual int | GetMapProjection (string varname, string &projstring) const |
Public Member Functions inherited from VetsUtil::MyBase | |
MyBase () | |
const string & | getClassName () const |
Static Public Member Functions | |
static string | GetDataDir (string path) |
static bool | DataDirExists (string path) |
Static Public Member Functions inherited from VAPoR::DC | |
static bool | ParseDimensions (const vector< DC::Dimension > &dimensions, vector< size_t > &sdims, size_t &numts) |
Static Public Member Functions inherited from VetsUtil::MyBase | |
static void | SetErrMsg (const char *format,...) |
Record a formatted error message. More... | |
static void | SetErrMsg (int errcode, const char *format,...) |
Record a formatted error message and an error code. More... | |
static const char * | GetErrMsg () |
static void | SetErrCode (int err_code) |
Record an error code. More... | |
static int | GetErrCode () |
Retrieve the current error code. More... | |
static void | SetErrMsgCB (ErrMsgCB_T cb) |
static ErrMsgCB_T | GetErrMsgCB () |
static void | SetErrMsgFilePtr (FILE *fp) |
static const FILE * | SetErrMsgFilePtr () |
static void | SetDiagMsg (const char *format,...) |
Record a formatted diagnostic message. More... | |
static const char * | GetDiagMsg () |
static void | SetDiagMsgCB (DiagMsgCB_T cb) |
static DiagMsgCB_T | GetDiagMsgCB () |
static void | SetDiagMsgFilePtr (FILE *fp) |
static bool | EnableErrMsg (bool enable) |
Additional Inherited Members | |
Public Types inherited from VAPoR::VDC | |
enum | AccessMode { R, W, A } |
Public Types inherited from VAPoR::DC | |
enum | XType { INVALID = -1, FLOAT, DOUBLE, INT32, INT64, TEXT } |
External storage types for primitive data. More... | |
Public Types inherited from VetsUtil::MyBase | |
typedef void(* | ErrMsgCB_T) (const char *msg, int err_code) |
typedef void(* | DiagMsgCB_T) (const char *msg) |
Static Public Attributes inherited from VetsUtil::MyBase | |
static char * | ErrMsg |
static int | ErrCode |
static int | ErrMsgSize |
static FILE * | ErrMsgFilePtr |
static ErrMsgCB_T | ErrMsgCB |
static char * | DiagMsg |
static int | DiagMsgSize |
static FILE * | DiagMsgFilePtr |
static DiagMsgCB_T | DiagMsgCB |
static bool | Enabled |
Protected Member Functions inherited from VetsUtil::MyBase | |
void | SetClassName (const string &name) |
Protected Attributes inherited from VAPoR::VDC | |
string | _master_path |
AccessMode | _mode |
bool | _defineMode |
std::vector< size_t > | _bs |
string | _wname |
std::vector< size_t > | _cratios |
vector< bool > | _periodic |
VAPoR::UDUnits | _udunits |
std::map< string, Dimension > | _dimsMap |
std::map< string, Attribute > | _atts |
std::map< string, CoordVar > | _coordVars |
std::map< string, DataVar > | _dataVars |
Implements the VDC abstract class, providing storage of VDC data in NetCDF files.
Implements the VDC abstract class, providing storage of VDC data in NetCDF files. Data (variables) are stored in multiple NetCDF files. The distribution of variables to files is described by GetPath().
Definition at line 27 of file VDCNetCDF.h.
VAPoR::VDCNetCDF::VDCNetCDF | ( | int | numthreads = 0 , |
size_t | master_theshold = 10 *1024 *1024 , |
||
size_t | variable_threshold = 100 *1024 *1024 |
||
) |
Class constructor
[in] | numthreads | Number of parallel execution threads to be run during encoding and decoding of compressed data. A value of 0, the default, indicates that the thread count should be determined by the environment in a platform-specific manner, for example using sysconf(_SC_NPROCESSORS_ONLN) under *nix OSes. |
[in] | master_theshold | Variables that are either compressed, or whose total number of elements are larger than master_theshold , will not be stored in the master file. Ignored if the file is open for appending or reading. |
[in] | variable_threshold | Variables not stored in the master file and whose total number of elements are larger than variable_threshold will be stored with one time step per file. Ignored if the file is open for appending or reading. |
|
virtual |
|
virtual |
Close the currently opened variable
Close the handle for variable opened with OpenVariableWrite(), or OpenVariableRead()
Implements VAPoR::VDC.
bool VAPoR::VDCNetCDF::CompressionInfo | ( | std::vector< size_t > | bs, |
string | wname, | ||
size_t & | nlevels, | ||
size_t & | maxcratio | ||
) | const |
This method computes and returns the depth (number of levels) in a a multi-resolution hierarch for a given wavelet, wname
, and decomposition block, bs
. It also computes the maximum compression ratio, cratio
, possible for the the specified combination of block size, bs
, and wavelet, wname
. The maximum compression ratio is cratio:1
.
[in] | bs | Dimensions of native decomposition block. The rank of bs may be less than or equal to the rank of dims . |
[in] | wname | wavelet name. Empty string if no compression is to be performed. |
[out] | nlevels | Number of levels in hierarchy |
[out] | maxcratio | Maximum compression ratio |
bool | If bs , wname , or the combination there of is invalid false is returned and the values of nlevels and maxcratio are undefined. Upon success true is returned. |
|
static |
Return true if a data directory exists for the master file named by path
[in] | path | Path to VDC master file |
|
static |
Return path to the data directory
Return the file path to the data directory associated with the master file named by path
. Data files, those NetCDF files containing coordinate and data variables, are stored in separate files from the VDC master file (See VDC::Initialize()). The data files reside under the directory returned by this command.
[in] | path | Path to VDC master file |
dir | : Path to the data directory |
|
virtual |
Return a variable's dimension lengths at a specified refinement level
Compressed variables have a multi-resolution grid representation. This method returns the variable's ordered spatial and temporal dimension lengths, and block dimensions at the multiresolution refinement level specified by level
.
If the variable named by varname
is not compressed the variable's native dimensions are returned.
[in] | varname | Data or coordinate variable name. |
[in] | level | Specifies a member of a multi-resolution variable's grid hierarchy as described above. |
[out] | dims_at_level | An ordered vector containing the variable's dimensions at the specified refinement level |
[out] | bs_at_level | An ordered vector containing the variable's block dimensions at the specified refinement level |
status | Zero is returned upon success, otherwise -1. |
|
inline |
Return the master file size threshold
Definition at line 131 of file VDCNetCDF.h.
|
virtual |
Return the path name and temporal offset for a variable
Data and coordinate variables in a VDC are in general distributed into multiple files. For example, for large variables only a single time step may be stored per file. This method returns the file path name, path
, of the file containing varname
at time step ts
. Also returned is the integer time offset of the variable within path
.
[in] | varname | Data or coordinate variable name. |
[in] | ts | Integer offset relative to a variable's temporal dimension |
[out] | path | Path to file containing variable varname at time step ts . |
[out] | file_ts | Temporal offset of variable varname in file path . |
[out] | max_ts | Maximum number of time steps stored in path |
status | A negative int is returned if varname or ts are invalid, or if the class object is in define mode. |
Implements VAPoR::VDC.
|
virtual |
Read an entire variable in one call
This method reads and entire variable (all time steps, all grid points) from a VDC. This is the simplest interface for reading data from a VDC. If the variable is split across multiple files GetVar() ensures that the data are correctly gathered and assembled into memory Any variables currently opened with OpenVariableRead() are first closed. Thus variables need not be opened with OpenVariableRead() prior to calling GetVar();
It is an error to call this method in define mode
[in] | varname | Name of the variable to write |
[in] | level | Refinement level of the variable. Ignored if the variable is not compressed. |
[in] | lod | Approximation level of the variable. A value of -1 indicates the maximum approximation level defined for the VDC. Ignored if the variable is not compressed. |
[out] | data | Pointer to where data will be copied. It is the caller's responsbility to ensure data points to sufficient memory. |
status | A negative int is returned on failure |
Implements VAPoR::VDC.
|
virtual |
Read an entire variable in one call
This method reads and entire variable (all time steps, all grid points) from a VDC. This is the simplest interface for reading data from a VDC. If the variable is split across multiple files GetVar() ensures that the data are correctly gathered and assembled into memory Any variables currently opened with OpenVariableRead() are first closed. Thus variables need not be opened with OpenVariableRead() prior to calling GetVar();
It is an error to call this method in define mode
[in] | varname | Name of the variable to write |
[in] | level | Refinement level of the variable. Ignored if the variable is not compressed. |
[in] | lod | Approximation level of the variable. A value of -1 indicates the maximum approximation level defined for the VDC. Ignored if the variable is not compressed. |
[out] | data | Pointer to where data will be copied. It is the caller's responsbility to ensure data points to sufficient memory. |
status | A negative int is returned on failure |
Implements VAPoR::VDC.
|
inline |
Return the variable size threshold
Definition at line 137 of file VDCNetCDF.h.
|
virtual |
Initialize the VDCNetCDF class Initialize the VDC class
Prepare a VDC for reading or writing/appending. This method prepares the master VDC file indicated by path
for reading or writing. The method should be called immediately after the constructor, before any other class methods. This method exists only because C++ constructors can not return error codes.
[in] | path | A single element vector that specifies the name of file that contains, or will contain, the VDC master file for this data collection |
[in] | mode | One of R, W, or A, indicating whether path will be opened for reading, writing, or appending, respectively. When mode is A underlying NetCDF files will be opened opened with nc_open(path, NC_WRITE)). When mode is W NetCDF files will be created (opened with nc_create(path)). When mode is A additional time steps may be added to an existing file. |
mode
controls the access to the master file indicated by path
and the variable data files in a somewhat unintuitive manner. If mode
is R or A the master file path
must already exist. If mode
is A or W the contents of the VDC master may be changed (written) and the VDC is put into define mode until EndDefine() is called. While in define mode metadata that will be contained in the VDC master file may be changed, but coordinate and data variables may not be accessed (read or written). Similarly, when not in define mode coordinate and data variables may be accessed (read or written), but metadata in the VDC master may not be changed. See OpenVariableRead() and OpenVariableWrite() for discussion on how mode
effects reading and writing of coordinate and data variables.status | A negative int is returned on failure |
[in] | chunksizehint | : NetCDF chunk size hint. A value of zero results in NC_SIZEHINT_DEFAULT being used. |
|
inlinevirtual |
Definition at line 119 of file VDCNetCDF.h.
|
virtual |
Open the named variable for reading
This method prepares a data or coordinate variable, indicated by a variable name and time step pair, for subsequent read operations by methods of this class. The value of the refinement levels parameter, level
, indicates the resolution of the volume in the multiresolution hierarchy as described by GetDimLensAtLevel().
The level-of-detail parameter, lod
, selects the approximation level. Valid values for lod
are integers in the range 0..n-1, where n is returned by DC::BaseVar::GetCRatios().size(), or the value -1 may be used to select the best approximation available.
An error occurs, indicated by a negative return value, if the volume identified by the {varname, timestep, level, lod} tupple is not available. Note the availability of a volume can be tested with the VariableExists() method.
[in] | ts | Time step of the variable to read. This is the integer offset into the variable's temporal dimension. If the variable does not have a temporal dimension ts is ignored. |
[in] | varname | Name of the variable to read |
[in] | level | Refinement level of the variable. Ignored if the variable is not compressed. |
[in] | lod | Approximation level of the variable. A value of -1 indicates the maximum approximation level defined for the VDC. Ignored if the variable is not compressed. |
status | Returns a non-negative value on success |
Implements VAPoR::VDC.
|
virtual |
Open the named variable for writing
This method prepares a data or coordinate variable, indicated by a variable name and time step pair, for subsequent write operations by methods of this class.
The behavior of this method is impacted somewhat by the setting of the Initialize() mode parameter. Coordinate or data variable files may be written regardless of the mode
setting. However, if mode
is W the first time a coordinate or data file is written it will be created (opened with nc_create(path), for example) regareless of whether the file previously existed. If mode
is A or R existing coordinate or data files will be opened for appending (e.g. opened with nc_open(path, NC_WRITE)). New files will be created (opened with nc_create(path)).
An error occurs, indicated by a negative return value, if the varible identified by the {varname, timestep, lod} tupple is not defined.
[in] | ts | Time step of the variable to read. This is the integer offset into the variable's temporal dimension. If the variable does not have a temporal dimension ts is ignored. |
[in] | varname | Name of the variable to read |
[in] | lod | Approximation level of the variable. A value of -1 indicates the maximum approximation level defined for the VDC. Ignored if the variable is not compressed. |
status | Returns a non-negative value on success |
Implements VAPoR::VDC.
|
virtual |
Write an entire variable in one call
This method writes and entire variable (all time steps, all grid points) into a VDC. This is the simplest interface for writing data into a VDC. If the variable is split across multiple files PutVar() ensures that the data are correctly distributed. Any variables currently opened with OpenVariableWrite() are first closed. Thus variables need not be opened with OpenVariableWrite() prior to calling PutVar();
It is an error to call this method in define mode
[in] | varname | Name of the variable to write |
[in] | lod | Approximation level of the variable. A value of -1 indicates the maximum approximation level defined for the VDC. Ignored if the variable is not compressed. |
[in] | data | Pointer from where the data will be copied |
status | A negative int is returned on failure |
Implements VAPoR::VDC.
|
virtual |
Write an entire variable in one call
This method writes and entire variable (all time steps, all grid points) into a VDC. This is the simplest interface for writing data into a VDC. If the variable is split across multiple files PutVar() ensures that the data are correctly distributed. Any variables currently opened with OpenVariableWrite() are first closed. Thus variables need not be opened with OpenVariableWrite() prior to calling PutVar();
It is an error to call this method in define mode
[in] | varname | Name of the variable to write |
[in] | lod | Approximation level of the variable. A value of -1 indicates the maximum approximation level defined for the VDC. Ignored if the variable is not compressed. |
[in] | data | Pointer from where the data will be copied |
status | A negative int is returned on failure |
Implements VAPoR::VDC.
|
virtual |
Read all spatial values of the currently opened variable
This method reads, and decompresses as necessary, the contents of the currently opened variable into the array data
. The number of values read into data
is given by the product of the spatial dimensions of the open variable at the refinement level specified.
It is the caller's responsibility to ensure data
points to adequate space.
[out] | data | An array of data to be written |
status | Returns a non-negative value on success |
Implements VAPoR::VDC.
int VAPoR::VDCNetCDF::ReadRegion | ( | const std::vector< size_t > & | min, |
const std::vector< size_t > & | max, | ||
float * | region | ||
) |
Read in and return a subregion from the currently opened variable
This method reads and returns a subset of variable data. The min
and max
vectors, whose dimensions must match the spatial rank of the currently opened variable, identify the minimum and maximum extents, in grid coordinates, of the subregion of interest. The minimum and maximum valid values of an element of min or max are 0 and n-1, respectively, where n is the length of the associated dimension at the opened refinement level.
The region returned is stored in the memory region pointed to by region
. It is the caller's responsbility to ensure adequate space is available.
[in] | min | Minimum region extents in grid coordinates |
[in] | max | Maximum region extents in grid coordinates |
[out] | region | The requested volume subregion |
status | Returns a non-negative value on success |
|
virtual |
Read in and return a blocked subregion from the currently opened variable.
This method is identical to ReadRegion() with the exceptions that for compressed variables:
start
and count
must be aligned with the underlying storage block of the variable. See DC::SetCompressionBlock()region
will preserve its underlying storage blocking (the data will not be contiguous) Implements VAPoR::VDC.
|
virtual |
Read a single slice of data from the currently opened variable
Decompress, as necessary, and read a single slice (2D array) of data from the variable indicated by the most recent call to OpenVariableRead(). The dimensions of a slices are NX by NY, where NX is the dimension of the array along the fastest varying spatial dimension, specified in grid points, and NY is the length of the second fastest varying dimension at the currently opened grid refinement level. See OpenVariableRead().
This method should be called exactly NZ times for each opened variable, where NZ is the dimension of third, and slowest varying dimension. In the case of a 2D variable, NZ is 1.
It is the caller's responsibility to ensure slice
points to adequate space.
[out] | slice | A 2D slice of data |
status | Returns a non-negative value on success |
Implements VAPoR::VDC.
|
virtual |
Returns true if indicated data volume is available
Returns true if the variable identified by the timestep, variable name, refinement level, and level-of-detail is present in the data set. Returns false if the variable is not available.
[in] | ts | A valid time step between 0 and GetNumTimesteps()-1 |
[in] | varname | A valid variable name |
[in] | reflevel | Refinement level requested. |
[in] | lod | Compression level of detail requested. refinement level contained in the VDC. |
Implements VAPoR::VDC.
|
virtual |
Write all spatial values to the currently opened variable
This method writes, and compresses as necessary, the contents of the array contained in data
to the currently opened variable. The number of values written from data
is given by the product of the spatial dimensions of the open variable.
[in] | data | An array of data to be written |
status | Returns a non-negative value on success |
Implements VAPoR::VDC.
|
virtual |
Write a single slice of data to the currently opened variable
Compress, and necessary, and write a single slice (2D array) of data to the variable indicated by the most recent call to OpenVariableWrite(). The dimensions of a slices are NX by NY, where NX is the dimension of the array along the fastest varying spatial dimension, specified in grid points, and NY is the length of the second fastest varying dimension at the currently opened refinement level. See OpenVariableWrite().
This method should be called exactly NZ times for each opened variable, where NZ is the dimension of third, and slowest varying dimension. In the case of a 2D variable, NZ is 1.
[in] | slice | A 2D slice of data |
status | Returns a non-negative value on success |
Implements VAPoR::VDC.