46 #ifndef _INCLUDED_Field3D_SparseFieldIO_H_
47 #define _INCLUDED_Field3D_SparseFieldIO_H_
86 typedef boost::intrusive_ptr<SparseFieldIO>
Ptr;
95 return "SparseFieldIO";
119 const std::string &layerPath,
128 {
return "SparseField"; }
135 template <
class Data_T>
139 template <
class Data_T>
142 const std::string &filename,
143 const std::string &layerPath,
171 template <
class Data_T>
184 int valuesPerBlock = (1 << (field->
m_blockOrder * 3)) * components;
187 size[0] = dw.max.x - dw.min.x + 1;
188 size[1] = dw.max.y - dw.min.y + 1;
189 size[2] = dw.max.z - dw.min.z + 1;
192 hsize_t totalSize[1];
193 totalSize[0] = size[0] * size[1] * size[2] * components;
198 { ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z };
208 { dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z };
234 int numBlocks = blockRes.x * blockRes.y * blockRes.z;
260 vector<char> isAllocated(numBlocks);
261 vector<char>::iterator i = isAllocated.begin();
262 typename vector<SparseBlock<Data_T> >::const_iterator b =
264 for (; i != isAllocated.end(); ++i, ++b)
265 *i = static_cast<char>(b->isAllocated);
266 writeSimpleData<char>(layerGroup,
"block_is_allocated_data", isAllocated);
271 vector<Data_T> emptyValue(numBlocks);
272 typename vector<Data_T>::iterator i = emptyValue.begin();
273 typename vector<SparseBlock<Data_T> >::const_iterator b =
275 for (; i != emptyValue.end(); ++i, ++b)
276 *i = static_cast<Data_T>(b->emptyValue);
277 writeSimpleData<Data_T>(layerGroup,
"block_empty_value_data", emptyValue);
281 int occupiedBlocks = 0;
282 typename vector<SparseBlock<Data_T> >::iterator b =
284 for (; b != field->
m_blocks.end(); ++b) {
289 throw WriteAttributeException(
"Couldn't add attribute " +
293 if (occupiedBlocks > 0) {
297 memDims[0] = valuesPerBlock;
299 H5Sset_extent_simple(memDataSpace.
id(), 1, memDims, NULL);
303 fileDims[0] = occupiedBlocks;
304 fileDims[1] = valuesPerBlock;
306 H5Sset_extent_simple(fileDataSpace.
id(), 2, fileDims, NULL);
310 hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
311 hsize_t chunkSize[2];
313 chunkSize[1] = valuesPerBlock;
315 herr_t status = H5Pset_deflate(dcpl, 9);
319 status = H5Pset_chunk(dcpl, 2, chunkSize);
329 H5P_DEFAULT, dcpl, H5P_DEFAULT);
330 if (dataSet.id() < 0)
331 throw CreateDataSetException(
"Couldn't create data set in "
332 "SparseFieldIO::writeInternal");
336 int nextBlockIdx = 0;
342 if (b->isAllocated) {
343 offset[0] = nextBlockIdx;
346 count[1] = valuesPerBlock;
347 status = H5Sselect_hyperslab(fileDataSpace.
id(), H5S_SELECT_SET,
348 offset, NULL, count, NULL);
350 throw WriteHyperSlabException(
351 "Couldn't select slab " +
352 boost::lexical_cast<std::string>(nextBlockIdx));
354 Data_T *data = &b->data[0];
357 fileDataSpace.
id(), H5P_DEFAULT, data);
359 throw WriteHyperSlabException(
360 "Couldn't write slab " +
361 boost::lexical_cast<std::string>(nextBlockIdx));
376 template <
class Data_T>
379 const std::string &filename,
380 const std::string &layerPath,
393 int valuesPerBlock = (1 << (result->
m_blockOrder * 3)) * components;
398 throw MissingAttributeException(
"Couldn't find attribute: " +
403 if (dynamicLoading) {
415 vector<char> isAllocated(numBlocks);
416 vector<char>::iterator i = isAllocated.begin();
417 readSimpleData<char>(location,
"block_is_allocated_data", isAllocated);
418 typename vector<SparseBlock<Data_T> >::iterator b =
420 typename vector<SparseBlock<Data_T> >::iterator bend =
423 for (; b != bend; ++b, ++i) {
424 b->isAllocated =
static_cast<bool>(*i);
425 if (*i && !dynamicLoading) {
426 b->data.resize(valuesPerBlock);
434 vector<Data_T> emptyValue(numBlocks);
435 readSimpleData<Data_T>(location,
"block_empty_value_data", emptyValue);
436 typename vector<SparseBlock<Data_T> >::iterator b =
438 typename vector<SparseBlock<Data_T> >::iterator bend =
440 typename vector<Data_T>::iterator i = emptyValue.begin();
442 for (; b != bend; ++b, ++i) {
449 if (occupiedBlocks > 0) {
451 if (dynamicLoading) {
457 typename vector<SparseBlock<Data_T> >::iterator b =
459 typename vector<SparseBlock<Data_T> >::iterator bend =
465 static const long maxMemPerPass = 50*1024*1024;
467 for (
int nextBlockIdx = 0;;) {
470 std::vector<Data_T*> memoryList;
472 for (; b != bend && mem < maxMemPerPass; ++b) {
473 if (b->isAllocated) {
474 mem +=
sizeof(Data_T)*valuesPerBlock;
475 memoryList.push_back(&b->data[0]);
480 if (!memoryList.size()) {
485 nextBlockIdx += memoryList.size();
#define FIELD3D_NAMESPACE_HEADER_CLOSE
void addReference(const std::string &filename, const std::string &layerPath, int valuesPerBlock, int occupiedBlocks)
Internal function to create a Reference for the current field, for use in dynamic reading...
V3i m_blockRes
Block array resolution.
Contains utility functions and classes for Hdf5 files.
const Box3i & extents() const
Returns the extents of the data. This signifies the relevant area that the data exists over...
Contains the Field3DFile classesOSS sanitized.
static int dataDims()
Dimensions of the given data type. i.e. 3 for V3f, 1 for float.
Namespace for Exception objects.
static const std::string k_dataStr
void readBlockList(int idx, const std::vector< Data_T * > &memoryList)
Reads a series of blocks, storing each block of data in memoryList, which is assumed to contain enoug...
virtual std::string className() const
Returns the class name.
static const std::string k_versionAttrName
Contains functions controlling the loading of sparse fields.
virtual bool write(hid_t layerGroup, FieldBase::Ptr field)
Writes the given field to disk.
static const std::string k_dataWindowStr
void setupReferenceBlocks()
Internal function to setup the Reference's block pointers, for use with dynamic reading.
boost::intrusive_ptr< FieldBase > Ptr
static const std::string k_numOccupiedBlocksStr
static const std::string k_extentsStr
virtual ~SparseFieldIO()
Dtor.
DEFINE_FIELD_RTTI_CONCRETE_CLASS
void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
int m_blockOrder
Block order (size = 2^blockOrder)
boost::intrusive_ptr< SparseField > Ptr
std::vector< Block > m_blocks
Information for all blocks in the field.
boost::intrusive_ptr< SparseFieldIO > Ptr
static const int k_versionNumber
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
bool readAttribute(hid_t location, const std::string &attrName, std::string &value)
Reads a string attribute.
bool checkHdf5Gzip()
Checks whether gzip is available in the current hdf5 library.
virtual FieldBase::Ptr read(hid_t layerGroup, const std::string &filename, const std::string &layerPath, DataTypeEnum typeEnum)
Reads the field at the given location and tries to create a SparseField object from it...
static const std::string k_blockOrderStr
static FieldIO::Ptr create()
Namespace for sparse field specifics.
static const std::string k_numBlocksStr
static const std::string k_componentsStr
const char * classType() const
static const std::string k_blockResStr
bool writeAttribute(hid_t location, const std::string &attrName, const std::string &value)
Writes a string attribute.
Contains functions controlling the loading of sparse fields.
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Scoped object - creates a dataspace on creation and closes it on destruction.
static const std::string k_bitsPerComponentStr
FieldIO base
Convenience typedef for referring to base class.
Contains the SparseField class.
Scoped object - creates a dataset on creation and closes it on destruction.
boost::intrusive_ptr< FieldIO > Ptr
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
const Box3i & dataWindow() const
Returns the data window. Any coordinate inside this window is safe to pass to value() in the Field su...
bool writeInternal(hid_t layerGroup, typename SparseField< Data_T >::Ptr field)
This call writes all the attributes and sets up the data space.
bool readData(hid_t location, int numBlocks, const std::string &filename, const std::string &layerPath, typename SparseField< Data_T >::Ptr result)
Reads the data that is dependent on the data type on disk.
hid_t id() const
Query the hid_t value.