44 #include <boost/intrusive_ptr.hpp> 46 #include <boost/thread/thread.hpp> 47 #include <boost/thread/mutex.hpp> 50 #include "SparseFieldIO.h" 55 using namespace boost;
77 template <
typename Data_T>
78 struct ReadThreadingState
80 ReadThreadingState(
const OgIGroup &i_location,
82 const size_t i_numVoxels,
83 const size_t i_numBlocks,
84 const size_t i_numOccupiedBlocks,
85 const bool i_isCompressed,
86 const std::vector<size_t> &i_blockIdxToDatasetIdx)
87 : location(i_location),
89 numVoxels(i_numVoxels),
90 numBlocks(i_numBlocks),
91 numOccupiedBlocks(i_numOccupiedBlocks),
92 isCompressed(i_isCompressed),
93 blockIdxToDatasetIdx(i_blockIdxToDatasetIdx),
97 const OgIGroup &location;
99 const size_t numVoxels;
100 const size_t numBlocks;
101 const size_t numOccupiedBlocks;
102 const bool isCompressed;
103 const std::vector<size_t> &blockIdxToDatasetIdx;
104 size_t nextBlockToRead;
106 boost::mutex readMutex;
111 template <
typename Data_T>
115 ReadBlockOp(ReadThreadingState<Data_T> &state,
const size_t threadId)
119 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
120 const uLong cmpLenBound = compressBound(srcLen);
121 m_cache.resize(cmpLenBound);
125 m_state.numOccupiedBlocks,
126 m_state.isCompressed));
127 m_reader = m_readerPtr.get();
129 m_reader->setThreadId(threadId);
136 boost::mutex::scoped_lock lock(m_state.readMutex);
137 blockIdx = m_state.nextBlockToRead;
138 m_state.nextBlockToRead++;
141 while (blockIdx < m_state.numBlocks) {
142 if (m_state.blocks[blockIdx].isAllocated) {
143 const size_t datasetIdx = m_state.blockIdxToDatasetIdx[blockIdx];
144 m_reader->readBlock(datasetIdx, m_state.blocks[blockIdx].data);
148 boost::mutex::scoped_lock lock(m_state.readMutex);
149 blockIdx = m_state.nextBlockToRead;
150 m_state.nextBlockToRead++;
156 ReadThreadingState<Data_T> &m_state;
157 std::vector<uint8_t> m_cache;
158 boost::shared_ptr<OgSparseDataReader<Data_T> > m_readerPtr;
164 template <
typename Data_T>
165 struct ThreadingState
167 ThreadingState(OgOCDataset<Data_T> &i_data,
169 const size_t i_numVoxels,
170 const size_t i_numBlocks,
171 const std::vector<uint8_t> &i_isAllocated)
174 numVoxels(i_numVoxels),
175 numBlocks(i_numBlocks),
176 isAllocated(i_isAllocated),
177 nextBlockToCompress(0),
181 for (
size_t i = 0; i < numBlocks; ++i) {
182 if (blocks[i].isAllocated) {
183 nextBlockToCompress = i;
184 nextBlockToWrite = i;
189 nextBlockToCompress = numBlocks;
190 nextBlockToWrite = numBlocks;
193 OgOCDataset<Data_T> &data;
195 const size_t numVoxels;
196 const size_t numBlocks;
197 const std::vector<uint8_t> isAllocated;
198 size_t nextBlockToCompress;
199 size_t nextBlockToWrite;
201 boost::mutex compressMutex;
206 template <
typename Data_T>
210 WriteBlockOp(ThreadingState<Data_T> &state,
const size_t threadId)
211 : m_state(state), m_threadId(threadId)
213 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
214 const uLong cmpLenBound = compressBound(srcLen);
215 m_cache.resize(cmpLenBound);
223 boost::mutex::scoped_lock lock(m_state.compressMutex);
224 blockIdx = m_state.nextBlockToCompress;
226 while (m_state.nextBlockToCompress < m_state.numBlocks) {
227 m_state.nextBlockToCompress++;
228 if (m_state.blocks[m_state.nextBlockToCompress].isAllocated) {
234 while (blockIdx < m_state.numBlocks) {
235 if (m_state.blocks[blockIdx].isAllocated) {
237 const uint8_t *srcData =
238 reinterpret_cast<const uint8_t *
>(m_state.blocks[blockIdx].data);
240 const uLong srcLen = m_state.numVoxels *
sizeof(Data_T);
241 const uLong cmpLenBound = compressBound(srcLen);
242 uLong cmpLen = cmpLenBound;
244 const int status = compress2(&m_cache[0], &cmpLen,
245 srcData, srcLen, level);
247 if (status != Z_OK) {
248 std::cout <<
"ERROR: Couldn't compress in SparseFieldIO." << std::endl
249 <<
" Level: " << level << std::endl
250 <<
" Status: " << status << std::endl
251 <<
" srcLen: " << srcLen << std::endl
252 <<
" cmpLenBound: " << cmpLenBound << std::endl
253 <<
" cmpLen: " << cmpLen << std::endl;
257 while (m_state.nextBlockToWrite != blockIdx) {
259 boost::this_thread::sleep(boost::posix_time::microseconds(1));
262 m_state.data.addData(cmpLen, &m_cache[0]);
264 while (m_state.nextBlockToWrite < m_state.numBlocks){
266 m_state.nextBlockToWrite++;
267 if (m_state.blocks[m_state.nextBlockToWrite].isAllocated) {
274 boost::mutex::scoped_lock lock(m_state.compressMutex);
275 blockIdx = m_state.nextBlockToCompress;
277 while (m_state.nextBlockToCompress < m_state.numBlocks) {
278 m_state.nextBlockToCompress++;
279 if (m_state.blocks[m_state.nextBlockToCompress].isAllocated) {
288 ThreadingState<Data_T> &m_state;
289 std::vector<uint8_t> m_cache;
290 const size_t m_threadId;
301 const int SparseFieldIO::k_versionNumber(1);
302 const std::string SparseFieldIO::k_versionAttrName(
"version");
303 const std::string SparseFieldIO::k_extentsStr(
"extents");
304 const std::string SparseFieldIO::k_extentsMinStr(
"extents_min");
305 const std::string SparseFieldIO::k_extentsMaxStr(
"extents_max");
306 const std::string SparseFieldIO::k_dataWindowStr(
"data_window");
307 const std::string SparseFieldIO::k_dataWindowMinStr(
"data_window_min");
308 const std::string SparseFieldIO::k_dataWindowMaxStr(
"data_window_max");
309 const std::string SparseFieldIO::k_componentsStr(
"components");
310 const std::string SparseFieldIO::k_dataStr(
"data");
311 const std::string SparseFieldIO::k_blockOrderStr(
"block_order");
312 const std::string SparseFieldIO::k_numBlocksStr(
"num_blocks");
313 const std::string SparseFieldIO::k_blockResStr(
"block_res");
314 const std::string SparseFieldIO::k_bitsPerComponentStr(
"bits_per_component");
315 const std::string SparseFieldIO::k_numOccupiedBlocksStr(
"num_occupied_blocks");
316 const std::string SparseFieldIO::k_isCompressed(
"data_is_compressed");
321 SparseFieldIO::read(hid_t layerGroup,
const std::string &filename,
322 const std::string &layerPath,
325 Box3i extents, dataW;
331 if (layerGroup == -1) {
337 if (!
readAttribute(layerGroup, k_versionAttrName, 1, version))
338 throw MissingAttributeException(
"Couldn't find attribute: " +
341 if (version != k_versionNumber)
342 throw UnsupportedVersionException(
"SparseField version not supported: " +
343 lexical_cast<std::string>(version));
345 if (!
readAttribute(layerGroup, k_extentsStr, 6, extents.min.x))
346 throw MissingAttributeException(
"Couldn't find attribute: " +
349 if (!
readAttribute(layerGroup, k_dataWindowStr, 6, dataW.min.x))
350 throw MissingAttributeException(
"Couldn't find attribute: " +
353 if (!
readAttribute(layerGroup, k_componentsStr, 1, components))
354 throw MissingAttributeException(
"Couldn't find attribute: " +
358 if (!
readAttribute(layerGroup, k_blockOrderStr, 1, blockOrder))
359 throw MissingAttributeException(
"Couldn't find attribute: " +
363 if (!
readAttribute(layerGroup, k_numBlocksStr, 1, numBlocks))
364 throw MissingAttributeException(
"Couldn't find attribute: " +
368 if (!
readAttribute(layerGroup, k_blockResStr, 3, blockRes.x))
369 throw MissingAttributeException(
"Couldn't find attribute: " +
374 int numCalculatedBlocks = blockRes.x * blockRes.y * blockRes.z;
375 if (numCalculatedBlocks != numBlocks)
376 throw FileIntegrityException(
"Incorrect block count in SparseFieldIO::read");
383 if (!
readAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks))
384 throw MissingAttributeException(
"Couldn't find attribute: " +
385 k_numOccupiedBlocksStr);
390 if (!
readAttribute(layerGroup, k_bitsPerComponentStr, 1, bits))
391 throw MissingAttributeException(
"Couldn't find attribute: " +
392 k_bitsPerComponentStr);
395 bool isFloat =
false;
396 bool isDouble =
false;
412 if (components == 1) {
415 field->setSize(extents, dataW);
416 field->setBlockOrder(blockOrder);
417 readData<half>(layerGroup, numBlocks, filename, layerPath, field);
421 field->setSize(extents, dataW);
422 field->setBlockOrder(blockOrder);
423 readData<float>(layerGroup, numBlocks, filename, layerPath, field);
427 field->setSize(extents, dataW);
428 field->setBlockOrder(blockOrder);
429 readData<double>(layerGroup, numBlocks, filename, layerPath, field);
432 }
else if (components == 3) {
435 field->setSize(extents, dataW);
436 field->setBlockOrder(blockOrder);
437 readData<V3h>(layerGroup, numBlocks, filename, layerPath, field);
441 field->setSize(extents, dataW);
442 field->setBlockOrder(blockOrder);
443 readData<V3f>(layerGroup, numBlocks, filename, layerPath, field);
447 field->setSize(extents, dataW);
448 field->setBlockOrder(blockOrder);
449 readData<V3d>(layerGroup, numBlocks, filename, layerPath, field);
460 SparseFieldIO::read(
const OgIGroup &layerGroup,
const std::string &filename,
461 const std::string &layerPath,
OgDataType typeEnum)
463 Box3i extents, dataW;
468 if (!layerGroup.isValid()) {
469 throw MissingGroupException(
"Invalid group in SparseFieldIO::read()");
475 layerGroup.findAttribute<
int>(k_versionAttrName);
476 if (!versionAttr.isValid()) {
477 throw MissingAttributeException(
"Couldn't find attribute: " +
480 const int version = versionAttr.value();
482 if (version != k_versionNumber) {
483 throw UnsupportedVersionException(
"SparseField version not supported: " +
484 lexical_cast<std::string>(version));
490 layerGroup.findAttribute<
veci32_t>(k_extentsMinStr);
492 layerGroup.findAttribute<
veci32_t>(k_extentsMaxStr);
493 if (!extMinAttr.isValid()) {
494 throw MissingAttributeException(
"Couldn't find attribute " +
497 if (!extMaxAttr.isValid()) {
498 throw MissingAttributeException(
"Couldn't find attribute " +
502 extents.min = extMinAttr.value();
503 extents.max = extMaxAttr.value();
508 layerGroup.findAttribute<
veci32_t>(k_dataWindowMinStr);
510 layerGroup.findAttribute<
veci32_t>(k_dataWindowMaxStr);
511 if (!dwMinAttr.isValid()) {
512 throw MissingAttributeException(
"Couldn't find attribute " +
515 if (!dwMaxAttr.isValid()) {
516 throw MissingAttributeException(
"Couldn't find attribute " +
520 dataW.min = dwMinAttr.value();
521 dataW.max = dwMaxAttr.value();
526 layerGroup.findAttribute<uint8_t>(k_componentsStr);
527 if (!numComponentsAttr.isValid()) {
528 throw MissingAttributeException(
"Couldn't find attribute " +
535 layerGroup.findAttribute<uint8_t>(k_blockOrderStr);
536 if (!blockOrderAttr.isValid()) {
537 throw MissingAttributeException(
"Couldn't find attribute: " +
540 blockOrder = blockOrderAttr.value();
545 layerGroup.findAttribute<uint32_t>(k_numBlocksStr);
546 if (!numBlocksAttr.isValid()) {
547 throw MissingAttributeException(
"Couldn't find attribute: " +
550 numBlocks = numBlocksAttr.value();
555 layerGroup.findAttribute<
veci32_t>(k_blockResStr);
556 if (!blockResAttr.isValid()) {
557 throw MissingAttributeException(
"Couldn't find attribute: " +
560 blockRes = blockResAttr.value();
564 int numCalculatedBlocks = blockRes.x * blockRes.y * blockRes.z;
565 if (numCalculatedBlocks != numBlocks) {
566 throw FileIntegrityException(
"Incorrect block count in " 567 "SparseFieldIO::read()");
573 layerGroup.findAttribute<uint32_t>(k_numOccupiedBlocksStr);
574 if (!occupiedBlocksAttr.isValid()) {
575 throw MissingAttributeException(
"Couldn't find attribute: " +
576 k_numOccupiedBlocksStr);
582 layerGroup.findAttribute<uint8_t>(k_isCompressed);
583 if (!isCompressedAttr.isValid()) {
584 throw MissingAttributeException(
"Couldn't find attribute: " +
594 if (isCompressedAttr.value() == 0) {
595 typeOnDisk = layerGroup.datasetType(k_dataStr);
597 typeOnDisk = layerGroup.compressedDatasetType(k_dataStr);
600 if (typeEnum == typeOnDisk) {
602 result = readData<float16_t>(layerGroup, extents, dataW, blockOrder,
603 numBlocks, filename, layerPath);
605 result = readData<float32_t>(layerGroup, extents, dataW, blockOrder,
606 numBlocks, filename, layerPath);
608 result = readData<float64_t>(layerGroup, extents, dataW, blockOrder,
609 numBlocks, filename, layerPath);
611 result = readData<vec16_t>(layerGroup, extents, dataW, blockOrder,
612 numBlocks, filename, layerPath);
614 result = readData<vec32_t>(layerGroup, extents, dataW, blockOrder,
615 numBlocks, filename, layerPath);
617 result = readData<vec64_t>(layerGroup, extents, dataW, blockOrder,
618 numBlocks, filename, layerPath);
630 if (layerGroup == -1) {
637 1, k_versionNumber)) {
657 success = writeInternal<half>(layerGroup, halfField);
658 }
else if (floatField) {
659 success = writeInternal<float>(layerGroup, floatField);
660 }
else if (doubleField) {
661 success = writeInternal<double>(layerGroup, doubleField);
662 }
else if (vecHalfField) {
663 success = writeInternal<V3h>(layerGroup, vecHalfField);
664 }
else if (vecFloatField) {
665 success = writeInternal<V3f>(layerGroup, vecFloatField);
666 }
else if (vecDoubleField) {
667 success = writeInternal<V3d>(layerGroup, vecDoubleField);
669 throw WriteLayerException(
"SparseFieldIO::write does not support the given " 670 "SparseField template parameter");
702 success = writeInternal<float>(layerGroup, floatField);
704 else if (halfField) {
705 success = writeInternal<half>(layerGroup, halfField);
707 else if (doubleField) {
708 success = writeInternal<double>(layerGroup, doubleField);
710 else if (vecFloatField) {
711 success = writeInternal<V3f>(layerGroup, vecFloatField);
713 else if (vecHalfField) {
714 success = writeInternal<V3h>(layerGroup, vecHalfField);
716 else if (vecDoubleField) {
717 success = writeInternal<V3d>(layerGroup, vecDoubleField);
720 throw WriteLayerException(
"SparseFieldIO does not support the given " 721 "SparseField template parameter");
729 template <
class Data_T>
731 SparseFieldIO::readData(
const OgIGroup &location,
const Box3i &extents,
732 const Box3i &dataW,
const size_t blockOrder,
733 const size_t numBlocks,
const std::string &filename,
734 const std::string &layerPath)
741 result->setSize(extents, dataW);
742 result->setBlockOrder(blockOrder);
746 const size_t numVoxels = (1 << (result->m_blockOrder * 3));
747 const int valuesPerBlock = (1 << (result->m_blockOrder * 3)) * components;
752 location.findAttribute<uint32_t>(k_numOccupiedBlocksStr);
753 if (!occupiedBlocksAttr.isValid()) {
754 throw MissingAttributeException(
"Couldn't find attribute: " +
755 k_numOccupiedBlocksStr);
757 const size_t occupiedBlocks = occupiedBlocksAttr.value();
761 if (dynamicLoading) {
764 result->addReference(filename, layerPath, valuesPerBlock, numVoxels,
773 std::vector<size_t> blockIdxToDatasetIdx(numBlocks);
777 vector<uint8_t> isAllocated(numBlocks);
779 location.findDataset<uint8_t>(
"block_is_allocated_data");
780 if (!isAllocatedData.isValid()) {
781 throw MissingGroupException(
"Couldn't find block_is_allocated_data: ");
783 isAllocatedData.getData(0, &isAllocated[0], OGAWA_THREAD);
785 for (
size_t i = 0, nextBlockOnDisk = 0; i < numBlocks; ++i) {
787 if (!dynamicLoading && isAllocated[i]) {
788 blocks[i].
resize(numVoxels);
790 blockIdxToDatasetIdx[i] = nextBlockOnDisk;
800 vector<Data_T> emptyValue(numBlocks);
802 location.findDataset<Data_T>(
"block_empty_value_data");
803 if (!emptyValueData.isValid()) {
804 throw MissingGroupException(
"Couldn't find block_empty_value_data: ");
806 emptyValueData.getData(0, &emptyValue[0], OGAWA_THREAD);
808 for (
size_t i = 0; i < numBlocks; ++i) {
817 location.findAttribute<uint8_t>(k_isCompressed);
818 const bool isCompressed = isCompressedAttr.value() != 0;
820 if (occupiedBlocks > 0) {
821 if (dynamicLoading) {
823 result->setupReferenceBlocks();
826 ReadThreadingState<Data_T> state(location, blocks, numVoxels, numBlocks,
827 occupiedBlocks, isCompressed,
828 blockIdxToDatasetIdx);
832 boost::thread_group threads;
833 for (
size_t i = 0; i < numThreads; ++i) {
834 threads.create_thread(ReadBlockOp<Data_T>(state, i));
848 template <
class Data_T>
849 bool SparseFieldIO::writeInternal(hid_t layerGroup,
861 int valuesPerBlock = (1 << (field->
m_blockOrder * 3)) * components;
866 { ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z };
876 { dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z };
878 if (!
writeAttribute(layerGroup, k_dataWindowStr, 6, dataWindow[0])) {
885 if (!
writeAttribute(layerGroup, k_componentsStr, 1, components)) {
894 if (!
writeAttribute(layerGroup, k_blockOrderStr, 1, blockOrder)) {
902 int numBlocks = blockRes.x * blockRes.y * blockRes.z;
919 if (!
writeAttribute(layerGroup, k_bitsPerComponentStr, 1, bits)) {
930 vector<char> isAllocated(numBlocks);
931 for (
int i = 0; i < numBlocks; ++i) {
932 isAllocated[i] =
static_cast<char>(blocks[i].
isAllocated);
934 writeSimpleData<char>(layerGroup,
"block_is_allocated_data", isAllocated);
939 vector<Data_T> emptyValue(numBlocks);
940 for (
int i = 0; i < numBlocks; ++i) {
941 emptyValue[i] =
static_cast<Data_T
>(blocks[i].
emptyValue);
943 writeSimpleData<Data_T>(layerGroup,
"block_empty_value_data", emptyValue);
947 int occupiedBlocks = 0;
948 for (
int i = 0; i < numBlocks; ++i) {
949 if (blocks[i].isAllocated) {
954 if (!
writeAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks)) {
955 throw WriteAttributeException(
"Couldn't add attribute " +
956 k_numOccupiedBlocksStr);
959 if (occupiedBlocks > 0) {
963 memDims[0] = valuesPerBlock;
965 H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
969 fileDims[0] = occupiedBlocks;
970 fileDims[1] = valuesPerBlock;
972 H5Sset_extent_simple(fileDataSpace.id(), 2, fileDims, NULL);
976 hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
977 hsize_t chunkSize[2];
979 chunkSize[1] = valuesPerBlock;
981 herr_t status = H5Pset_deflate(dcpl, 9);
985 status = H5Pset_chunk(dcpl, 2, chunkSize);
995 H5P_DEFAULT, dcpl, H5P_DEFAULT);
996 if (dataSet.id() < 0)
997 throw CreateDataSetException(
"Couldn't create data set in " 998 "SparseFieldIO::writeInternal");
1002 int nextBlockIdx = 0;
1007 for (
int i = 0; i < numBlocks; ++i) {
1008 if (blocks[i].isAllocated) {
1009 offset[0] = nextBlockIdx;
1012 count[1] = valuesPerBlock;
1013 status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
1014 offset, NULL, count, NULL);
1016 throw WriteHyperSlabException(
1017 "Couldn't select slab " +
1018 boost::lexical_cast<std::string>(nextBlockIdx));
1023 fileDataSpace.id(), H5P_DEFAULT, data);
1025 throw WriteHyperSlabException(
1026 "Couldn't write slab " +
1027 boost::lexical_cast<std::string>(nextBlockIdx));
1042 template <
class Data_T>
1043 bool SparseFieldIO::writeInternal(OgOGroup &layerGroup,
1046 using namespace Exc;
1054 const size_t numBlocks = blockRes.x * blockRes.y * blockRes.z;
1055 const size_t numVoxels = (1 << (field->
m_blockOrder * 3));
1081 std::vector<uint8_t> isAllocated(numBlocks);
1082 for (
size_t i = 0; i < numBlocks; ++i) {
1083 isAllocated[i] =
static_cast<uint8_t
>(blocks[i].
isAllocated);
1086 isAllocatedData.addData(numBlocks, &isAllocated[0]);
1089 std::vector<Data_T> emptyValue(numBlocks);
1090 for (
size_t i = 0; i < numBlocks; ++i) {
1091 emptyValue[i] =
static_cast<Data_T
>(blocks[i].
emptyValue);
1094 emptyValueData.addData(numBlocks, &emptyValue[0]);
1097 int occupiedBlocks = 0;
1098 for (
size_t i = 0; i < numBlocks; ++i) {
1099 if (blocks[i].isAllocated) {
1104 k_numOccupiedBlocksStr,
1111 OgOCDataset<Data_T> data(layerGroup, k_dataStr);
1113 if (occupiedBlocks > 0) {
1115 ThreadingState<Data_T> state(data, blocks, numVoxels, numBlocks,
1120 boost::thread_group threads;
1121 for (
size_t i = 0; i < numThreads; ++i) {
1122 threads.create_thread(WriteBlockOp<Data_T>(state, i));
1132 template <
class Data_T>
1133 bool SparseFieldIO::readData(hid_t location,
1135 const std::string &filename,
1136 const std::string &layerPath,
1139 using namespace std;
1140 using namespace Exc;
1150 int valuesPerBlock = numVoxels * components;
1154 if (!
readAttribute(location, k_numOccupiedBlocksStr, 1, occupiedBlocks))
1155 throw MissingAttributeException(
"Couldn't find attribute: " +
1156 k_numOccupiedBlocksStr);
1160 if (dynamicLoading) {
1163 valuesPerBlock, numVoxels,
1174 vector<char> isAllocated(numBlocks);
1175 readSimpleData<char>(location,
"block_is_allocated_data", isAllocated);
1176 for (
int i = 0; i < numBlocks; ++i) {
1178 if (!dynamicLoading && isAllocated[i]) {
1179 blocks[i].
resize(numVoxels);
1187 vector<Data_T> emptyValue(numBlocks);
1188 readSimpleData<Data_T>(location,
"block_empty_value_data", emptyValue);
1189 for (
int i = 0; i < numBlocks; ++i) {
1196 if (occupiedBlocks > 0) {
1198 if (dynamicLoading) {
1204 size_t b = 0, bend = b + numBlocks;
1209 static const long maxMemPerPass = 50*1024*1024;
1211 for (
int nextBlockIdx = 0;;) {
1214 std::vector<Data_T*> memoryList;
1216 for (; b != bend && mem < maxMemPerPass; ++b) {
1217 if (blocks[b].isAllocated) {
1218 mem +=
sizeof(Data_T)*numVoxels;
1219 memoryList.push_back(blocks[b].data);
1224 if (!memoryList.size()) {
1228 reader.readBlockList(nextBlockIdx, memoryList);
1229 nextBlockIdx += memoryList.size();
Field_T::Ptr field_dynamic_cast(RefBase::Ptr field)
Dynamic cast that uses string-comparison in order to be safe even after an object crosses a shared li...
V3i m_blockRes
Block array resolution.
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Contains utility functions and classes for Hdf5 files.
Contains typedefs for the commonly used types in Field3D.
FIELD3D_API bool writeAttribute(hid_t location, const std::string &attrName, const std::string &value)
Writes a string attribute.
#define FIELD3D_NAMESPACE_SOURCE_CLOSE
Namespace for Exception objects.
const Box3i & extents() const
Returns the extents of the data. This signifies the relevant area that the data exists over...
FIELD3D_API bool readAttribute(hid_t location, const std::string &attrName, std::string &value)
Reads a string attribute.
void setupReferenceBlocks()
Internal function to setup the Reference's block pointers, for use with dynamic reading.
boost::intrusive_ptr< FieldBase > Ptr
Data_T * data
Pointer to data. Null if block is unallocated.
FIELD3D_API size_t numIOThreads()
Returns the number of I/O threads to use.
FIELD3D_API void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
int m_blockOrder
Block order (size = 2^blockOrder)
boost::intrusive_ptr< SparseField > Ptr
OgDataType
Enumerates the various uses for Ogawa-level groups.
Data_T emptyValue
The value to use if the block isn't allocated. We allow setting this per block so that we for example...
const Box3i & dataWindow() const
Returns the data window. Any coordinate inside this window is safe to pass to value() in the Field su...
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
void addReference(const std::string &filename, const std::string &layerPath, int valuesPerBlock, int numVoxels, int occupiedBlocks)
Internal function to create a Reference for the current field, for use in dynamic reading...
FIELD3D_API bool checkHdf5Gzip()
Checks whether gzip is available in the current hdf5 library.
Namespace for sparse field specifics.
void resize(int n)
Alloc data.
bool isAllocated
Whether the block is allocated or not.
Storage for one individual block of a SparseField.
Scoped object - creates a dataspace on creation and closes it on destruction.
Contains the initIO function.
Scoped object - creates a dataset on creation and closes it on destruction.
This Field subclass stores voxel data in block-allocated arrays.
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
Block * m_blocks
Array of blocks. Not using std::vector since SparseBlock is noncopyable.