OpenVDB: openvdb/io/Compression.h Source File

OpenVDB 9.0.0

Compression.h
Go to the documentation of this file.
1// Copyright Contributors to the OpenVDB Project
2// SPDX-License-Identifier: MPL-2.0
3
4#ifndef OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
5#define OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
6
7#include <openvdb/Types.h>
8#include <openvdb/MetaMap.h>
9#include <openvdb/math/Math.h> // for negative()
10#include "io.h" // for getDataCompression(), etc.
11#include "DelayedLoadMetadata.h"
12#include <algorithm>
13#include <iostream>
14#include <memory>
15#include <string>
16#include <vector>
17
18
19namespace openvdb {
20 OPENVDB_USE_VERSION_NAMESPACE
21namespace OPENVDB_VERSION_NAME {
22namespace io {
23
24/// @brief OR-able bit flags for compression options on input and output streams
25/// @details
26/// <dl>
27/// <dt><tt>COMPRESS_NONE</tt>
28/// <dd>On write, don't compress data.<br>
29/// On read, the input stream contains uncompressed data.
30///
31/// <dt><tt>COMPRESS_ZIP</tt>
32/// <dd>When writing grids other than level sets or fog volumes, apply
33/// ZLIB compression to internal and leaf node value buffers.<br>
34/// When reading grids other than level sets or fog volumes, indicate that
35/// the value buffers of internal and leaf nodes are ZLIB-compressed.<br>
36/// ZLIB compresses well but is slow.
37///
38/// <dt><tt>COMPRESS_ACTIVE_MASK</tt>
39/// <dd>When writing a grid of any class, don't output a node's inactive values
40/// if it has two or fewer distinct values. Instead, output minimal information
41/// to permit the lossless reconstruction of inactive values.<br>
42/// On read, nodes might have been stored without inactive values.
43/// Where necessary, reconstruct inactive values from available information.
44///
45/// <dt><tt>COMPRESS_BLOSC</tt>
46/// <dd>When writing grids other than level sets or fog volumes, apply
47/// Blosc compression to internal and leaf node value buffers.<br>
48/// When reading grids other than level sets or fog volumes, indicate that
49/// the value buffers of internal and leaf nodes are Blosc-compressed.<br>
50/// Blosc is much faster than ZLIB and produces comparable file sizes.
51/// </dl>
52enum {
53 COMPRESS_NONE = 0,
54 COMPRESS_ZIP = 0x1,
55 COMPRESS_ACTIVE_MASK = 0x2,
56 COMPRESS_BLOSC = 0x4
57};
58
59/// Return a string describing the given compression flags.
60 OPENVDB_API std::string compressionToString(uint32_t flags);
61
62
63////////////////////////////////////////
64
65
66/// @internal Per-node indicator byte that specifies what additional metadata
67/// is stored to permit reconstruction of inactive values
68enum {
69 /*0*/ NO_MASK_OR_INACTIVE_VALS, // no inactive vals, or all inactive vals are +background
70 /*1*/ NO_MASK_AND_MINUS_BG, // all inactive vals are -background
71 /*2*/ NO_MASK_AND_ONE_INACTIVE_VAL, // all inactive vals have the same non-background val
72 /*3*/ MASK_AND_NO_INACTIVE_VALS, // mask selects between -background and +background
73 /*4*/ MASK_AND_ONE_INACTIVE_VAL, // mask selects between backgd and one other inactive val
74 /*5*/ MASK_AND_TWO_INACTIVE_VALS, // mask selects between two non-background inactive vals
75 /*6*/ NO_MASK_AND_ALL_VALS // > 2 inactive vals, so no mask compression at all
76};
77
78
79template <typename ValueT, typename MaskT>
80 struct MaskCompress
81{
82 // Comparison function for values
83 static inline bool eq(const ValueT& a, const ValueT& b) {
84 return math::isExactlyEqual(a, b);
85 }
86
87 MaskCompress(
88 const MaskT& valueMask, const MaskT& childMask,
89 const ValueT* srcBuf, const ValueT& background)
90 {
91 /// @todo Consider all values, not just inactive values?
92 inactiveVal[0] = inactiveVal[1] = background;
93 int numUniqueInactiveVals = 0;
94 for (typename MaskT::OffIterator it = valueMask.beginOff();
95 numUniqueInactiveVals < 3 && it; ++it)
96 {
97 const Index32 idx = it.pos();
98
99 // Skip inactive values that are actually child node pointers.
100 if (childMask.isOn(idx)) continue;
101
102 const ValueT& val = srcBuf[idx];
103 const bool unique = !(
104 (numUniqueInactiveVals > 0 && MaskCompress::eq(val, inactiveVal[0])) ||
105 (numUniqueInactiveVals > 1 && MaskCompress::eq(val, inactiveVal[1]))
106 );
107 if (unique) {
108 if (numUniqueInactiveVals < 2) inactiveVal[numUniqueInactiveVals] = val;
109 ++numUniqueInactiveVals;
110 }
111 }
112
113 metadata = NO_MASK_OR_INACTIVE_VALS;
114
115 if (numUniqueInactiveVals == 1) {
116 if (!MaskCompress::eq(inactiveVal[0], background)) {
117 if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
118 metadata = NO_MASK_AND_MINUS_BG;
119 } else {
120 metadata = NO_MASK_AND_ONE_INACTIVE_VAL;
121 }
122 }
123 } else if (numUniqueInactiveVals == 2) {
124 metadata = NO_MASK_OR_INACTIVE_VALS;
125 if (!MaskCompress::eq(inactiveVal[0], background) && !MaskCompress::eq(inactiveVal[1], background)) {
126 // If neither inactive value is equal to the background, both values
127 // need to be saved, along with a mask that selects between them.
128 metadata = MASK_AND_TWO_INACTIVE_VALS;
129
130 } else if (MaskCompress::eq(inactiveVal[1], background)) {
131 if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
132 // If the second inactive value is equal to the background and
133 // the first is equal to -background, neither value needs to be saved,
134 // but save a mask that selects between -background and +background.
135 metadata = MASK_AND_NO_INACTIVE_VALS;
136 } else {
137 // If the second inactive value is equal to the background, only
138 // the first value needs to be saved, along with a mask that selects
139 // between it and the background.
140 metadata = MASK_AND_ONE_INACTIVE_VAL;
141 }
142 } else if (MaskCompress::eq(inactiveVal[0], background)) {
143 if (MaskCompress::eq(inactiveVal[1], math::negative(background))) {
144 // If the first inactive value is equal to the background and
145 // the second is equal to -background, neither value needs to be saved,
146 // but save a mask that selects between -background and +background.
147 metadata = MASK_AND_NO_INACTIVE_VALS;
148 std::swap(inactiveVal[0], inactiveVal[1]);
149 } else {
150 // If the first inactive value is equal to the background, swap it
151 // with the second value and save only that value, along with a mask
152 // that selects between it and the background.
153 std::swap(inactiveVal[0], inactiveVal[1]);
154 metadata = MASK_AND_ONE_INACTIVE_VAL;
155 }
156 }
157 } else if (numUniqueInactiveVals > 2) {
158 metadata = NO_MASK_AND_ALL_VALS;
159 }
160 }
161
162 int8_t metadata = NO_MASK_AND_ALL_VALS;
163 ValueT inactiveVal[2];
164};
165
166
167////////////////////////////////////////
168
169
170/// @brief RealToHalf and its specializations define a mapping from
171/// floating-point data types to analogous half float types.
172template<typename T>
173 struct RealToHalf {
174 enum { isReal = false }; // unless otherwise specified, type T is not a floating-point type
175 using HalfT = T; // type T's half float analogue is T itself
176 static HalfT convert(const T& val) { return val; }
177};
178 template<> struct RealToHalf<float> {
179 enum { isReal = true };
180 using HalfT = math::half;
181 static HalfT convert(float val) { return HalfT(val); }
182};
183 template<> struct RealToHalf<double> {
184 enum { isReal = true };
185 using HalfT = math::half;
186 // A half can only be constructed from a float, so cast the value to a float first.
187 static HalfT convert(double val) { return HalfT(float(val)); }
188};
189 template<> struct RealToHalf<Vec2s> {
190 enum { isReal = true };
191 using HalfT = Vec2H;
192 static HalfT convert(const Vec2s& val) { return HalfT(val); }
193};
194 template<> struct RealToHalf<Vec2d> {
195 enum { isReal = true };
196 using HalfT = Vec2H;
197 // A half can only be constructed from a float, so cast the vector's elements to floats first.
198 static HalfT convert(const Vec2d& val) { return HalfT(Vec2s(val)); }
199};
200 template<> struct RealToHalf<Vec3s> {
201 enum { isReal = true };
202 using HalfT = Vec3H;
203 static HalfT convert(const Vec3s& val) { return HalfT(val); }
204};
205 template<> struct RealToHalf<Vec3d> {
206 enum { isReal = true };
207 using HalfT = Vec3H;
208 // A half can only be constructed from a float, so cast the vector's elements to floats first.
209 static HalfT convert(const Vec3d& val) { return HalfT(Vec3s(val)); }
210};
211
212
213/// Return the given value truncated to 16-bit float precision.
214template<typename T>
215inline T
216truncateRealToHalf(const T& val)
217{
218 return T(RealToHalf<T>::convert(val));
219}
220
221
222////////////////////////////////////////
223
224
225 OPENVDB_API size_t zipToStreamSize(const char* data, size_t numBytes);
226 OPENVDB_API void zipToStream(std::ostream&, const char* data, size_t numBytes);
227 OPENVDB_API void unzipFromStream(std::istream&, char* data, size_t numBytes);
228 OPENVDB_API size_t bloscToStreamSize(const char* data, size_t valSize, size_t numVals);
229 OPENVDB_API void bloscToStream(std::ostream&, const char* data, size_t valSize, size_t numVals);
230 OPENVDB_API void bloscFromStream(std::istream&, char* data, size_t numBytes);
231
232/// @brief Read data from a stream.
233/// @param is the input stream
234/// @param data the contiguous array of data to read in
235/// @param count the number of elements to read in
236/// @param compression whether and how the data is compressed (either COMPRESS_NONE,
237/// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
238/// @param metadata optional pointer to a DelayedLoadMetadata object that stores
239/// the size of the compressed buffer
240/// @param metadataOffset offset into DelayedLoadMetadata, ignored if pointer is null
241/// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
242/// without Blosc support.
243/// @details This default implementation is instantiated only for types
244/// whose size can be determined by the sizeof() operator.
245template<typename T>
246inline void
247readData(std::istream& is, T* data, Index count, uint32_t compression,
248 DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0))
249{
250 const bool seek = data == nullptr;
251 if (seek) {
252 assert(!getStreamMetadataPtr(is) || getStreamMetadataPtr(is)->seekable());
253 }
254 const bool hasCompression = compression & (COMPRESS_BLOSC | COMPRESS_ZIP);
255
256 if (metadata && seek && hasCompression) {
257 size_t compressedSize = metadata->getCompressedSize(metadataOffset);
258 is.seekg(compressedSize, std::ios_base::cur);
259 } else if (compression & COMPRESS_BLOSC) {
260 bloscFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
261 } else if (compression & COMPRESS_ZIP) {
262 unzipFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
263 } else if (seek) {
264 is.seekg(sizeof(T) * count, std::ios_base::cur);
265 } else {
266 is.read(reinterpret_cast<char*>(data), sizeof(T) * count);
267 }
268}
269
270/// Specialization for std::string input
271template<>
272 inline void
273readData<std::string>(std::istream& is, std::string* data, Index count, uint32_t /*compression*/,
274 DelayedLoadMetadata* /*metadata*/, size_t /*metadataOffset*/)
275{
276 for (Index i = 0; i < count; ++i) {
277 size_t len = 0;
278 is >> len;
279 //data[i].resize(len);
280 //is.read(&(data[i][0]), len);
281
282 std::string buffer(len+1, ' ');
283 is.read(&buffer[0], len+1);
284 if (data != nullptr) data[i].assign(buffer, 0, len);
285 }
286}
287
288/// HalfReader wraps a static function, read(), that is analogous to readData(), above,
289/// except that it is partially specialized for floating-point types in order to promote
290/// 16-bit half float values to full float. A wrapper class is required because
291/// only classes, not functions, can be partially specialized.
292 template<bool IsReal, typename T> struct HalfReader;
293/// Partial specialization for non-floating-point types (no half to float promotion)
294template<typename T>
295 struct HalfReader</*IsReal=*/false, T> {
296 static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
297 DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
298 readData(is, data, count, compression, metadata, metadataOffset);
299 }
300};
301/// Partial specialization for floating-point types
302template<typename T>
303 struct HalfReader</*IsReal=*/true, T> {
304 using HalfT = typename RealToHalf<T>::HalfT;
305 static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
306 DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
307 if (count < 1) return;
308 if (data == nullptr) {
309 // seek mode - pass through null pointer
310 readData<HalfT>(is, nullptr, count, compression, metadata, metadataOffset);
311 } else {
312 std::vector<HalfT> halfData(count); // temp buffer into which to read half float values
313 readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compression,
314 metadata, metadataOffset);
315 // Copy half float values from the temporary buffer to the full float output array.
316 std::copy(halfData.begin(), halfData.end(), data);
317 }
318 }
319};
320
321
322template<typename T>
323inline size_t
324writeDataSize(const T *data, Index count, uint32_t compression)
325{
326 if (compression & COMPRESS_BLOSC) {
327 return bloscToStreamSize(reinterpret_cast<const char*>(data), sizeof(T), count);
328 } else if (compression & COMPRESS_ZIP) {
329 return zipToStreamSize(reinterpret_cast<const char*>(data), sizeof(T) * count);
330 } else {
331 return sizeof(T) * count;
332 }
333}
334
335
336/// Specialization for std::string output
337template<>
338 inline size_t
339writeDataSize<std::string>(const std::string* data, Index count,
340 uint32_t /*compression*/) ///< @todo add compression
341{
342 size_t size(0);
343 for (Index i = 0; i < count; ++i) {
344 const size_t len = data[i].size();
345 size += sizeof(size_t) + (len+1);
346 }
347 return size;
348}
349
350
351/// Write data to a stream.
352/// @param os the output stream
353/// @param data the contiguous array of data to write
354/// @param count the number of elements to write out
355/// @param compression whether and how to compress the data (either COMPRESS_NONE,
356/// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
357/// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
358/// without Blosc support.
359/// @details This default implementation is instantiated only for types
360/// whose size can be determined by the sizeof() operator.
361template<typename T>
362inline void
363writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
364{
365 if (compression & COMPRESS_BLOSC) {
366 bloscToStream(os, reinterpret_cast<const char*>(data), sizeof(T), count);
367 } else if (compression & COMPRESS_ZIP) {
368 zipToStream(os, reinterpret_cast<const char*>(data), sizeof(T) * count);
369 } else {
370 os.write(reinterpret_cast<const char*>(data), sizeof(T) * count);
371 }
372}
373
374/// Specialization for std::string output
375template<>
376 inline void
377writeData<std::string>(std::ostream& os, const std::string* data, Index count,
378 uint32_t /*compression*/) ///< @todo add compression
379{
380 for (Index i = 0; i < count; ++i) {
381 const size_t len = data[i].size();
382 os << len;
383 os.write(data[i].c_str(), len+1);
384 //os.write(&(data[i][0]), len );
385 }
386}
387
388/// HalfWriter wraps a static function, write(), that is analogous to writeData(), above,
389/// except that it is partially specialized for floating-point types in order to quantize
390/// floating-point values to 16-bit half float. A wrapper class is required because
391/// only classes, not functions, can be partially specialized.
392 template<bool IsReal, typename T> struct HalfWriter;
393/// Partial specialization for non-floating-point types (no float to half quantization)
394template<typename T>
395 struct HalfWriter</*IsReal=*/false, T> {
396 static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
397 return writeDataSize(data, count, compression);
398 }
399 static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
400 writeData(os, data, count, compression);
401 }
402};
403/// Partial specialization for floating-point types
404template<typename T>
405 struct HalfWriter</*IsReal=*/true, T> {
406 using HalfT = typename RealToHalf<T>::HalfT;
407 static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
408 if (count < 1) return size_t(0);
409 // Convert full float values to half float, then output the half float array.
410 std::vector<HalfT> halfData(count);
411 for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
412 return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
413 }
414 static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
415 if (count < 1) return;
416 // Convert full float values to half float, then output the half float array.
417 std::vector<HalfT> halfData(count);
418 for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
419 writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
420 }
421};
422#ifdef _MSC_VER
423/// Specialization to avoid double to float warnings in MSVC
424template<>
425struct HalfWriter</*IsReal=*/true, double> {
426 using HalfT = RealToHalf<double>::HalfT;
427 static inline size_t writeSize(const double* data, Index count, uint32_t compression)
428 {
429 if (count < 1) return size_t(0);
430 // Convert full float values to half float, then output the half float array.
431 std::vector<HalfT> halfData(count);
432 for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
433 return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
434 }
435 static inline void write(std::ostream& os, const double* data, Index count,
436 uint32_t compression)
437 {
438 if (count < 1) return;
439 // Convert full float values to half float, then output the half float array.
440 std::vector<HalfT> halfData(count);
441 for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
442 writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
443 }
444};
445#endif // _MSC_VER
446
447
448////////////////////////////////////////
449
450
451/// Populate the given buffer with @a destCount values of type @c ValueT
452/// read from the given stream, taking into account that the stream might
453/// have been compressed via one of several supported schemes.
454/// [Mainly for internal use]
455/// @param is a stream from which to read data (possibly compressed,
456/// depending on the stream's compression settings)
457/// @param destBuf a buffer into which to read values of type @c ValueT
458/// @param destCount the number of values to be stored in the buffer
459/// @param valueMask a bitmask (typically, a node's value mask) indicating
460/// which positions in the buffer correspond to active values
461/// @param fromHalf if true, read 16-bit half floats from the input stream
462/// and convert them to full floats
463template<typename ValueT, typename MaskT>
464inline void
465readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
466 const MaskT& valueMask, bool fromHalf)
467{
468 // Get the stream's compression settings.
469 auto meta = getStreamMetadataPtr(is);
470 const uint32_t compression = getDataCompression(is);
471 const bool maskCompressed = compression & COMPRESS_ACTIVE_MASK;
472
473 const bool seek = (destBuf == nullptr);
474 assert(!seek || (!meta || meta->seekable()));
475
476 // Get delayed load metadata if it exists
477
478 DelayedLoadMetadata::Ptr delayLoadMeta;
479 uint64_t leafIndex(0);
480 if (seek && meta && meta->delayedLoadMeta()) {
481 delayLoadMeta =
482 meta->gridMetadata().getMetadata<DelayedLoadMetadata>("file_delayed_load");
483 leafIndex = meta->leaf();
484 }
485
486 int8_t metadata = NO_MASK_AND_ALL_VALS;
487 if (getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) {
488 // Read the flag that specifies what, if any, additional metadata
489 // (selection mask and/or inactive value(s)) is saved.
490 if (seek && !maskCompressed) {
491 is.seekg(/*bytes=*/1, std::ios_base::cur);
492 } else if (seek && delayLoadMeta) {
493 metadata = delayLoadMeta->getMask(leafIndex);
494 is.seekg(/*bytes=*/1, std::ios_base::cur);
495 } else {
496 is.read(reinterpret_cast<char*>(&metadata), /*bytes=*/1);
497 }
498 }
499
500 ValueT background = zeroVal<ValueT>();
501 if (const void* bgPtr = getGridBackgroundValuePtr(is)) {
502 background = *static_cast<const ValueT*>(bgPtr);
503 }
504 ValueT inactiveVal1 = background;
505 ValueT inactiveVal0 =
506 ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : math::negative(background));
507
508 if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
509 metadata == MASK_AND_ONE_INACTIVE_VAL ||
510 metadata == MASK_AND_TWO_INACTIVE_VALS)
511 {
512 // Read one of at most two distinct inactive values.
513 if (seek) {
514 is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
515 } else {
516 is.read(reinterpret_cast<char*>(&inactiveVal0), /*bytes=*/sizeof(ValueT));
517 }
518 if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
519 // Read the second of two distinct inactive values.
520 if (seek) {
521 is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
522 } else {
523 is.read(reinterpret_cast<char*>(&inactiveVal1), /*bytes=*/sizeof(ValueT));
524 }
525 }
526 }
527
528 MaskT selectionMask;
529 if (metadata == MASK_AND_NO_INACTIVE_VALS ||
530 metadata == MASK_AND_ONE_INACTIVE_VAL ||
531 metadata == MASK_AND_TWO_INACTIVE_VALS)
532 {
533 // For use in mask compression (only), read the bitmask that selects
534 // between two distinct inactive values.
535 if (seek) {
536 is.seekg(/*bytes=*/selectionMask.memUsage(), std::ios_base::cur);
537 } else {
538 selectionMask.load(is);
539 }
540 }
541
542 ValueT* tempBuf = destBuf;
543 std::unique_ptr<ValueT[]> scopedTempBuf;
544
545 Index tempCount = destCount;
546
547 if (maskCompressed && metadata != NO_MASK_AND_ALL_VALS
548 && getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION)
549 {
550 tempCount = valueMask.countOn();
551 if (!seek && tempCount != destCount) {
552 // If this node has inactive voxels, allocate a temporary buffer
553 // into which to read just the active values.
554 scopedTempBuf.reset(new ValueT[tempCount]);
555 tempBuf = scopedTempBuf.get();
556 }
557 }
558
559 // Read in the buffer.
560 if (fromHalf) {
561 HalfReader<RealToHalf<ValueT>::isReal, ValueT>::read(
562 is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
563 } else {
564 readData<ValueT>(
565 is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
566 }
567
568 // If mask compression is enabled and the number of active values read into
569 // the temp buffer is smaller than the size of the destination buffer,
570 // then there are missing (inactive) values.
571 if (!seek && maskCompressed && tempCount != destCount) {
572 // Restore inactive values, using the background value and, if available,
573 // the inside/outside mask. (For fog volumes, the destination buffer is assumed
574 // to be initialized to background value zero, so inactive values can be ignored.)
575 for (Index destIdx = 0, tempIdx = 0; destIdx < MaskT::SIZE; ++destIdx) {
576 if (valueMask.isOn(destIdx)) {
577 // Copy a saved active value into this node's buffer.
578 destBuf[destIdx] = tempBuf[tempIdx];
579 ++tempIdx;
580 } else {
581 // Reconstruct an unsaved inactive value and copy it into this node's buffer.
582 destBuf[destIdx] = (selectionMask.isOn(destIdx) ? inactiveVal1 : inactiveVal0);
583 }
584 }
585 }
586}
587
588
589template<typename ValueT, typename MaskT>
590inline size_t
591writeCompressedValuesSize(ValueT* srcBuf, Index srcCount,
592 const MaskT& valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
593{
594 using NonConstValueT = typename std::remove_const<ValueT>::type;
595
596 const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
597
598 Index tempCount = srcCount;
599 ValueT* tempBuf = srcBuf;
600 std::unique_ptr<NonConstValueT[]> scopedTempBuf;
601
602 if (maskCompress && maskMetadata != NO_MASK_AND_ALL_VALS) {
603
604 tempCount = 0;
605
606 Index64 onVoxels = valueMask.countOn();
607 if (onVoxels > Index64(0)) {
608 // Create a new array to hold just the active values.
609 scopedTempBuf.reset(new NonConstValueT[onVoxels]);
610 NonConstValueT* localTempBuf = scopedTempBuf.get();
611
612 // Copy active values to a new, contiguous array.
613 for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
614 localTempBuf[tempCount] = srcBuf[it.pos()];
615 }
616
617 tempBuf = scopedTempBuf.get();
618 }
619 }
620
621 // Return the buffer size.
622 if (toHalf) {
623 return HalfWriter<RealToHalf<NonConstValueT>::isReal, NonConstValueT>::writeSize(
624 tempBuf, tempCount, compress);
625 } else {
626 return writeDataSize<NonConstValueT>(tempBuf, tempCount, compress);
627 }
628}
629
630
631/// Write @a srcCount values of type @c ValueT to the given stream, optionally
632/// after compressing the values via one of several supported schemes.
633/// [Mainly for internal use]
634/// @param os a stream to which to write data (possibly compressed, depending
635/// on the stream's compression settings)
636/// @param srcBuf a buffer containing values of type @c ValueT to be written
637/// @param srcCount the number of values stored in the buffer
638/// @param valueMask a bitmask (typically, a node's value mask) indicating
639/// which positions in the buffer correspond to active values
640/// @param childMask a bitmask (typically, a node's child mask) indicating
641/// which positions in the buffer correspond to child node pointers
642/// @param toHalf if true, convert floating-point values to 16-bit half floats
643template<typename ValueT, typename MaskT>
644inline void
645writeCompressedValues(std::ostream& os, ValueT* srcBuf, Index srcCount,
646 const MaskT& valueMask, const MaskT& childMask, bool toHalf)
647{
648 // Get the stream's compression settings.
649 const uint32_t compress = getDataCompression(os);
650 const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
651
652 Index tempCount = srcCount;
653 ValueT* tempBuf = srcBuf;
654 std::unique_ptr<ValueT[]> scopedTempBuf;
655
656 int8_t metadata = NO_MASK_AND_ALL_VALS;
657
658 if (!maskCompress) {
659 os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
660 } else {
661 // A valid level set's inactive values are either +background (outside)
662 // or -background (inside), and a fog volume's inactive values are all zero.
663 // Rather than write out all of these values, we can store just the active values
664 // (given that the value mask specifies their positions) and, if necessary,
665 // an inside/outside bitmask.
666
667 const ValueT zero = zeroVal<ValueT>();
668 ValueT background = zero;
669 if (const void* bgPtr = getGridBackgroundValuePtr(os)) {
670 background = *static_cast<const ValueT*>(bgPtr);
671 }
672
673 MaskCompress<ValueT, MaskT> maskCompressData(valueMask, childMask, srcBuf, background);
674 metadata = maskCompressData.metadata;
675
676 os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
677
678 if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
679 metadata == MASK_AND_ONE_INACTIVE_VAL ||
680 metadata == MASK_AND_TWO_INACTIVE_VALS)
681 {
682 if (!toHalf) {
683 // Write one of at most two distinct inactive values.
684 os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[0]), sizeof(ValueT));
685 if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
686 // Write the second of two distinct inactive values.
687 os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[1]), sizeof(ValueT));
688 }
689 } else {
690 // Write one of at most two distinct inactive values.
691 ValueT truncatedVal = static_cast<ValueT>(truncateRealToHalf(maskCompressData.inactiveVal[0]));
692 os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
693 if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
694 // Write the second of two distinct inactive values.
695 truncatedVal = truncateRealToHalf(maskCompressData.inactiveVal[1]);
696 os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
697 }
698 }
699 }
700
701 if (metadata == NO_MASK_AND_ALL_VALS) {
702 // If there are more than two unique inactive values, the entire input buffer
703 // needs to be saved (both active and inactive values).
704 /// @todo Save the selection mask as long as most of the inactive values
705 /// are one of two values?
706 } else {
707 // Create a new array to hold just the active values.
708 scopedTempBuf.reset(new ValueT[srcCount]);
709 tempBuf = scopedTempBuf.get();
710
711 if (metadata == NO_MASK_OR_INACTIVE_VALS ||
712 metadata == NO_MASK_AND_MINUS_BG ||
713 metadata == NO_MASK_AND_ONE_INACTIVE_VAL)
714 {
715 // Copy active values to the contiguous array.
716 tempCount = 0;
717 for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
718 tempBuf[tempCount] = srcBuf[it.pos()];
719 }
720 } else {
721 // Copy active values to a new, contiguous array and populate a bitmask
722 // that selects between two distinct inactive values.
723 MaskT selectionMask;
724 tempCount = 0;
725 for (Index srcIdx = 0; srcIdx < srcCount; ++srcIdx) {
726 if (valueMask.isOn(srcIdx)) { // active value
727 tempBuf[tempCount] = srcBuf[srcIdx];
728 ++tempCount;
729 } else { // inactive value
730 if (MaskCompress<ValueT, MaskT>::eq(srcBuf[srcIdx], maskCompressData.inactiveVal[1])) {
731 selectionMask.setOn(srcIdx); // inactive value 1
732 } // else inactive value 0
733 }
734 }
735 assert(tempCount == valueMask.countOn());
736
737 // Write out the mask that selects between two inactive values.
738 selectionMask.save(os);
739 }
740 }
741 }
742
743 // Write out the buffer.
744 if (toHalf) {
745 HalfWriter<RealToHalf<ValueT>::isReal, ValueT>::write(os, tempBuf, tempCount, compress);
746 } else {
747 writeData(os, tempBuf, tempCount, compress);
748 }
749}
750
751} // namespace io
752} // namespace OPENVDB_VERSION_NAME
753} // namespace openvdb
754
755#endif // OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
DelayedLoadMetadata.h
Math.h
General-purpose arithmetic and comparison routines, most of which accept arbitrary value types (or at...
MetaMap.h
OPENVDB_API
#define OPENVDB_API
Definition: Platform.h:254
openvdb::v9_0::io::DelayedLoadMetadata
Store a buffer of data that can be optionally used during reading for faster delayed-load I/O perform...
Definition: DelayedLoadMetadata.h:22
openvdb::v9_0::io::DelayedLoadMetadata::Ptr
SharedPtr< DelayedLoadMetadata > Ptr
Definition: DelayedLoadMetadata.h:24
openvdb::v9_0::math::Vec2
Definition: Vec2.h:24
openvdb::v9_0::math::Vec3
Definition: Vec3.h:24
openvdb::v9_0::math::internal::half
Definition: Half.h:99
io.h
nanovdb::io::Internal::read
static void read(std::istream &is, GridHandle< BufferT > &handle, Codec codec)
nanovdb::io::Internal::write
static fileSize_t write(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec)
nanovdb::Vec3d
Vec3< double > Vec3d
Definition: NanoVDB.h:1174
openvdb::v9_0::io::readData
void readData(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Read data from a stream.
Definition: Compression.h:247
openvdb::v9_0::io::compressionToString
OPENVDB_API std::string compressionToString(uint32_t flags)
Return a string describing the given compression flags.
openvdb::v9_0::io::bloscFromStream
OPENVDB_API void bloscFromStream(std::istream &, char *data, size_t numBytes)
openvdb::v9_0::io::bloscToStream
OPENVDB_API void bloscToStream(std::ostream &, const char *data, size_t valSize, size_t numVals)
openvdb::v9_0::io::COMPRESS_BLOSC
@ COMPRESS_BLOSC
Definition: Compression.h:56
openvdb::v9_0::io::COMPRESS_NONE
@ COMPRESS_NONE
Definition: Compression.h:53
openvdb::v9_0::io::COMPRESS_ACTIVE_MASK
@ COMPRESS_ACTIVE_MASK
Definition: Compression.h:55
openvdb::v9_0::io::COMPRESS_ZIP
@ COMPRESS_ZIP
Definition: Compression.h:54
openvdb::v9_0::io::unzipFromStream
OPENVDB_API void unzipFromStream(std::istream &, char *data, size_t numBytes)
openvdb::v9_0::io::MASK_AND_NO_INACTIVE_VALS
@ MASK_AND_NO_INACTIVE_VALS
Definition: Compression.h:72
openvdb::v9_0::io::MASK_AND_TWO_INACTIVE_VALS
@ MASK_AND_TWO_INACTIVE_VALS
Definition: Compression.h:74
openvdb::v9_0::io::NO_MASK_AND_MINUS_BG
@ NO_MASK_AND_MINUS_BG
Definition: Compression.h:70
openvdb::v9_0::io::NO_MASK_AND_ONE_INACTIVE_VAL
@ NO_MASK_AND_ONE_INACTIVE_VAL
Definition: Compression.h:71
openvdb::v9_0::io::MASK_AND_ONE_INACTIVE_VAL
@ MASK_AND_ONE_INACTIVE_VAL
Definition: Compression.h:73
openvdb::v9_0::io::NO_MASK_AND_ALL_VALS
@ NO_MASK_AND_ALL_VALS
Definition: Compression.h:75
openvdb::v9_0::io::NO_MASK_OR_INACTIVE_VALS
@ NO_MASK_OR_INACTIVE_VALS
Definition: Compression.h:69
openvdb::v9_0::io::truncateRealToHalf
T truncateRealToHalf(const T &val)
Return the given value truncated to 16-bit float precision.
Definition: Compression.h:216
openvdb::v9_0::io::zipToStream
OPENVDB_API void zipToStream(std::ostream &, const char *data, size_t numBytes)
openvdb::v9_0::io::bloscToStreamSize
OPENVDB_API size_t bloscToStreamSize(const char *data, size_t valSize, size_t numVals)
openvdb::v9_0::io::getFormatVersion
OPENVDB_API uint32_t getFormatVersion(std::ios_base &)
Return the file format version number associated with the given input stream.
openvdb::v9_0::io::writeCompressedValuesSize
size_t writeCompressedValuesSize(ValueT *srcBuf, Index srcCount, const MaskT &valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
Definition: Compression.h:591
openvdb::v9_0::io::writeCompressedValues
void writeCompressedValues(std::ostream &os, ValueT *srcBuf, Index srcCount, const MaskT &valueMask, const MaskT &childMask, bool toHalf)
Definition: Compression.h:645
openvdb::v9_0::io::getDataCompression
OPENVDB_API uint32_t getDataCompression(std::ios_base &)
Return a bitwise OR of compression option flags (COMPRESS_ZIP, COMPRESS_ACTIVE_MASK,...
openvdb::v9_0::io::writeDataSize
size_t writeDataSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:324
openvdb::v9_0::io::readCompressedValues
void readCompressedValues(std::istream &is, ValueT *destBuf, Index destCount, const MaskT &valueMask, bool fromHalf)
Definition: Compression.h:465
openvdb::v9_0::io::writeData
void writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:363
openvdb::v9_0::io::getGridBackgroundValuePtr
OPENVDB_API const void * getGridBackgroundValuePtr(std::ios_base &)
Return a pointer to the background value of the grid currently being read from or written to the give...
openvdb::v9_0::io::zipToStreamSize
OPENVDB_API size_t zipToStreamSize(const char *data, size_t numBytes)
openvdb::v9_0::io::getStreamMetadataPtr
OPENVDB_API SharedPtr< StreamMetadata > getStreamMetadataPtr(std::ios_base &)
Return a shared pointer to an object that stores metadata (file format, compression scheme,...
openvdb::v9_0::math::Vec2d
Vec2< double > Vec2d
Definition: Vec2.h:537
openvdb::v9_0::math::isExactlyEqual
bool isExactlyEqual(const T0 &a, const T1 &b)
Return true if a is exactly equal to b.
Definition: Math.h:444
openvdb::v9_0::math::Vec2s
Vec2< float > Vec2s
Definition: Vec2.h:536
openvdb::v9_0::math::half
internal::half half
Definition: Types.h:29
openvdb::v9_0::math::Vec3s
Vec3< float > Vec3s
Definition: Vec3.h:667
openvdb::v9_0::math::negative
T negative(const T &val)
Return the unary negation of the given value.
Definition: Math.h:127
openvdb::v9_0::Vec2H
math::Vec2< math::half > Vec2H
Definition: Types.h:66
openvdb::v9_0::Index
Index32 Index
Definition: Types.h:54
openvdb::v9_0::Vec3H
math::Vec3< math::half > Vec3H
Definition: Types.h:75
openvdb::v9_0::Index32
uint32_t Index32
Definition: Types.h:52
openvdb::v9_0::Index64
uint64_t Index64
Definition: Types.h:53
openvdb::v9_0::OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION
@ OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION
Definition: version.h.in:246
openvdb
Definition: Exceptions.h:13
Types.h
openvdb::v9_0::io::HalfReader< false, T >::read
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition: Compression.h:296
openvdb::v9_0::io::HalfReader< true, T >::HalfT
typename RealToHalf< T >::HalfT HalfT
Definition: Compression.h:304
openvdb::v9_0::io::HalfReader< true, T >::read
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition: Compression.h:305
openvdb::v9_0::io::HalfReader
Definition: Compression.h:292
openvdb::v9_0::io::HalfWriter< false, T >::writeSize
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:396
openvdb::v9_0::io::HalfWriter< false, T >::write
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:399
openvdb::v9_0::io::HalfWriter< true, T >::writeSize
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:407
openvdb::v9_0::io::HalfWriter< true, T >::HalfT
typename RealToHalf< T >::HalfT HalfT
Definition: Compression.h:406
openvdb::v9_0::io::HalfWriter< true, T >::write
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:414
openvdb::v9_0::io::HalfWriter
Definition: Compression.h:392
openvdb::v9_0::io::MaskCompress
Definition: Compression.h:81
openvdb::v9_0::io::MaskCompress::metadata
int8_t metadata
Definition: Compression.h:162
openvdb::v9_0::io::MaskCompress::MaskCompress
MaskCompress(const MaskT &valueMask, const MaskT &childMask, const ValueT *srcBuf, const ValueT &background)
Definition: Compression.h:87
openvdb::v9_0::io::MaskCompress::eq
static bool eq(const ValueT &a, const ValueT &b)
Definition: Compression.h:83
openvdb::v9_0::io::MaskCompress::inactiveVal
ValueT inactiveVal[2]
Definition: Compression.h:163
openvdb::v9_0::io::RealToHalf< Vec2d >::convert
static HalfT convert(const Vec2d &val)
Definition: Compression.h:198
openvdb::v9_0::io::RealToHalf< Vec2s >::convert
static HalfT convert(const Vec2s &val)
Definition: Compression.h:192
openvdb::v9_0::io::RealToHalf< Vec3d >::convert
static HalfT convert(const Vec3d &val)
Definition: Compression.h:209
openvdb::v9_0::io::RealToHalf< Vec3s >::convert
static HalfT convert(const Vec3s &val)
Definition: Compression.h:203
openvdb::v9_0::io::RealToHalf< double >::convert
static HalfT convert(double val)
Definition: Compression.h:187
openvdb::v9_0::io::RealToHalf< float >::convert
static HalfT convert(float val)
Definition: Compression.h:181
openvdb::v9_0::io::RealToHalf
RealToHalf and its specializations define a mapping from floating-point data types to analogous half ...
Definition: Compression.h:173
openvdb::v9_0::io::RealToHalf::HalfT
T HalfT
Definition: Compression.h:175
openvdb::v9_0::io::RealToHalf::convert
static HalfT convert(const T &val)
Definition: Compression.h:176
OPENVDB_VERSION_NAME
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition: version.h.in:116
OPENVDB_USE_VERSION_NAMESPACE
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h.in:202