OpenVDB: nanovdb/util/IO.h Source File

OpenVDB 10.0.1

IO.h
Go to the documentation of this file.
1// Copyright Contributors to the OpenVDB Project
2// SPDX-License-Identifier: MPL-2.0
3
4/*!
5 \file IO.h
6
7 \author Ken Museth
8
9 \date May 1, 2020
10
11 \brief Implements I/O for NanoVDB grids. Features optional BLOSC and ZIP
12 file compression, support for multiple grids per file as well as
13 multiple grid types.
14
15 \note This file does NOT depend on OpenVDB, but optionally on ZIP and BLOSC
16*/
17
18#ifndef NANOVDB_IO_H_HAS_BEEN_INCLUDED
19#define NANOVDB_IO_H_HAS_BEEN_INCLUDED
20
21#include "../NanoVDB.h"
22#include "GridHandle.h"
23
24#include <fstream> // for std::ifstream
25#include <iostream> // for std::cerr/cout
26#include <string> // for std::string
27#include <sstream> // for std::stringstream
28#include <cstring> // for std::strcmp
29#include <memory> // for std::unique_ptr
30#include <vector> // for std::vector
31#ifdef NANOVDB_USE_ZIP
32#include <zlib.h> // for ZIP compression
33#endif
34#ifdef NANOVDB_USE_BLOSC
35#include <blosc.h> // for BLOSC compression
36#endif
37
38// Due to a bug in older versions of gcc, including fstream might
39// define "major" and "minor" which are used as member data below.
40// See https://bugzilla.redhat.com/show_bug.cgi?id=130601
41#if defined(major) || defined(minor)
42#undef major
43#undef minor
44#endif
45
46namespace nanovdb {
47
48namespace io {
49
50/// We fix a specific size for counting bytes in files so that they
51/// are saved the same regardless of machine precision. (Note there are
52/// still little/bigendian issues, however)
53 using fileSize_t = uint64_t;
54
55/// @brief Optional compression codecs
56///
57/// @note NONE is the default, ZIP is slow but compact and BLOSC offers a great balance.
58///
59/// @warning NanoVDB optionally supports ZIP and BLOSC compression and will throw an exception
60/// if it support is required but missing.
61 enum class Codec : uint16_t { NONE = 0,
62 ZIP = 1,
63 BLOSC = 2,
64 END = 3 };
65
66 inline __hostdev__ const char* toStr(Codec codec)
67{
68 static const char * LUT[] = { "NONE", "ZIP", "BLOSC" , "END" };
69 return LUT[static_cast<int>(codec)];
70}
71
72/// @brief Internal functions for compressed read/write of a NanoVDB GridHandle into a stream
73///
74/// @warning These functions should never be called directly by client code
75 namespace Internal {
76 static constexpr fileSize_t MAX_SIZE = 1UL << 30; // size is 1 GB
77
78template<typename BufferT>
79 static fileSize_t write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec);
80
81template<typename BufferT>
82 static void read(std::istream& is, GridHandle<BufferT>& handle, Codec codec);
83}; // namespace Internal
84
85/// @brief Standard hash function to use on strings; std::hash may vary by
86/// platform/implementation and is know to produce frequent collisions.
87uint64_t stringHash(const char* cstr);
88
89/// @brief Return a uint64_t hash key of a std::string
90 inline uint64_t stringHash(const std::string& str)
91{
92 return stringHash(str.c_str());
93}
94
95/// @brief Return a uint64_t with its bytes reversed so we can check for endianness
96 inline uint64_t reverseEndianness(uint64_t val)
97{
98 return (((val) >> 56) & 0x00000000000000FF) | (((val) >> 40) & 0x000000000000FF00) |
99 (((val) >> 24) & 0x0000000000FF0000) | (((val) >> 8) & 0x00000000FF000000) |
100 (((val) << 8) & 0x000000FF00000000) | (((val) << 24) & 0x0000FF0000000000) |
101 (((val) << 40) & 0x00FF000000000000) | (((val) << 56) & 0xFF00000000000000);
102}
103
104/// @brief Data encoded at the head of each segment of a file or stream.
105///
106/// @note A file or stream is composed of one or more segments that each contain
107// one or more grids.
108// Magic number of NanoVDB files (uint64_t) |
109// Version numbers of this file (uint32_t) | one header for each segment
110// Number of grids in this segment (uint16_t) |
111// Compression mode (uint16_t) |
112 struct Header
113{// 16 bytes
114 uint64_t magic; // 8 bytes
115 Version version;// 4 bytes version numbers
116 uint16_t gridCount; // 2 bytes
117 Codec codec; // 2 bytes
118 Header(Codec c = Codec::NONE)
119 : magic(NANOVDB_MAGIC_NUMBER) // Magic number: "NanoVDB" in hex
120 , version()// major, minor and patch version numbers
121 , gridCount(0)
122 , codec(c)
123 {
124 }
125}; // Header ( 16 bytes = 2 words )
126
127/// @brief Data encoded for each of the grids associated with a segment.
128// Grid size in memory (uint64_t) |
129// Grid size on disk (uint64_t) |
130// Grid name hash key (uint64_t) |
131// Numer of active voxels (uint64_t) |
132// Grid type (uint32_t) |
133// Grid class (uint32_t) |
134// Characters in grid name (uint32_t) |
135// AABB in world space (2*3*double) | one per grid in file
136// AABB in index space (2*3*int) |
137// Size of a voxel in world units (3*double) |
138// Byte size of the grid name (uint32_t) |
139// Number of nodes per level (4*uint32_t) |
140// Numer of active tiles per level (3*uint32_t) |
141// Codec for file compression (uint16_t) |
142// Padding due to 8B alignment (uint16_t) |
143// Version number (uint32_t) |
144 struct MetaData
145{// 176 bytes
146 uint64_t gridSize, fileSize, nameKey, voxelCount; // 4 * 8 = 32B.
147 GridType gridType; // 4B.
148 GridClass gridClass; // 4B.
149 BBox<Vec3d> worldBBox; // 2 * 3 * 8 = 48B.
150 CoordBBox indexBBox; // 2 * 3 * 4 = 24B.
151 Vec3R voxelSize; // 24B.
152 uint32_t nameSize; // 4B.
153 uint32_t nodeCount[4]; //4 x 4 = 16B
154 uint32_t tileCount[3];// 3 x 4 = 12B
155 Codec codec; // 2B
156 uint16_t padding;// 2B, due to 8B alignment from uint64_t
157 Version version;// 4B
158}; // MetaData
159
160 struct GridMetaData : public MetaData
161{
162 static_assert(sizeof(MetaData) == 176, "Unexpected sizeof(MetaData)");
163 std::string gridName;
164 void read(std::istream& is);
165 void write(std::ostream& os) const;
166 GridMetaData() {}
167 template<typename ValueT>
168 GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid);
169 uint64_t memUsage() const { return sizeof(MetaData) + nameSize; }
170}; // GridMetaData
171
172 struct Segment
173{
174 // Check assumptions made during read and write of Header and MetaData
175 static_assert(sizeof(Header) == 16u, "Unexpected sizeof(Header)");
176 Header header;
177 std::vector<GridMetaData> meta;
178 Segment(Codec c = Codec::NONE)
179 : header(c)
180 , meta()
181 {
182 }
183 template<typename BufferT>
184 void add(const GridHandle<BufferT>& h);
185 bool read(std::istream& is);
186 void write(std::ostream& os) const;
187 uint64_t memUsage() const;
188}; // Segment
189
190/// @brief Write a single grid to file (over-writing existing content of the file)
191template<typename BufferT>
192void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE, int verbose = 0);
193
194/// @brief Write a single grid to stream (starting at the current position)
195///
196/// @note This method can be used to append grid to an existing stream
197template<typename BufferT>
198void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE);
199
200/// @brief Write multiple grids to file (over-writing existing content of the file)
201template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
202void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE, int verbose = 0);
203
204/// @brief Writes multiple grids to stream (starting at its current position)
205///
206/// @note This method can be used to append multiple grids to an existing stream
207template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
208void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE);
209
210/// @brief Read the n'th grid from file (defaults to first grid)
211///
212/// @throw If n exceeds the number of grids in the file
213template<typename BufferT = HostBuffer>
214 GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n = 0, int verbose = 0, const BufferT& buffer = BufferT());
215
216/// @brief Read the n'th grid from stream (defaults to first grid)
217///
218/// @throw If n exceeds the number of grids in the stream
219template<typename BufferT = HostBuffer>
220 GridHandle<BufferT> readGrid(std::istream& is, uint64_t n = 0, const BufferT& buffer = BufferT());
221
222/// @brief Read the first grid with a specific name
223///
224/// @warning If not grid exists with the specified name the resulting GridHandle is empty
225template<typename BufferT = HostBuffer>
226 GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose = 0, const BufferT& buffer = BufferT());
227
228/// @brief Read the first grid with a specific name
229template<typename BufferT = HostBuffer>
230 GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer = BufferT());
231
232/// @brief Read all the grids in the file
233template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
234VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose = 0, const BufferT& buffer = BufferT());
235
236/// @brief Real all grids at the current position of the input stream
237template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
238VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer = BufferT());
239
240/// @brief Return true if the file contains a grid with the specified name
241bool hasGrid(const std::string& fileName, const std::string& gridName);
242
243/// @brief Return true if the stream contains a grid with the specified name
244bool hasGrid(std::istream& is, const std::string& gridName);
245
246/// @brief Reads and returns a vector of meta data for all the grids found in the specified file
247std::vector<GridMetaData> readGridMetaData(const std::string& fileName);
248
249/// @brief Reads and returns a vector of meta data for all the grids found in the specified stream
250std::vector<GridMetaData> readGridMetaData(std::istream& is);
251
252// --------------------------> Implementations for Internal <------------------------------------
253
254template<typename BufferT>
255 fileSize_t Internal::write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
256{
257 const char* data = reinterpret_cast<const char*>(handle.data());
258 fileSize_t total = 0, residual = handle.size();
259
260 switch (codec) {
261 case Codec::ZIP: {
262#ifdef NANOVDB_USE_ZIP
263 uLongf size = compressBound(residual); // Get an upper bound on the size of the compressed data.
264 std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
265 const int status = compress(tmp.get(), &size, reinterpret_cast<const Bytef*>(data), residual);
266 if (status != Z_OK)
267 std::runtime_error("Internal write error in ZIP");
268 if (size > residual)
269 std::cerr << "\nWarning: Unexpected ZIP compression from " << residual << " to " << size << " bytes\n";
270 const fileSize_t outBytes = size;
271 os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
272 os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
273 total += sizeof(fileSize_t) + outBytes;
274#else
275 throw std::runtime_error("ZIP compression codec was disabled during build");
276#endif
277 break;
278 }
279 case Codec::BLOSC: {
280#ifdef NANOVDB_USE_BLOSC
281 do {
282 fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE, size = chunk + BLOSC_MAX_OVERHEAD;
283 std::unique_ptr<char[]> tmp(new char[size]);
284 const int count = blosc_compress_ctx(9, 1, sizeof(float), chunk, data, tmp.get(), size, BLOSC_LZ4_COMPNAME, 1 << 18, 1);
285 if (count <= 0)
286 std::runtime_error("Internal write error in BLOSC");
287 const fileSize_t outBytes = count;
288 os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
289 os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
290 total += sizeof(fileSize_t) + outBytes;
291 data += chunk;
292 residual -= chunk;
293 } while (residual > 0);
294#else
295 throw std::runtime_error("BLOSC compression codec was disabled during build");
296#endif
297 break;
298 }
299 default:
300 os.write(data, residual);
301 total += residual;
302 }
303 if (!os) {
304 throw std::runtime_error("Failed to write Tree to file");
305 }
306 return total;
307} // Internal::write
308
309template<typename BufferT>
310void Internal::read(std::istream& is, GridHandle<BufferT>& handle, Codec codec)
311{
312 char* data = reinterpret_cast<char*>(handle.buffer().data());
313 fileSize_t residual = handle.buffer().size();
314
315 // read tree using optional compression
316 switch (codec) {
317 case Codec::ZIP: {
318#ifdef NANOVDB_USE_ZIP
319 fileSize_t size;
320 is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
321 std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
322 is.read(reinterpret_cast<char*>(tmp.get()), size);
323 uLongf numBytes = residual;
324 int status = uncompress(reinterpret_cast<Bytef*>(data), &numBytes, tmp.get(), static_cast<uLongf>(size));
325 if (status != Z_OK)
326 std::runtime_error("Internal read error in ZIP");
327 if (fileSize_t(numBytes) != residual)
328 throw std::runtime_error("UNZIP failed on byte size");
329#else
330 throw std::runtime_error("ZIP compression codec was disabled during build");
331#endif
332 break;
333 }
334 case Codec::BLOSC: {
335#ifdef NANOVDB_USE_BLOSC
336 do {
337 fileSize_t size;
338 is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
339 std::unique_ptr<char[]> tmp(new char[size]);
340 is.read(reinterpret_cast<char*>(tmp.get()), size);
341 const fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE;
342 const int count = blosc_decompress_ctx(tmp.get(), data, size_t(chunk), 1); //fails with more threads :(
343 if (count < 1)
344 std::runtime_error("Internal read error in BLOSC");
345 if (count != int(chunk))
346 throw std::runtime_error("BLOSC failed on byte size");
347 data += size_t(chunk);
348 residual -= chunk;
349 } while (residual > 0);
350#else
351 throw std::runtime_error("BLOSC compression codec was disabled during build");
352#endif
353 break;
354 }
355 default:
356 is.read(data, residual);
357 }
358 if (!is) {
359 throw std::runtime_error("Failed to read Tree from file");
360 }
361} // Internal::read
362
363// --------------------------> Implementations for GridMetaData <------------------------------------
364
365template<typename ValueT>
366 inline GridMetaData::GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid)
367 : MetaData{size, // gridSize
368 0, // fileSize
369 0, // nameKey
370 grid.activeVoxelCount(), // voxelCount
371 grid.gridType(), // gridType
372 grid.gridClass(), // gridClass
373 grid.worldBBox(), // worldBBox
374 grid.tree().bbox(), // indexBBox
375 grid.voxelSize(), // voxelSize
376 0, // nameSize
377 {0, 0, 0, 1}, // nodeCount[4]
378 {0, 0, 0}, // tileCount[3]
379 c, // codec
380 0, // padding
381 Version()}// version
382 , gridName(grid.gridName())
383{
384 nameKey = stringHash(gridName);
385 nameSize = static_cast<uint32_t>(gridName.size() + 1); // include '\0'
386 const uint32_t* ptr = reinterpret_cast<const TreeData<3>*>(&grid.tree())->mNodeCount;
387 for (int i = 0; i < 3; ++i) {
388 MetaData::nodeCount[i] = *ptr++;
389 }
390 //MetaData::nodeCount[3] = 1;// one root node
391 for (int i = 0; i < 3; ++i) {
392 MetaData::tileCount[i] = *ptr++;
393 }
394}
395
396 inline void GridMetaData::write(std::ostream& os) const
397{
398 os.write(reinterpret_cast<const char*>(this), sizeof(MetaData));
399 os.write(gridName.c_str(), nameSize);
400 if (!os) {
401 throw std::runtime_error("Failed writing GridMetaData");
402 }
403}
404
405 inline void GridMetaData::read(std::istream& is)
406{
407 is.read(reinterpret_cast<char*>(this), sizeof(MetaData));
408 std::unique_ptr<char[]> tmp(new char[nameSize]);
409 is.read(reinterpret_cast<char*>(tmp.get()), nameSize);
410 gridName.assign(tmp.get());
411 if (!is) {
412 throw std::runtime_error("Failed reading GridMetaData");
413 }
414}
415
416// --------------------------> Implementations for Segment <------------------------------------
417
418 inline uint64_t Segment::memUsage() const
419{
420 uint64_t sum = sizeof(Header);
421 for (auto& m : meta) {
422 sum += m.memUsage();
423 }
424 return sum;
425}
426
427template<typename BufferT>
428 inline void Segment::add(const GridHandle<BufferT>& h)
429{
430 if (auto* grid = h.template grid<float>()) { // most common
431 meta.emplace_back(h.size(), header.codec, *grid);
432 } else if (auto* grid = h.template grid<Vec3f>()) {
433 meta.emplace_back(h.size(), header.codec, *grid);
434 } else if (auto* grid = h.template grid<double>()) {
435 meta.emplace_back(h.size(), header.codec, *grid);
436 } else if (auto* grid = h.template grid<int32_t>()) {
437 meta.emplace_back(h.size(), header.codec, *grid);
438 } else if (auto* grid = h.template grid<uint32_t>()) {
439 meta.emplace_back(h.size(), header.codec, *grid);
440 } else if (auto* grid = h.template grid<int64_t>()) {
441 meta.emplace_back(h.size(), header.codec, *grid);
442 } else if (auto* grid = h.template grid<int16_t>()) {
443 meta.emplace_back(h.size(), header.codec, *grid);
444 } else if (auto* grid = h.template grid<Vec3d>()) {
445 meta.emplace_back(h.size(), header.codec, *grid);
446 } else if (auto* grid = h.template grid<ValueMask>()) {
447 meta.emplace_back(h.size(), header.codec, *grid);
448 } else if (auto* grid = h.template grid<ValueIndex>()) {
449 meta.emplace_back(h.size(), header.codec, *grid);
450 } else if (auto* grid = h.template grid<bool>()) {
451 meta.emplace_back(h.size(), header.codec, *grid);
452 } else if (auto* grid = h.template grid<Rgba8>()) {
453 meta.emplace_back(h.size(), header.codec, *grid);
454 } else if (auto* grid = h.template grid<Fp4>()) {
455 meta.emplace_back(h.size(), header.codec, *grid);
456 } else if (auto* grid = h.template grid<Fp8>()) {
457 meta.emplace_back(h.size(), header.codec, *grid);
458 } else if (auto* grid = h.template grid<Fp16>()) {
459 meta.emplace_back(h.size(), header.codec, *grid);
460 } else if (auto* grid = h.template grid<FpN>()) {
461 meta.emplace_back(h.size(), header.codec, *grid);
462 } else if (auto* grid = h.template grid<Vec4f>()) {
463 meta.emplace_back(h.size(), header.codec, *grid);
464 } else if (auto* grid = h.template grid<Vec4d>()) {
465 meta.emplace_back(h.size(), header.codec, *grid);
466 } else {
467 throw std::runtime_error("nanovdb::io::Segment::add Cannot write grid of unknown type to file");
468 }
469 header.gridCount += 1;
470}
471
472 inline void Segment::write(std::ostream& os) const
473{
474 if (header.gridCount == 0) {
475 throw std::runtime_error("Segment contains no grids");
476 } else if (!os.write(reinterpret_cast<const char*>(&header), sizeof(Header))) {
477 throw std::runtime_error("Failed to write Header of Segment");
478 }
479 for (auto& m : meta) {
480 m.write(os);
481 }
482}
483
484 inline bool Segment::read(std::istream& is)
485{
486 is.read(reinterpret_cast<char*>(&header), sizeof(Header));
487 if (is.eof()) {
488 return false;
489 }
490 if (!is || header.magic != NANOVDB_MAGIC_NUMBER) {
491 // first check for byte-swapped header magic.
492 if (header.magic == reverseEndianness(NANOVDB_MAGIC_NUMBER))
493 throw std::runtime_error("This nvdb file has reversed endianness");
494 throw std::runtime_error("Magic number error: This is not a valid nvdb file");
495 } else if ( header.version.getMajor() != NANOVDB_MAJOR_VERSION_NUMBER) {
496 std::stringstream ss;
497 if (header.version.getMajor() < NANOVDB_MAJOR_VERSION_NUMBER) {
498 ss << "The file contains an older version of NanoVDB: " << std::string(header.version.c_str()) << "!\n\t"
499 << "Recommendation: Re-generate this NanoVDB file with this version: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X of NanoVDB";
500 } else {
501 ss << "This tool was compiled against an older version of NanoVDB: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X!\n\t"
502 << "Recommendation: Re-compile this tool against the newer version: " << header.version.getMajor() << ".X of NanoVDB";
503 }
504 throw std::runtime_error("An unrecoverable error in nanovdb::Segment::read:\n\tIncompatible file format: " + ss.str());
505 }
506 meta.resize(header.gridCount);
507 for (auto& m : meta) {
508 m.read(is);
509 m.version = header.version;
510 }
511 return true;
512}
513
514// --------------------------> Implementations for read/write <------------------------------------
515
516template<typename BufferT>
517 void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec, int verbose)
518{
519 std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
520 if (!os.is_open()) {
521 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
522 }
523 writeGrid<BufferT>(os, handle, codec);
524 if (verbose) {
525 std::cout << "Wrote nanovdb::Grid to file named \"" << fileName << "\"" << std::endl;
526 }
527}
528
529template<typename BufferT>
530 void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
531{
532 Segment s(codec);
533 s.add(handle);
534 const uint64_t headerSize = s.memUsage();
535 std::streamoff seek = headerSize;
536 os.seekp(seek, std::ios_base::cur); // skip forward from the current position
537 s.meta[0].fileSize = Internal::write(os, handle, codec);
538 seek += s.meta[0].fileSize;
539 os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
540 s.write(os); // write header
541 os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
542}
543
544template<typename BufferT, template<typename...> class VecT>
545 void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec, int verbose)
546{
547 std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
548 if (!os.is_open()) {
549 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
550 }
551 writeGrids<BufferT, VecT>(os, handles, codec);
552 if (verbose) {
553 std::cout << "Wrote " << handles.size() << " nanovdb::Grid(s) to file named \"" << fileName << "\"" << std::endl;
554 }
555}
556
557template<typename BufferT, template<typename...> class VecT>
558 void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec)
559{
560 Segment s(codec);
561 for (auto& h : handles) {
562 s.add(h);
563 }
564 const uint64_t headerSize = s.memUsage();
565 std::streamoff seek = headerSize;
566 os.seekp(seek, std::ios_base::cur); // skip forward from the current position
567 for (size_t i = 0; i < handles.size(); ++i) {
568 s.meta[i].fileSize = Internal::write(os, handles[i], codec);
569 seek += s.meta[i].fileSize;
570 }
571 os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
572 s.write(os); // write header
573 os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
574}
575
576/// @brief Read the n'th grid
577template<typename BufferT>
578 GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n, int verbose, const BufferT& buffer)
579{
580 std::ifstream is(fileName, std::ios::in | std::ios::binary);
581 if (!is.is_open()) {
582 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
583 }
584 auto handle = readGrid<BufferT>(is, n, buffer);
585 if (verbose) {
586 std::cout << "Read NanoGrid # " << n << " from the file named \"" << fileName << "\"" << std::endl;
587 }
588 return handle; // is converted to r-value and return value is move constructed.
589}
590
591template<typename BufferT>
592 GridHandle<BufferT> readGrid(std::istream& is, uint64_t n, const BufferT& buffer)
593{
594 Segment s;
595 uint64_t counter = 0;
596 while (s.read(is)) {
597 std::streamoff seek = 0;
598 for (auto& m : s.meta) {
599 if (counter == n) {
600 GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
601 is.seekg(seek, std::ios_base::cur); // skip forward from the current position
602 Internal::read(is, handle, s.header.codec);
603 return handle; // is converted to r-value and return value is move constructed.
604 } else {
605 seek += m.fileSize;
606 }
607 ++counter;
608 }
609 is.seekg(seek, std::ios_base::cur); // skip forward from the current position
610 }
611 throw std::runtime_error("Grid index exceeds grid count in file");
612}
613
614/// @brief Read the first grid with a specific name
615template<typename BufferT>
616 GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose, const BufferT& buffer)
617{
618 std::ifstream is(fileName, std::ios::in | std::ios::binary);
619 if (!is.is_open()) {
620 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
621 }
622 auto handle = readGrid<BufferT>(is, gridName, buffer);
623 if (verbose) {
624 if (handle) {
625 std::cout << "Read NanoGrid named \"" << gridName << "\" from the file named \"" << fileName << "\"" << std::endl;
626 } else {
627 std::cout << "File named \"" << fileName << "\" does not contain a grid named \"" + gridName + "\"" << std::endl;
628 }
629 }
630 return handle; // is converted to r-value and return value is move constructed.
631}
632
633template<typename BufferT>
634 GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer)
635{
636 const auto key = stringHash(gridName);
637 Segment s;
638 while (s.read(is)) {
639 std::streamoff seek = 0;
640 for (auto& m : s.meta) {
641 if (m.nameKey == key && m.gridName == gridName) { // check for hash key collision
642 GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
643 is.seekg(seek, std::ios_base::cur); // rewind
644 Internal::read(is, handle, s.header.codec);
645 return handle; // is converted to r-value and return value is move constructed.
646 } else {
647 seek += m.fileSize;
648 }
649 }
650 is.seekg(seek, std::ios_base::cur); // skip forward from the current position
651 }
652 return GridHandle<BufferT>(); // empty handle
653}
654
655/// @brief Read all the grids
656template<typename BufferT, template<typename...> class VecT>
657VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose, const BufferT& buffer)
658{
659 std::ifstream is(fileName, std::ios::in | std::ios::binary);
660 if (!is.is_open()) {
661 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
662 }
663 auto handles = readGrids<BufferT, VecT>(is, buffer);
664 if (verbose) {
665 std::cout << "Read " << handles.size() << " NanoGrid(s) from the file named \"" << fileName << "\"" << std::endl;
666 }
667 return handles; // is converted to r-value and return value is move constructed.
668}
669
670template<typename BufferT, template<typename...> class VecT>
671VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer)
672{
673 VecT<GridHandle<BufferT>> handles;
674 Segment seg;
675 while (seg.read(is)) {
676 for (auto& m : seg.meta) {
677 GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
678 Internal::read(is, handle, seg.header.codec);
679 handles.push_back(std::move(handle)); // force move copy assignment
680 }
681 }
682 return handles; // is converted to r-value and return value is move constructed.
683}
684
685 inline std::vector<GridMetaData> readGridMetaData(const std::string& fileName)
686{
687 std::ifstream is(fileName, std::ios::in | std::ios::binary);
688 if (!is.is_open()) {
689 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
690 }
691 return readGridMetaData(is); // is converted to r-value and return value is move constructed.
692}
693
694 inline std::vector<GridMetaData> readGridMetaData(std::istream& is)
695{
696 std::vector<GridMetaData> meta;
697 Segment seg;
698 while (seg.read(is)) {
699 std::streamoff seek = 0;
700 for (auto& m : seg.meta) {
701 meta.push_back(m);
702 seek += m.fileSize;
703 }
704 is.seekg(seek, std::ios_base::cur);
705 }
706 return meta; // is converted to r-value and return value is move constructed.
707}
708
709 inline bool hasGrid(const std::string& fileName, const std::string& gridName)
710{
711 std::ifstream is(fileName, std::ios::in | std::ios::binary);
712 if (!is.is_open()) {
713 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
714 }
715 return hasGrid(is, gridName);
716}
717
718 inline bool hasGrid(std::istream& is, const std::string& gridName)
719{
720 const auto key = stringHash(gridName);
721 Segment s;
722 while (s.read(is)) {
723 std::streamoff seek = 0;
724 for (auto& m : s.meta) {
725 if (m.nameKey == key && m.gridName == gridName) {
726 return true; // check for hash key collision
727 }
728 seek += m.fileSize;
729 }
730 is.seekg(seek, std::ios_base::cur);
731 }
732 return false;
733}
734
735 inline uint64_t stringHash(const char* cstr)
736{
737 uint64_t hash = 0;
738 if (!cstr) {
739 return hash;
740 }
741 for (auto* str = reinterpret_cast<const unsigned char*>(cstr); *str; ++str) {
742 uint64_t overflow = hash >> (64 - 8);
743 hash *= 67; // Next-ish prime after 26 + 26 + 10
744 hash += *str + overflow;
745 }
746 return hash;
747}
748
749}
750} // namespace nanovdb::io
751
752#endif // NANOVDB_IO_H_HAS_BEEN_INCLUDED
GridHandle.h
Defines two classes, a GridRegister the defines the value type (e.g. Double, Float etc) of a NanoVDB ...
__hostdev__
#define __hostdev__
Definition: NanoVDB.h:192
NANOVDB_MAJOR_VERSION_NUMBER
#define NANOVDB_MAJOR_VERSION_NUMBER
Definition: NanoVDB.h:123
NANOVDB_MAGIC_NUMBER
#define NANOVDB_MAGIC_NUMBER
Definition: NanoVDB.h:121
nanovdb::GridHandle
This class serves to manage a raw memory buffer of a NanoVDB Grid.
Definition: GridHandle.h:71
nanovdb::GridHandle::size
uint64_t size() const override
Returns the size in bytes of the raw memory buffer managed by this GridHandle's allocator.
Definition: GridHandle.h:125
nanovdb::GridHandle::data
uint8_t * data() override
Returns a non-const pointer to the data.
Definition: GridHandle.h:117
nanovdb::Grid
Highest level of the data structure. Contains a tree and a world->index transform (that currently onl...
Definition: NanoVDB.h:2556
nanovdb::HostBuffer
This is a buffer that contains a shared or private pool to either externally or internally managed ho...
Definition: HostBuffer.h:115
nanovdb::Vec3< double >
nanovdb::Version
Bit-compacted representation of all three version numbers.
Definition: NanoVDB.h:648
hash
__hostdev__ uint32_t hash(uint32_t x)
Definition: common.h:13
nanovdb::io::Internal::read
static void read(std::istream &is, GridHandle< BufferT > &handle, Codec codec)
nanovdb::io::Internal::write
static fileSize_t write(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec)
nanovdb::io::Internal::MAX_SIZE
static constexpr fileSize_t MAX_SIZE
Definition: IO.h:76
nanovdb::io::hasGrid
bool hasGrid(std::istream &is, const std::string &gridName)
Return true if the stream contains a grid with the specified name.
Definition: IO.h:718
nanovdb::io::writeGrid
void writeGrid(const std::string &fileName, const GridHandle< BufferT > &handle, Codec codec=Codec::NONE, int verbose=0)
Write a single grid to file (over-writing existing content of the file)
Definition: IO.h:517
nanovdb::io::readGrids
VecT< GridHandle< BufferT > > readGrids(const std::string &fileName, int verbose=0, const BufferT &buffer=BufferT())
Read all the grids in the file.
Definition: IO.h:657
nanovdb::io::writeGrids
void writeGrids(const std::string &fileName, const VecT< GridHandle< BufferT > > &handles, Codec codec=Codec::NONE, int verbose=0)
Write multiple grids to file (over-writing existing content of the file)
Definition: IO.h:545
nanovdb::io::fileSize_t
uint64_t fileSize_t
Definition: IO.h:53
nanovdb::io::toStr
const char * toStr(Codec codec)
Definition: IO.h:66
nanovdb::io::stringHash
uint64_t stringHash(const char *cstr)
Standard hash function to use on strings; std::hash may vary by platform/implementation and is know t...
Definition: IO.h:735
nanovdb::io::readGridMetaData
std::vector< GridMetaData > readGridMetaData(const std::string &fileName)
Reads and returns a vector of meta data for all the grids found in the specified file.
Definition: IO.h:685
nanovdb::io::reverseEndianness
uint64_t reverseEndianness(uint64_t val)
Return a uint64_t with its bytes reversed so we can check for endianness.
Definition: IO.h:96
nanovdb::io::readGrid
GridHandle< BufferT > readGrid(std::istream &is, const std::string &gridName, const BufferT &buffer=BufferT())
Read the first grid with a specific name.
Definition: IO.h:634
nanovdb::io::readGridMetaData
std::vector< GridMetaData > readGridMetaData(std::istream &is)
Reads and returns a vector of meta data for all the grids found in the specified stream.
Definition: IO.h:694
nanovdb::io::writeGrid
void writeGrid(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec=Codec::NONE)
Write a single grid to stream (starting at the current position)
Definition: IO.h:530
nanovdb::io::stringHash
uint64_t stringHash(const std::string &str)
Return a uint64_t hash key of a std::string.
Definition: IO.h:90
nanovdb::io::readGrids
VecT< GridHandle< BufferT > > readGrids(std::istream &is, const BufferT &buffer=BufferT())
Real all grids at the current position of the input stream.
Definition: IO.h:671
nanovdb::io::Codec
Codec
Optional compression codecs.
Definition: IO.h:61
nanovdb::io::Codec::ZIP
@ ZIP
nanovdb::io::Codec::BLOSC
@ BLOSC
nanovdb::io::Codec::END
@ END
nanovdb::io::Codec::NONE
@ NONE
nanovdb::io::writeGrids
void writeGrids(std::ostream &os, const VecT< GridHandle< BufferT > > &handles, Codec codec=Codec::NONE)
Writes multiple grids to stream (starting at its current position)
Definition: IO.h:558
nanovdb::io::hasGrid
bool hasGrid(const std::string &fileName, const std::string &gridName)
Return true if the file contains a grid with the specified name.
Definition: IO.h:709
nanovdb::io::readGrid
GridHandle< BufferT > readGrid(const std::string &fileName, uint64_t n=0, int verbose=0, const BufferT &buffer=BufferT())
Read the n'th grid from file (defaults to first grid)
Definition: IO.h:578
nanovdb
Definition: Camera.h:16
nanovdb::GridClass
GridClass
Classes (defined in OpenVDB) that are currently supported by NanoVDB.
Definition: NanoVDB.h:281
nanovdb::GridType
GridType
List of types that are currently supported by NanoVDB.
Definition: NanoVDB.h:243
openvdb::v10_0::tools::memUsage
Index64 memUsage(const TreeT &tree, bool threaded=true)
Return the total amount of memory in bytes occupied by this tree.
Definition: Count.h:493
nanovdb::BBox
Definition: NanoVDB.h:1655
nanovdb::io::GridMetaData
Definition: IO.h:161
nanovdb::io::GridMetaData::GridMetaData
GridMetaData()
Definition: IO.h:166
nanovdb::io::GridMetaData::write
void write(std::ostream &os) const
Definition: IO.h:396
nanovdb::io::GridMetaData::memUsage
uint64_t memUsage() const
Definition: IO.h:169
nanovdb::io::GridMetaData::gridName
std::string gridName
Definition: IO.h:162
nanovdb::io::GridMetaData::read
void read(std::istream &is)
Definition: IO.h:405
nanovdb::io::Header
Data encoded at the head of each segment of a file or stream.
Definition: IO.h:113
nanovdb::io::Header::gridCount
uint16_t gridCount
Definition: IO.h:116
nanovdb::io::Header::codec
Codec codec
Definition: IO.h:117
nanovdb::io::Header::magic
uint64_t magic
Definition: IO.h:114
nanovdb::io::Header::version
Version version
Definition: IO.h:115
nanovdb::io::Header::Header
Header(Codec c=Codec::NONE)
Definition: IO.h:118
nanovdb::io::MetaData
Data encoded for each of the grids associated with a segment.
Definition: IO.h:145
nanovdb::io::MetaData::worldBBox
BBox< Vec3d > worldBBox
Definition: IO.h:149
nanovdb::io::MetaData::nameKey
uint64_t nameKey
Definition: IO.h:146
nanovdb::io::MetaData::padding
uint16_t padding
Definition: IO.h:156
nanovdb::io::MetaData::fileSize
uint64_t fileSize
Definition: IO.h:146
nanovdb::io::MetaData::tileCount
uint32_t tileCount[3]
Definition: IO.h:154
nanovdb::io::MetaData::gridClass
GridClass gridClass
Definition: IO.h:148
nanovdb::io::MetaData::indexBBox
CoordBBox indexBBox
Definition: IO.h:150
nanovdb::io::MetaData::nodeCount
uint32_t nodeCount[4]
Definition: IO.h:153
nanovdb::io::MetaData::voxelSize
Vec3R voxelSize
Definition: IO.h:151
nanovdb::io::MetaData::codec
Codec codec
Definition: IO.h:155
nanovdb::io::MetaData::voxelCount
uint64_t voxelCount
Definition: IO.h:146
nanovdb::io::MetaData::version
Version version
Definition: IO.h:157
nanovdb::io::MetaData::gridType
GridType gridType
Definition: IO.h:147
nanovdb::io::MetaData::nameSize
uint32_t nameSize
Definition: IO.h:152
nanovdb::io::MetaData::gridSize
uint64_t gridSize
Definition: IO.h:146
nanovdb::io::Segment
Definition: IO.h:173
nanovdb::io::Segment::read
bool read(std::istream &is)
Definition: IO.h:484
nanovdb::io::Segment::add
void add(const GridHandle< BufferT > &h)
Definition: IO.h:428
nanovdb::io::Segment::write
void write(std::ostream &os) const
Definition: IO.h:472
nanovdb::io::Segment::meta
std::vector< GridMetaData > meta
Definition: IO.h:177
nanovdb::io::Segment::header
Header header
Definition: IO.h:175
nanovdb::io::Segment::memUsage
uint64_t memUsage() const
Definition: IO.h:418
nanovdb::io::Segment::Segment
Segment(Codec c=Codec::NONE)
Definition: IO.h:178