OpenVDB: nanovdb/util/IO.h Source File

OpenVDB 9.1.0

IO.h
Go to the documentation of this file.
1// Copyright Contributors to the OpenVDB Project
2// SPDX-License-Identifier: MPL-2.0
3
4/*!
5 \file IO.h
6
7 \author Ken Museth
8
9 \date May 1, 2020
10
11 \brief Implements I/O for NanoVDB grids. Features optional BLOSC and ZIP
12 file compression, support for multiple grids per file as well as
13 multiple grid types.
14
15 \note This file does NOT depend on OpenVDB, but optionally on ZIP and BLOSC
16*/
17
18#ifndef NANOVDB_IO_H_HAS_BEEN_INCLUDED
19#define NANOVDB_IO_H_HAS_BEEN_INCLUDED
20
21#include "../NanoVDB.h"
22#include "GridHandle.h"
23
24#include <fstream> // for std::ifstream
25#include <iostream> // for std::cerr/cout
26#include <string> // for std::string
27#include <sstream> // for std::stringstream
28#include <cstring> // for std::strcmp
29#include <memory> // for std::unique_ptr
30#include <vector> // for std::vector
31#ifdef NANOVDB_USE_ZIP
32#include <zlib.h> // for ZIP compression
33#endif
34#ifdef NANOVDB_USE_BLOSC
35#include <blosc.h> // for BLOSC compression
36#endif
37
38// Due to a bug in older versions of gcc, including fstream might
39// define "major" and "minor" which are used as member data below.
40// See https://bugzilla.redhat.com/show_bug.cgi?id=130601
41#if defined(major) || defined(minor)
42#undef major
43#undef minor
44#endif
45
46namespace nanovdb {
47
48 namespace io {
49
50/// We fix a specific size for counting bytes in files so that they
51/// are saved the same regardless of machine precision. (Note there are
52/// still little/bigendian issues, however)
53 using fileSize_t = uint64_t;
54
55/// @brief Optional compression codecs
56///
57/// @note NONE is the default, ZIP is slow but compact and BLOSC offers a great balance.
58///
59/// @warning NanoVDB optionally supports ZIP and BLOSC compression and will throw an exception
60/// if it support is required but missing.
61 enum class Codec : uint16_t { NONE = 0,
62 ZIP = 1,
63 BLOSC = 2,
64 END = 3 };
65
66 inline __hostdev__ const char* toStr(Codec codec)
67{
68 static const char * LUT[] = { "NONE", "ZIP", "BLOSC" , "END" };
69 return LUT[static_cast<int>(codec)];
70}
71
72/// @brief Internal functions for compressed read/write of a NanoVDB GridHandle into a stream
73///
74/// @warning These functions should never be called directly by client code
75 namespace Internal {
76 static constexpr fileSize_t MAX_SIZE = 1UL << 30; // size is 1 GB
77
78template<typename BufferT>
79 static fileSize_t write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec);
80
81template<typename BufferT>
82 static void read(std::istream& is, GridHandle<BufferT>& handle, Codec codec);
83}; // namespace Internal
84
85/// @brief Standard hash function to use on strings; std::hash may vary by
86/// platform/implementation and is know to produce frequent collisions.
87uint64_t stringHash(const char* cstr);
88
89/// @brief Return a uint64_t hash key of a std::string
90 inline uint64_t stringHash(const std::string& str)
91{
92 return stringHash(str.c_str());
93}
94
95/// @brief Return a uint64_t with its bytes reversed so we can check for endianness
96 inline uint64_t reverseEndianness(uint64_t val)
97{
98 return (((val) >> 56) & 0x00000000000000FF) | (((val) >> 40) & 0x000000000000FF00) |
99 (((val) >> 24) & 0x0000000000FF0000) | (((val) >> 8) & 0x00000000FF000000) |
100 (((val) << 8) & 0x000000FF00000000) | (((val) << 24) & 0x0000FF0000000000) |
101 (((val) << 40) & 0x00FF000000000000) | (((val) << 56) & 0xFF00000000000000);
102}
103
104/// @brief Data encoded at the head of each segment of a file or stream.
105///
106/// @note A file or stream is composed of one or more segments that each contain
107// one or more grids.
108// Magic number of NanoVDB files (uint64_t) |
109// Version numbers of this file (uint32_t) | one header for each segment
110// Number of grids in this segment (uint16_t) |
111// Compression mode (uint16_t) |
112 struct Header
113{// 16 bytes
114 uint64_t magic; // 8 bytes
115 Version version;// 4 bytes version numbers
116 uint16_t gridCount; // 2 bytes
117 Codec codec; // 2 bytes
118 Header(Codec c = Codec::NONE)
119 : magic(NANOVDB_MAGIC_NUMBER) // Magic number: "NanoVDB" in hex
120 , version()// major, minor and patch version numbers
121 , gridCount(0)
122 , codec(c)
123 {
124 }
125}; // Header ( 16 bytes = 2 words )
126
127/// @brief Data encoded for each of the grids associated with a segment.
128// Grid size in memory (uint64_t) |
129// Grid size on disk (uint64_t) |
130// Grid name hash key (uint64_t) |
131// Numer of active voxels (uint64_t) |
132// Grid type (uint32_t) |
133// Grid class (uint32_t) |
134// Characters in grid name (uint32_t) |
135// AABB in world space (2*3*double) | one per grid in file
136// AABB in index space (2*3*int) |
137// Size of a voxel in world units (3*double) |
138// Byte size of the grid name (uint32_t) |
139// Number of nodes per level (4*uint32_t) |
140// Numer of active tiles per level (3*uint32_t) |
141// Codec for file compression (uint16_t) |
142// Padding due to 8B alignment (uint16_t) |
143// Version number (uint32_t) |
144 struct MetaData
145{// 176 bytes
146 uint64_t gridSize, fileSize, nameKey, voxelCount; // 4 * 8 = 32B.
147 GridType gridType; // 4B.
148 GridClass gridClass; // 4B.
149 BBox<Vec3d> worldBBox; // 2 * 3 * 8 = 48B.
150 CoordBBox indexBBox; // 2 * 3 * 4 = 24B.
151 Vec3R voxelSize; // 24B.
152 uint32_t nameSize; // 4B.
153 uint32_t nodeCount[4]; //4 x 4 = 16B
154 uint32_t tileCount[3];// 3 x 4 = 12B
155 Codec codec; // 2B
156 uint16_t padding;// 2B, due to 8B alignment from uint64_t
157 Version version;// 4B
158}; // MetaData
159
160 struct GridMetaData : public MetaData
161{
162 static_assert(sizeof(MetaData) == 176, "Unexpected sizeof(MetaData)");
163 std::string gridName;
164 void read(std::istream& is);
165 void write(std::ostream& os) const;
166 GridMetaData() {}
167 template<typename ValueT>
168 GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid);
169 uint64_t memUsage() const { return sizeof(MetaData) + nameSize; }
170}; // GridMetaData
171
172 struct Segment
173{
174 // Check assumptions made during read and write of Header and MetaData
175 static_assert(sizeof(Header) == 16u, "Unexpected sizeof(Header)");
176 Header header;
177 std::vector<GridMetaData> meta;
178 Segment(Codec c = Codec::NONE)
179 : header(c)
180 , meta()
181 {
182 }
183 template<typename BufferT>
184 void add(const GridHandle<BufferT>& h);
185 bool read(std::istream& is);
186 void write(std::ostream& os) const;
187 uint64_t memUsage() const;
188}; // Segment
189
190/// @brief Write a single grid to file (over-writing existing content of the file)
191template<typename BufferT>
192void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE, int verbose = 0);
193
194/// @brief Write a single grid to stream (starting at the current position)
195///
196/// @note This method can be used to append grid to an existing stream
197template<typename BufferT>
198void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec = Codec::NONE);
199
200/// @brief Write multiple grids to file (over-writing existing content of the file)
201template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
202void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE, int verbose = 0);
203
204/// @brief Writes multiple grids to stream (starting at its current position)
205///
206/// @note This method can be used to append multiple grids to an existing stream
207template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
208void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec = Codec::NONE);
209
210/// @brief Read the n'th grid from file (defaults to first grid)
211///
212/// @throw If n exceeds the number of grids in the file
213template<typename BufferT = HostBuffer>
214 GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n = 0, int verbose = 0, const BufferT& buffer = BufferT());
215
216/// @brief Read the n'th grid from stream (defaults to first grid)
217///
218/// @throw If n exceeds the number of grids in the stream
219template<typename BufferT = HostBuffer>
220 GridHandle<BufferT> readGrid(std::istream& is, uint64_t n = 0, const BufferT& buffer = BufferT());
221
222/// @brief Read the first grid with a specific name
223///
224/// @warning If not grid exists with the specified name the resulting GridHandle is empty
225template<typename BufferT = HostBuffer>
226 GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose = 0, const BufferT& buffer = BufferT());
227
228/// @brief Read the first grid with a specific name
229template<typename BufferT = HostBuffer>
230 GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer = BufferT());
231
232/// @brief Read all the grids in the file
233template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
234VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose = 0, const BufferT& buffer = BufferT());
235
236/// @brief Real all grids at the current position of the input stream
237template<typename BufferT = HostBuffer, template<typename...> class VecT = std::vector>
238VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer = BufferT());
239
240/// @brief Return true if the file contains a grid with the specified name
241bool hasGrid(const std::string& fileName, const std::string& gridName);
242
243/// @brief Return true if the stream contains a grid with the specified name
244bool hasGrid(std::istream& is, const std::string& gridName);
245
246/// @brief Reads and returns a vector of meta data for all the grids found in the specified file
247std::vector<GridMetaData> readGridMetaData(const std::string& fileName);
248
249/// @brief Reads and returns a vector of meta data for all the grids found in the specified stream
250std::vector<GridMetaData> readGridMetaData(std::istream& is);
251
252// --------------------------> Implementations for Internal <------------------------------------
253
254template<typename BufferT>
255 fileSize_t Internal::write(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
256{
257 const char* data = reinterpret_cast<const char*>(handle.data());
258 fileSize_t total = 0, residual = handle.size();
259
260 switch (codec) {
261 case Codec::ZIP: {
262#ifdef NANOVDB_USE_ZIP
263 uLongf size = compressBound(residual); // Get an upper bound on the size of the compressed data.
264 std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
265 const int status = compress(tmp.get(), &size, reinterpret_cast<const Bytef*>(data), residual);
266 if (status != Z_OK)
267 std::runtime_error("Internal write error in ZIP");
268 if (size > residual)
269 std::cerr << "\nWarning: Unexpected ZIP compression from " << residual << " to " << size << " bytes\n";
270 const fileSize_t outBytes = size;
271 os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
272 os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
273 total += sizeof(fileSize_t) + outBytes;
274#else
275 throw std::runtime_error("ZIP compression codec was disabled during build");
276#endif
277 break;
278 }
279 case Codec::BLOSC: {
280#ifdef NANOVDB_USE_BLOSC
281 do {
282 fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE, size = chunk + BLOSC_MAX_OVERHEAD;
283 std::unique_ptr<char[]> tmp(new char[size]);
284 const int count = blosc_compress_ctx(9, 1, sizeof(float), chunk, data, tmp.get(), size, BLOSC_LZ4_COMPNAME, 1 << 18, 1);
285 if (count <= 0)
286 std::runtime_error("Internal write error in BLOSC");
287 const fileSize_t outBytes = count;
288 os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
289 os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
290 total += sizeof(fileSize_t) + outBytes;
291 data += chunk;
292 residual -= chunk;
293 } while (residual > 0);
294#else
295 throw std::runtime_error("BLOSC compression codec was disabled during build");
296#endif
297 break;
298 }
299 default:
300 os.write(data, residual);
301 total += residual;
302 }
303 if (!os) {
304 throw std::runtime_error("Failed to write Tree to file");
305 }
306 return total;
307} // Internal::write
308
309template<typename BufferT>
310void Internal::read(std::istream& is, GridHandle<BufferT>& handle, Codec codec)
311{
312 char* data = reinterpret_cast<char*>(handle.buffer().data());
313 fileSize_t residual = handle.buffer().size();
314
315 // read tree using optional compression
316 switch (codec) {
317 case Codec::ZIP: {
318#ifdef NANOVDB_USE_ZIP
319 fileSize_t size;
320 is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
321 std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
322 is.read(reinterpret_cast<char*>(tmp.get()), size);
323 uLongf numBytes = residual;
324 int status = uncompress(reinterpret_cast<Bytef*>(data), &numBytes, tmp.get(), static_cast<uLongf>(size));
325 if (status != Z_OK)
326 std::runtime_error("Internal read error in ZIP");
327 if (fileSize_t(numBytes) != residual)
328 throw std::runtime_error("UNZIP failed on byte size");
329#else
330 throw std::runtime_error("ZIP compression codec was disabled during build");
331#endif
332 break;
333 }
334 case Codec::BLOSC: {
335#ifdef NANOVDB_USE_BLOSC
336 do {
337 fileSize_t size;
338 is.read(reinterpret_cast<char*>(&size), sizeof(fileSize_t));
339 std::unique_ptr<char[]> tmp(new char[size]);
340 is.read(reinterpret_cast<char*>(tmp.get()), size);
341 const fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE;
342 const int count = blosc_decompress_ctx(tmp.get(), data, size_t(chunk), 1); //fails with more threads :(
343 if (count < 1)
344 std::runtime_error("Internal read error in BLOSC");
345 if (count != int(chunk))
346 throw std::runtime_error("BLOSC failed on byte size");
347 data += size_t(chunk);
348 residual -= chunk;
349 } while (residual > 0);
350#else
351 throw std::runtime_error("BLOSC compression codec was disabled during build");
352#endif
353 break;
354 }
355 default:
356 is.read(data, residual);
357 }
358 if (!is) {
359 throw std::runtime_error("Failed to read Tree from file");
360 }
361} // Internal::read
362
363// --------------------------> Implementations for GridMetaData <------------------------------------
364
365template<typename ValueT>
366 inline GridMetaData::GridMetaData(uint64_t size, Codec c, const NanoGrid<ValueT>& grid)
367 : MetaData{size, // gridSize
368 0, // fileSize
369 0, // nameKey
370 grid.activeVoxelCount(), // voxelCount
371 grid.gridType(), // gridType
372 grid.gridClass(), // gridClass
373 grid.worldBBox(), // worldBBox
374 grid.tree().bbox(), // indexBBox
375 grid.voxelSize(), // voxelSize
376 0, // nameSize
377 {0, 0, 0, 1}, // nodeCount[4]
378 {0, 0, 0}, // tileCount[3]
379 c, // codec
380 0, // padding
381 Version()}// version
382 , gridName(grid.gridName())
383{
384 nameKey = stringHash(gridName);
385 nameSize = static_cast<uint32_t>(gridName.size() + 1); // include '\0'
386 const uint32_t* ptr = reinterpret_cast<const TreeData<3>*>(&grid.tree())->mNodeCount;
387 for (int i = 0; i < 3; ++i) {
388 MetaData::nodeCount[i] = *ptr++;
389 }
390 //MetaData::nodeCount[3] = 1;// one root node
391 for (int i = 0; i < 3; ++i) {
392 MetaData::tileCount[i] = *ptr++;
393 }
394}
395
396 inline void GridMetaData::write(std::ostream& os) const
397{
398 os.write(reinterpret_cast<const char*>(this), sizeof(MetaData));
399 os.write(gridName.c_str(), nameSize);
400 if (!os) {
401 throw std::runtime_error("Failed writing GridMetaData");
402 }
403}
404
405 inline void GridMetaData::read(std::istream& is)
406{
407 is.read(reinterpret_cast<char*>(this), sizeof(MetaData));
408 std::unique_ptr<char[]> tmp(new char[nameSize]);
409 is.read(reinterpret_cast<char*>(tmp.get()), nameSize);
410 gridName.assign(tmp.get());
411 if (!is) {
412 throw std::runtime_error("Failed reading GridMetaData");
413 }
414}
415
416// --------------------------> Implementations for Segment <------------------------------------
417
418 inline uint64_t Segment::memUsage() const
419{
420 uint64_t sum = sizeof(Header);
421 for (auto& m : meta) {
422 sum += m.memUsage();
423 }
424 return sum;
425}
426
427template<typename BufferT>
428 inline void Segment::add(const GridHandle<BufferT>& h)
429{
430 if (auto* grid = h.template grid<float>()) { // most common
431 meta.emplace_back(h.size(), header.codec, *grid);
432 } else if (auto* grid = h.template grid<Vec3f>()) {
433 meta.emplace_back(h.size(), header.codec, *grid);
434 } else if (auto* grid = h.template grid<double>()) {
435 meta.emplace_back(h.size(), header.codec, *grid);
436 } else if (auto* grid = h.template grid<int32_t>()) {
437 meta.emplace_back(h.size(), header.codec, *grid);
438 } else if (auto* grid = h.template grid<uint32_t>()) {
439 meta.emplace_back(h.size(), header.codec, *grid);
440 } else if (auto* grid = h.template grid<int64_t>()) {
441 meta.emplace_back(h.size(), header.codec, *grid);
442 } else if (auto* grid = h.template grid<int16_t>()) {
443 meta.emplace_back(h.size(), header.codec, *grid);
444 } else if (auto* grid = h.template grid<Vec3d>()) {
445 meta.emplace_back(h.size(), header.codec, *grid);
446 } else if (auto* grid = h.template grid<ValueMask>()) {
447 meta.emplace_back(h.size(), header.codec, *grid);
448 } else if (auto* grid = h.template grid<bool>()) {
449 meta.emplace_back(h.size(), header.codec, *grid);
450 } else if (auto* grid = h.template grid<Rgba8>()) {
451 meta.emplace_back(h.size(), header.codec, *grid);
452 } else if (auto* grid = h.template grid<Fp4>()) {
453 meta.emplace_back(h.size(), header.codec, *grid);
454 } else if (auto* grid = h.template grid<Fp8>()) {
455 meta.emplace_back(h.size(), header.codec, *grid);
456 } else if (auto* grid = h.template grid<Fp16>()) {
457 meta.emplace_back(h.size(), header.codec, *grid);
458 } else if (auto* grid = h.template grid<FpN>()) {
459 meta.emplace_back(h.size(), header.codec, *grid);
460 } else if (auto* grid = h.template grid<Vec4f>()) {
461 meta.emplace_back(h.size(), header.codec, *grid);
462 } else if (auto* grid = h.template grid<Vec4d>()) {
463 meta.emplace_back(h.size(), header.codec, *grid);
464 } else {
465 throw std::runtime_error("nanovdb::io::Segment::add Cannot write grid of unknown type to file");
466 }
467 header.gridCount += 1;
468}
469
470 inline void Segment::write(std::ostream& os) const
471{
472 if (header.gridCount == 0) {
473 throw std::runtime_error("Segment contains no grids");
474 } else if (!os.write(reinterpret_cast<const char*>(&header), sizeof(Header))) {
475 throw std::runtime_error("Failed to write Header of Segment");
476 }
477 for (auto& m : meta) {
478 m.write(os);
479 }
480}
481
482 inline bool Segment::read(std::istream& is)
483{
484 is.read(reinterpret_cast<char*>(&header), sizeof(Header));
485 if (is.eof()) {
486 return false;
487 }
488 if (!is || header.magic != NANOVDB_MAGIC_NUMBER) {
489 // first check for byte-swapped header magic.
490 if (header.magic == reverseEndianness(NANOVDB_MAGIC_NUMBER))
491 throw std::runtime_error("This nvdb file has reversed endianness");
492 throw std::runtime_error("Magic number error: This is not a valid nvdb file");
493 } else if ( header.version.getMajor() != NANOVDB_MAJOR_VERSION_NUMBER) {
494 std::stringstream ss;
495 if (header.version.getMajor() < NANOVDB_MAJOR_VERSION_NUMBER) {
496 ss << "The file contains an older version of NanoVDB: " << std::string(header.version.c_str()) << "!\n\t"
497 << "Recommendation: Re-generate this NanoVDB file with this version: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X of NanoVDB";
498 } else {
499 ss << "This tool was compiled against an older version of NanoVDB: " << NANOVDB_MAJOR_VERSION_NUMBER << ".X!\n\t"
500 << "Recommendation: Re-compile this tool against the newer version: " << header.version.getMajor() << ".X of NanoVDB";
501 }
502 throw std::runtime_error("An unrecoverable error in nanovdb::Segment::read:\n\tIncompatible file format: " + ss.str());
503 }
504 meta.resize(header.gridCount);
505 for (auto& m : meta) {
506 m.read(is);
507 m.version = header.version;
508 }
509 return true;
510}
511
512// --------------------------> Implementations for read/write <------------------------------------
513
514template<typename BufferT>
515 void writeGrid(const std::string& fileName, const GridHandle<BufferT>& handle, Codec codec, int verbose)
516{
517 std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
518 if (!os.is_open()) {
519 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
520 }
521 writeGrid<BufferT>(os, handle, codec);
522 if (verbose) {
523 std::cout << "Wrote nanovdb::Grid to file named \"" << fileName << "\"" << std::endl;
524 }
525}
526
527template<typename BufferT>
528 void writeGrid(std::ostream& os, const GridHandle<BufferT>& handle, Codec codec)
529{
530 Segment s(codec);
531 s.add(handle);
532 const uint64_t headerSize = s.memUsage();
533 std::streamoff seek = headerSize;
534 os.seekp(seek, std::ios_base::cur); // skip forward from the current position
535 s.meta[0].fileSize = Internal::write(os, handle, codec);
536 seek += s.meta[0].fileSize;
537 os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
538 s.write(os); // write header
539 os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
540}
541
542template<typename BufferT, template<typename...> class VecT>
543 void writeGrids(const std::string& fileName, const VecT<GridHandle<BufferT>>& handles, Codec codec, int verbose)
544{
545 std::ofstream os(fileName, std::ios::out | std::ios::binary | std::ios::trunc);
546 if (!os.is_open()) {
547 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for output");
548 }
549 writeGrids<BufferT, VecT>(os, handles, codec);
550 if (verbose) {
551 std::cout << "Wrote " << handles.size() << " nanovdb::Grid(s) to file named \"" << fileName << "\"" << std::endl;
552 }
553}
554
555template<typename BufferT, template<typename...> class VecT>
556 void writeGrids(std::ostream& os, const VecT<GridHandle<BufferT>>& handles, Codec codec)
557{
558 Segment s(codec);
559 for (auto& h : handles) {
560 s.add(h);
561 }
562 const uint64_t headerSize = s.memUsage();
563 std::streamoff seek = headerSize;
564 os.seekp(seek, std::ios_base::cur); // skip forward from the current position
565 for (size_t i = 0; i < handles.size(); ++i) {
566 s.meta[i].fileSize = Internal::write(os, handles[i], codec);
567 seek += s.meta[i].fileSize;
568 }
569 os.seekp(-seek, std::ios_base::cur); // rewind to start of stream
570 s.write(os); // write header
571 os.seekp(seek - headerSize, std::ios_base::cur); // skip to end
572}
573
574/// @brief Read the n'th grid
575template<typename BufferT>
576 GridHandle<BufferT> readGrid(const std::string& fileName, uint64_t n, int verbose, const BufferT& buffer)
577{
578 std::ifstream is(fileName, std::ios::in | std::ios::binary);
579 if (!is.is_open()) {
580 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
581 }
582 auto handle = readGrid<BufferT>(is, n, buffer);
583 if (verbose) {
584 std::cout << "Read NanoGrid # " << n << " from the file named \"" << fileName << "\"" << std::endl;
585 }
586 return handle; // is converted to r-value and return value is move constructed.
587}
588
589template<typename BufferT>
590 GridHandle<BufferT> readGrid(std::istream& is, uint64_t n, const BufferT& buffer)
591{
592 Segment s;
593 uint64_t counter = 0;
594 while (s.read(is)) {
595 std::streamoff seek = 0;
596 for (auto& m : s.meta) {
597 if (counter == n) {
598 GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
599 is.seekg(seek, std::ios_base::cur); // skip forward from the current position
600 Internal::read(is, handle, s.header.codec);
601 return handle; // is converted to r-value and return value is move constructed.
602 } else {
603 seek += m.fileSize;
604 }
605 ++counter;
606 }
607 is.seekg(seek, std::ios_base::cur); // skip forward from the current position
608 }
609 throw std::runtime_error("Grid index exceeds grid count in file");
610}
611
612/// @brief Read the first grid with a specific name
613template<typename BufferT>
614 GridHandle<BufferT> readGrid(const std::string& fileName, const std::string& gridName, int verbose, const BufferT& buffer)
615{
616 std::ifstream is(fileName, std::ios::in | std::ios::binary);
617 if (!is.is_open()) {
618 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
619 }
620 auto handle = readGrid<BufferT>(is, gridName, buffer);
621 if (verbose) {
622 if (handle) {
623 std::cout << "Read NanoGrid named \"" << gridName << "\" from the file named \"" << fileName << "\"" << std::endl;
624 } else {
625 std::cout << "File named \"" << fileName << "\" does not contain a grid named \"" + gridName + "\"" << std::endl;
626 }
627 }
628 return handle; // is converted to r-value and return value is move constructed.
629}
630
631template<typename BufferT>
632 GridHandle<BufferT> readGrid(std::istream& is, const std::string& gridName, const BufferT& buffer)
633{
634 const auto key = stringHash(gridName);
635 Segment s;
636 while (s.read(is)) {
637 std::streamoff seek = 0;
638 for (auto& m : s.meta) {
639 if (m.nameKey == key && m.gridName == gridName) { // check for hask key collision
640 GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
641 is.seekg(seek, std::ios_base::cur); // rewind
642 Internal::read(is, handle, s.header.codec);
643 return handle; // is converted to r-value and return value is move constructed.
644 } else {
645 seek += m.fileSize;
646 }
647 }
648 is.seekg(seek, std::ios_base::cur); // skip forward from the current position
649 }
650 return GridHandle<BufferT>(); // empty handle
651}
652
653/// @brief Read all the grids
654template<typename BufferT, template<typename...> class VecT>
655VecT<GridHandle<BufferT>> readGrids(const std::string& fileName, int verbose, const BufferT& buffer)
656{
657 std::ifstream is(fileName, std::ios::in | std::ios::binary);
658 if (!is.is_open()) {
659 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
660 }
661 auto handles = readGrids<BufferT, VecT>(is, buffer);
662 if (verbose) {
663 std::cout << "Read " << handles.size() << " NanoGrid(s) from the file named \"" << fileName << "\"" << std::endl;
664 }
665 return handles; // is converted to r-value and return value is move constructed.
666}
667
668template<typename BufferT, template<typename...> class VecT>
669VecT<GridHandle<BufferT>> readGrids(std::istream& is, const BufferT& buffer)
670{
671 VecT<GridHandle<BufferT>> handles;
672 Segment seg;
673 while (seg.read(is)) {
674 for (auto& m : seg.meta) {
675 GridHandle<BufferT> handle(BufferT::create(m.gridSize, &buffer));
676 Internal::read(is, handle, seg.header.codec);
677 handles.push_back(std::move(handle)); // force move copy assignment
678 }
679 }
680 return handles; // is converted to r-value and return value is move constructed.
681}
682
683 inline std::vector<GridMetaData> readGridMetaData(const std::string& fileName)
684{
685 std::ifstream is(fileName, std::ios::in | std::ios::binary);
686 if (!is.is_open()) {
687 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
688 }
689 return readGridMetaData(is); // is converted to r-value and return value is move constructed.
690}
691
692 inline std::vector<GridMetaData> readGridMetaData(std::istream& is)
693{
694 std::vector<GridMetaData> meta;
695 Segment seg;
696 while (seg.read(is)) {
697 std::streamoff seek = 0;
698 for (auto& m : seg.meta) {
699 meta.push_back(m);
700 seek += m.fileSize;
701 }
702 is.seekg(seek, std::ios_base::cur);
703 }
704 return meta; // is converted to r-value and return value is move constructed.
705}
706
707 inline bool hasGrid(const std::string& fileName, const std::string& gridName)
708{
709 std::ifstream is(fileName, std::ios::in | std::ios::binary);
710 if (!is.is_open()) {
711 throw std::runtime_error("Unable to open file named \"" + fileName + "\" for input");
712 }
713 return hasGrid(is, gridName);
714}
715
716 inline bool hasGrid(std::istream& is, const std::string& gridName)
717{
718 const auto key = stringHash(gridName);
719 Segment s;
720 while (s.read(is)) {
721 std::streamoff seek = 0;
722 for (auto& m : s.meta) {
723 if (m.nameKey == key && m.gridName == gridName) {
724 return true; // check for hash key collision
725 }
726 seek += m.fileSize;
727 }
728 is.seekg(seek, std::ios_base::cur);
729 }
730 return false;
731}
732
733 inline uint64_t stringHash(const char* cstr)
734{
735 uint64_t hash = 0;
736 if (!cstr) {
737 return hash;
738 }
739 for (auto* str = reinterpret_cast<const unsigned char*>(cstr); *str; ++str) {
740 uint64_t overflow = hash >> (64 - 8);
741 hash *= 67; // Next-ish prime after 26 + 26 + 10
742 hash += *str + overflow;
743 }
744 return hash;
745}
746
747}
748} // namespace nanovdb::io
749
750#endif // NANOVDB_IO_H_HAS_BEEN_INCLUDED
GridHandle.h
Defines two classes, a GridRegister the defines the value type (e.g. Double, Float etc) of a NanoVDB ...
__hostdev__
#define __hostdev__
Definition: NanoVDB.h:168
NANOVDB_MAJOR_VERSION_NUMBER
#define NANOVDB_MAJOR_VERSION_NUMBER
Definition: NanoVDB.h:104
NANOVDB_MAGIC_NUMBER
#define NANOVDB_MAGIC_NUMBER
Definition: NanoVDB.h:102
nanovdb::GridHandle
This class serves to manage a raw memory buffer of a NanoVDB Grid.
Definition: GridHandle.h:71
nanovdb::GridHandle::size
uint64_t size() const override
Returns the size in bytes of the raw memory buffer managed by this GridHandle's allocator.
Definition: GridHandle.h:123
nanovdb::GridHandle::data
uint8_t * data() override
Returns a non-const pointer to the data.
Definition: GridHandle.h:115
nanovdb::Grid
Highest level of the data structure. Contains a tree and a world->index transform (that currently onl...
Definition: NanoVDB.h:2309
nanovdb::HostBuffer
This is a buffer that contains a shared or private pool to either externally or internally managed ho...
Definition: HostBuffer.h:110
nanovdb::Vec3< double >
nanovdb::Version
Bit-compacted representation of all three version numbers.
Definition: NanoVDB.h:541
hash
__hostdev__ uint32_t hash(uint32_t x)
Definition: common.h:13
nanovdb::io::Internal::read
static void read(std::istream &is, GridHandle< BufferT > &handle, Codec codec)
nanovdb::io::Internal::write
static fileSize_t write(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec)
nanovdb::io::Internal::MAX_SIZE
static constexpr fileSize_t MAX_SIZE
Definition: IO.h:76
nanovdb::io::hasGrid
bool hasGrid(std::istream &is, const std::string &gridName)
Return true if the stream contains a grid with the specified name.
Definition: IO.h:716
nanovdb::io::writeGrid
void writeGrid(const std::string &fileName, const GridHandle< BufferT > &handle, Codec codec=Codec::NONE, int verbose=0)
Write a single grid to file (over-writing existing content of the file)
Definition: IO.h:515
nanovdb::io::readGrids
VecT< GridHandle< BufferT > > readGrids(const std::string &fileName, int verbose=0, const BufferT &buffer=BufferT())
Read all the grids in the file.
Definition: IO.h:655
nanovdb::io::writeGrids
void writeGrids(const std::string &fileName, const VecT< GridHandle< BufferT > > &handles, Codec codec=Codec::NONE, int verbose=0)
Write multiple grids to file (over-writing existing content of the file)
Definition: IO.h:543
nanovdb::io::fileSize_t
uint64_t fileSize_t
Definition: IO.h:53
nanovdb::io::toStr
const char * toStr(Codec codec)
Definition: IO.h:66
nanovdb::io::stringHash
uint64_t stringHash(const char *cstr)
Standard hash function to use on strings; std::hash may vary by platform/implementation and is know t...
Definition: IO.h:733
nanovdb::io::readGridMetaData
std::vector< GridMetaData > readGridMetaData(const std::string &fileName)
Reads and returns a vector of meta data for all the grids found in the specified file.
Definition: IO.h:683
nanovdb::io::reverseEndianness
uint64_t reverseEndianness(uint64_t val)
Return a uint64_t with its bytes reversed so we can check for endianness.
Definition: IO.h:96
nanovdb::io::readGrid
GridHandle< BufferT > readGrid(std::istream &is, const std::string &gridName, const BufferT &buffer=BufferT())
Read the first grid with a specific name.
Definition: IO.h:632
nanovdb::io::readGridMetaData
std::vector< GridMetaData > readGridMetaData(std::istream &is)
Reads and returns a vector of meta data for all the grids found in the specified stream.
Definition: IO.h:692
nanovdb::io::writeGrid
void writeGrid(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec=Codec::NONE)
Write a single grid to stream (starting at the current position)
Definition: IO.h:528
nanovdb::io::stringHash
uint64_t stringHash(const std::string &str)
Return a uint64_t hash key of a std::string.
Definition: IO.h:90
nanovdb::io::readGrids
VecT< GridHandle< BufferT > > readGrids(std::istream &is, const BufferT &buffer=BufferT())
Real all grids at the current position of the input stream.
Definition: IO.h:669
nanovdb::io::Codec
Codec
Optional compression codecs.
Definition: IO.h:61
nanovdb::io::Codec::ZIP
@ ZIP
nanovdb::io::Codec::BLOSC
@ BLOSC
nanovdb::io::Codec::END
@ END
nanovdb::io::Codec::NONE
@ NONE
nanovdb::io::writeGrids
void writeGrids(std::ostream &os, const VecT< GridHandle< BufferT > > &handles, Codec codec=Codec::NONE)
Writes multiple grids to stream (starting at its current position)
Definition: IO.h:556
nanovdb::io::hasGrid
bool hasGrid(const std::string &fileName, const std::string &gridName)
Return true if the file contains a grid with the specified name.
Definition: IO.h:707
nanovdb::io::readGrid
GridHandle< BufferT > readGrid(const std::string &fileName, uint64_t n=0, int verbose=0, const BufferT &buffer=BufferT())
Read the n'th grid from file (defaults to first grid)
Definition: IO.h:576
nanovdb
Definition: NanoVDB.h:184
nanovdb::GridClass
GridClass
Classes (defined in OpenVDB) that are currently supported by NanoVDB.
Definition: NanoVDB.h:253
nanovdb::GridType
GridType
List of types that are currently supported by NanoVDB.
Definition: NanoVDB.h:216
openvdb::v9_1::tools::memUsage
Index64 memUsage(const TreeT &tree, bool threaded=true)
Return the total amount of memory in bytes occupied by this tree.
Definition: Count.h:493
nanovdb::BBox
Definition: NanoVDB.h:1524
nanovdb::io::GridMetaData
Definition: IO.h:161
nanovdb::io::GridMetaData::GridMetaData
GridMetaData()
Definition: IO.h:166
nanovdb::io::GridMetaData::write
void write(std::ostream &os) const
Definition: IO.h:396
nanovdb::io::GridMetaData::memUsage
uint64_t memUsage() const
Definition: IO.h:169
nanovdb::io::GridMetaData::gridName
std::string gridName
Definition: IO.h:162
nanovdb::io::GridMetaData::read
void read(std::istream &is)
Definition: IO.h:405
nanovdb::io::Header
Data encoded at the head of each segment of a file or stream.
Definition: IO.h:113
nanovdb::io::Header::gridCount
uint16_t gridCount
Definition: IO.h:116
nanovdb::io::Header::codec
Codec codec
Definition: IO.h:117
nanovdb::io::Header::magic
uint64_t magic
Definition: IO.h:114
nanovdb::io::Header::version
Version version
Definition: IO.h:115
nanovdb::io::Header::Header
Header(Codec c=Codec::NONE)
Definition: IO.h:118
nanovdb::io::MetaData
Data encoded for each of the grids associated with a segment.
Definition: IO.h:145
nanovdb::io::MetaData::worldBBox
BBox< Vec3d > worldBBox
Definition: IO.h:149
nanovdb::io::MetaData::nameKey
uint64_t nameKey
Definition: IO.h:146
nanovdb::io::MetaData::padding
uint16_t padding
Definition: IO.h:156
nanovdb::io::MetaData::fileSize
uint64_t fileSize
Definition: IO.h:146
nanovdb::io::MetaData::tileCount
uint32_t tileCount[3]
Definition: IO.h:154
nanovdb::io::MetaData::gridClass
GridClass gridClass
Definition: IO.h:148
nanovdb::io::MetaData::indexBBox
CoordBBox indexBBox
Definition: IO.h:150
nanovdb::io::MetaData::nodeCount
uint32_t nodeCount[4]
Definition: IO.h:153
nanovdb::io::MetaData::voxelSize
Vec3R voxelSize
Definition: IO.h:151
nanovdb::io::MetaData::codec
Codec codec
Definition: IO.h:155
nanovdb::io::MetaData::voxelCount
uint64_t voxelCount
Definition: IO.h:146
nanovdb::io::MetaData::version
Version version
Definition: IO.h:157
nanovdb::io::MetaData::gridType
GridType gridType
Definition: IO.h:147
nanovdb::io::MetaData::nameSize
uint32_t nameSize
Definition: IO.h:152
nanovdb::io::MetaData::gridSize
uint64_t gridSize
Definition: IO.h:146
nanovdb::io::Segment
Definition: IO.h:173
nanovdb::io::Segment::read
bool read(std::istream &is)
Definition: IO.h:482
nanovdb::io::Segment::add
void add(const GridHandle< BufferT > &h)
Definition: IO.h:428
nanovdb::io::Segment::write
void write(std::ostream &os) const
Definition: IO.h:470
nanovdb::io::Segment::meta
std::vector< GridMetaData > meta
Definition: IO.h:177
nanovdb::io::Segment::header
Header header
Definition: IO.h:175
nanovdb::io::Segment::memUsage
uint64_t memUsage() const
Definition: IO.h:418
nanovdb::io::Segment::Segment
Segment(Codec c=Codec::NONE)
Definition: IO.h:178