OpenVDB: StreamCompression.h Source File

OpenVDB 7.0.0

StreamCompression.h
Go to the documentation of this file.
1 // Copyright Contributors to the OpenVDB Project
2 // SPDX-License-Identifier: MPL-2.0
3
18
19 #ifndef OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
20 #define OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
21
22 #include <openvdb/io/io.h>
23 #include <tbb/spin_mutex.h>
24 #include <memory>
25 #include <string>
26
27
28 class TestStreamCompression;
29
30 namespace openvdb {
31 OPENVDB_USE_VERSION_NAMESPACE
32 namespace OPENVDB_VERSION_NAME {
33 namespace compression {
34
35
36 // This is the minimum number of bytes below which Blosc compression is not used to
37 // avoid unecessary computation, as Blosc offers minimal compression until this limit
38 static const int BLOSC_MINIMUM_BYTES = 48;
39
40 // This is the minimum number of bytes below which the array is padded with zeros up
41 // to this number of bytes to allow Blosc to perform compression with small arrays
42 static const int BLOSC_PAD_BYTES = 128;
43
44
46 OPENVDB_API bool bloscCanCompress();
47
51 OPENVDB_API size_t bloscUncompressedSize(const char* buffer);
52
60 OPENVDB_API void bloscCompress(char* compressedBuffer, size_t& compressedBytes,
61 const size_t bufferBytes, const char* uncompressedBuffer, const size_t uncompressedBytes);
62
71 OPENVDB_API std::unique_ptr<char[]> bloscCompress(const char* buffer,
72 const size_t uncompressedBytes, size_t& compressedBytes, const bool resize = true);
73
78 OPENVDB_API size_t bloscCompressedSize(const char* buffer, const size_t uncompressedBytes);
79
87 OPENVDB_API void bloscDecompress(char* uncompressedBuffer, const size_t expectedBytes,
88 const size_t bufferBytes, const char* compressedBuffer);
89
97 OPENVDB_API std::unique_ptr<char[]> bloscDecompress(const char* buffer,
98 const size_t expectedBytes, const bool resize = true);
99
100
102
103
104 // 1MB = 1048576 Bytes
105 static const int PageSize = 1024 * 1024;
106
107
111 class OPENVDB_API Page
112 {
113 private:
114 struct Info
115 {
116 io::MappedFile::Ptr mappedFile;
117 SharedPtr<io::StreamMetadata> meta;
118 std::streamoff filepos;
119 long compressedBytes;
120 long uncompressedBytes;
121 }; // Info
122
123 public:
124 using Ptr = std::shared_ptr<Page>;
125
126 Page() = default;
127
129 void load() const;
130
133 long uncompressedBytes() const;
134
137 const char* buffer(const int index) const;
138
140 void readHeader(std::istream&);
141
144 void readBuffers(std::istream&, bool delayed);
145
147 bool isOutOfCore() const;
148
149 private:
151 void copy(const std::unique_ptr<char[]>& temp, int pageSize);
152
154 void decompress(const std::unique_ptr<char[]>& temp);
155
157 void doLoad() const;
158
159 std::unique_ptr<Info> mInfo = std::unique_ptr<Info>(new Info);
160 std::unique_ptr<char[]> mData;
161 tbb::spin_mutex mMutex;
162 }; // class Page
163
164
167 class OPENVDB_API PageHandle
168 {
169 public:
170 #if OPENVDB_ABI_VERSION_NUMBER >= 6
171 using Ptr = std::unique_ptr<PageHandle>;
172 #else
173 using Ptr = std::shared_ptr<PageHandle>;
174 #endif
175
180 PageHandle(const Page::Ptr& page, const int index, const int size);
181
183 Page& page();
184
186 int size() const { return mSize; }
187
190 std::unique_ptr<char[]> read();
191
193 Ptr copy() { return Ptr(new PageHandle(mPage, mIndex, mSize)); }
194
195 protected:
196 friend class ::TestStreamCompression;
197
198 private:
199 Page::Ptr mPage;
200 int mIndex = -1;
201 int mSize = 0;
202 }; // class PageHandle
203
204
208 class OPENVDB_API PagedInputStream
209 {
210 public:
211 using Ptr = std::shared_ptr<PagedInputStream>;
212
213 PagedInputStream() = default;
214
215 explicit PagedInputStream(std::istream& is);
216
218 void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
219 bool sizeOnly() const { return mSizeOnly; }
220
221 // @brief Set and get the input stream
222 std::istream& getInputStream() { assert(mIs); return *mIs; }
223 void setInputStream(std::istream& is) { mIs = &is; }
224
226 PageHandle::Ptr createHandle(std::streamsize n);
227
231 void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true);
232
233 private:
234 int mByteIndex = 0;
235 int mUncompressedBytes = 0;
236 std::istream* mIs = nullptr;
237 Page::Ptr mPage;
238 bool mSizeOnly = false;
239 }; // class PagedInputStream
240
241
245 class OPENVDB_API PagedOutputStream
246 {
247 public:
248 using Ptr = std::shared_ptr<PagedOutputStream>;
249
250 PagedOutputStream();
251
252 explicit PagedOutputStream(std::ostream& os);
253
255 void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
256 bool sizeOnly() const { return mSizeOnly; }
257
259 std::ostream& getOutputStream() { assert(mOs); return *mOs; }
260 void setOutputStream(std::ostream& os) { mOs = &os; }
261
263 PagedOutputStream& write(const char* str, std::streamsize n);
264
266 void flush();
267
268 private:
271 void compressAndWrite(const char* buffer, size_t size);
272
274 void resize(size_t size);
275
276 std::unique_ptr<char[]> mData = std::unique_ptr<char[]>(new char[PageSize]);
277 std::unique_ptr<char[]> mCompressedData = nullptr;
278 size_t mCapacity = PageSize;
279 int mBytes = 0;
280 std::ostream* mOs = nullptr;
281 bool mSizeOnly = false;
282 }; // class PagedOutputStream
283
284
285 } // namespace compression
286 } // namespace OPENVDB_VERSION_NAME
287 } // namespace openvdb
288
289 #endif // OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
openvdb::v7_0::compression::PagedOutputStream::setOutputStream
void setOutputStream(std::ostream &os)
Definition: StreamCompression.h:260
openvdb::v7_0::compression::bloscCompress
OPENVDB_API std::unique_ptr< char[]> bloscCompress(const char *buffer, const size_t uncompressedBytes, size_t &compressedBytes, const bool resize=true)
Compress and return the heap-allocated compressed buffer.
openvdb::v7_0::io::MappedFile::Ptr
SharedPtr< MappedFile > Ptr
Definition: io.h:136
openvdb::v7_0::compression::bloscDecompress
OPENVDB_API std::unique_ptr< char[]> bloscDecompress(const char *buffer, const size_t expectedBytes, const bool resize=true)
Decompress and return the the heap-allocated uncompressed buffer.
openvdb::v7_0::compression::bloscCanCompress
OPENVDB_API bool bloscCanCompress()
Returns true if compression is available.
openvdb::v7_0::compression::PagedInputStream
A Paging wrapper to std::istream that is responsible for reading from a given input stream and creati...
Definition: StreamCompression.h:208
openvdb::v7_0::compression::Page
Stores a variable-size, compressed, delayed-load Page of data that is loaded into memory when accesse...
Definition: StreamCompression.h:111
openvdb::v7_0::compression::PageHandle::size
int size() const
Return the size of the buffer.
Definition: StreamCompression.h:186
openvdb::v7_0::compression::PagedInputStream::Ptr
std::shared_ptr< PagedInputStream > Ptr
Definition: StreamCompression.h:211
openvdb::v7_0::SharedPtr
std::shared_ptr< T > SharedPtr
Definition: Types.h:91
openvdb::v7_0::compression::bloscCompressedSize
OPENVDB_API size_t bloscCompressedSize(const char *buffer, const size_t uncompressedBytes)
Convenience wrapper to retrieve the compressed size of buffer when compressed.
openvdb::v7_0::compression::PagedOutputStream::Ptr
std::shared_ptr< PagedOutputStream > Ptr
Definition: StreamCompression.h:248
openvdb::v7_0::compression::PageHandle::copy
Ptr copy()
Return a copy of this PageHandle.
Definition: StreamCompression.h:193
openvdb::v7_0::compression::PageSize
static const int PageSize
Definition: StreamCompression.h:105
OPENVDB_API
#define OPENVDB_API
Helper macros for defining library symbol visibility.
Definition: Platform.h:230
openvdb::v7_0::compression::bloscUncompressedSize
OPENVDB_API size_t bloscUncompressedSize(const char *buffer)
Retrieves the uncompressed size of buffer when uncompressed.
openvdb::v7_0::compression::PagedInputStream::setSizeOnly
void setSizeOnly(bool sizeOnly)
Size-only mode tags the stream as only reading size data.
Definition: StreamCompression.h:218
openvdb::v7_0::compression::PagedInputStream::setInputStream
void setInputStream(std::istream &is)
Definition: StreamCompression.h:223
openvdb::v7_0::compression::PageHandle::Ptr
std::unique_ptr< PageHandle > Ptr
Definition: StreamCompression.h:171
openvdb::v7_0::compression::BLOSC_MINIMUM_BYTES
static const int BLOSC_MINIMUM_BYTES
Definition: StreamCompression.h:38
openvdb::v7_0::compression::PageHandle
A PageHandle holds a unique ptr to a Page and a specific stream pointer to a point within the decompr...
Definition: StreamCompression.h:167
openvdb::v7_0::compression::BLOSC_PAD_BYTES
static const int BLOSC_PAD_BYTES
Definition: StreamCompression.h:42
openvdb::v7_0::compression::Page::Ptr
std::shared_ptr< Page > Ptr
Definition: StreamCompression.h:124
openvdb::v7_0::compression::PagedOutputStream::sizeOnly
bool sizeOnly() const
Definition: StreamCompression.h:256
OPENVDB_USE_VERSION_NAMESPACE
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h:154
openvdb::v7_0::compression::PagedInputStream::sizeOnly
bool sizeOnly() const
Definition: StreamCompression.h:219
openvdb::v7_0::compression::PagedOutputStream
A Paging wrapper to std::ostream that is responsible for writing from a given output stream at interv...
Definition: StreamCompression.h:245
io.h
OPENVDB_VERSION_NAME
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition: version.h:102
openvdb::v7_0::compression::PagedOutputStream::setSizeOnly
void setSizeOnly(bool sizeOnly)
Size-only mode tags the stream as only writing size data.
Definition: StreamCompression.h:255
openvdb::v7_0::compression::PagedInputStream::getInputStream
std::istream & getInputStream()
Definition: StreamCompression.h:222
openvdb
Definition: Exceptions.h:13
openvdb::v7_0::compression::PagedOutputStream::getOutputStream
std::ostream & getOutputStream()
Set and get the output stream.
Definition: StreamCompression.h:259