OpenVDB: StreamCompression.h Source File

OpenVDB 6.1.0

StreamCompression.h
Go to the documentation of this file.
1 //
3 // Copyright (c) 2012-2018 DreamWorks Animation LLC
4 //
5 // All rights reserved. This software is distributed under the
6 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
7 //
8 // Redistributions of source code must retain the above copyright
9 // and license notice and the following restrictions and disclaimer.
10 //
11 // * Neither the name of DreamWorks Animation nor the names of
12 // its contributors may be used to endorse or promote products derived
13 // from this software without specific prior written permission.
14 //
15 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL,
20 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 // IN NO EVENT SHALL THE COPYRIGHT HOLDERS' AND CONTRIBUTORS' AGGREGATE
27 // LIABILITY FOR ALL CLAIMS REGARDLESS OF THEIR BASIS EXCEED US$250.00.
28 //
30
45
46 #ifndef OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
47 #define OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
48
49 #include <openvdb/io/io.h>
50 #include <tbb/spin_mutex.h>
51 #include <memory>
52 #include <string>
53
54
55 class TestStreamCompression;
56
57 namespace openvdb {
58 OPENVDB_USE_VERSION_NAMESPACE
59 namespace OPENVDB_VERSION_NAME {
60 namespace compression {
61
62
63 // This is the minimum number of bytes below which Blosc compression is not used to
64 // avoid unecessary computation, as Blosc offers minimal compression until this limit
65 static const int BLOSC_MINIMUM_BYTES = 48;
66
67 // This is the minimum number of bytes below which the array is padded with zeros up
68 // to this number of bytes to allow Blosc to perform compression with small arrays
69 static const int BLOSC_PAD_BYTES = 128;
70
71
73 OPENVDB_API bool bloscCanCompress();
74
78 OPENVDB_API size_t bloscUncompressedSize(const char* buffer);
79
87 OPENVDB_API void bloscCompress(char* compressedBuffer, size_t& compressedBytes,
88 const size_t bufferBytes, const char* uncompressedBuffer, const size_t uncompressedBytes);
89
98 OPENVDB_API std::unique_ptr<char[]> bloscCompress(const char* buffer,
99 const size_t uncompressedBytes, size_t& compressedBytes, const bool resize = true);
100
105 OPENVDB_API size_t bloscCompressedSize(const char* buffer, const size_t uncompressedBytes);
106
114 OPENVDB_API void bloscDecompress(char* uncompressedBuffer, const size_t expectedBytes,
115 const size_t bufferBytes, const char* compressedBuffer);
116
124 OPENVDB_API std::unique_ptr<char[]> bloscDecompress(const char* buffer,
125 const size_t expectedBytes, const bool resize = true);
126
127
129
130
131 // 1MB = 1048576 Bytes
132 static const int PageSize = 1024 * 1024;
133
134
138 class OPENVDB_API Page
139 {
140 private:
141 struct Info
142 {
143 io::MappedFile::Ptr mappedFile;
144 SharedPtr<io::StreamMetadata> meta;
145 std::streamoff filepos;
146 long compressedBytes;
147 long uncompressedBytes;
148 }; // Info
149
150 public:
151 using Ptr = std::shared_ptr<Page>;
152
153 Page() = default;
154
156 void load() const;
157
160 long uncompressedBytes() const;
161
164 const char* buffer(const int index) const;
165
167 void readHeader(std::istream&);
168
171 void readBuffers(std::istream&, bool delayed);
172
174 bool isOutOfCore() const;
175
176 private:
178 void copy(const std::unique_ptr<char[]>& temp, int pageSize);
179
181 void decompress(const std::unique_ptr<char[]>& temp);
182
184 void doLoad() const;
185
186 std::unique_ptr<Info> mInfo = std::unique_ptr<Info>(new Info);
187 std::unique_ptr<char[]> mData;
188 tbb::spin_mutex mMutex;
189 }; // class Page
190
191
194 class OPENVDB_API PageHandle
195 {
196 public:
197 #if OPENVDB_ABI_VERSION_NUMBER >= 6
198 using Ptr = std::unique_ptr<PageHandle>;
199 #else
200 using Ptr = std::shared_ptr<PageHandle>;
201 #endif
202
207 PageHandle(const Page::Ptr& page, const int index, const int size);
208
210 Page& page();
211
213 int size() const { return mSize; }
214
217 std::unique_ptr<char[]> read();
218
220 Ptr copy() { return Ptr(new PageHandle(mPage, mIndex, mSize)); }
221
222 protected:
223 friend class ::TestStreamCompression;
224
225 private:
226 Page::Ptr mPage;
227 int mIndex = -1;
228 int mSize = 0;
229 }; // class PageHandle
230
231
235 class OPENVDB_API PagedInputStream
236 {
237 public:
238 using Ptr = std::shared_ptr<PagedInputStream>;
239
240 PagedInputStream() = default;
241
242 explicit PagedInputStream(std::istream& is);
243
245 void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
246 bool sizeOnly() const { return mSizeOnly; }
247
248 // @brief Set and get the input stream
249 std::istream& getInputStream() { assert(mIs); return *mIs; }
250 void setInputStream(std::istream& is) { mIs = &is; }
251
253 PageHandle::Ptr createHandle(std::streamsize n);
254
258 void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true);
259
260 private:
261 int mByteIndex = 0;
262 int mUncompressedBytes = 0;
263 std::istream* mIs = nullptr;
264 Page::Ptr mPage;
265 bool mSizeOnly = false;
266 }; // class PagedInputStream
267
268
272 class OPENVDB_API PagedOutputStream
273 {
274 public:
275 using Ptr = std::shared_ptr<PagedOutputStream>;
276
277 PagedOutputStream();
278
279 explicit PagedOutputStream(std::ostream& os);
280
282 void setSizeOnly(bool sizeOnly) { mSizeOnly = sizeOnly; }
283 bool sizeOnly() const { return mSizeOnly; }
284
286 std::ostream& getOutputStream() { assert(mOs); return *mOs; }
287 void setOutputStream(std::ostream& os) { mOs = &os; }
288
290 PagedOutputStream& write(const char* str, std::streamsize n);
291
293 void flush();
294
295 private:
298 void compressAndWrite(const char* buffer, size_t size);
299
301 void resize(size_t size);
302
303 std::unique_ptr<char[]> mData = std::unique_ptr<char[]>(new char[PageSize]);
304 std::unique_ptr<char[]> mCompressedData = nullptr;
305 size_t mCapacity = PageSize;
306 int mBytes = 0;
307 std::ostream* mOs = nullptr;
308 bool mSizeOnly = false;
309 }; // class PagedOutputStream
310
311
312 } // namespace compression
313 } // namespace OPENVDB_VERSION_NAME
314 } // namespace openvdb
315
316 #endif // OPENVDB_TOOLS_STREAM_COMPRESSION_HAS_BEEN_INCLUDED
317
318 // Copyright (c) 2012-2018 DreamWorks Animation LLC
319 // All rights reserved. This software is distributed under the
320 // Mozilla Public License 2.0 ( http://www.mozilla.org/MPL/2.0/ )
openvdb::v6_1::compression::Page::Ptr
std::shared_ptr< Page > Ptr
Definition: StreamCompression.h:151
OPENVDB_API
#define OPENVDB_API
Helper macros for defining library symbol visibility.
Definition: Platform.h:288
openvdb::v6_1::compression::PageSize
static const int PageSize
Definition: StreamCompression.h:132
openvdb::v6_1::io::MappedFile::Ptr
SharedPtr< MappedFile > Ptr
Definition: io.h:153
openvdb::v6_1::compression::PagedOutputStream::getOutputStream
std::ostream & getOutputStream()
Set and get the output stream.
Definition: StreamCompression.h:286
openvdb::v6_1::compression::PagedInputStream::sizeOnly
bool sizeOnly() const
Definition: StreamCompression.h:246
openvdb::v6_1::compression::PagedInputStream::setInputStream
void setInputStream(std::istream &is)
Definition: StreamCompression.h:250
openvdb::v6_1::compression::bloscCompressedSize
OPENVDB_API size_t bloscCompressedSize(const char *buffer, const size_t uncompressedBytes)
Convenience wrapper to retrieve the compressed size of buffer when compressed.
openvdb::v6_1::compression::PageHandle
A PageHandle holds a unique ptr to a Page and a specific stream pointer to a point within the decompr...
Definition: StreamCompression.h:194
openvdb::v6_1::compression::PageHandle::Ptr
std::unique_ptr< PageHandle > Ptr
Definition: StreamCompression.h:198
openvdb::v6_1::compression::bloscCompress
OPENVDB_API std::unique_ptr< char[]> bloscCompress(const char *buffer, const size_t uncompressedBytes, size_t &compressedBytes, const bool resize=true)
Compress and return the heap-allocated compressed buffer.
openvdb::v6_1::compression::bloscCanCompress
OPENVDB_API bool bloscCanCompress()
Returns true if compression is available.
openvdb::v6_1::compression::PagedOutputStream::Ptr
std::shared_ptr< PagedOutputStream > Ptr
Definition: StreamCompression.h:275
io.h
openvdb::v6_1::SharedPtr
std::shared_ptr< T > SharedPtr
Definition: Types.h:139
openvdb::v6_1::compression::BLOSC_PAD_BYTES
static const int BLOSC_PAD_BYTES
Definition: StreamCompression.h:69
OPENVDB_VERSION_NAME
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition: version.h:125
openvdb::v6_1::compression::PageHandle::size
int size() const
Return the size of the buffer.
Definition: StreamCompression.h:213
openvdb::v6_1::compression::PageHandle::copy
Ptr copy()
Return a copy of this PageHandle.
Definition: StreamCompression.h:220
openvdb::v6_1::compression::BLOSC_MINIMUM_BYTES
static const int BLOSC_MINIMUM_BYTES
Definition: StreamCompression.h:65
openvdb
Definition: Exceptions.h:40
openvdb::v6_1::compression::PagedOutputStream
A Paging wrapper to std::ostream that is responsible for writing from a given output stream at interv...
Definition: StreamCompression.h:272
openvdb::v6_1::compression::PagedOutputStream::setSizeOnly
void setSizeOnly(bool sizeOnly)
Size-only mode tags the stream as only writing size data.
Definition: StreamCompression.h:282
openvdb::v6_1::compression::PagedOutputStream::sizeOnly
bool sizeOnly() const
Definition: StreamCompression.h:283
openvdb::v6_1::compression::PagedInputStream
A Paging wrapper to std::istream that is responsible for reading from a given input stream and creati...
Definition: StreamCompression.h:235
openvdb::v6_1::compression::PagedOutputStream::setOutputStream
void setOutputStream(std::ostream &os)
Definition: StreamCompression.h:287
openvdb::v6_1::compression::PagedInputStream::Ptr
std::shared_ptr< PagedInputStream > Ptr
Definition: StreamCompression.h:238
openvdb::v6_1::compression::bloscDecompress
OPENVDB_API std::unique_ptr< char[]> bloscDecompress(const char *buffer, const size_t expectedBytes, const bool resize=true)
Decompress and return the the heap-allocated uncompressed buffer.
openvdb::v6_1::compression::PagedInputStream::getInputStream
std::istream & getInputStream()
Definition: StreamCompression.h:249
openvdb::v6_1::compression::Page
Stores a variable-size, compressed, delayed-load Page of data that is loaded into memory when accesse...
Definition: StreamCompression.h:138
OPENVDB_USE_VERSION_NAMESPACE
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h:177
openvdb::v6_1::compression::PagedInputStream::setSizeOnly
void setSizeOnly(bool sizeOnly)
Size-only mode tags the stream as only reading size data.
Definition: StreamCompression.h:245
openvdb::v6_1::compression::bloscUncompressedSize
OPENVDB_API size_t bloscUncompressedSize(const char *buffer)
Retrieves the uncompressed size of buffer when uncompressed.