36 #ifndef VIGRA_MULTI_ARRAY_CHUNKED_HDF5_HXX 
   37 #define VIGRA_MULTI_ARRAY_CHUNKED_HDF5_HXX 
   41 #include "multi_array_chunked.hxx" 
   42 #include "hdf5impex.hxx" 
   45 #ifdef VIGRA_CHECK_BOUNDS 
   46 #define VIGRA_ASSERT_INSIDE(diff) \ 
   47   vigra_precondition(this->isInside(diff), "Index out of bounds") 
   49 #define VIGRA_ASSERT_INSIDE(diff) 
   71 template <
unsigned int N, 
class T, 
class Alloc = std::allocator<T> >
 
   87     : 
public ChunkBase<N, T>
 
   92         typedef value_type * pointer;
 
   93         typedef value_type & reference;
 
   97         : ChunkBase<N, T>(detail::defaultStride(shape))
 
  109         std::size_t 
size()
 const 
  114         void write(
bool deallocate = 
true)
 
  116             if(this->pointer_ != 0)
 
  118                 if(!array_->file_.isReadOnly())
 
  120                     herr_t status = array_->file_.writeBlock(array_->dataset_, start_,
 
  122                     vigra_postcondition(status >= 0,
 
  123                         "ChunkedArrayHDF5: write to dataset failed.");
 
  127                     alloc_.deallocate(this->pointer_, this->
size());
 
  135             if(this->pointer_ == 0)
 
  137                 this->pointer_ = alloc_.allocate(this->
size());
 
  138                 herr_t status = array_->file_.readBlock(array_->dataset_, start_, shape_,
 
  140                 vigra_postcondition(status >= 0,
 
  141                     "ChunkedArrayHDF5: read from dataset failed.");
 
  143             return this->pointer_;
 
  151         Chunk & operator=(Chunk 
const &);
 
  157     typedef T value_type;
 
  158     typedef value_type * pointer;
 
  159     typedef value_type & reference;
 
  192                      Alloc 
const & alloc = Alloc())
 
  195       dataset_name_(dataset),
 
  197       compression_(options.compression_method),
 
  228                      Alloc 
const & alloc = Alloc())
 
  233       dataset_name_(dataset),
 
  235       compression_(options.compression_method),
 
  246     dataset_name_(src.dataset_name_),
 
  247     compression_(src.compression_),
 
  250         if( file_.isReadOnly() )
 
  251             init(HDF5File::ReadOnly);
 
  253             init(HDF5File::ReadWrite);
 
  260         if(mode == HDF5File::Replace)
 
  262             mode = HDF5File::New;
 
  264         else if(mode == HDF5File::Default)
 
  267                 mode = HDF5File::ReadOnly;
 
  269                 mode = HDF5File::New;
 
  272         if(mode == HDF5File::ReadOnly)
 
  275             vigra_precondition(!file_.isReadOnly(),
 
  276                 "ChunkedArrayHDF5(): 'mode' is incompatible with read-only file.");
 
  278         vigra_precondition(exists || !file_.isReadOnly(),
 
  279             "ChunkedArrayHDF5(): dataset does not exist, but file is read-only.");
 
  281         if(!exists || mode == HDF5File::New)
 
  300             if(compression_ == DEFAULT_COMPRESSION)
 
  301                 compression_ = ZLIB_FAST;
 
  302             vigra_precondition(compression_ != LZ4,
 
  303                 "ChunkedArrayHDF5(): HDF5 does not support LZ4 compression.");
 
  305             vigra_precondition(this->
size() > 0,
 
  306                 "ChunkedArrayHDF5(): invalid shape.");
 
  307             typename detail::HDF5TypeTraits<T>::value_type init(this->fill_scalar_);
 
  320             typedef detail::HDF5TypeTraits<T> TypeTraits;
 
  321             if(TypeTraits::numberOfBands() > 1)
 
  323                 vigra_precondition(fileShape.size() == N+1,
 
  324                     "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
 
  325                 vigra_precondition(fileShape[0] == static_cast<unsigned>(TypeTraits::numberOfBands()),
 
  326                     "ChunkedArrayHDF5(file, dataset): dataset has wrong number of bands.");
 
  327                 shape_type 
shape(fileShape.begin()+1);
 
  330                     vigra_precondition(shape == this->shape_,
 
  331                         "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
 
  335                     this->shape_ = 
shape;
 
  340                 vigra_precondition(fileShape.size() == N,
 
  341                     "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
 
  342                 shape_type 
shape(fileShape.begin());
 
  345                     vigra_precondition(shape == this->shape_,
 
  346                         "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
 
  350                     this->shape_ = 
shape;
 
  351                     ChunkStorage(detail::computeChunkArrayShape(shape, this->bits_, this->mask_)).swap(this->handle_array_);
 
  355                                             end = this->handle_array_.
end();
 
  358                 i->chunk_state_.store(base_type::chunk_asleep);
 
  373     void closeImpl(
bool force_destroy)
 
  375         flushToDiskImpl(
true, force_destroy);
 
  381         flushToDiskImpl(
false, 
false);
 
  384     void flushToDiskImpl(
bool destroy, 
bool force_destroy)
 
  386         if(file_.isReadOnly())
 
  389         threading::lock_guard<threading::mutex> guard(*this->chunk_lock_);
 
  391                                         end = this->handle_array_.
end();
 
  392         if(destroy && !force_destroy)
 
  396                 vigra_precondition(i->chunk_state_.load() <= 0,
 
  397                     "ChunkedArrayHDF5::close(): cannot close file because there are active chunks.");
 
  399             i   = this->handle_array_.
begin();
 
  403             Chunk * chunk = 
static_cast<Chunk*
>(i->pointer_);
 
  419     virtual bool isReadOnly()
 const 
  421         return file_.isReadOnly();
 
  424     virtual pointer loadChunk(ChunkBase<N, T> ** p, shape_type 
const & index)
 
  426         vigra_precondition(file_.isOpen(),
 
  427             "ChunkedArrayHDF5::loadChunk(): file was already closed.");
 
  430             *p = 
new Chunk(this->
chunkShape(index), index*this->chunk_shape_, 
this, alloc_);
 
  431             this->overhead_bytes_ += 
sizeof(Chunk);
 
  433         return static_cast<Chunk *
>(*p)->read();
 
  436     virtual bool unloadChunk(ChunkBase<N, T> * chunk, 
bool )
 
  440         static_cast<Chunk *
>(chunk)->write();
 
  444     virtual std::string backend()
 const 
  446         return "ChunkedArrayHDF5<'" + file_.
filename() + 
"/" + dataset_name_ + 
"'>";
 
  449     virtual std::size_t 
dataBytes(ChunkBase<N,T> * c)
 const 
  451         return c->pointer_ == 0
 
  453                  : 
static_cast<Chunk*
>(c)->
size()*
sizeof(T);
 
  458         return sizeof(Chunk) + 
sizeof(SharedChunkHandle<N, T>);
 
  461     std::string fileName()
 const 
  466     std::string datasetName()
 const 
  468         return dataset_name_;
 
  472     std::string dataset_name_;
 
  473     HDF5HandleShared dataset_;
 
  474     CompressionMethod compression_;
 
  482 #undef VIGRA_ASSERT_INSIDE 
iterator end()
Definition: multi_array.hxx:1937
std::size_t dataBytes() const 
Bytes of main memory occupied by the array's data. 
Definition: multi_array_chunked.hxx:1674
Option object for ChunkedArray construction. 
Definition: multi_array_chunked.hxx:1267
ArrayVector< hsize_t > getDatasetShape(std::string datasetName) const 
Get the shape of each dimension of a certain dataset. 
Definition: hdf5impex.hxx:1394
MultiArrayIndex size() const 
Return the number of elements in this array. 
ChunkedArrayHDF5(HDF5File const &file, std::string const &dataset, HDF5File::OpenMode mode=HDF5File::ReadOnly, ChunkedArrayOptions const &options=ChunkedArrayOptions(), Alloc const &alloc=Alloc())
Construct for an already existing dataset with given 'options', using 'alloc' to manage the in-memory...
Definition: multi_array_chunked_hdf5.hxx:225
view_type::iterator iterator
Definition: multi_array.hxx:2548
iterator begin()
Create a scan-order iterator for the entire chunked array. 
Definition: multi_array_chunked.hxx:2381
iterator begin()
Definition: multi_array.hxx:1921
Main MultiArray class containing the memory management. 
Definition: multi_array.hxx:2474
shape_type const & shape() const 
Return the shape in this array. 
HDF5HandleShared getDatasetHandleShared(std::string const &datasetName) const 
Obtain a shared HDF5 handle of a dataset. 
Definition: hdf5impex.hxx:1527
Interface and base class for chunked arrays. 
Definition: multi_array_chunked.hxx:463
HDF5HandleShared createDataset(std::string datasetName, TinyVector< MultiArrayIndex, N > const &shape, typename detail::HDF5TypeTraits< T >::value_type init=typename detail::HDF5TypeTraits< T >::value_type(), TinyVector< MultiArrayIndex, N > const &chunkSize=(TinyVector< MultiArrayIndex, N >()), int compressionParameter=0)
Create a new dataset. This function can be used to create a dataset filled with a default value init...
Definition: hdf5impex.hxx:2761
view_type::difference_type difference_type
Definition: multi_array.hxx:2522
shape_type const & chunkShape() const 
Return the global chunk shape. 
std::string filename() const 
Get the name of the associated file. 
Definition: hdf5impex.hxx:1347
NumericTraits< V >::Promote prod(TinyVectorBase< V, SIZE, D1, D2 > const &l)
product of the vector's elements 
Definition: tinyvector.hxx:2097
Definition: multi_array_chunked_hdf5.hxx:72
ChunkedArrayHDF5(HDF5File const &file, std::string const &dataset, HDF5File::OpenMode mode, shape_type const &shape, shape_type const &chunk_shape=shape_type(), ChunkedArrayOptions const &options=ChunkedArrayOptions(), Alloc const &alloc=Alloc())
Construct with given 'shape', 'chunk_shape' and 'options', using 'alloc' to manage the in-memory vers...
Definition: multi_array_chunked_hdf5.hxx:187
iterator end()
Create the end iterator for scan-order iteration over the entire chunked array. 
Definition: multi_array_chunked.hxx:2389
OpenMode
Set how a file is opened. 
Definition: hdf5impex.hxx:1031
void flushToDisk()
Immediately write all data to disk. 
Definition: hdf5impex.hxx:2234
Class for fixed size vectors.This class contains an array of size SIZE of the specified VALUETYPE...
Definition: accessor.hxx:940
void close()
Close the current file. 
Definition: hdf5impex.hxx:1199
Base class for, and view to, vigra::MultiArray. 
Definition: multi_array.hxx:704
virtual std::size_t overheadBytesPerChunk() const 
Bytes of main memory needed to manage a single chunk. 
Definition: multi_array_chunked_hdf5.hxx:456
bool existsDataset(std::string datasetName) const 
Check if given datasetName exists. 
Definition: hdf5impex.hxx:1354
UInt32 ceilPower2(UInt32 x)
Round up to the nearest power of 2. 
Definition: mathutil.hxx:294
Access to HDF5 files. 
Definition: hdf5impex.hxx:974