73:
public ChunkedArray<N, T>
87 :
public ChunkBase<N, T>
92 typedef value_type * pointer;
93 typedef value_type & reference;
95 Chunk(shape_type
const &
shape, shape_type
const & start,
97 : ChunkBase<N, T>(detail::defaultStride(
shape))
109 std::size_t
size()
const
114 void write(
bool deallocate =
true)
116 if(this->pointer_ != 0)
118 if(!array_->file_.isReadOnly())
120 herr_t status = array_->file_.
writeBlock(array_->dataset_, start_,
122 vigra_postcondition(status >= 0,
123 "ChunkedArrayHDF5: write to dataset failed.");
127 alloc_.deallocate(this->pointer_, this->
size());
135 if(this->pointer_ == 0)
137 this->pointer_ = alloc_.allocate(this->
size());
138 herr_t status = array_->file_.
readBlock(array_->dataset_, start_, shape_,
140 vigra_postcondition(status >= 0,
141 "ChunkedArrayHDF5: read from dataset failed.");
143 return this->pointer_;
146 shape_type shape_, start_;
151 Chunk & operator=(Chunk
const &);
154 typedef ChunkedArray<N, T> base_type;
157 typedef T value_type;
158 typedef value_type * pointer;
159 typedef value_type & reference;
189 shape_type
const &
shape,
190 shape_type
const & chunk_shape=shape_type(),
192 Alloc
const & alloc = Alloc())
193 : ChunkedArray<N, T>(
shape, chunk_shape, options),
195 dataset_name_(dataset),
197 compression_(options.compression_method),
228 Alloc
const & alloc = Alloc())
229 : ChunkedArray<N, T>(shape_type(),
233 dataset_name_(dataset),
235 compression_(options.compression_method),
246 dataset_name_(src.dataset_name_),
247 compression_(src.compression_),
250 if( file_.isReadOnly() )
251 init(HDF5File::ReadOnly);
253 init(HDF5File::ReadWrite);
260 if(mode == HDF5File::Replace)
262 mode = HDF5File::New;
264 else if(mode == HDF5File::Default)
267 mode = HDF5File::ReadOnly;
269 mode = HDF5File::New;
272 if(mode == HDF5File::ReadOnly)
275 vigra_precondition(!file_.isReadOnly(),
276 "ChunkedArrayHDF5(): 'mode' is incompatible with read-only file.");
278 vigra_precondition(exists || !file_.isReadOnly(),
279 "ChunkedArrayHDF5(): dataset does not exist, but file is read-only.");
281 if(!exists || mode == HDF5File::New)
300 if(compression_ == DEFAULT_COMPRESSION)
301 compression_ = ZLIB_FAST;
302 vigra_precondition(compression_ != LZ4,
303 "ChunkedArrayHDF5(): HDF5 does not support LZ4 compression.");
305 vigra_precondition(this->
size() > 0,
306 "ChunkedArrayHDF5(): invalid shape.");
307 typename detail::HDF5TypeTraits<T>::value_type init(this->fill_scalar_);
308 dataset_ = file_.createDataset<N, T>(dataset_name_,
316 dataset_ = file_.getDatasetHandleShared(dataset_name_);
319 ArrayVector<hsize_t> fileShape(file_.getDatasetShape(dataset_name_));
320 typedef detail::HDF5TypeTraits<T> TypeTraits;
321 if(TypeTraits::numberOfBands() > 1)
323 vigra_precondition(fileShape.size() == N+1,
324 "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
325 vigra_precondition(fileShape[0] ==
static_cast<unsigned>(TypeTraits::numberOfBands()),
326 "ChunkedArrayHDF5(file, dataset): dataset has wrong number of bands.");
327 shape_type
shape(fileShape.begin()+1);
330 vigra_precondition(shape == this->shape_,
331 "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
335 this->shape_ = shape;
340 vigra_precondition(fileShape.size() == N,
341 "ChunkedArrayHDF5(file, dataset): dataset has wrong dimension.");
342 shape_type
shape(fileShape.begin());
345 vigra_precondition(shape == this->shape_,
346 "ChunkedArrayHDF5(file, dataset, shape): shape mismatch between dataset and shape argument.");
350 this->shape_ = shape;
351 ChunkStorage(detail::computeChunkArrayShape(shape, this->bits_, this->mask_)).swap(this->handle_array_);
355 end = this->handle_array_.end();
358 i->chunk_state_.store(base_type::chunk_asleep);
373 void closeImpl(
bool force_destroy)
375 flushToDiskImpl(
true, force_destroy);
381 flushToDiskImpl(
false,
false);
384 void flushToDiskImpl(
bool destroy,
bool force_destroy)
386 if(file_.isReadOnly())
389 threading::lock_guard<threading::mutex> guard(*this->chunk_lock_);
391 end = this->handle_array_.end();
392 if(destroy && !force_destroy)
396 vigra_precondition(i->chunk_state_.load() <= 0,
397 "ChunkedArrayHDF5::close(): cannot close file because there are active chunks.");
399 i = this->handle_array_.begin();
403 Chunk * chunk =
static_cast<Chunk*
>(i->pointer_);
419 virtual bool isReadOnly()
const
421 return file_.isReadOnly();
424 virtual pointer loadChunk(ChunkBase<N, T> ** p, shape_type
const & index)
426 vigra_precondition(file_.isOpen(),
427 "ChunkedArrayHDF5::loadChunk(): file was already closed.");
430 *p =
new Chunk(this->
chunkShape(index), index*this->chunk_shape_,
this, alloc_);
431 this->overhead_bytes_ +=
sizeof(Chunk);
433 return static_cast<Chunk *
>(*p)->read();
436 virtual bool unloadChunk(ChunkBase<N, T> * chunk,
bool )
440 static_cast<Chunk *
>(chunk)->write();
444 virtual std::string backend()
const
446 return "ChunkedArrayHDF5<'" + file_.filename() +
"/" + dataset_name_ +
"'>";
449 virtual std::size_t
dataBytes(ChunkBase<N,T> * c)
const
451 return c->pointer_ == 0
453 :
static_cast<Chunk*
>(c)->
size()*
sizeof(T);
458 return sizeof(Chunk) +
sizeof(SharedChunkHandle<N, T>);
461 std::string fileName()
const
466 std::string datasetName()
const
468 return dataset_name_;
472 std::string dataset_name_;
473 HDF5HandleShared dataset_;
474 CompressionMethod compression_;
ChunkedArrayHDF5(HDF5File const &file, std::string const &dataset, HDF5File::OpenMode mode, shape_type const &shape, shape_type const &chunk_shape=shape_type(), ChunkedArrayOptions const &options=ChunkedArrayOptions(), Alloc const &alloc=Alloc())
Construct with given 'shape', 'chunk_shape' and 'options', using 'alloc' to manage the in-memory vers...
Definition multi_array_chunked_hdf5.hxx:187
void readBlock(std::string datasetName, typename MultiArrayShape< N >::type blockOffset, typename MultiArrayShape< N >::type blockShape, MultiArrayView< N, T, Stride > array)
Read a block of data into a multi array. This function allows to read a small block out of a larger v...
Definition hdf5impex.hxx:2068