: blob_id(id), blob_location_table(1)
104blob_location_table[0].offset =
offset;
105blob_location_table[0].size =
size;
120 boolHasBlob(
Uint4blob_id)
const;
126 size_t Size()
const{
returnm_BlobMap.size(); }
129 voidGetBlobIdRange(
Uint4* min_id,
Uint4* max_id)
const;
141 Uint8buf_offset = 0)
const;
145 Uint8buf_offset = 0);
148 size_tComputeSerializationSize()
const;
159 size_tx_ComputeSerializationSize(
unsigned* bits_used,
160 bool* is_single_chunk)
const;
212 Uint8buf_offset = 0)
const;
215 Uint8buf_offset = 0);
218 size_tComputeSerializationSize()
const;
305 template<
classTBV>
335 void Open(
const string& storage_name,
412 template<
classTBV>
419m_ContainerMax(64 * 1024),
428 template<
classTBV>
439 catch(std::exception& ex)
442 "Exception in ~CBDB_ExtBlobStore "<< ex.what());
445 deletem_AttrContainer;
452 template<
classTBV>
456m_Compressor.reset(compressor, own);
459 template<
classTBV>
465 template<
classTBV>
471 template<
classTBV>
474 deletem_BlobAttrDB; m_BlobAttrDB = 0;
475 deletem_ExtStore; m_ExtStore = 0;
478 template<
classTBV>
483 typenameTBitVector::statistics st1;
484m_BlobIds.optimize(0, TBV::opt_compress, &st1);
485m_CompressBuffer.resize_mem(st1.max_serialize_mem);
488(
unsigned char*)&m_CompressBuffer[0],
491m_CompressBuffer.resize(
size);
495m_BlobAttrDB->id_from = 0;
496m_BlobAttrDB->id_to = 0;
498 EBDB_ErrCoderet = m_BlobAttrDB->CBDB_BLobFile::UpdateInsert(m_CompressBuffer);
500 BDB_THROW(eInvalidOperation,
"Cannot save ext. blob summary");
505 template<
classTBV>
511m_OpenMode = open_mode;
514 if(!m_StoreAttrDir.empty() && m_Env == 0) {
515 CDirdir(m_StoreAttrDir);
520 if(!m_StoreDataDir.empty()) {
521 CDirdir(m_StoreDataDir);
530 const stringattr_fname_postf =
"_ext.db";
533m_BlobAttrDB->
SetEnv(*m_Env);
534attr_fname = storage_name + attr_fname_postf;
536 if(!m_StoreAttrDir.empty()) {
537attr_fname = m_StoreAttrDir + storage_name + attr_fname_postf;
540attr_fname = storage_name + attr_fname_postf;
543m_BlobAttrDB->Open(attr_fname, open_mode);
549 const stringext_fname_postf =
"_store.blob";
550 if(!m_StoreDataDir.empty()) {
551ext_fname = m_StoreDataDir + storage_name + ext_fname_postf;
554ext_fname = storage_name + ext_fname_postf;
557IOS_BASE::openmode
om= IOS_BASE::binary;
574 if(!m_ExtStore->is_open() || m_ExtStore->bad()) {
575 deletem_ExtStore; m_ExtStore = 0;
576 BDB_THROW(eFileIO,
"Cannot open file "+ ext_fname);
580m_BlobAttrDB->id_from = 0;
581m_BlobAttrDB->id_to = 0;
583 EBDB_ErrCoderet = m_BlobAttrDB->ReadRealloc(m_CompressBuffer);
586od.
deserialize(m_BlobIds, m_CompressBuffer.data(), m_STmpBlock,
594 template<
classTBV>
600 if(m_BlobIds[blob_id]) {
602 "External store already has BLOB id=" 608 if(m_LastOp ==
eRead&& m_AttrContainer) {
609 deletem_AttrContainer; m_AttrContainer = 0;
614 if(!m_AttrContainer) {
616m_BlobContainer.resize_mem(
buf.size());
619::memcpy(m_BlobContainer.data(),
buf.data(),
buf.size());
625 if(m_BlobContainer.size() > m_ContainerMax) {
628m_BlobIds.set(blob_id);
632 if(
buf.size() + m_BlobContainer.size() > m_ContainerMax) {
634this->StoreBlob(blob_id,
buf);
638 offset= (unsigned)m_BlobContainer.size();
639m_BlobContainer.resize(m_BlobContainer.size() +
buf.size());
641::memcpy(m_BlobContainer.data() +
offset,
buf.data(),
buf.size());
645m_BlobIds.set(blob_id);
648 template<
classTBV>
651 if(!m_AttrContainer) {
664 if(m_Compressor.get()) {
665m_CompressBuffer.resize_mem(m_BlobContainer.size() * 2);
669m_Compressor->CompressBuffer(m_BlobContainer.data(),
670m_BlobContainer.size(),
671m_CompressBuffer.data(),
672m_CompressBuffer.size(),
675 BDB_THROW(eInvalidOperation,
"Cannot compress BLOB");
678m_ExtStore->write((
char*)m_CompressBuffer.data(),
682m_ExtStore->write((
char*)m_BlobContainer.data(),
683m_BlobContainer.size());
684m_AttrContainer->SetLoc(stream_offset, m_BlobContainer.size());
687 if(m_ExtStore->bad()) {
688 BDB_THROW(eFileIO,
"Cannot write to external store file ");
691 EBDB_ErrCoderet = m_BlobAttrDB->UpdateInsert(*m_AttrContainer);
692 deletem_AttrContainer; m_AttrContainer = 0;
694 BDB_THROW(eInvalidOperation,
"Cannot store BLOB metainfo");
699 template<
classTBV>
704 if(m_AttrContainer) {
708 if(blob_id >= m_LastFromBlobId && blob_id <= m_LastToBlobId) {
715 buf.resize_mem((
size_t)
size);
718 buf.resize_mem((
size_t)
size);
719 if(m_Compressor.get()) {
724::memcpy(
buf.data(),
725m_CompressBuffer.data() + (
size_t)
offset,
728::memcpy(
buf.data(),
729m_BlobContainer.data() + (
size_t)
offset,
740 template<
classTBV>
749 if(m_LastOp == eWrite && m_AttrContainer) {
750 BDB_THROW(eInvalidOperation,
"Cannot read on unflushed data. ");
754 if(!m_BlobIds[blob_id]) {
759 if(m_AttrContainer) {
760 if(x_ReadCache(blob_id,
buf)) {
763 deletem_AttrContainer; m_AttrContainer = 0;
770ret = m_BlobAttrDB->FetchMeta(blob_id,
775 deletem_AttrContainer; m_AttrContainer = 0;
786m_ExtStore->seekg(pos, IOS_BASE::beg);
787 if(m_ExtStore->bad()) {
788 BDB_THROW(eFileIO,
"Cannot read from external store file ");
790m_BlobContainer.resize_mem((
size_t)
size);
791m_ExtStore->read((
char*)m_BlobContainer.data(), (
size_t)
size);
792 if(m_ExtStore->bad()) {
793 BDB_THROW(eFileIO,
"Cannot read from external store file ");
796 if(m_Compressor.get()) {
797m_CompressBuffer.resize_mem((
size_t)m_ContainerMax * 10);
799 bool ok= m_Compressor->DecompressBuffer(m_BlobContainer.data(),
800m_BlobContainer.size(),
801m_CompressBuffer.data(),
802m_CompressBuffer.size(),
805 BDB_THROW(eInvalidOperation,
"Cannot decompress BLOB");
807m_CompressBuffer.resize(dst_len);
812 if(x_ReadCache(blob_id,
buf)) {
815 deletem_AttrContainer; m_AttrContainer = 0;
BDB library BLOB support.
Berkeley BDB file cursor.
Serialization / compression of bvector<>. Set theoretical operations on compressed BLOBs.
Berkeley DB BLOB File class.
BDB environment object a collection including support for some or all of caching, locking,...
BLOB map, encapsulates collection of BLOB ids and BLOB locations.
Raw file class wraps up basic Berkeley DB operations.
Reallocable memory buffer (no memory copy overhead) Mimics vector<>, without the overhead of explicit...
void reserve(size_type new_size)
Deserializer, performs logical operations between bit-vector and serialized bit-vector.
size_type deserialize(bvector_type &bv, const unsigned char *buf, set_operation op, bool exit_on_one=false)
Deserialize bvector using buffer as set operation argument.
Include a standard set of the NCBI C++ Toolkit most basic headers.
The NCBI C++ standard methods for dealing with std::string.
static ulg compressed_len
std::ofstream out("events_result.xml")
main entry point for tests
@ eTakeOwnership
An object can take ownership of another.
CBDB_RawFile::TBuffer m_BlobContainer
Blob container.
void SetEnv(CBDB_Env &env)
CBDB_RawFile::EOpenMode m_OpenMode
SBlobLoc(Uint4 id, Uint8 offset, Uint8 size)
Construct one-chunk blob locator.
unsigned m_ContainerMax
Max size of a BLOB container.
bm::word_t * m_STmpBlock
temp block for bitvector serialization
vector< SBlobLoc > TBlobMap
Collection of BLOBs (id + allocation table)
CBDB_ExtBlobMap::TBlobChunkVec m_Loc
Super BLOB location vector.
CBDB_FieldUint4 id_from
Id range from.
CBDB_ExtBlobStore & operator=(const CBDB_ExtBlobStore &)
CBDB_ExtBlobStore(const CBDB_ExtBlobStore &)
const CBDB_ExtBlobMap::TBlobChunkVec & GetSuperLoc() const
Get location table of a super BLOB Location table is used to reassemble BLOB from chunks.
CBlobMetaDB(const CBlobMetaDB &)
CBlobMetaDB * m_BlobAttrDB
unsigned m_LastFromBlobId
Recently read id interval.
void SetStoreAttrDir(const string &dir_name)
Store attributes DB location (by default it is the same as SetStoreDataDir).
ELastOp m_LastOp
Last operation status.
CBDB_ExtBlobMap m_BlobMap
Blob attributes (super BLOB content)
void Flush()
Flush current container to disk.
void Add(Uint4 blob_id, Uint8 offset, Uint8 size)
Add BLOB. BLOB consists of one single chunk.
vector< SBlobChunkLoc > TBlobChunkVec
BLOB location table (list of chunks and sizes)
Uint8 offset
chunk offset
EBDB_ErrCode ReadBlob(unsigned blob_id, CBDB_RawFile::TBuffer &buf)
Read blob from external store.
bool GetBlobLoc(Uint4 blob_id, Uint8 *offset, Uint8 *size) const
Get BLOB location.
TBitVector m_BlobIds
List of BLOB ids stored.
TBlobChunkVec blob_location_table
CBDB_Env * GetEnv(void) const
CBlobMetaDB & operator=(const CBlobMetaDB &)
AutoPtr< ICompression > m_Compressor
Record compressor.
CBDB_ExtBlobMap::TBlobChunkVec & SetSuperLoc()
Get Edit access to location table.
void Save()
Save all changes (flush buffers, store attributes, etc.)
unsigned m_LastToBlobId
Recently read id interval.
CNcbiFstream * m_ExtStore
void Close()
Close store.
size_t Size() const
Number of BLOBs registered in the map.
void SetCompressor(ICompression *compressor, EOwnership own=eTakeOwnership)
Set compressor for external BLOB.
const CBDB_ExtBlobMap & GetBlobMap() const
void SetStoreDataDir(const string &dir_name)
External store location.
void Open(const string &storage_name, CBDB_RawFile::EOpenMode open_mode)
Open external store.
ELastOp
Last operation type.
unsigned GetContainerMaxSize() const
Get container max size.
CBDB_FieldUint4 id_to
Id range to.
void StoreBlob(unsigned blob_id, const CBDB_RawFile::TBuffer &buf)
Add blob to external store.
SBlobChunkLoc(Uint8 off, Uint8 s)
bool x_ReadCache(unsigned blob_id, CBDB_RawFile::TBuffer &buf)
Try to read BLOB from the recently loaded container.
CBDB_RawFile::TBuffer m_CompressBuffer
void SetContainerMaxSize(unsigned max_size)
Set maximum size of BLOB container.
CBDB_BlobMetaContainer * m_AttrContainer
Blob attributes container.
CBDB_RawFile::TBuffer TBuffer
CBDB_ExtBlobMap & SetBlobMap()
EOpenMode
BDB file open mode.
void SetEnv(CBDB_Env &env)
Associate file with environment.
EBDB_ErrCode
BDB Return codes.
@ eReadWriteCreate
read-write, create if it doesn't exist
@ eCreate
implies 'eReadWrite' too
#define BDB_THROW(errcode, message)
#define ERR_POST_XX(error_name, err_subcode, message)
Error posting with error code having given name and with given error subcode.
void Error(CExceptionArgs_Base &args)
static string AddTrailingPathSeparator(const string &path)
Add trailing path separator, if needed.
virtual bool Exists(void) const
Check if directory "dirname" exists.
bool Create(TCreateFlags flags=fCreate_Default) const
Create the directory using "dirname" passed in the constructor.
uint32_t Uint4
4-byte (32-bit) unsigned integer
int64_t Int8
8-byte (64-bit) signed integer
uint64_t Uint8
8-byte (64-bit) unsigned integer
#define END_NCBI_SCOPE
End previously defined NCBI scope.
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
Int8 NcbiStreamposToInt8(NCBI_NS_STD::char_traits< char >::pos_type stream_pos)
Convert stream position to 64-bit int.
NCBI_NS_STD::char_traits< char >::pos_type NcbiInt8ToStreampos(Int8 pos)
Convert plain numeric stream position (offset) into stream position usable with STL stream library.
IO_PREFIX::streampos CNcbiStreampos
Portable alias for streampos.
IO_PREFIX::fstream CNcbiFstream
Portable alias for fstream.
static string UIntToString(unsigned int value, TNumToStringFlags flags=0, int base=10)
Convert UInt to string.
enum ENcbiOwnership EOwnership
Ownership relations between objects.
size_t serialize(const BV &bv, unsigned char *buf, bm::word_t *temp_block=0, unsigned serialization_flags=0)
Saves bitvector into memory.
@ BM_NO_BYTE_ORDER
save no byte-order info (save some space)
Definition of all error codes used in bdb library (bdb.lib and ncbi_xcache_bdb.lib).
void aligned_free(void *ptr) BMNOEXCEPT
Aligned free.
void * aligned_new_malloc(size_t size)
Aligned malloc (unlike classic malloc it throws bad_alloc exception)
const unsigned set_block_alloc_size
const struct ncbi::grid::netcache::search::fields::SIZE size
Compressed bitset (entry point to bm.h)
Defines classes: CDirEntry, CFile, CDir, CSymLink, CMemoryFile, CFileUtil, CFileLock,...
Multi-threading â mutexes; rw-locks; semaphore.
NCBI C++ stream class wrappers for triggering between "new" and "old" C++ stream libraries.
std::istream & in(std::istream &in_, double &x_)
int GetLoc(const string &acc, const string &pat, CSeq_loc &loc, CScope &scope)
static SLJIT_INLINE sljit_ins msg(sljit_gpr r, sljit_s32 d, sljit_gpr x, sljit_gpr b)
CRef< objects::CObjectManager > om
BLOB chunk location: offset in file + chunk size.
Blob id + blob location table (list of chunks and sizes)
void Serialize(CNcbiOstream &, const CRawScoreVector< Key, Score > &)
Generics These throw an exception; we must implement serialization for each type.
void Deserialize(CNcbiIstream &istr, CRawScoreVector< Key, Score > &)
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4