= path + cache +
'_'+
key+
'_' 131 m_BlobLock(blob_lock.GetLockVector(), blob_lock.GetTimeout())
144 m_BlobLock(blob_lock.GetLockVector(), blob_lock.GetTimeout())
153 ERR_POST(
"CBDB_CacheIReader: detected unread input "<<
163 ERR_POST(
"CBDB_CacheIReader: detected unread input "<<
164(end-pos)<<
": "<<pos<<
" of "<<end);
195 if(*bytes_read == 0) {
241 size_treserve = requested_size / 2;
242 if(reserve > 1 * 1024 * 1024) {
243reserve = 1 * 1024 * 1024;
245 returnrequested_size + reserve;
260 unsignedblob_id_ext,
261 const string& blob_key,
267 const string& owner,
287 m_BlobLock(blob_lock.GetLockVector(), blob_lock.GetTimeout())
306 _TRACE(
"CBDB_CacheIWriter::~CBDB_CacheIWriter point 2");
335}
catch(exception& ) {
355}
catch(exception& ) {
362}
catch(exception & ex) {
363 ERR_POST_X(1,
"Exception in ~CBDB_CacheIWriter() : "<< ex.what()
373 size_t* bytes_written = 0)
398 string msg(
"BLOB larger than allowed. size=");
400 msg.append(
" quota=");
416*bytes_written =
count;
429 catch(exception& ) {
448*bytes_written =
count;
504 "Error trying to flush an overflow file");
549 stringerr =
"LC: Cannot create overflow file ";
551 BDB_THROW(eCannotOpenOverflowFile, err);
562 "LC: Attempt to write to a non-open overflow file");
638 unsignedhist_value = 512;
639 for(
unsigned i= 0;
i< 100; ++
i) {
640(*hist)[hist_value] = 0;
649 if(hist->
empty()) {
653 if(it == hist->
end()) {
662 void s_GetDayHour(time_t time_in_secs,
unsigned* day,
unsigned* hour)
664*day =
static_cast<unsigned>(time_in_secs / (24 * 60 * 60));
665 unsignedsecs_in_day =
static_cast<unsigned>(time_in_secs % (24 * 60 * 60));
666*hour = secs_in_day / 3600;
691 if(ta_stat.
day== day && ta_stat.
hour== hour) {
712 if(ta_stat.
day== day && ta_stat.
hour== hour) {
781<<
"Current database number of records(BLOBs) "<<
"\t"<<
blobs_db<<
"\n" 782<<
"Current size of all BLOBs "<<
"\t"<<
blobs_size_db<<
"\n" 784<<
"Number of NetCache protocol errors "<<
"\t"<<
err_protocol<<
"\n" 786<<
"Number of NetCache server internal errors "<<
"\t"<<
err_internal<<
"\n" 787<<
"Number of BLOB not found situations "<<
"\t"<<
err_no_blob<<
"\n" 788<<
"Number of errors when getting BLOBs "<<
"\t"<<
err_blob_get<<
"\n" 789<<
"Number of errors when storing BLOBs "<<
"\t"<<
err_blob_put<<
"\n" 790<<
"Number of errors when BLOB is over the size limit "<<
"\t"<<
err_blob_over_quota<<
"\n";
795 out<<
"# Time access statistics:"<<
"\n"<<
"\n";
796 out<<
"# Hour \t Puts \t Gets"<<
"\n";
808 out<<
"# BLOB size histogram:"<<
"\n"<<
"\n";
809 out<<
"# Size \t Count"<<
"\n";
816 if(it->second > 0) {
821 out<< it->first <<
"\t"<< it->second <<
"\n";
822 if(it == hist_end) {
831 const string& sect_name_postfix)
const 833 stringpostfix(sect_name_postfix);
839 stringsect_stat(
"bdb_stat");
840 if(!postfix.empty()) {
841sect_stat.append(
"_");
842sect_stat.append(postfix);
845reg->
Set(sect_stat,
"blobs_stored_total",
847 "Total number of blobs ever stored");
848reg->
Set(sect_stat,
"blobs_overflow_total",
850 "Total number of overflow blobs (large size)");
851reg->
Set(sect_stat,
"blobs_updates_total",
853 "Total number of blobs updates");
854reg->
Set(sect_stat,
"blobs_never_read_total",
856 "Total number of blobs stored but never read");
857reg->
Set(sect_stat,
"blobs_read_total",
859 "Total number of reads");
860reg->
Set(sect_stat,
"blobs_expl_deleted_total",
862 "Total number of explicit deletes");
863reg->
Set(sect_stat,
"blobs_purge_deleted_total",
865 "Total number of BLOBs deletes by garbage collector");
866reg->
Set(sect_stat,
"blobs_size_total",
868 "Total size of all BLOBs ever stored");
869reg->
Set(sect_stat,
"blob_size_max_total",
871 "Size of the largest BLOB ever stored");
874reg->
Set(sect_stat,
"blobs_db",
876 "Current database number of records(BLOBs)");
877reg->
Set(sect_stat,
"blobs_size_db",
879 "Current size of all BLOBs");
881reg->
Set(sect_stat,
"err_protocol",
883 "Number of NetCache protocol errors");
884reg->
Set(sect_stat,
"err_communication",
886 "Number of communication errors");
887reg->
Set(sect_stat,
"err_internal",
889 "Number of NetCache server internal errors");
890reg->
Set(sect_stat,
"err_no_blob",
892 "Number of BLOB not found situations");
893reg->
Set(sect_stat,
"err_blob_get",
895 "Number of errors when getting BLOBs");
896reg->
Set(sect_stat,
"err_blob_put",
898 "Number of errors when storing BLOBs");
899reg->
Set(sect_stat,
"err_blob_over_quota",
901 "Number of errors when BLOB is over the size limit");
918pair.append(put_count);
920 if(!put_history.empty()) {
921put_history.append(
";");
923put_history.append(pair);
928pair.append(get_count);
930 if(!get_history.empty()) {
931get_history.append(
";");
933get_history.append(pair);
937reg->
Set(sect_stat,
"get_access",
939 "Read access by hours (hour=count)");
940reg->
Set(sect_stat,
"put_access",
942 "Write access by hours (hour=count)");
952 stringsect_hist(
"bdb_stat_hist");
953 if(!postfix.empty()) {
954sect_hist.append(
"_");
955sect_hist.append(postfix);
963 if(it->second > 0) {
968 stringvar_name =
"size_";
973 if(it == hist_end) {
1088 const string&
client= it->first;
1096 out<<
"## "<<
"\n" 1097<<
"## Global statistics"<<
"\n" 1098<<
"## "<<
"\n\n";
1103 const string&
client= it->first;
1106 out<<
"## "<<
"\n" 1107<<
"## Owner statistics:"<<
client<<
"\n" 1108<<
"## "<<
"\n\n";
1118m_ReadOnly(
false),
1119m_InitIfDirty(
false),
1120m_JoinedEnv(
false),
1121m_LockTimeout(20 * 1000),
1124m_BLOB_SplitStore(0),
1127m_CacheAttrDB_RO1(0),
1128m_CacheAttrDB_RO2(0),
1132m_VersionFlag(eDropOlder),
1133m_WSync(eWriteNoSync),
1134m_PurgeBatchSize(150),
1136m_PurgeStopSignal(0, 100),
1137m_CleanLogOnPurge(0),
1140m_PurgeNowRunning(
false),
1141m_RunPurgeThread(
false),
1142m_PurgeThreadDelay(10),
1143m_CheckPointInterval(24 * (1024 * 1024)),
1144m_OverflowLimit(512 * 1024),
1145m_MaxTTL_prolong(0),
1146m_SaveStatistics(
false),
1148m_RoundRobinVolumes(0),
1166 ERR_POST(
"~CBDB_Cache(): exception: "<<exc);
1168 catch(exception& exc) {
1169 ERR_POST(
"~CBDB_Cache(): exception: "<<exc.
what());
1181 stringlock_file_path =
m_Path+ lock_file;
1234 const string& cache_name,
1236 Uint8cache_ram_size,
1238 unsigned intlog_mem_size)
1260 if((!fl.empty() || fl_clean.
Exists())
1261&& s_OpenedDirs.count(
m_Path) == 0)
1264 ERR_POST(
"Database was closed uncleanly. Removing directory " 1270 ERR_POST(
"Database was closed uncleanly. Running recovery...");
1282 if(!fl_clean.
Exists()) {
1285writer.
Write(pid.data(), pid.size());
1291 stringerr_file =
m_Path+
"err"+
string(cache_name) +
".log";
1295 boolneeds_recovery =
false;
1302 "LC: '"<< cache_name <<
1303 "' Joined transactional environment ");
1306 "LC: '"<< cache_name <<
1307 "' Warning: Joined non-transactional environment ");
1315 "LC: '"<< cache_name <<
1316 "'Warning: DB_ENV returned DB_RUNRECOVERY code." 1317 " Running the recovery procedure.");
1319needs_recovery =
true;
1323needs_recovery =
true;
1328 if(log_mem_size == 0) {
1343 if(cache_ram_size) {
1351 if(cache_ram_size > (500 * 1024 * 1024)) {
1352cache_num = (
int)((cache_ram_size) /
Uint8(250 * 1024 * 1024));
1357 if(cache_num > 10) {
1364 if(checkpoint_KB == 0) {
1377 if(needs_recovery) {
1385 LOG_POST_X(5,
Info<<
"BDB_Cache: Creating locking environment");
1420 stringcache_blob_db_name =
1422 stringattr_db_name =
1424 stringid_idx_name =
1444 unsignedtimeline_precision = 5 * 60;
1448 if(timeline_precision > (5 * 60)) {
1449timeline_precision = 5 * 60;
1451 if(timeline_precision < (1 * 60)) {
1452timeline_precision = 1 * 60;
1469 unsignedmax_blob_id = 0;
1472max_blob_id =
max(max_blob_id, blob_id);
1532 "LC: '"<< cache_name <<
1533 "' Cache mount at: "<< cache_path);
1548 # ifdef NCBI_THREADS 1568 "Cannot run background thread in non-MT configuration.");
1577 # ifdef NCBI_THREADS 1591 const string& cache_name,
1610 stringcache_blob_db_name =
1612 stringattr_db_name =
1623 "LC: '"<< cache_name <<
1624 "' Cache mount read-only at: "<< cache_path);
1665 "LC: '"<<
m_Name<<
"' Unmounted. BDB ENV deleted.");
1668<<
"' environment still in use.");
1671 catch(exception& ex) {
1673<<
"' Exception in Close() "<< ex.what()
1709 unsigned inttimeout,
1710 unsigned intmax_timeout)
1718 m_MaxTimeout= max_timeout > timeout ? max_timeout : timeout;
1843 unsignedsplit_coord[2];
1849 if(coord[0] != split_coord[0] || coord[1] != split_coord[1]) {
1871 unsignedtime_to_live,
1872 const string& owner)
1874time_t curr = time(0);
1876 unsignedblob_id = 0;
1877 unsignedcoord[2]={0,};
1897ret = cur.
Fetch();
1964 unsignedsplit_coord[2];
1968 if(coord[0] != split_coord[0] || coord[1] != split_coord[1]) {
1986 const string&
key,
1991 unsigned inttime_to_live,
1992 const string& owner,
1999 unsignedcoord[2] = {0,};
2000 unsignedoverflow = 0, old_overflow = 0;
2003time_t curr = time(0);
2016 string msg(
"BLOB larger than allowed. size=");
2018 msg.append(
" quota=");
2043blob_lock, do_blob_lock,
2044&coord[0], &coord[1],
2050blob_id = blob_lock.
GetId();
2066 if(!oveflow_file.is_open() || oveflow_file.bad()) {
2067 stringerr =
"LC: Cannot create overflow file ";
2069 BDB_THROW(eCannotOpenOverflowFile, err);
2092 unsignedsplit_coord[2];
2096 if(coord[0] != split_coord[0] || coord[1] != split_coord[1]) {
2110coord[0] = coord[1] = 0;
2170ret = cur.
Fetch();
2186 if(blob_id != old_blob_id) {
2188 "BLOB id mutation detected!");
2202 string msg=
"BLOB insert-delete race detected!";
2222 unsignedblob_updated = 0;
2223 unsignedblob_stored = 0;
2234blob_stored, blob_updated, (
unsigned)
size, overflow);
2253 const string&
key,
2258 unsigned inttime_to_live,
2259 const string& owner)
2272 unsigned inttime_to_live,
2273 const string& owner)
2286 size_tblob_size = 0;
2288 unsigned intttl, blob_id, volume_id, split_id;
2291 if(!blob_id)
return false;
2301blob_id, volume_id, split_id);
2302 if(!rec_exists)
return false;
2321blob_size = (size_t)entry.
GetLength();
2324 if(blob_id == 0)
return false;
2327coords[0] = volume_id;
2328coords[1] = split_id;
2334 if(ret !=
eBDB_Ok)
return false;
2372time_t curr = time(0);
2397 if(
key!= found_key )
2437time_t curr = time(0);
2441 unsignedvolume_id = 0, split_id = 0;
2444 if(!blob_id)
return false;
2460ret = cur.
Fetch();
2471 if(max_time == 0 || max_time >= (
unsigned)curr) {
2510unique_ptr<CNcbiIfstream>
2512 if(!overflow_file->is_open()) {
2516overflow_file->read((
char*)
buf, buf_size);
2517 if(!*overflow_file) {
2530 if(coords[0] != volume_id ||
2531coords[1] != split_id) {
2538coords[0] = volume_id;
2539coords[1] = split_id;
2653 size_t& file_length,
2658unique_ptr<CNcbiIfstream>
2661 if(!overflow_file->is_open()) {
2665file_length = (size_t) entry.
GetLength();
2677time_t curr = time(0);
2680 unsignedvolume_id = 0, split_id = 0;
2702ret = cur.
Fetch();
2713 if(max_time == 0 || max_time >= (
unsigned) curr) {
2748unique_ptr<IReader> rd;
2751 returnrd.release();
2762 if(coords[0] != volume_id ||
2763coords[1] != split_id) {
2770coords[0] = volume_id;
2771coords[1] = split_id;
2877 "key, subkey, &version, &validity) is not implemented");
2886 "key, subkey, version) is not implemented");
2897blob_descr->
reader.reset();
2903time_t curr = time(0);
2906 unsignedvolume_id = 0, split_id = 0;
2931ret = cur.
Fetch();
2943 if(max_time == 0 || max_time >= (
unsigned) curr) {
2982blob_descr->
reader.reset(
2985 if(blob_descr->
reader.get()) {
3003 if(coords[0] != volume_id ||
3004coords[1] != split_id) {
3012coords[0] = volume_id;
3013coords[1] = split_id;
3019 char** ptr = &blob_descr->
buf;
3038unique_ptr<CBDB_RawFile::TBuffer>
buffer(
3047blob_descr->
reader.reset(
3199 unsigned inttime_to_live,
3200 const string& owner)
3203time_to_live, owner);
3208 const string&
key,
3212 unsigned inttime_to_live,
3213 const string& owner)
3235time_t curr = time(0);
3244 unsignedcoord[2] = {0,};
3250blob_lock, do_id_lock,
3251&coord[0], &coord[1],
3260blob_lock.
GetId(),
3278vector<SCacheDescr> cache_elements;
3297cache_elements.push_back(
3304 if(0 == read_count) {
3320 ITERATE(vector<SCacheDescr>, it, cache_elements) {
3356vector<SCacheDescr> cache_elements;
3370cache_elements.push_back(
3377 if(0 == read_count) {
3394 ITERATE(vector<SCacheDescr>, it, cache_elements) {
3470 const string& overflow_file_path,
3474overflow_file.write(
buf,
count);
3475 if(overflow_file.bad()) {
3477overflow_file.close();
3479 stringerr(
"Overflow file IO error ");
3480err += overflow_file_path;
3526time_t curr = time(0);
3531 string msg=
"Purge: Timeline evaluation skiped ";
3532 msg+=
"(early wakeup for this precision) remains=";
3536 msg+=
" precision=";
3551 if(!delete_candidates.
any()) {
3554 "Purge: Timeline evaluation exits " 3555 "(no candidates) \n";
3564 if(batch_size < 10) {
3568 unsignedcandidates = delete_candidates.
count();
3571 unsignedcleaned_candidates = delete_candidates.
count();
3572 if(cleaned_candidates != candidates) {
3575 msg=
"Purge: Timeline total timeline candidates=";
3577 msg+=
" cleaned_candidates=";
3584candidates = cleaned_candidates;
3589 doubleid_trans_time = 0.0;
3590 doubleexp_check_time = 0.0;
3591 doubledel_time = 0.0;
3592 unsigneddeleted_cnt = 0;
3594vector<unsigned> blob_id_vect(batch_size);
3595vector<SCacheDescr> blob_batch_vect(batch_size);
3596vector<SCacheDescr> blob_exp_vect(batch_size);
3602blob_id_vect.resize(0);
3603blob_batch_vect.resize(0);
3604blob_exp_vect.resize(0);
3605 for(
unsigned i= 0; en.valid() &&
i< batch_size; ++
i) {
3606 unsignedblob_id = *en;
3607blob_id_vect.push_back(blob_id);
3613 if(blob_id_vect.size()) {
3615vector<unsigned>::const_iterator it = blob_id_vect.begin();
3616 unsignedblob_id = *it;
3622cur.
From<< blob_id;
3627 if(blob_id != bid) {
3628 if((++it) == blob_id_vect.end())
3631cur.
From<< (blob_id = *it);
3637blob_batch_vect.push_back(
3642 if(it == blob_id_vect.end()) {
3648 if((blob_id + 1) == *it) {
3654cur.
From<< (blob_id = *it);
3664 if(blob_batch_vect.size()) {
3674 for(
size_t i= 0;
i< blob_batch_vect.size(); ++
i) {
3688blob_exp_vect.push_back(blob_descr);
3703 unsigneddeleted_batch_cnt = 0;
3706 if(blob_exp_vect.size()) {
3708 for(
size_t i= 0;
i< blob_exp_vect.size(); ++
i) {
3717 msg+=
" Purge: DELETING \"";
3733deleted_batch_cnt += blob_deleted;
3748deleted_cnt += deleted_batch_cnt;
3753 msg=
"Purge: Timeline deleted_batch_cnt=";
3755 msg+=
" deleted_cnt=";
3757 msg+=
" candidates=";
3760 msg+=
" id_trans_time=";
3762 msg+=
" exp_check_time=";
3764 msg+=
" del_time=";
3779*interrupted =
true;
3881 if(access_timeout == 0) {
3889 boolscan_interrupted =
false;
3891 if(scan_interrupted) {
3895time_t gc_start = time(0);
3927 unsignedsplit_store_blobs = bv.count();
3934 unsignedtl_count = timeline_blobs.
count();
3937 "Purge: scan skipped(nothing todo) remains=" 3973vector<SCacheDescr> cache_entries;
3974cache_entries.reserve(1000);
3976 unsigneddb_recs = 0;
3980 stringfirst_key, last_key;
3981 stringnext_exp_key;
3983 for(
boolflag =
true; flag;) {
3987time_t curr = time(0);
3997cur.
From<< last_key;
3999 for(rec_cnt = 0; rec_cnt < batch_size; ++rec_cnt) {
4030cache_entries.push_back(
4050next_exp_key =
key;
4056next_exp_key =
key;
4062first_key = last_key;
4064 if(rec_cnt == (batch_size - 1)) {
4070 if(first_key == last_key) {
4079 string msg(
"Purge: Inspected ");
4081 msg+=
" records.\n";
4088 if(cache_entries.size() > 0) {
4093 for(
size_t i= 0;
i< cache_entries.size(); ++
i) {
4103 msg+=
" Purge: DELETING \"";
4142<<
"Purge suppressed exception when deleting BLOB " 4149 catch(exception& ex) {
4151<<
"Purge suppressed exception when deleting BLOB " 4168cache_entries.resize(0);
4173 LOG_POST_X(24,
Info<<
"BDB Cache: Stopping Purge execution.");
4178 if(scan_interrupted) {
4210 unsignedtime_in_purge = 0;
4214time_t curr = time(0);
4218time_in_purge = (unsigned)curr - (
unsigned)gc_start;
4241time_t access_timeout)
4247 if(
key.empty() && access_timeout == 0) {
4255vector<SCacheDescr> cache_entries;
4257 unsignedrecs_scanned = 0;
4269time_t curr = time(0);
4271 if(access_timeout && access_timeout < timeout) {
4272timeout = (
int)access_timeout;
4286 unsignedto = timeout;
4301 if(access_timeout && (!(curr - to > db_time_stamp))) {
4306cache_entries.push_back(
4313 if(0 == read_count) {
4329 ITERATE(vector<SCacheDescr>, it, cache_entries) {
4349 const string& cache_name,
4350 const string& err_file,
4362 m_Env->
Open(cache_path, DB_INIT_MPOOL | DB_USE_ENVIRON);
4367DB_CREATE | DB_INIT_MPOOL | DB_PRIVATE | DB_USE_ENVIRON);
4370 ERR_POST_X(26,
"Cache location: "+
string(cache_path));
4372 stringcache_blob_db_name =
4374 stringattr_db_name =
4380 stringbak =
m_Path+ attr_db_name +
".bak";
4381FILE* fl = fopen(bak.c_str(),
"wb");
4383 ERR_POST_X(27,
"Running verification for: "+ attr_db_name);
4395 ERR_POST_X(28,
"Cannot delete the environment (it is busy by another process)");
4422exp_time = time_stamp + timeout;
4435 unsigned intttl = attr_db.
ttl;
4449*exp_time = db_time_stamp + timeout;
4452 if(curr - timeout > db_time_stamp) {
4470 unsignedtimeout = (unsigned)time(0);
4480 if(old_ts < timeout) {
4482 if(max_time == 0 || max_time >= timeout) {
4485 switch(access_type) {
4525 unsignedblob_id, volume_id, split_id;
4531 unsignedcoords[2] = { volume_id, split_id };
4542 LOG_POST_X(29,
Info<<
"CBDB_BLOB_Cache:: cache database truncated");
4550 stringov_(
".ov_");
4554 if( (*it)->IsFile() ) {
4555ext = (*it)->GetExt();
4570 unsigned int& blob_id,
4571 unsigned int& volume_id,
4572 unsigned int& split_id)
4620}
catch(exception& ex) {
4621 ERR_POST_X(30,
"Blob Store: Cannot remove file: "<< path
4622<<
" "<< ex.what());
4633}
catch(exception& ex) {
4634 ERR_POST_X(31,
"Blob Store: Cannot remove file: "<< file_path
4635<<
" "<< ex.what());
4650 BDB_THROW(eInvalidOperation,
"Cannot lock new BLOB ID");
4687time_t curr = time(0);
4689 unsignedcoords[2] = {0,};
4690 unsignedsplit_coord[2] = {0,};
4694 boolblob_expired =
false;
4710blob_expired =
true;
4716blob_expired =
false;
4722 if(!blob_expired) {
4737 if(overflow == 1) {
4774 if(split_coord[0] != coords[0] ||
4775split_coord[1] != coords[1]) {
4803 if(overflow == 1) {
4807 unsignedcoords[2] = {0,};
4808 unsignedsplit_coord[2] = {0,};
4811 booldelete_split =
false;
4825 if(split_coord[0] != coords[0] ||
4826split_coord[1] != coords[1]) {
4827delete_split =
true;
4868 if(!blob_id)
return false;
4875 const string&
key,
4881 unsigned* volume_id,
4882 unsigned* split_id,
4883 unsigned* overflow)
4890 unsignedblob_id = 0;
4892 while(blob_id == 0) {
4901ret = cur.
Fetch();
4913blob_lock.
Lock(blob_id);
4915blob_lock.
SetId(blob_id);
4920*overflow = *volume_id = *split_id = 0;
4927 if(blob_id_ext == 0) {
4930blob_id = blob_id_ext;
4972blob_lock.
Lock(blob_id);
4974blob_lock.
SetId(blob_id);
4979 "Cannot update blob id index");
5085 const string& owner)
5095 const string& owner)
5105 const string& owner)
5115 const string& owner)
5132 const string& x_subkey)
5177 ICache* x_CreateInstance(
5229 if( !driver_params ) {
5233 stringstr_path = path ?
5236 if(!path || str_path !=
m_Path) {
5245 const string& driver,
5249unique_ptr<CBDB_Cache> drv;
5250 if(driver.empty() || driver == m_DriverName) {
5260 returndrv.release();
5264 const string& path =
5275 unsignedoverflow_limit = (unsigned)
5277 if(overflow_limit) {
5278drv->SetOverflowLimit(overflow_limit);
5286 Uint8log_mem_size =
5289 unsignedcheckpoint_bytes = (unsigned)
5291 false, 24 * 1024 * 1024);
5292drv->SetCheckpoint(checkpoint_bytes);
5294 unsignedcheckpoint_delay =
5296drv->SetCheckpointDelay(checkpoint_delay);
5298 unsignedlog_file_max = (unsigned)
5300 false, 200 * 1024 * 1024);
5301drv->SetLogFileMax(log_file_max);
5303 stringtransaction_log_path =
5305 if(transaction_log_path != path) {
5306drv->SetLogDir(transaction_log_path);
5315drv->SetWriteSync(w_sync ?
5318 unsignedttl_prolong =
5320drv->SetTTL_Prolongation(ttl_prolong);
5322 unsignedmax_blob_size =(unsigned)
5324drv->SetMaxBlobSize(max_blob_size);
5326 unsignedrr_volumes =
5328drv->SetRR_Volumes(rr_volumes);
5330 unsignedmemp_trickle =
5332drv->SetMempTrickle(memp_trickle);
5334ConfigureICache(drv.get(), params);
5339 unsignedbatch_size =
5341drv->SetPurgeBatchSize(batch_size);
5343 unsignedbatch_sleep =
5345drv->SetBatchSleep(batch_sleep);
5347 unsignedpurge_clean_factor =
5349drv->CleanLogOnPurge(purge_clean_factor);
5353 unsignedpurge_thread_delay =
5356 unsignedtas_spins =
5360drv->OpenReadOnly(path.c_str(), name.c_str(), (
unsigned)mem_size);
5362drv->Open(path, name,
5365(
unsigned)log_mem_size);
5368 if(!drv->IsJoinedEnv()) {
5375 env->SetDirectDB(direct_db);
5376 env->SetDirectLog(direct_log);
5378 env->SetTasSpins(tas_spins);
5381 env->SetMpMaxWrite(0, 0);
5385drv->RunPurgeThread(purge_thread_delay);
5386drv->StartPurgeThread();
5389 returndrv.release();
5411: m_BlobCache(blob_cache),
5423 const string& path,
5424 const string& name,
5437timeout = 24 * 60 * 60;
5443bdb_cache.
Open(path.c_str(),
static const char * kCFParam_log_mem_size
static const char * kCFParam_lock_default
static const char * kCFParam_write_sync
static void s_GetDayHour(time_t time_in_secs, unsigned *day, unsigned *hour)
Compute current hour of day.
static const char * kCFParam_direct_db
static const char * kCFParam_transaction_log_path
const char * kBDBCacheStartedFileName
static const char * kCFParam_path
static const char * kCFParam_max_blob_size
static void s_MakeOverflowFileName(string &buf, const string &path, const string &cache, const string &key, int version, const string &subkey)
static const char * kCFParam_ttl_prolong
static const char * kCFParam_mem_size
static const char * kCFParam_lock_pid_lock
static const char * kCFParam_drop_if_dirty
static const char * kCFParam_purge_thread
static const char * kCFParam_direct_log
static const char * kCFParam_purge_thread_delay
static const char * kCFParam_checkpoint_bytes
static const char * kCFParam_purge_clean_log
static const char * kCFParam_lock
static const char * kCFParam_TAS_spins
static const char * kCFParam_purge_batch_size
static const char * kCFParam_read_only
static const char * kCFParam_memp_trickle
static const char * kCFParam_checkpoint_delay
static const char * kCFParam_overflow_limit
static const char * kCFParam_rr_volumes
static const char * kCFParam_name
static const char * kCFParam_log_file_max
static const char * kCFParam_use_transactions
static const char * kCFParam_purge_batch_sleep
ICache interface implementation on top of Berkeley DB.
Berkeley BDB file cursor.
Wrapper around Berkeley DB transaction structure.
ncbi::TMaskedQueryRegions mask
BLOB demultiplexer implements round-robin volume rotation.
CBDB_RawFile::TBuffer * m_RawBuffer
CBDB_Cache::TBlobLock m_BlobLock
virtual ERW_Result PendingCount(size_t *count)
Via parameter "count" (which is guaranteed to be supplied non-NULL) return the number of bytes that a...
unsigned char * m_BufferPtr
virtual ~CBDB_CacheIReader()
CNcbiIfstream * m_OverflowFile
CBDB_CacheIReader & operator=(const CBDB_CacheIReader &)
CBDB_CacheIReader(CBDB_Cache &bdb_cache, CNcbiIfstream *overflow_file, CBDB_Cache::TBlobLock &blob_lock)
CBDB_CacheIReader(CBDB_Cache &bdb_cache, CBDB_RawFile::TBuffer *raw_buffer, CBDB_Cache::TBlobLock &blob_lock)
virtual ERW_Result Read(void *buf, size_t count, size_t *bytes_read)
Read as many as "count" bytes into a buffer pointed to by the "buf" argument.
CBDB_CacheIReader(const CBDB_CacheIReader &)
bool m_Flushed
FALSE until Flush() called.
virtual ~CBDB_CacheIWriter()
CBDB_CacheIWriter(CBDB_Cache &bdb_cache, const char *path, unsigned blob_id_ext, const string &blob_key, int version, const string &subkey, SCache_AttrDB &attr_db, unsigned int ttl, time_t request_time, const string &owner, CBDB_Cache::TBlobLock &blob_lock)
CSimpleBufferT< unsigned char, CCacheBufferResizeStrategy > TBuffer
CNcbiOfstream * m_OverflowFile
virtual ERW_Result Flush(void)
Flush pending data (if any) down to output device.
CBDB_Cache::TBlobLock m_BlobLock
unsigned m_BlobSize
Number of bytes written.
void x_WriteOverflow(const char *buf, streamsize count)
CBDB_CacheIWriter(const CBDB_CacheIWriter &)
CBDB_Cache::EWriteSyncMode m_WSync
virtual ERW_Result Write(const void *buf, size_t count, size_t *bytes_written=0)
Write up to "count" bytes from the buffer pointed to by the "buf" argument onto the output device.
unsigned m_Overflow
Overflow file created.
string m_OverflowFilePath
CBDB_CacheIWriter & operator=(const CBDB_CacheIWriter &)
Class factory for BDB implementation of ICache.
virtual ICache * x_CreateInstance(const string &driver=kEmptyStr, CVersionInfo version=NCBI_INTERFACE_VERSION(ICache), const TPluginManagerParamTree *params=0) const
CICacheCF< CBDB_Cache > TParent
CBDB_Cache_OnAppExit(CBDB_Cache &bdb_cache)
static void AddOnExitCallback(CBDB_Cache &bdb_cache)
shared_ptr< bool > m_StopThreadFlag
void operator()(void) const
BDB cache implementation.
BDB environment object a collection including support for some or all of caching, locking,...
BDB errno exception class.
Base BDB exception class.
Berkeley DB file cursor class.
Buffer resize strategy, to balance memory reallocs and heap consumption.
static size_t GetNewCapacity(size_t, size_t requested_size)
Thread class, peridically calls ICache::Purge to remove obsolete elements.
Utility class for ICache class factories.
CPIDGuard â Process guard.
Reallocable memory buffer (no memory copy overhead) Mimics vector<>, without the overhead of explicit...
const value_type * data() const
void resize(size_type new_size)
void reserve(size_type new_size)
void RequestStop()
Schedule thread Stop.
void AddObject(time_t tm, unsigned object_id)
Add object to the timeline.
bool RemoveObject(time_t object_time, unsigned object_id)
Remove object from the time line, object_time defines time slot.
void EnumerateObjects(TBitVector *objects) const
Enumerate all objects registered in the timeline.
void ExtractObjects(time_t tm, TBitVector *objects)
Extracts all objects up to 'tm' and puts them into 'objects' vector.
unsigned GetDiscrFactor() const
Time discretization factor.
definition of a Culling tree
BLOB cache read/write/maintenance interface.
EKeepVersions
If to keep already cached versions of the BLOB when storing another version of it (not necessarily a ...
@ eDropOlder
Delete the earlier (than the one being stored) versions of the BLOB.
@ eDropAll
Delete all versions of the BLOB, even those which are newer than the one being stored.
@ eKeepAll
Do not delete other versions of cache entries.
int TFlags
Bitwise OR of "EFlags" flags.
EBlobVersionValidity
BLOB version existence and validity â from the point of view of the underlying cache implementation.
@ fTimeStampOnCreate
Cache element is created with a certain timestamp (default)
@ fTimeStampOnRead
Timestamp is updated every on every access (read or write)
@ fExpireLeastFrequentlyUsed
Expire objects older than a certain time frame Example: If object is not accessed within a week it is...
@ fPurgeOnStartup
Expired objects should be deleted on cache mount (Open)
@ fCheckExpirationAlways
Expiration timeout is checked on any access to cache element.
int TTimeStampFlags
Holds a bitwise OR of "ETimeStampFlags".
A very basic data-read interface.
A very basic data-write interface.
Constant iterator designed to enumerate "ON" bits.
bool valid() const noexcept
Checks if iterator is still valid.
Bitvector Bit-vector container with runtime compression of bits.
bool any() const noexcept
Returns true if any bits in this bitset are set, and otherwise returns false.
bvector< Alloc > & set(size_type n, bool val=true)
Sets bit n if val is true, clears bit n if val is false.
void optimize(bm::word_t *temp_block=0, optmode opt_mode=opt_compress, statistics *stat=0)
Optimize memory bitvector's memory allocation.
enumerator first() const
Returns enumerator pointing on the first non-zero bit.
void clear(const size_type *ids, size_type ids_size, bm::sort_order so=bm::BM_UNKNOWN)
clear list of bits in this bitset
size_type count() const noexcept
population count (count of ON bits)
container_type::const_iterator const_iterator
container_type::iterator iterator
const_iterator begin() const
const_iterator end() const
const_iterator upper_bound(const key_type &key) const
iterator_bool insert(const value_type &val)
std::ofstream out("events_result.xml")
main entry point for tests
void AddOnExitAction(TFunc func)
Add callback to be executed from CNcbiApplicationAPI destructor.
static CNcbiApplicationGuard InstanceGuard(void)
Singleton method.
#define ITERATE(Type, Var, Cont)
ITERATE macro to sequence through container elements.
TOwnerStatMap m_OwnerStatMap
time_t ComputeMaxTime(time_t start) const
Compute maximum TTL.
void DropBlob(const string &key, int version, const string &subkey, bool for_update, unsigned *blob_id, unsigned *coord)
Delete BLOB.
void RunPurgeThread(unsigned purge_delay=30)
Schedule a background Purge thread (cleans the cache from the obsolete entries) SetPurgeBatchSize and...
EErrGetPut
Put/Get errors.
CBDB_FieldUint4 ttl
time-to-live
void SetOverflowLimit(unsigned limit)
CFastMutex m_DB_Lock
Database lock.
unsigned m_CheckPointDelay
Trnasaction checkpoint interval (minutes)
CBDB_BlobSplitStore< TBitVector, CBDB_BlobDeMux_RoundRobin, CFastMutex > TSplitStore
Split store for BLOBs.
unsigned m_PurgeCount
Number of times we run purge.
CBDB_FieldUint4 volume_id
demux coord[0]
bool m_RunPurgeThread
Run a background purge thread.
unsigned m_CleanLogOnPurge
Number of bytes stored in cache since last checkpoint Clean log on Purge (factor)
virtual bool SameCacheParams(const TCacheParams *params) const
SCache_AttrDB * m_CacheAttrDB_RO1
Cache attributes r-only1.
CTimeLine< TBitVector > TTimeLine
void AddInternalError(const string &client, SBDB_CacheUnitStatistics::EErrGetPut operation)
CBDB_FieldUint4 blob_id
BLOB counter.
TTimeStampFlags m_TimeStampFlag
Time stamp flag.
void AddNoBlobError(EErrGetPut operation)
virtual void SetBlobVersionAsCurrent(const string &key, const string &subkey, int version)
Set current valid version for a BLOB.
bool m_JoinedEnv
Joined environment.
SCache_AttrDB * m_CacheAttrDB
Cache attributes database.
void AddExplDelete(const string &client)
unsigned GetBatchSleep() const
void AddStore(time_t tm, unsigned store, unsigned update, unsigned blob_size, unsigned overflow)
void x_Store(unsigned blob_id, const string &key, int version, const string &subkey, const void *data, size_t size, unsigned int time_to_live, const string &owner, bool do_blob_lock)
void AddProtocolError(const string &client, SBDB_CacheUnitStatistics::EErrGetPut operation)
time_t m_LastTimeLineCheck
CPIDGuard * m_PidGuard
Cache lock.
CBDB_FieldString owner_name
owner's name
SBDB_CacheUnitStatistics & GlobalStatistics()
unsigned GetNextBlobId(bool lock_id)
Get next BLOB id out from the atomic couter.
unsigned err_communication
Communication error.
void x_AddErrGetPut(EErrGetPut operation)
CFastLocalTime m_LocalTimer
Fast local timer.
EWriteSyncMode GetWriteSync() const
double blobs_size_db
Current size of all BLOBs.
ETRansact
Underlying BDB database can be configured using transactional or non-transactional API.
unsigned m_MempTrickle
memp_trickle parameter (percent)
IServer_Monitor * m_Monitor
Pointer to monitoring interface.
CFastMutex m_CARO1_Lock
Database lock.
void NCBI_BDB_ICacheEntryPoint(CPluginManager< ICache >::TDriverInfoList &info_list, CPluginManager< ICache >::EEntryPointRequest method)
void RegisterOverflow(const string &key, int version, const string &subkey, unsigned time_to_live, const string &owner)
Register BLOB object as overflow file.
void AddToTimeLine(unsigned blob_id, time_t exp_time)
Add BLOB to expiration timeline.
TBitVector m_GC_Deleted
BLOB ids deleted by GC.
void InitHistorgam(TBlobSizeHistogram *hist)
bool m_ReadOnly
read-only flag
EBlobCheckinMode
BLOB check-in mode.
friend class CBDB_CacheIReader
unsigned blobs_read_total
Number of reads.
unsigned m_PurgeBatchSize
Number of records to process in Purge() (with locking)
bool GetSizeEx(const string &key, int version, const string &subkey, size_t *size)
void RegisterProtocolError(SBDB_CacheUnitStatistics::EErrGetPut operation, const string &owner)
virtual void GetBlobOwner(const string &key, int version, const string &subkey, string *owner)
Retrieve BLOB owner.
void StopPurge()
Request to stop Purge.
unsigned m_MaxTTL_prolong
Number of m_MaxTimeout values object lives (read-prolongation)
void Store(unsigned blob_id_ext, const string &key, int version, const string &subkey, const void *data, size_t size, unsigned int time_to_live=0, const string &owner=kEmptyStr)
void AddNeverRead(const string &client)
CFastMutex m_CARO2_Lock
Database lock.
void NCBI_EntryPoint_xcache_bdb(CPluginManager< ICache >::TDriverInfoList &info_list, CPluginManager< ICache >::EEntryPointRequest method)
void x_UpdateOwnerStatOnDelete(const string &owner, bool expl_delete)
update BLOB owners' statistics on BLOB delete
void RegisterNoBlobError(SBDB_CacheUnitStatistics::EErrGetPut operation, const string &owner)
void RegisterCommError(SBDB_CacheUnitStatistics::EErrGetPut operation, const string &owner)
TLockVector m_LockVector
BLOB lock vector.
void AddInternalError(EErrGetPut operation)
virtual IReader * GetReadStream(const string &key, int version, const string &subkey)
Return sequential stream interface to read BLOB data.
void AddStore(const string &client, time_t tm, unsigned store, unsigned update, unsigned blob_size, unsigned overflow)
time_t m_NextExpTime
Time of the next blob expiration.
unsigned err_blob_get
retrive errors
EWriteSyncMode
Suggest page size.
bool m_Closed
Double-close safeguard flag.
virtual bool Read(const string &key, int version, const string &subkey, void *buf, size_t buf_size)
Fetch the BLOB.
EBlobCheckinRes BlobCheckIn(unsigned blob_id_ext, const string &key, int version, const string &subkey, EBlobCheckinMode mode, TBlobLock &blob_lock, bool do_id_lock, unsigned *volume_id, unsigned *split_id, unsigned *overflow)
Check if BLOB exists, create registration record if necessary.
void SetTTL_Prolongation(unsigned ttl_prolong)
Maximum limit for read updates (blob expires eventually even if it is accessed) 0 - unlimited prolong...
virtual void Remove(const string &key)
unsigned err_blob_put
store errors
unsigned m_LockTimeout
Lock timeout (ms)
CSemaphore m_PurgeStopSignal
TRUE when Purge processing requested to stop.
unsigned m_MaxTimeout
Maximum time to live.
void Init()
Drop all collected statistics.
unsigned m_OverflowLimit
Overflow limit (objects lower than that stored as BLOBs)
void EvaluateTimeLine(bool *interrupted)
Evaluate timeline BLOBs as deletion candidates.
const SBDB_CacheStatistics & GetStatistics() const
Get cache operations statistics.
void x_PidLock(ELockMode lm)
CBDB_FieldUint4 upd_count
update counter
void RegisterInternalError(SBDB_CacheUnitStatistics::EErrGetPut operation, const string &owner)
virtual void SetVersionRetention(EKeepVersions policy)
Set version retention policy.
virtual bool HasBlobs(const string &key, const string &subkey)
Check if any BLOB exists (any version)
bool DropBlobWithExpCheck(const string &key, int version, const string &subkey, CBDB_Transaction &trans)
Drop BLOB with time expiration check.
bool m_InitIfDirty
Delete DB if it wasn't closed clean.
virtual size_t GetSize(const string &key, int version, const string &subkey)
Check if BLOB exists, return BLOB size.
void PrintStatistics(CNcbiOstream &out) const
SCache_IdIDX * m_CacheIdIDX_RO
Cache id index r-only.
unsigned err_protocol
Protocol errors.
void KillBlob(const string &key, int version, const string &subkey, int overflow, unsigned blob_id)
const char * kBDBCacheDriverName
string m_LogDir
Transaction log location.
bool m_PurgeNowRunning
Flag that Purge is already running.
void WriteOverflow(CNcbiOfstream &overflow_file, const string &overflow_file_path, const char *buf, streamsize count)
Write to the overflow file with error checking If write failes, method deletes the file to avoid file...
shared_ptr< bool > m_StopThreadFlag
EBlobCheckinRes
Result of BLOB check-in process.
CBDB_Env * m_Env
Common environment for cache DBs.
unsigned m_LogSizeMax
transaction log size
virtual time_t GetAccessTime(const string &key, int version, const string &subkey)
Return last access time for the specified cache entry.
CBDB_FieldUint4 max_time
max ttl limit for BLOB
IWriter * GetWriteStream(unsigned blob_id_ext, const string &key, int version, const string &subkey, bool do_id_lock, unsigned int time_to_live=0, const string &owner=kEmptyStr)
IReader * x_CreateOverflowReader(const string &key, int version, const string &subkey, size_t &file_length, TBlobLock &blob_lock)
void StartPurgeThread()
Start background thread.
unsigned GetMaxBlobSize() const
Get max allowed BLOB size.
void x_DropOverflow(const string &key, int version, const string &subkey)
time_t x_ComputeExpTime(int time_stamp, unsigned ttl, int timeout)
bool x_RetrieveBlobAttributes(const string &key, int version, const string &subkey, int &overflow, unsigned int &ttl, unsigned int &blob_id, unsigned int &volume_id, unsigned int &split_id)
1.
unsigned GetBlobId(const string &key, int version, const string &subkey)
Determines BLOB size (requires fetched attribute record)
virtual void SetFlags(TFlags flags)
Pass flags to the underlying storage.
bool IsSaveStatistics(void) const
SBDB_CacheUnitStatistics m_GlobalStat
TBlobSizeHistogram blob_size_hist
Blob size historgam.
void AddCommError(EErrGetPut operation)
ELockMode
Locking modes to protect cache instance.
void x_DropBlob(const string &key, int version, const string &subkey, int overflow, unsigned blob_id, CBDB_Transaction &trans)
friend class CBDB_CacheIWriter
TTimeLine * m_TimeLine
BLOB expiration timeline.
unsigned blobs_db
Current database number of records.
void AddBlobQuotaError(const string &client)
void OpenReadOnly(const string &cache_path, const string &cache_name, unsigned int cache_ram_size=0)
Open local cache in read-only mode.
bool IsLocked(unsigned blob_id)
Check if BLOB is locked.
unsigned err_no_blob
BLOB not found errors.
CFastMutex m_IDIDX_Lock_RO
Database lock.
unsigned m_CheckPointInterval
Trnasaction checkpoint interval (bytes)
bool IsReadOnly() const
Return TRUE if cache is read-only.
const string & GetName() const
unsigned blobs_stored_total
Total number of blobs.
virtual void Purge(time_t access_timeout)
Delete all BLOBs older than specified.
unsigned blobs_expl_deleted_total
BLOBs explicitly removed.
SBDB_CacheUnitStatistics()
void AddCommError(const string &client, SBDB_CacheUnitStatistics::EErrGetPut operation)
void x_PerformCheckPointNoLock(unsigned bytes_written)
SCache_AttrDB * m_CacheAttrDB_RO2
Cache attributes r-only2.
void AddNoBlobError(const string &client, SBDB_CacheUnitStatistics::EErrGetPut operation)
virtual void SetTimeStampPolicy(TTimeStampFlags policy, unsigned int timeout, unsigned int max_timeout=0)
Set timestamp update policy.
void ConvertToRegistry(IRWRegistry *reg, const string §_name_postfix) const
Convert statistics into registry sections and entries.
unsigned m_Timeout
Timeout expiration policy.
CBDB_FieldUint4 read_count
read counter
unsigned err_blob_over_quota
BLOB max quota.
unsigned blobs_overflow_total
number of overflow blobs
TTimeAccess time_access
Hourly access history.
void Open(const string &cache_path, const string &cache_name, ELockMode lm=eNoLock, Uint8 cache_ram_size=0, ETRansact use_trans=eUseTrans, unsigned int log_mem_size=0)
Open local cache instance (read-write access) If cache does not exists it is created.
unsigned blob_size_max_total
Largest BLOB ever.
CacheKey(const string &x_key, int x_version, const string &x_subkey)
void x_UpdateAccessTime(const string &key, int version, const string &subkey, EBlobAccessType access_type, CBDB_Transaction &trans)
Transactional update of access time attributes.
void SetPurgeBatchSize(unsigned batch_size)
Number of records in scanned at once by Purge Cache is exclusively locks an internal mutex while scan...
TSplitStore * m_BLOB_SplitStore
Cache BLOB storage.
void PrintStatistics(CNcbiOstream &out) const
virtual TFlags GetFlags()
Retrieve the effective combination of flags from the underlying storage.
unsigned m_BatchSleep
Sleep interval (milliseconds) between batches.
virtual TTimeStampFlags GetTimeStampPolicy() const
Get timestamp policy.
bool x_CheckTimeStampExpired(SCache_AttrDB &attr_db, time_t curr, time_t *exp_time=0)
CBDB_FieldInt4 overflow
overflow flag
EKeepVersions m_VersionFlag
Version retention policy.
SBDB_CacheStatistics m_Statistics
Stat counters.
EBlobAccessType
access type for "UpdateAccessTime" methods
bool operator<(const CacheKey &cache_key) const
void BDB_ConfigureCache(CBDB_Cache &bdb_cache, const string &path, const string &name, unsigned timeout, ICache::TTimeStampFlags tflags)
Utility for simple cache configuration.
void StopPurgeThread()
Stop background thread.
unsigned err_internal
Internal errors of all sorts.
void AddProtocolError(EErrGetPut operation)
unsigned GetPurgeBatchSize() const
SCache_IdIDX * m_CacheIdIDX
Cache id index.
void SetBatchSleep(unsigned sleep)
Set sleep in milliseconds between Purge batches (for low priority Purge processes)
void AddRead(const string &client, time_t tm)
CRef< CCacheCleanerThread > m_PurgeThread
Purge thread.
void ConvertToRegistry(IRWRegistry *reg) const
unsigned blobs_updates_total
How many updates registered.
unsigned blobs_purge_deleted_total
BLOBs garbage collected.
unsigned GetOverflowLimit() const
double blobs_size_total
Size of BLOBs total.
void BDB_Register_Cache(void)
Register NCBI_BDB_ICacheEntryPoint.
unsigned m_RoundRobinVolumes
Number of rotated volumes.
void Verify(const string &cache_path, const string &cache_name, const string &err_file=0, bool force_remove=false)
Run verification of the cache database.
CAtomicCounter m_BlobIdCounter
Atomic counter for BLOB ids.
virtual bool IsOpen() const
string m_Path
Path to storage.
CBDB_FieldUint4 split_id
demux coord[1]
bool x_FetchBlobAttributes(const string &key, int version, const string &subkey)
void CleanLog()
Remove all non-active LOG files.
virtual EKeepVersions GetVersionRetention() const
Get version retention.
void Unlock()
Unlock cache access.
void AddPurgeDelete(const string &client)
virtual void GetBlobAccess(const string &key, int version, const string &subkey, SBlobAccessDescr *blob_descr)
Get BLOB access using BlobAccessDescr.
virtual int GetTimeout() const
Get expiration timeout.
CBDB_FieldUint4 time_stamp
access timestamp
void InitStatistics()
Drop the previously collected statistics.
void Lock()
Lock cache access.
string m_Name
Cache name.
CFastMutex m_TimeLine_Lock
static void AddToHistogram(TBlobSizeHistogram *hist, unsigned size)
unsigned m_PurgeThreadDelay
Delay in seconds between Purge runs.
deque< SBDB_TimeAccessStatistics > TTimeAccess
CBDB_CacheHolder(ICache *blob_cache, ICache *id_cache)
unsigned blobs_never_read_total
BLOBs never read before.
@ eErr_Unknown
no info on operation
@ eNoTrans
Non-transactional environment.
@ eUseTrans
Use transaction environment.
@ eBlobCheckIn_Create
If record does not exist - create.
@ eBlobCheckIn
Check if record exists.
@ eBlobCheckIn_Created
BLOB record created.
@ eBlobCheckIn_Found
Existing BLOB.
@ EBlobCheckIn_NotFound
BLOB does not exist.
@ eNoLock
Do not lock-protect cache instance.
@ ePidLock
Create PID lock on cache (exception if failed)
void OpenProjections()
Try to open all storage files in all projections This is only possible when object de-mux has fixed n...
EBDB_ErrCode Delete(unsigned id, CBDB_RawFile::EIgnoreError on_error=CBDB_RawFile::eThrowOnError)
Delete BLOB.
virtual void SetTransaction(ITransaction *trans)
Establish transaction association.
void FreeUnusedMem()
Reclaim unused memory.
EBDB_ErrCode ReadRealloc(unsigned id, CBDB_RawFile::TBuffer &buffer)
Read BLOB into vector.
EBDB_ErrCode UpdateInsert(unsigned id, const void *data, size_t size, unsigned *coord)
Update or insert BLOB.
void Save(typename TDeMuxStore::ECompact compact_vectors=TDeMuxStore::eCompact)
Save storage dictionary (demux disposition).
EBDB_ErrCode Fetch(unsigned id, void **buf, size_t buf_size, CBDB_RawFile::EReallocMode allow_realloc, size_t *blob_size)
Fetch LOB record directly into the provided '*buf'.
void SetEnv(CBDB_Env &env)
Associate with the environment. Should be called before opening.
EBDB_ErrCode BlobSize(unsigned id, size_t *blob_size)
Get size of the BLOB.
void GetIdVector(TBitVector *bv) const
Get all id of all BLOBs stored.
EBDB_ErrCode GetCoordinates(unsigned id, unsigned *coords)
Find (demux) coordinates by BLOB id.
void Open(const string &storage_name, CBDB_RawFile::EOpenMode open_mode, CBDB_RawFile::EDBType db_type=CBDB_RawFile::eBtree)
Open storage (reads storage dictionary into memory)
void AssignCoordinates(unsigned id, const unsigned *coords)
Assing de-mux coordinates.
unsigned int Truncate()
Empty the database. Return number of records removed.
EBDB_ErrCode Delete(EIgnoreError on_error=eThrowOnError)
Delete record corresponding to the current key value.
void SetCondition(ECondition cond_from, ECondition cond_to=eNotSet)
Set search condition(type of interval)
EBDB_ErrCode FetchFirst()
EBDB_ErrCode Update(CBDB_File::EAfterWrite write_flag=CBDB_File::eDiscardData)
void Verify(const char *filename, const char *database, FILE *backup)
Run database verification (DB->verify)
void Open(const string &filename, EOpenMode open_mode, bool support_dirty_read=false, unsigned rec_len=0)
Open file with specified access mode.
EBDB_ErrCode Insert(EAfterWrite write_flag=eDiscardData)
Insert new record.
unsigned CountRecs(bool bFast=false)
Compute database statistic, return number of records.
void SetEnv(CBDB_Env &env)
Associate file with environment.
virtual void SetTransaction(ITransaction *trans)
Establish transaction association.
void InitMultiFetch(size_t buffer_size, EMultiFetchMode mfm=eFetchAll)
Init multi-row fetch.
EBDB_ErrCode
BDB Return codes.
EBDB_ErrCode Fetch(EFetchDirection fdir=eDefault)
Fetch record.
CBDB_ConditionHandle From
EBDB_ErrCode Delete(CBDB_File::EIgnoreError on_error=CBDB_File::eThrowOnError)
EBDB_ErrCode Fetch()
Fetches the record corresponding to the current key value.
@ eReadWriteCreate
read-write, create if it doesn't exist
@ eReadModifyUpdate
Use DB_RMW (write locking) on fetch.
virtual string GetString() const
virtual void ToString(string &str) const
void BDB_RecoverEnv(const string &path, bool fatal_recover)
Run Berkeley DB recovery in private environment.
void SetCacheSize(Uint8 cache_size, int num_caches=1)
Set cache size for the environment.
virtual void Commit()
Commit transaction.
void CleanLog()
Remove all non-active log files.
bool CheckRemove()
Try to Remove the environment, if DB_ENV::remove returns 0, but fails files ramain on disk anyway cal...
void SetLockTimeout(unsigned timeout)
Set timeout value for locks in microseconds (1 000 000 in sec)
void SetLogInMemory(bool on_off)
Configure environment for non-durable in-memory logging.
void TransactionCheckpoint()
Flush the underlying memory pools, logs and data bases.
void SetMaxLocks(unsigned locks)
Set max number of locks in the database.
void StopBackgroundWriterThread()
Stop transaction checkpoint thread.
void SetLogAutoRemove(bool on_off)
If set, Berkeley DB will automatically remove log files that are no longer needed.
void SetLogDir(const string &log_dir)
Path to directory where transaction logs are to be stored By default it is the same directory as envi...
void SetLogFileMax(unsigned int lg_max)
Set maximum size of LOG files.
void RunBackgroundWriter(TBackgroundFlags flags, unsigned thread_delay=30, int memp_trickle=0, unsigned err_max=0)
Schedule background maintenance thread which will do:
void SetTransactionTimeout(unsigned timeout)
Set timeout value for transactions in microseconds (1 000 000 in sec)
#define BDB_THROW(errcode, message)
unsigned int TEnvOpenFlags
OR-ed combination of EEnvOptions.
unsigned TBackgroundFlags
Background work flags (combination of EBackgroundWork)
void OpenWithLocks(const string &db_home)
Open environment with database locking (DB_INIT_LOCK)
void SetTransactionSync(CBDB_Transaction::ETransSync sync)
Set default syncronicity level.
bool Remove()
Non-force removal of BDB environment.
void SetCheckPointKB(unsigned kb)
If the kb parameter is non-zero, a checkpoint will be done if more than kbyte kilobytes of log data h...
void ForceRemove()
Force remove BDB environment.
void OpenErrFile(const string &file_name)
Open error reporting file for the environment.
void Open(const string &db_home, int flags)
Open environment.
void OpenWithTrans(const string &db_home, TEnvOpenFlags opt=0)
Open environment using transaction.
bool IsTransactional() const
Return TRUE if environment has been open as transactional.
void ForceTransactionCheckpoint()
Forced checkpoint.
bool IsRecovery() const
If it is DB_RUNRECOVERY error.
void SetLogBSize(unsigned lg_bsize)
Set the size of the in-memory log buffer, in bytes.
void JoinEnv(const string &db_home, TEnvOpenFlags opt=0, ETransactionDiscovery trans_test=eTestTransactions)
Join the existing environment.
@ eTransASync
Non-durable asyncronous transaction.
@ eEnvDefault
Use default from CBDB_Env.
@ eTransSync
Syncronous transaction.
@ eBackground_DeadLockDetect
@ eBackground_MempTrickle
@ eNoAssociation
No association tracking.
@ eRunRecovery
Run DB recovery first.
@ eThreaded
corresponds to DB_THREAD
@ eDeadLock_Default
Default deadlock detector.
void Set(TValue new_value) THROWS_NONE
Set atomic counter value.
TValue Add(int delta) THROWS_NONE
Atomically add value (=delta), and return new counter value.
#define LOG_POST_X(err_subcode, message)
#define ERR_POST_X(err_subcode, message)
Error posting with default error code and given error subcode.
#define ERR_POST(message)
Error posting with file, line number information but without error codes.
#define NCBI_USER_THROW(message)
Throw a quick-and-dirty runtime exception of type 'CException' with the given error message and error...
void Error(CExceptionArgs_Base &args)
void Warning(CExceptionArgs_Base &args)
virtual const char * what(void) const noexcept
Standard report (includes full backlog).
void Info(CExceptionArgs_Base &args)
TEntries GetEntries(const string &mask=kEmptyStr, TGetEntriesFlags flags=0) const
Get directory entries based on the specified "mask".
static string CreateAbsolutePath(const string &path, ERelativeToWhat rtw=eRelativeToCwd)
Get an absolute path from some, possibly relative, path.
virtual bool Exists(void) const
Check the entry existence.
Int8 GetLength(void) const
Get size of file.
virtual bool Remove(TRemoveFlags flags=eRecursive) const
Remove a directory entry.
static string MakePath(const string &dir=kEmptyStr, const string &base=kEmptyStr, const string &ext=kEmptyStr)
Assemble a path from basic components.
static string AddTrailingPathSeparator(const string &path)
Add trailing path separator, if needed.
virtual bool Exists(void) const
Check if directory "dirname" exists.
virtual ERW_Result Write(const void *buf, size_t count, size_t *bytes_written=0)
Virtual methods from IWriter.
bool Create(TCreateFlags flags=fCreate_Default) const
Create the directory using "dirname" passed in the constructor.
virtual bool Remove(TRemoveFlags flags=eRecursive) const
Delete existing directory.
const string & GetPath(void) const
Get entry path.
virtual bool Exists(void) const
Check existence of file.
void Reset(void)
Reset reference object.
bool Empty(void) const THROWS_NONE
Check if CRef is empty â not pointing to any object, which means having a null value.
static void NCBI_EntryPointImpl(TDriverInfoList &info_list, EEntryPointRequest method)
Entry point implementation.
#define NCBI_INTERFACE_VERSION(iface)
Macro to construct CVersionInfo class using interface name (relies on CInterfaceVersion class)
list< SDriverInfo > TDriverInfoList
List of driver information.
EEntryPointRequest
Actions performed by the entry point.
uint32_t Uint4
4-byte (32-bit) unsigned integer
uint64_t Uint8
8-byte (64-bit) unsigned integer
static TPid GetPid(void)
Get process identifier (pid) for the current process.
bool Set(const string §ion, const string &name, const string &value, TFlags flags=0, const string &comment=kEmptyStr)
Set the configuration parameter value.
#define END_NCBI_SCOPE
End previously defined NCBI scope.
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
ERW_Result
Result codes for I/O operations.
IO_PREFIX::ofstream CNcbiOfstream
Portable alias for ofstream.
IO_PREFIX::ostream CNcbiOstream
Portable alias for ostream.
IO_PREFIX::ifstream CNcbiIfstream
Portable alias for ifstream.
IO_PREFIX::streampos CNcbiStreampos
Portable alias for streampos.
@ eRW_Eof
End of data, should be considered permanent.
@ eRW_Error
Unrecoverable error, no retry possible.
@ eRW_Success
Everything is okay, I/O completed.
static string SizetToString(size_t value, TNumToStringFlags flags=0, int base=10)
Convert size_t to string.
static string DoubleToString(double value, int precision=-1, TNumToStringFlags flags=0)
Convert double to string.
static int CompareNocase(const CTempString s1, SIZE_TYPE pos, SIZE_TYPE n, const char *s2)
Case-insensitive compare of a substring with another string.
static string IntToString(int value, TNumToStringFlags flags=0, int base=10)
Convert int to string.
static string & Replace(const string &src, const string &search, const string &replace, string &dst, SIZE_TYPE start_pos=0, SIZE_TYPE max_replace=0, SIZE_TYPE *num_replace=0)
Replace occurrences of a substring within a string.
static int Compare(const CTempString s1, SIZE_TYPE pos, SIZE_TYPE n, const char *s2, ECase use_case=eCase)
Compare of a substring with another string.
static string UIntToString(unsigned int value, TNumToStringFlags flags=0, int base=10)
Convert UInt to string.
static string ULongToString(unsigned long value, TNumToStringFlags flags=0, int base=10)
Convert unsigned long to string.
static enable_if< is_arithmetic< TNumeric >::value||is_convertible< TNumeric, Int8 >::value, string >::type NumericToString(TNumeric value, TNumToStringFlags flags=0, int base=10)
Convert numeric value to string.
virtual bool IsActive()=0
Check if monitoring is active.
virtual void Send(const char *msg, size_t length)=0
Send message.
bool Run(TRunMode flags=fRunDefault)
Run the thread.
bool TryLock(unsigned id)
Try to acquire lock of specified id.
void TakeFrom(CLockVectorGuard &lg)
Transfer lock ownership from another lock.
void Lock(void)
Acquire mutex for the current thread with no nesting checks.
void FreeUnusedMem()
Reclaim unused memory.
void Lock(unsigned id)
Acquire lock.
unsigned GetId() const
Get BLOB id.
bool IsLocked(unsigned id) const
Check if id is locked or not.
void Unlock()
Unlocks the lock.
void SetId(unsigned id)
Assign Id (no locking)
bool TryWait(unsigned int timeout_sec=0, unsigned int timeout_nsec=0)
Timed wait.
void Post(unsigned int count=1)
Increment the semaphore by "count".
void Join(void **exit_data=0)
Wait for the thread termination.
void Unlock(void)
Release mutex with no owner or nesting checks.
double Elapsed(void) const
Return time elapsed since first Start() or last Restart() call (in seconds).
string AsString(const CTimeFormat &format=kEmptyStr, TSeconds out_tz=eCurrentTimeZone) const
Transform time to string.
int GetLocalTimezone(void)
Get difference in seconds between UTC and current local time (daylight information included)
CTime GetFastLocalTime(void)
Quick and dirty getter of local time.
@ eStart
Start timer immediately after creating.
const TTreeType * FindNode(const TKeyType &key, TNodeSearchMode sflag=eImmediateAndTop) const
Search for node.
const TValue & GetValue(void) const
Return node's value.
@ eNonCompatible
major, minor does not match
operation
Bit operations.
unsigned int
A callback function used to compare two keys in a database.
Definition of all error codes used in bdb library (bdb.lib and ncbi_xcache_bdb.lib).
if(yy_accept[yy_current_state])
const string version
version string
const struct ncbi::grid::netcache::search::fields::SIZE size
const struct ncbi::grid::netcache::search::fields::KEY key
const struct ncbi::grid::netcache::search::fields::SUBKEY subkey
Defines process management classes.
Defines NCBI C++ exception handling.
Defines classes: CDirEntry, CFile, CDir, CSymLink, CMemoryFile, CFileUtil, CFileLock,...
Multi-threading â mutexes; rw-locks; semaphore.
Defines: CTimeFormat - storage class for time format.
std::istream & in(std::istream &in_, double &x_)
Int4 delta(size_t dimension_, const Int4 *score_)
Helper classes and templates to implement plugins.
static SLJIT_INLINE sljit_ins msg(sljit_gpr r, sljit_s32 d, sljit_gpr x, sljit_gpr b)
static CNamedPipeClient * client
Cache accession for internal in-memory storage.
unique_ptr< IReader > reader
BDB cache statistics (covers one elemntary unit of measurements)
BDB cache access statistics.
SCacheDescr(string x_key, int x_version, string x_subkey, int x_overflow, unsigned x_blob_id)
BLOB ID -> key, value, subkey index.
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4