A RetroSearch Logo

Home - News ( United States | United Kingdom | Italy | Germany ) - Football scores

Search Query:

Showing content from http://www.ncbi.nlm.nih.gov/IEB/ToolBox/CPP_DOC/doxyhtml/bdb__blobcache_8cpp_source.html below:

NCBI C++ ToolKit: src/db/bdb/bdb_blobcache.cpp Source File

63 #define NCBI_USE_ERRCODE_X Db_Bdb_BlobCache 90  buf

= path + cache +

'_'

+

key

+

'_' 131  m_BlobLock

(blob_lock.GetLockVector(), blob_lock.GetTimeout())

144  m_BlobLock

(blob_lock.GetLockVector(), blob_lock.GetTimeout())

153  ERR_POST

(

"CBDB_CacheIReader: detected unread input "

<<

163  ERR_POST

(

"CBDB_CacheIReader: detected unread input "

<<

164

(end-pos)<<

": "

<<pos<<

" of "

<<end);

195  if

(*bytes_read == 0) {

241  size_t

reserve = requested_size / 2;

242  if

(reserve > 1 * 1024 * 1024) {

243

reserve = 1 * 1024 * 1024;

245  return

requested_size + reserve;

260  unsigned

blob_id_ext,

261  const string

& blob_key,

267  const string

& owner,

287  m_BlobLock

(blob_lock.GetLockVector(), blob_lock.GetTimeout())

306  _TRACE

(

"CBDB_CacheIWriter::~CBDB_CacheIWriter point 2"

);

335

}

catch

(exception& ) {

355

}

catch

(exception& ) {

362

}

catch

(exception & ex) {

363  ERR_POST_X

(1,

"Exception in ~CBDB_CacheIWriter() : "

<< ex.what()

373  size_t

* bytes_written = 0)

398  string msg

(

"BLOB larger than allowed. size="

);

400  msg

.append(

" quota="

);

416

*bytes_written =

count

;

429  catch

(exception& ) {

448

*bytes_written =

count

;

504  "Error trying to flush an overflow file"

);

549  string

err =

"LC: Cannot create overflow file "

;

551  BDB_THROW

(eCannotOpenOverflowFile, err);

562  "LC: Attempt to write to a non-open overflow file"

);

638  unsigned

hist_value = 512;

639  for

(

unsigned i

= 0;

i

< 100; ++

i

) {

640

(*hist)[hist_value] = 0;

649  if

(hist->

empty

()) {

653  if

(it == hist->

end

()) {

662 void s_GetDayHour

(time_t time_in_secs,

unsigned

* day,

unsigned

* hour)

664

*day =

static_cast<unsigned>

(time_in_secs / (24 * 60 * 60));

665  unsigned

secs_in_day =

static_cast<unsigned>

(time_in_secs % (24 * 60 * 60));

666

*hour = secs_in_day / 3600;

691  if

(ta_stat.

day

== day && ta_stat.

hour

== hour) {

712  if

(ta_stat.

day

== day && ta_stat.

hour

== hour) {

781

<<

"Current database number of records(BLOBs) "

<<

"\t"

<<

blobs_db

<<

"\n" 782

<<

"Current size of all BLOBs "

<<

"\t"

<<

blobs_size_db

<<

"\n" 784

<<

"Number of NetCache protocol errors "

<<

"\t"

<<

err_protocol

<<

"\n" 786

<<

"Number of NetCache server internal errors "

<<

"\t"

<<

err_internal

<<

"\n" 787

<<

"Number of BLOB not found situations "

<<

"\t"

<<

err_no_blob

<<

"\n" 788

<<

"Number of errors when getting BLOBs "

<<

"\t"

<<

err_blob_get

<<

"\n" 789

<<

"Number of errors when storing BLOBs "

<<

"\t"

<<

err_blob_put

<<

"\n" 790

<<

"Number of errors when BLOB is over the size limit "

<<

"\t"

<<

err_blob_over_quota

<<

"\n"

;

795  out

<<

"# Time access statistics:"

<<

"\n"

<<

"\n"

;

796  out

<<

"# Hour \t Puts \t Gets"

<<

"\n"

;

808  out

<<

"# BLOB size histogram:"

<<

"\n"

<<

"\n"

;

809  out

<<

"# Size \t Count"

<<

"\n"

;

816  if

(it->second > 0) {

821  out

<< it->first <<

"\t"

<< it->second <<

"\n"

;

822  if

(it == hist_end) {

831  const string

& sect_name_postfix)

const 833  string

postfix(sect_name_postfix);

839  string

sect_stat(

"bdb_stat"

);

840  if

(!postfix.empty()) {

841

sect_stat.append(

"_"

);

842

sect_stat.append(postfix);

845

reg->

Set

(sect_stat,

"blobs_stored_total"

,

847  "Total number of blobs ever stored"

);

848

reg->

Set

(sect_stat,

"blobs_overflow_total"

,

850  "Total number of overflow blobs (large size)"

);

851

reg->

Set

(sect_stat,

"blobs_updates_total"

,

853  "Total number of blobs updates"

);

854

reg->

Set

(sect_stat,

"blobs_never_read_total"

,

856  "Total number of blobs stored but never read"

);

857

reg->

Set

(sect_stat,

"blobs_read_total"

,

859  "Total number of reads"

);

860

reg->

Set

(sect_stat,

"blobs_expl_deleted_total"

,

862  "Total number of explicit deletes"

);

863

reg->

Set

(sect_stat,

"blobs_purge_deleted_total"

,

865  "Total number of BLOBs deletes by garbage collector"

);

866

reg->

Set

(sect_stat,

"blobs_size_total"

,

868  "Total size of all BLOBs ever stored"

);

869

reg->

Set

(sect_stat,

"blob_size_max_total"

,

871  "Size of the largest BLOB ever stored"

);

874

reg->

Set

(sect_stat,

"blobs_db"

,

876  "Current database number of records(BLOBs)"

);

877

reg->

Set

(sect_stat,

"blobs_size_db"

,

879  "Current size of all BLOBs"

);

881

reg->

Set

(sect_stat,

"err_protocol"

,

883  "Number of NetCache protocol errors"

);

884

reg->

Set

(sect_stat,

"err_communication"

,

886  "Number of communication errors"

);

887

reg->

Set

(sect_stat,

"err_internal"

,

889  "Number of NetCache server internal errors"

);

890

reg->

Set

(sect_stat,

"err_no_blob"

,

892  "Number of BLOB not found situations"

);

893

reg->

Set

(sect_stat,

"err_blob_get"

,

895  "Number of errors when getting BLOBs"

);

896

reg->

Set

(sect_stat,

"err_blob_put"

,

898  "Number of errors when storing BLOBs"

);

899

reg->

Set

(sect_stat,

"err_blob_over_quota"

,

901  "Number of errors when BLOB is over the size limit"

);

918

pair.append(put_count);

920  if

(!put_history.empty()) {

921

put_history.append(

";"

);

923

put_history.append(pair);

928

pair.append(get_count);

930  if

(!get_history.empty()) {

931

get_history.append(

";"

);

933

get_history.append(pair);

937

reg->

Set

(sect_stat,

"get_access"

,

939  "Read access by hours (hour=count)"

);

940

reg->

Set

(sect_stat,

"put_access"

,

942  "Write access by hours (hour=count)"

);

952  string

sect_hist(

"bdb_stat_hist"

);

953  if

(!postfix.empty()) {

954

sect_hist.append(

"_"

);

955

sect_hist.append(postfix);

963  if

(it->second > 0) {

968  string

var_name =

"size_"

;

973  if

(it == hist_end) {

1088  const string

&

client

= it->first;

1096  out

<<

"## "

<<

"\n" 1097

<<

"## Global statistics"

<<

"\n" 1098

<<

"## "

<<

"\n\n"

;

1103  const string

&

client

= it->first;

1106  out

<<

"## "

<<

"\n" 1107

<<

"## Owner statistics:"

<<

client

<<

"\n" 1108

<<

"## "

<<

"\n\n"

;

1118

m_ReadOnly(

false

),

1119

m_InitIfDirty(

false

),

1120

m_JoinedEnv(

false

),

1121

m_LockTimeout(20 * 1000),

1124

m_BLOB_SplitStore(0),

1127

m_CacheAttrDB_RO1(0),

1128

m_CacheAttrDB_RO2(0),

1132

m_VersionFlag(eDropOlder),

1133

m_WSync(eWriteNoSync),

1134

m_PurgeBatchSize(150),

1136

m_PurgeStopSignal(0, 100),

1137

m_CleanLogOnPurge(0),

1140

m_PurgeNowRunning(

false

),

1141

m_RunPurgeThread(

false

),

1142

m_PurgeThreadDelay(10),

1143

m_CheckPointInterval(24 * (1024 * 1024)),

1144

m_OverflowLimit(512 * 1024),

1145

m_MaxTTL_prolong(0),

1146

m_SaveStatistics(

false

),

1148

m_RoundRobinVolumes(0),

1166  ERR_POST

(

"~CBDB_Cache(): exception: "

<<exc);

1168  catch

(exception& exc) {

1169  ERR_POST

(

"~CBDB_Cache(): exception: "

<<exc.

what

());

1181  string

lock_file_path =

m_Path

+ lock_file;

1234  const string

& cache_name,

1236  Uint8

cache_ram_size,

1238  unsigned int

log_mem_size)

1260  if

((!fl.empty() || fl_clean.

Exists

())

1261

&& s_OpenedDirs.count(

m_Path

) == 0)

1264  ERR_POST

(

"Database was closed uncleanly. Removing directory " 1270  ERR_POST

(

"Database was closed uncleanly. Running recovery..."

);

1282  if

(!fl_clean.

Exists

()) {

1285

writer.

Write

(pid.data(), pid.size());

1291  string

err_file =

m_Path

+

"err"

+

string

(cache_name) +

".log"

;

1295  bool

needs_recovery =

false

;

1302  "LC: '"

<< cache_name <<

1303  "' Joined transactional environment "

);

1306  "LC: '"

<< cache_name <<

1307  "' Warning: Joined non-transactional environment "

);

1315  "LC: '"

<< cache_name <<

1316  "'Warning: DB_ENV returned DB_RUNRECOVERY code." 1317  " Running the recovery procedure."

);

1319

needs_recovery =

true

;

1323

needs_recovery =

true

;

1328  if

(log_mem_size == 0) {

1343  if

(cache_ram_size) {

1351  if

(cache_ram_size > (500 * 1024 * 1024)) {

1352

cache_num = (

int

)((cache_ram_size) /

Uint8

(250 * 1024 * 1024));

1357  if

(cache_num > 10) {

1364  if

(checkpoint_KB == 0) {

1377  if

(needs_recovery) {

1385  LOG_POST_X

(5,

Info

<<

"BDB_Cache: Creating locking environment"

);

1420  string

cache_blob_db_name =

1422  string

attr_db_name =

1424  string

id_idx_name =

1444  unsigned

timeline_precision = 5 * 60;

1448  if

(timeline_precision > (5 * 60)) {

1449

timeline_precision = 5 * 60;

1451  if

(timeline_precision < (1 * 60)) {

1452

timeline_precision = 1 * 60;

1469  unsigned

max_blob_id = 0;

1472

max_blob_id =

max

(max_blob_id, blob_id);

1532  "LC: '"

<< cache_name <<

1533  "' Cache mount at: "

<< cache_path);

1548 # ifdef NCBI_THREADS 1568  "Cannot run background thread in non-MT configuration."

);

1577 # ifdef NCBI_THREADS 1591  const string

& cache_name,

1610  string

cache_blob_db_name =

1612  string

attr_db_name =

1623  "LC: '"

<< cache_name <<

1624  "' Cache mount read-only at: "

<< cache_path);

1665  "LC: '"

<<

m_Name

<<

"' Unmounted. BDB ENV deleted."

);

1668

<<

"' environment still in use."

);

1671  catch

(exception& ex) {

1673

<<

"' Exception in Close() "

<< ex.what()

1709  unsigned int

timeout,

1710  unsigned int

max_timeout)

1718  m_MaxTimeout

= max_timeout > timeout ? max_timeout : timeout;

1843  unsigned

split_coord[2];

1849  if

(coord[0] != split_coord[0] || coord[1] != split_coord[1]) {

1871  unsigned

time_to_live,

1872  const string

& owner)

1874

time_t curr = time(0);

1876  unsigned

blob_id = 0;

1877  unsigned

coord[2]={0,};

1897

ret = cur.

Fetch

();

1964  unsigned

split_coord[2];

1968  if

(coord[0] != split_coord[0] || coord[1] != split_coord[1]) {

1986  const string

&

key

,

1991  unsigned int

time_to_live,

1992  const string

& owner,

1999  unsigned

coord[2] = {0,};

2000  unsigned

overflow = 0, old_overflow = 0;

2003

time_t curr = time(0);

2016  string msg

(

"BLOB larger than allowed. size="

);

2018  msg

.append(

" quota="

);

2043

blob_lock, do_blob_lock,

2044

&coord[0], &coord[1],

2050

blob_id = blob_lock.

GetId

();

2066  if

(!oveflow_file.is_open() || oveflow_file.bad()) {

2067  string

err =

"LC: Cannot create overflow file "

;

2069  BDB_THROW

(eCannotOpenOverflowFile, err);

2092  unsigned

split_coord[2];

2096  if

(coord[0] != split_coord[0] || coord[1] != split_coord[1]) {

2110

coord[0] = coord[1] = 0;

2170

ret = cur.

Fetch

();

2186  if

(blob_id != old_blob_id) {

2188  "BLOB id mutation detected!"

);

2202  string msg

=

"BLOB insert-delete race detected!"

;

2222  unsigned

blob_updated = 0;

2223  unsigned

blob_stored = 0;

2234

blob_stored, blob_updated, (

unsigned

)

size

, overflow);

2253  const string

&

key

,

2258  unsigned int

time_to_live,

2259  const string

& owner)

2272  unsigned int

time_to_live,

2273  const string

& owner)

2286  size_t

blob_size = 0;

2288  unsigned int

ttl, blob_id, volume_id, split_id;

2291  if

(!blob_id)

return false

;

2301

blob_id, volume_id, split_id);

2302  if

(!rec_exists)

return false

;

2321

blob_size = (size_t)entry.

GetLength

();

2324  if

(blob_id == 0)

return false

;

2327

coords[0] = volume_id;

2328

coords[1] = split_id;

2334  if

(ret !=

eBDB_Ok

)

return false

;

2372

time_t curr = time(0);

2397  if

(

key

!= found_key )

2437

time_t curr = time(0);

2441  unsigned

volume_id = 0, split_id = 0;

2444  if

(!blob_id)

return false

;

2460

ret = cur.

Fetch

();

2471  if

(max_time == 0 || max_time >= (

unsigned

)curr) {

2510

unique_ptr<CNcbiIfstream>

2512  if

(!overflow_file->is_open()) {

2516

overflow_file->read((

char

*)

buf

, buf_size);

2517  if

(!*overflow_file) {

2530  if

(coords[0] != volume_id ||

2531

coords[1] != split_id) {

2538

coords[0] = volume_id;

2539

coords[1] = split_id;

2653  size_t

& file_length,

2658

unique_ptr<CNcbiIfstream>

2661  if

(!overflow_file->is_open()) {

2665

file_length = (size_t) entry.

GetLength

();

2677

time_t curr = time(0);

2680  unsigned

volume_id = 0, split_id = 0;

2702

ret = cur.

Fetch

();

2713  if

(max_time == 0 || max_time >= (

unsigned

) curr) {

2748

unique_ptr<IReader> rd;

2751  return

rd.release();

2762  if

(coords[0] != volume_id ||

2763

coords[1] != split_id) {

2770

coords[0] = volume_id;

2771

coords[1] = split_id;

2877  "key, subkey, &version, &validity) is not implemented"

);

2886  "key, subkey, version) is not implemented"

);

2897

blob_descr->

reader

.reset();

2903

time_t curr = time(0);

2906  unsigned

volume_id = 0, split_id = 0;

2931

ret = cur.

Fetch

();

2943  if

(max_time == 0 || max_time >= (

unsigned

) curr) {

2982

blob_descr->

reader

.reset(

2985  if

(blob_descr->

reader

.get()) {

3003  if

(coords[0] != volume_id ||

3004

coords[1] != split_id) {

3012

coords[0] = volume_id;

3013

coords[1] = split_id;

3019  char

** ptr = &blob_descr->

buf

;

3038

unique_ptr<CBDB_RawFile::TBuffer>

buffer

(

3047

blob_descr->

reader

.reset(

3199  unsigned int

time_to_live,

3200  const string

& owner)

3203

time_to_live, owner);

3208  const string

&

key

,

3212  unsigned int

time_to_live,

3213  const string

& owner)

3235

time_t curr = time(0);

3244  unsigned

coord[2] = {0,};

3250

blob_lock, do_id_lock,

3251

&coord[0], &coord[1],

3260

blob_lock.

GetId

(),

3278

vector<SCacheDescr> cache_elements;

3297

cache_elements.push_back(

3304  if

(0 == read_count) {

3320  ITERATE

(vector<SCacheDescr>, it, cache_elements) {

3356

vector<SCacheDescr> cache_elements;

3370

cache_elements.push_back(

3377  if

(0 == read_count) {

3394  ITERATE

(vector<SCacheDescr>, it, cache_elements) {

3470  const string

& overflow_file_path,

3474

overflow_file.write(

buf

,

count

);

3475  if

(overflow_file.bad()) {

3477

overflow_file.close();

3479  string

err(

"Overflow file IO error "

);

3480

err += overflow_file_path;

3526

time_t curr = time(0);

3531  string msg

=

"Purge: Timeline evaluation skiped "

;

3532  msg

+=

"(early wakeup for this precision) remains="

;

3536  msg

+=

" precision="

;

3551  if

(!delete_candidates.

any

()) {

3554  "Purge: Timeline evaluation exits " 3555  "(no candidates) \n"

;

3564  if

(batch_size < 10) {

3568  unsigned

candidates = delete_candidates.

count

();

3571  unsigned

cleaned_candidates = delete_candidates.

count

();

3572  if

(cleaned_candidates != candidates) {

3575  msg

=

"Purge: Timeline total timeline candidates="

;

3577  msg

+=

" cleaned_candidates="

;

3584

candidates = cleaned_candidates;

3589  double

id_trans_time = 0.0;

3590  double

exp_check_time = 0.0;

3591  double

del_time = 0.0;

3592  unsigned

deleted_cnt = 0;

3594

vector<unsigned> blob_id_vect(batch_size);

3595

vector<SCacheDescr> blob_batch_vect(batch_size);

3596

vector<SCacheDescr> blob_exp_vect(batch_size);

3602

blob_id_vect.resize(0);

3603

blob_batch_vect.resize(0);

3604

blob_exp_vect.resize(0);

3605  for

(

unsigned i

= 0; en.valid() &&

i

< batch_size; ++

i

) {

3606  unsigned

blob_id = *en;

3607

blob_id_vect.push_back(blob_id);

3613  if

(blob_id_vect.size()) {

3615

vector<unsigned>::const_iterator it = blob_id_vect.begin();

3616  unsigned

blob_id = *it;

3622

cur.

From

<< blob_id;

3627  if

(blob_id != bid) {

3628  if

((++it) == blob_id_vect.end())

3631

cur.

From

<< (blob_id = *it);

3637

blob_batch_vect.push_back(

3642  if

(it == blob_id_vect.end()) {

3648  if

((blob_id + 1) == *it) {

3654

cur.

From

<< (blob_id = *it);

3664  if

(blob_batch_vect.size()) {

3674  for

(

size_t i

= 0;

i

< blob_batch_vect.size(); ++

i

) {

3688

blob_exp_vect.push_back(blob_descr);

3703  unsigned

deleted_batch_cnt = 0;

3706  if

(blob_exp_vect.size()) {

3708  for

(

size_t i

= 0;

i

< blob_exp_vect.size(); ++

i

) {

3717  msg

+=

" Purge: DELETING \""

;

3733

deleted_batch_cnt += blob_deleted;

3748

deleted_cnt += deleted_batch_cnt;

3753  msg

=

"Purge: Timeline deleted_batch_cnt="

;

3755  msg

+=

" deleted_cnt="

;

3757  msg

+=

" candidates="

;

3760  msg

+=

" id_trans_time="

;

3762  msg

+=

" exp_check_time="

;

3764  msg

+=

" del_time="

;

3779

*interrupted =

true

;

3881  if

(access_timeout == 0) {

3889  bool

scan_interrupted =

false

;

3891  if

(scan_interrupted) {

3895

time_t gc_start = time(0);

3927  unsigned

split_store_blobs = bv.count();

3934  unsigned

tl_count = timeline_blobs.

count

();

3937  "Purge: scan skipped(nothing todo) remains=" 3973

vector<SCacheDescr> cache_entries;

3974

cache_entries.reserve(1000);

3976  unsigned

db_recs = 0;

3980  string

first_key, last_key;

3981  string

next_exp_key;

3983  for

(

bool

flag =

true

; flag;) {

3987

time_t curr = time(0);

3997

cur.

From

<< last_key;

3999  for

(rec_cnt = 0; rec_cnt < batch_size; ++rec_cnt) {

4030

cache_entries.push_back(

4050

next_exp_key =

key

;

4056

next_exp_key =

key

;

4062

first_key = last_key;

4064  if

(rec_cnt == (batch_size - 1)) {

4070  if

(first_key == last_key) {

4079  string msg

(

"Purge: Inspected "

);

4081  msg

+=

" records.\n"

;

4088  if

(cache_entries.size() > 0) {

4093  for

(

size_t i

= 0;

i

< cache_entries.size(); ++

i

) {

4103  msg

+=

" Purge: DELETING \""

;

4142

<<

"Purge suppressed exception when deleting BLOB " 4149  catch

(exception& ex) {

4151

<<

"Purge suppressed exception when deleting BLOB " 4168

cache_entries.resize(0);

4173  LOG_POST_X

(24,

Info

<<

"BDB Cache: Stopping Purge execution."

);

4178  if

(scan_interrupted) {

4210  unsigned

time_in_purge = 0;

4214

time_t curr = time(0);

4218

time_in_purge = (unsigned)curr - (

unsigned

)gc_start;

4241

time_t access_timeout)

4247  if

(

key

.empty() && access_timeout == 0) {

4255

vector<SCacheDescr> cache_entries;

4257  unsigned

recs_scanned = 0;

4269

time_t curr = time(0);

4271  if

(access_timeout && access_timeout < timeout) {

4272

timeout = (

int

)access_timeout;

4286  unsigned

to = timeout;

4301  if

(access_timeout && (!(curr - to > db_time_stamp))) {

4306

cache_entries.push_back(

4313  if

(0 == read_count) {

4329  ITERATE

(vector<SCacheDescr>, it, cache_entries) {

4349  const string

& cache_name,

4350  const string

& err_file,

4362  m_Env

->

Open

(cache_path, DB_INIT_MPOOL | DB_USE_ENVIRON);

4367

DB_CREATE | DB_INIT_MPOOL | DB_PRIVATE | DB_USE_ENVIRON);

4370  ERR_POST_X

(26,

"Cache location: "

+

string

(cache_path));

4372  string

cache_blob_db_name =

4374  string

attr_db_name =

4380  string

bak =

m_Path

+ attr_db_name +

".bak"

;

4381

FILE* fl = fopen(bak.c_str(),

"wb"

);

4383  ERR_POST_X

(27,

"Running verification for: "

+ attr_db_name);

4395  ERR_POST_X

(28,

"Cannot delete the environment (it is busy by another process)"

);

4422

exp_time = time_stamp + timeout;

4435  unsigned int

ttl = attr_db.

ttl

;

4449

*exp_time = db_time_stamp + timeout;

4452  if

(curr - timeout > db_time_stamp) {

4470  unsigned

timeout = (unsigned)time(0);

4480  if

(old_ts < timeout) {

4482  if

(max_time == 0 || max_time >= timeout) {

4485  switch

(access_type) {

4525  unsigned

blob_id, volume_id, split_id;

4531  unsigned

coords[2] = { volume_id, split_id };

4542  LOG_POST_X

(29,

Info

<<

"CBDB_BLOB_Cache:: cache database truncated"

);

4550  string

ov_(

".ov_"

);

4554  if

( (*it)->IsFile() ) {

4555

ext = (*it)->GetExt();

4570  unsigned int

& blob_id,

4571  unsigned int

& volume_id,

4572  unsigned int

& split_id)

4620

}

catch

(exception& ex) {

4621  ERR_POST_X

(30,

"Blob Store: Cannot remove file: "

<< path

4622

<<

" "

<< ex.what());

4633

}

catch

(exception& ex) {

4634  ERR_POST_X

(31,

"Blob Store: Cannot remove file: "

<< file_path

4635

<<

" "

<< ex.what());

4650  BDB_THROW

(eInvalidOperation,

"Cannot lock new BLOB ID"

);

4687

time_t curr = time(0);

4689  unsigned

coords[2] = {0,};

4690  unsigned

split_coord[2] = {0,};

4694  bool

blob_expired =

false

;

4710

blob_expired =

true

;

4716

blob_expired =

false

;

4722  if

(!blob_expired) {

4737  if

(overflow == 1) {

4774  if

(split_coord[0] != coords[0] ||

4775

split_coord[1] != coords[1]) {

4803  if

(overflow == 1) {

4807  unsigned

coords[2] = {0,};

4808  unsigned

split_coord[2] = {0,};

4811  bool

delete_split =

false

;

4825  if

(split_coord[0] != coords[0] ||

4826

split_coord[1] != coords[1]) {

4827

delete_split =

true

;

4868  if

(!blob_id)

return false

;

4875  const string

&

key

,

4881  unsigned

* volume_id,

4882  unsigned

* split_id,

4883  unsigned

* overflow)

4890  unsigned

blob_id = 0;

4892  while

(blob_id == 0) {

4901

ret = cur.

Fetch

();

4913

blob_lock.

Lock

(blob_id);

4915

blob_lock.

SetId

(blob_id);

4920

*overflow = *volume_id = *split_id = 0;

4927  if

(blob_id_ext == 0) {

4930

blob_id = blob_id_ext;

4972

blob_lock.

Lock

(blob_id);

4974

blob_lock.

SetId

(blob_id);

4979  "Cannot update blob id index"

);

5085  const string

& owner)

5095  const string

& owner)

5105  const string

& owner)

5115  const string

& owner)

5132  const string

& x_subkey)

5177  ICache

* x_CreateInstance(

5229  if

( !driver_params ) {

5233  string

str_path = path ?

5236  if

(!path || str_path !=

m_Path

) {

5245  const string

& driver,

5249

unique_ptr<CBDB_Cache> drv;

5250  if

(driver.empty() || driver == m_DriverName) {

5260  return

drv.release();

5264  const string

& path =

5275  unsigned

overflow_limit = (unsigned)

5277  if

(overflow_limit) {

5278

drv->SetOverflowLimit(overflow_limit);

5286  Uint8

log_mem_size =

5289  unsigned

checkpoint_bytes = (unsigned)

5291  false

, 24 * 1024 * 1024);

5292

drv->SetCheckpoint(checkpoint_bytes);

5294  unsigned

checkpoint_delay =

5296

drv->SetCheckpointDelay(checkpoint_delay);

5298  unsigned

log_file_max = (unsigned)

5300  false

, 200 * 1024 * 1024);

5301

drv->SetLogFileMax(log_file_max);

5303  string

transaction_log_path =

5305  if

(transaction_log_path != path) {

5306

drv->SetLogDir(transaction_log_path);

5315

drv->SetWriteSync(w_sync ?

5318  unsigned

ttl_prolong =

5320

drv->SetTTL_Prolongation(ttl_prolong);

5322  unsigned

max_blob_size =(unsigned)

5324

drv->SetMaxBlobSize(max_blob_size);

5326  unsigned

rr_volumes =

5328

drv->SetRR_Volumes(rr_volumes);

5330  unsigned

memp_trickle =

5332

drv->SetMempTrickle(memp_trickle);

5334

ConfigureICache(drv.get(), params);

5339  unsigned

batch_size =

5341

drv->SetPurgeBatchSize(batch_size);

5343  unsigned

batch_sleep =

5345

drv->SetBatchSleep(batch_sleep);

5347  unsigned

purge_clean_factor =

5349

drv->CleanLogOnPurge(purge_clean_factor);

5353  unsigned

purge_thread_delay =

5356  unsigned

tas_spins =

5360

drv->OpenReadOnly(path.c_str(), name.c_str(), (

unsigned

)mem_size);

5362

drv->Open(path, name,

5365

(

unsigned

)log_mem_size);

5368  if

(!drv->IsJoinedEnv()) {

5375  env

->SetDirectDB(direct_db);

5376  env

->SetDirectLog(direct_log);

5378  env

->SetTasSpins(tas_spins);

5381  env

->SetMpMaxWrite(0, 0);

5385

drv->RunPurgeThread(purge_thread_delay);

5386

drv->StartPurgeThread();

5389  return

drv.release();

5411

: m_BlobCache(blob_cache),

5423  const string

& path,

5424  const string

& name,

5437

timeout = 24 * 60 * 60;

5443

bdb_cache.

Open

(path.c_str(),

static const char * kCFParam_log_mem_size

static const char * kCFParam_lock_default

static const char * kCFParam_write_sync

static void s_GetDayHour(time_t time_in_secs, unsigned *day, unsigned *hour)

Compute current hour of day.

static const char * kCFParam_direct_db

static const char * kCFParam_transaction_log_path

const char * kBDBCacheStartedFileName

static const char * kCFParam_path

static const char * kCFParam_max_blob_size

static void s_MakeOverflowFileName(string &buf, const string &path, const string &cache, const string &key, int version, const string &subkey)

static const char * kCFParam_ttl_prolong

static const char * kCFParam_mem_size

static const char * kCFParam_lock_pid_lock

static const char * kCFParam_drop_if_dirty

static const char * kCFParam_purge_thread

static const char * kCFParam_direct_log

static const char * kCFParam_purge_thread_delay

static const char * kCFParam_checkpoint_bytes

static const char * kCFParam_purge_clean_log

static const char * kCFParam_lock

static const char * kCFParam_TAS_spins

static const char * kCFParam_purge_batch_size

static const char * kCFParam_read_only

static const char * kCFParam_memp_trickle

static const char * kCFParam_checkpoint_delay

static const char * kCFParam_overflow_limit

static const char * kCFParam_rr_volumes

static const char * kCFParam_name

static const char * kCFParam_log_file_max

static const char * kCFParam_use_transactions

static const char * kCFParam_purge_batch_sleep

ICache interface implementation on top of Berkeley DB.

Berkeley BDB file cursor.

Wrapper around Berkeley DB transaction structure.

ncbi::TMaskedQueryRegions mask

BLOB demultiplexer implements round-robin volume rotation.

CBDB_RawFile::TBuffer * m_RawBuffer

CBDB_Cache::TBlobLock m_BlobLock

virtual ERW_Result PendingCount(size_t *count)

Via parameter "count" (which is guaranteed to be supplied non-NULL) return the number of bytes that a...

unsigned char * m_BufferPtr

virtual ~CBDB_CacheIReader()

CNcbiIfstream * m_OverflowFile

CBDB_CacheIReader & operator=(const CBDB_CacheIReader &)

CBDB_CacheIReader(CBDB_Cache &bdb_cache, CNcbiIfstream *overflow_file, CBDB_Cache::TBlobLock &blob_lock)

CBDB_CacheIReader(CBDB_Cache &bdb_cache, CBDB_RawFile::TBuffer *raw_buffer, CBDB_Cache::TBlobLock &blob_lock)

virtual ERW_Result Read(void *buf, size_t count, size_t *bytes_read)

Read as many as "count" bytes into a buffer pointed to by the "buf" argument.

CBDB_CacheIReader(const CBDB_CacheIReader &)

bool m_Flushed

FALSE until Flush() called.

virtual ~CBDB_CacheIWriter()

CBDB_CacheIWriter(CBDB_Cache &bdb_cache, const char *path, unsigned blob_id_ext, const string &blob_key, int version, const string &subkey, SCache_AttrDB &attr_db, unsigned int ttl, time_t request_time, const string &owner, CBDB_Cache::TBlobLock &blob_lock)

CSimpleBufferT< unsigned char, CCacheBufferResizeStrategy > TBuffer

CNcbiOfstream * m_OverflowFile

virtual ERW_Result Flush(void)

Flush pending data (if any) down to output device.

CBDB_Cache::TBlobLock m_BlobLock

unsigned m_BlobSize

Number of bytes written.

void x_WriteOverflow(const char *buf, streamsize count)

CBDB_CacheIWriter(const CBDB_CacheIWriter &)

CBDB_Cache::EWriteSyncMode m_WSync

virtual ERW_Result Write(const void *buf, size_t count, size_t *bytes_written=0)

Write up to "count" bytes from the buffer pointed to by the "buf" argument onto the output device.

unsigned m_Overflow

Overflow file created.

string m_OverflowFilePath

CBDB_CacheIWriter & operator=(const CBDB_CacheIWriter &)

Class factory for BDB implementation of ICache.

virtual ICache * x_CreateInstance(const string &driver=kEmptyStr, CVersionInfo version=NCBI_INTERFACE_VERSION(ICache), const TPluginManagerParamTree *params=0) const

CICacheCF< CBDB_Cache > TParent

CBDB_Cache_OnAppExit(CBDB_Cache &bdb_cache)

static void AddOnExitCallback(CBDB_Cache &bdb_cache)

shared_ptr< bool > m_StopThreadFlag

void operator()(void) const

BDB cache implementation.

BDB environment object a collection including support for some or all of caching, locking,...

BDB errno exception class.

Base BDB exception class.

Berkeley DB file cursor class.

Buffer resize strategy, to balance memory reallocs and heap consumption.

static size_t GetNewCapacity(size_t, size_t requested_size)

Thread class, peridically calls ICache::Purge to remove obsolete elements.

Utility class for ICache class factories.

CPIDGuard – Process guard.

Reallocable memory buffer (no memory copy overhead) Mimics vector<>, without the overhead of explicit...

const value_type * data() const

void resize(size_type new_size)

void reserve(size_type new_size)

void RequestStop()

Schedule thread Stop.

void AddObject(time_t tm, unsigned object_id)

Add object to the timeline.

bool RemoveObject(time_t object_time, unsigned object_id)

Remove object from the time line, object_time defines time slot.

void EnumerateObjects(TBitVector *objects) const

Enumerate all objects registered in the timeline.

void ExtractObjects(time_t tm, TBitVector *objects)

Extracts all objects up to 'tm' and puts them into 'objects' vector.

unsigned GetDiscrFactor() const

Time discretization factor.

definition of a Culling tree

BLOB cache read/write/maintenance interface.

EKeepVersions

If to keep already cached versions of the BLOB when storing another version of it (not necessarily a ...

@ eDropOlder

Delete the earlier (than the one being stored) versions of the BLOB.

@ eDropAll

Delete all versions of the BLOB, even those which are newer than the one being stored.

@ eKeepAll

Do not delete other versions of cache entries.

int TFlags

Bitwise OR of "EFlags" flags.

EBlobVersionValidity

BLOB version existence and validity – from the point of view of the underlying cache implementation.

@ fTimeStampOnCreate

Cache element is created with a certain timestamp (default)

@ fTimeStampOnRead

Timestamp is updated every on every access (read or write)

@ fExpireLeastFrequentlyUsed

Expire objects older than a certain time frame Example: If object is not accessed within a week it is...

@ fPurgeOnStartup

Expired objects should be deleted on cache mount (Open)

@ fCheckExpirationAlways

Expiration timeout is checked on any access to cache element.

int TTimeStampFlags

Holds a bitwise OR of "ETimeStampFlags".

A very basic data-read interface.

A very basic data-write interface.

Constant iterator designed to enumerate "ON" bits.

bool valid() const noexcept

Checks if iterator is still valid.

Bitvector Bit-vector container with runtime compression of bits.

bool any() const noexcept

Returns true if any bits in this bitset are set, and otherwise returns false.

bvector< Alloc > & set(size_type n, bool val=true)

Sets bit n if val is true, clears bit n if val is false.

void optimize(bm::word_t *temp_block=0, optmode opt_mode=opt_compress, statistics *stat=0)

Optimize memory bitvector's memory allocation.

enumerator first() const

Returns enumerator pointing on the first non-zero bit.

void clear(const size_type *ids, size_type ids_size, bm::sort_order so=bm::BM_UNKNOWN)

clear list of bits in this bitset

size_type count() const noexcept

population count (count of ON bits)

container_type::const_iterator const_iterator

container_type::iterator iterator

const_iterator begin() const

const_iterator end() const

const_iterator upper_bound(const key_type &key) const

iterator_bool insert(const value_type &val)

std::ofstream out("events_result.xml")

main entry point for tests

void AddOnExitAction(TFunc func)

Add callback to be executed from CNcbiApplicationAPI destructor.

static CNcbiApplicationGuard InstanceGuard(void)

Singleton method.

#define ITERATE(Type, Var, Cont)

ITERATE macro to sequence through container elements.

TOwnerStatMap m_OwnerStatMap

time_t ComputeMaxTime(time_t start) const

Compute maximum TTL.

void DropBlob(const string &key, int version, const string &subkey, bool for_update, unsigned *blob_id, unsigned *coord)

Delete BLOB.

void RunPurgeThread(unsigned purge_delay=30)

Schedule a background Purge thread (cleans the cache from the obsolete entries) SetPurgeBatchSize and...

EErrGetPut

Put/Get errors.

CBDB_FieldUint4 ttl

time-to-live

void SetOverflowLimit(unsigned limit)

CFastMutex m_DB_Lock

Database lock.

unsigned m_CheckPointDelay

Trnasaction checkpoint interval (minutes)

CBDB_BlobSplitStore< TBitVector, CBDB_BlobDeMux_RoundRobin, CFastMutex > TSplitStore

Split store for BLOBs.

unsigned m_PurgeCount

Number of times we run purge.

CBDB_FieldUint4 volume_id

demux coord[0]

bool m_RunPurgeThread

Run a background purge thread.

unsigned m_CleanLogOnPurge

Number of bytes stored in cache since last checkpoint Clean log on Purge (factor)

virtual bool SameCacheParams(const TCacheParams *params) const

SCache_AttrDB * m_CacheAttrDB_RO1

Cache attributes r-only1.

CTimeLine< TBitVector > TTimeLine

void AddInternalError(const string &client, SBDB_CacheUnitStatistics::EErrGetPut operation)

CBDB_FieldUint4 blob_id

BLOB counter.

TTimeStampFlags m_TimeStampFlag

Time stamp flag.

void AddNoBlobError(EErrGetPut operation)

virtual void SetBlobVersionAsCurrent(const string &key, const string &subkey, int version)

Set current valid version for a BLOB.

bool m_JoinedEnv

Joined environment.

SCache_AttrDB * m_CacheAttrDB

Cache attributes database.

void AddExplDelete(const string &client)

unsigned GetBatchSleep() const

void AddStore(time_t tm, unsigned store, unsigned update, unsigned blob_size, unsigned overflow)

void x_Store(unsigned blob_id, const string &key, int version, const string &subkey, const void *data, size_t size, unsigned int time_to_live, const string &owner, bool do_blob_lock)

void AddProtocolError(const string &client, SBDB_CacheUnitStatistics::EErrGetPut operation)

time_t m_LastTimeLineCheck

CPIDGuard * m_PidGuard

Cache lock.

CBDB_FieldString owner_name

owner's name

SBDB_CacheUnitStatistics & GlobalStatistics()

unsigned GetNextBlobId(bool lock_id)

Get next BLOB id out from the atomic couter.

unsigned err_communication

Communication error.

void x_AddErrGetPut(EErrGetPut operation)

CFastLocalTime m_LocalTimer

Fast local timer.

EWriteSyncMode GetWriteSync() const

double blobs_size_db

Current size of all BLOBs.

ETRansact

Underlying BDB database can be configured using transactional or non-transactional API.

unsigned m_MempTrickle

memp_trickle parameter (percent)

IServer_Monitor * m_Monitor

Pointer to monitoring interface.

CFastMutex m_CARO1_Lock

Database lock.

void NCBI_BDB_ICacheEntryPoint(CPluginManager< ICache >::TDriverInfoList &info_list, CPluginManager< ICache >::EEntryPointRequest method)

void RegisterOverflow(const string &key, int version, const string &subkey, unsigned time_to_live, const string &owner)

Register BLOB object as overflow file.

void AddToTimeLine(unsigned blob_id, time_t exp_time)

Add BLOB to expiration timeline.

TBitVector m_GC_Deleted

BLOB ids deleted by GC.

void InitHistorgam(TBlobSizeHistogram *hist)

bool m_ReadOnly

read-only flag

EBlobCheckinMode

BLOB check-in mode.

friend class CBDB_CacheIReader

unsigned blobs_read_total

Number of reads.

unsigned m_PurgeBatchSize

Number of records to process in Purge() (with locking)

bool GetSizeEx(const string &key, int version, const string &subkey, size_t *size)

void RegisterProtocolError(SBDB_CacheUnitStatistics::EErrGetPut operation, const string &owner)

virtual void GetBlobOwner(const string &key, int version, const string &subkey, string *owner)

Retrieve BLOB owner.

void StopPurge()

Request to stop Purge.

unsigned m_MaxTTL_prolong

Number of m_MaxTimeout values object lives (read-prolongation)

void Store(unsigned blob_id_ext, const string &key, int version, const string &subkey, const void *data, size_t size, unsigned int time_to_live=0, const string &owner=kEmptyStr)

void AddNeverRead(const string &client)

CFastMutex m_CARO2_Lock

Database lock.

void NCBI_EntryPoint_xcache_bdb(CPluginManager< ICache >::TDriverInfoList &info_list, CPluginManager< ICache >::EEntryPointRequest method)

void x_UpdateOwnerStatOnDelete(const string &owner, bool expl_delete)

update BLOB owners' statistics on BLOB delete

void RegisterNoBlobError(SBDB_CacheUnitStatistics::EErrGetPut operation, const string &owner)

void RegisterCommError(SBDB_CacheUnitStatistics::EErrGetPut operation, const string &owner)

TLockVector m_LockVector

BLOB lock vector.

void AddInternalError(EErrGetPut operation)

virtual IReader * GetReadStream(const string &key, int version, const string &subkey)

Return sequential stream interface to read BLOB data.

void AddStore(const string &client, time_t tm, unsigned store, unsigned update, unsigned blob_size, unsigned overflow)

time_t m_NextExpTime

Time of the next blob expiration.

unsigned err_blob_get

retrive errors

EWriteSyncMode

Suggest page size.

bool m_Closed

Double-close safeguard flag.

virtual bool Read(const string &key, int version, const string &subkey, void *buf, size_t buf_size)

Fetch the BLOB.

EBlobCheckinRes BlobCheckIn(unsigned blob_id_ext, const string &key, int version, const string &subkey, EBlobCheckinMode mode, TBlobLock &blob_lock, bool do_id_lock, unsigned *volume_id, unsigned *split_id, unsigned *overflow)

Check if BLOB exists, create registration record if necessary.

void SetTTL_Prolongation(unsigned ttl_prolong)

Maximum limit for read updates (blob expires eventually even if it is accessed) 0 - unlimited prolong...

virtual void Remove(const string &key)

unsigned err_blob_put

store errors

unsigned m_LockTimeout

Lock timeout (ms)

CSemaphore m_PurgeStopSignal

TRUE when Purge processing requested to stop.

unsigned m_MaxTimeout

Maximum time to live.

void Init()

Drop all collected statistics.

unsigned m_OverflowLimit

Overflow limit (objects lower than that stored as BLOBs)

void EvaluateTimeLine(bool *interrupted)

Evaluate timeline BLOBs as deletion candidates.

const SBDB_CacheStatistics & GetStatistics() const

Get cache operations statistics.

void x_PidLock(ELockMode lm)

CBDB_FieldUint4 upd_count

update counter

void RegisterInternalError(SBDB_CacheUnitStatistics::EErrGetPut operation, const string &owner)

virtual void SetVersionRetention(EKeepVersions policy)

Set version retention policy.

virtual bool HasBlobs(const string &key, const string &subkey)

Check if any BLOB exists (any version)

bool DropBlobWithExpCheck(const string &key, int version, const string &subkey, CBDB_Transaction &trans)

Drop BLOB with time expiration check.

bool m_InitIfDirty

Delete DB if it wasn't closed clean.

virtual size_t GetSize(const string &key, int version, const string &subkey)

Check if BLOB exists, return BLOB size.

void PrintStatistics(CNcbiOstream &out) const

SCache_IdIDX * m_CacheIdIDX_RO

Cache id index r-only.

unsigned err_protocol

Protocol errors.

void KillBlob(const string &key, int version, const string &subkey, int overflow, unsigned blob_id)

const char * kBDBCacheDriverName

string m_LogDir

Transaction log location.

bool m_PurgeNowRunning

Flag that Purge is already running.

void WriteOverflow(CNcbiOfstream &overflow_file, const string &overflow_file_path, const char *buf, streamsize count)

Write to the overflow file with error checking If write failes, method deletes the file to avoid file...

shared_ptr< bool > m_StopThreadFlag

EBlobCheckinRes

Result of BLOB check-in process.

CBDB_Env * m_Env

Common environment for cache DBs.

unsigned m_LogSizeMax

transaction log size

virtual time_t GetAccessTime(const string &key, int version, const string &subkey)

Return last access time for the specified cache entry.

CBDB_FieldUint4 max_time

max ttl limit for BLOB

IWriter * GetWriteStream(unsigned blob_id_ext, const string &key, int version, const string &subkey, bool do_id_lock, unsigned int time_to_live=0, const string &owner=kEmptyStr)

IReader * x_CreateOverflowReader(const string &key, int version, const string &subkey, size_t &file_length, TBlobLock &blob_lock)

void StartPurgeThread()

Start background thread.

unsigned GetMaxBlobSize() const

Get max allowed BLOB size.

void x_DropOverflow(const string &key, int version, const string &subkey)

time_t x_ComputeExpTime(int time_stamp, unsigned ttl, int timeout)

bool x_RetrieveBlobAttributes(const string &key, int version, const string &subkey, int &overflow, unsigned int &ttl, unsigned int &blob_id, unsigned int &volume_id, unsigned int &split_id)

1.

unsigned GetBlobId(const string &key, int version, const string &subkey)

Determines BLOB size (requires fetched attribute record)

virtual void SetFlags(TFlags flags)

Pass flags to the underlying storage.

bool IsSaveStatistics(void) const

SBDB_CacheUnitStatistics m_GlobalStat

TBlobSizeHistogram blob_size_hist

Blob size historgam.

void AddCommError(EErrGetPut operation)

ELockMode

Locking modes to protect cache instance.

void x_DropBlob(const string &key, int version, const string &subkey, int overflow, unsigned blob_id, CBDB_Transaction &trans)

friend class CBDB_CacheIWriter

TTimeLine * m_TimeLine

BLOB expiration timeline.

unsigned blobs_db

Current database number of records.

void AddBlobQuotaError(const string &client)

void OpenReadOnly(const string &cache_path, const string &cache_name, unsigned int cache_ram_size=0)

Open local cache in read-only mode.

bool IsLocked(unsigned blob_id)

Check if BLOB is locked.

unsigned err_no_blob

BLOB not found errors.

CFastMutex m_IDIDX_Lock_RO

Database lock.

unsigned m_CheckPointInterval

Trnasaction checkpoint interval (bytes)

bool IsReadOnly() const

Return TRUE if cache is read-only.

const string & GetName() const

unsigned blobs_stored_total

Total number of blobs.

virtual void Purge(time_t access_timeout)

Delete all BLOBs older than specified.

unsigned blobs_expl_deleted_total

BLOBs explicitly removed.

SBDB_CacheUnitStatistics()

void AddCommError(const string &client, SBDB_CacheUnitStatistics::EErrGetPut operation)

void x_PerformCheckPointNoLock(unsigned bytes_written)

SCache_AttrDB * m_CacheAttrDB_RO2

Cache attributes r-only2.

void AddNoBlobError(const string &client, SBDB_CacheUnitStatistics::EErrGetPut operation)

virtual void SetTimeStampPolicy(TTimeStampFlags policy, unsigned int timeout, unsigned int max_timeout=0)

Set timestamp update policy.

void ConvertToRegistry(IRWRegistry *reg, const string &sect_name_postfix) const

Convert statistics into registry sections and entries.

unsigned m_Timeout

Timeout expiration policy.

CBDB_FieldUint4 read_count

read counter

unsigned err_blob_over_quota

BLOB max quota.

unsigned blobs_overflow_total

number of overflow blobs

TTimeAccess time_access

Hourly access history.

void Open(const string &cache_path, const string &cache_name, ELockMode lm=eNoLock, Uint8 cache_ram_size=0, ETRansact use_trans=eUseTrans, unsigned int log_mem_size=0)

Open local cache instance (read-write access) If cache does not exists it is created.

unsigned blob_size_max_total

Largest BLOB ever.

CacheKey(const string &x_key, int x_version, const string &x_subkey)

void x_UpdateAccessTime(const string &key, int version, const string &subkey, EBlobAccessType access_type, CBDB_Transaction &trans)

Transactional update of access time attributes.

void SetPurgeBatchSize(unsigned batch_size)

Number of records in scanned at once by Purge Cache is exclusively locks an internal mutex while scan...

TSplitStore * m_BLOB_SplitStore

Cache BLOB storage.

void PrintStatistics(CNcbiOstream &out) const

virtual TFlags GetFlags()

Retrieve the effective combination of flags from the underlying storage.

unsigned m_BatchSleep

Sleep interval (milliseconds) between batches.

virtual TTimeStampFlags GetTimeStampPolicy() const

Get timestamp policy.

bool x_CheckTimeStampExpired(SCache_AttrDB &attr_db, time_t curr, time_t *exp_time=0)

CBDB_FieldInt4 overflow

overflow flag

EKeepVersions m_VersionFlag

Version retention policy.

SBDB_CacheStatistics m_Statistics

Stat counters.

EBlobAccessType

access type for "UpdateAccessTime" methods

bool operator<(const CacheKey &cache_key) const

void BDB_ConfigureCache(CBDB_Cache &bdb_cache, const string &path, const string &name, unsigned timeout, ICache::TTimeStampFlags tflags)

Utility for simple cache configuration.

void StopPurgeThread()

Stop background thread.

unsigned err_internal

Internal errors of all sorts.

void AddProtocolError(EErrGetPut operation)

unsigned GetPurgeBatchSize() const

SCache_IdIDX * m_CacheIdIDX

Cache id index.

void SetBatchSleep(unsigned sleep)

Set sleep in milliseconds between Purge batches (for low priority Purge processes)

void AddRead(const string &client, time_t tm)

CRef< CCacheCleanerThread > m_PurgeThread

Purge thread.

void ConvertToRegistry(IRWRegistry *reg) const

unsigned blobs_updates_total

How many updates registered.

unsigned blobs_purge_deleted_total

BLOBs garbage collected.

unsigned GetOverflowLimit() const

double blobs_size_total

Size of BLOBs total.

void BDB_Register_Cache(void)

Register NCBI_BDB_ICacheEntryPoint.

unsigned m_RoundRobinVolumes

Number of rotated volumes.

void Verify(const string &cache_path, const string &cache_name, const string &err_file=0, bool force_remove=false)

Run verification of the cache database.

CAtomicCounter m_BlobIdCounter

Atomic counter for BLOB ids.

virtual bool IsOpen() const

string m_Path

Path to storage.

CBDB_FieldUint4 split_id

demux coord[1]

bool x_FetchBlobAttributes(const string &key, int version, const string &subkey)

void CleanLog()

Remove all non-active LOG files.

virtual EKeepVersions GetVersionRetention() const

Get version retention.

void Unlock()

Unlock cache access.

void AddPurgeDelete(const string &client)

virtual void GetBlobAccess(const string &key, int version, const string &subkey, SBlobAccessDescr *blob_descr)

Get BLOB access using BlobAccessDescr.

virtual int GetTimeout() const

Get expiration timeout.

CBDB_FieldUint4 time_stamp

access timestamp

void InitStatistics()

Drop the previously collected statistics.

void Lock()

Lock cache access.

string m_Name

Cache name.

CFastMutex m_TimeLine_Lock

static void AddToHistogram(TBlobSizeHistogram *hist, unsigned size)

unsigned m_PurgeThreadDelay

Delay in seconds between Purge runs.

deque< SBDB_TimeAccessStatistics > TTimeAccess

CBDB_CacheHolder(ICache *blob_cache, ICache *id_cache)

unsigned blobs_never_read_total

BLOBs never read before.

@ eErr_Unknown

no info on operation

@ eNoTrans

Non-transactional environment.

@ eUseTrans

Use transaction environment.

@ eBlobCheckIn_Create

If record does not exist - create.

@ eBlobCheckIn

Check if record exists.

@ eBlobCheckIn_Created

BLOB record created.

@ eBlobCheckIn_Found

Existing BLOB.

@ EBlobCheckIn_NotFound

BLOB does not exist.

@ eNoLock

Do not lock-protect cache instance.

@ ePidLock

Create PID lock on cache (exception if failed)

void OpenProjections()

Try to open all storage files in all projections This is only possible when object de-mux has fixed n...

EBDB_ErrCode Delete(unsigned id, CBDB_RawFile::EIgnoreError on_error=CBDB_RawFile::eThrowOnError)

Delete BLOB.

virtual void SetTransaction(ITransaction *trans)

Establish transaction association.

void FreeUnusedMem()

Reclaim unused memory.

EBDB_ErrCode ReadRealloc(unsigned id, CBDB_RawFile::TBuffer &buffer)

Read BLOB into vector.

EBDB_ErrCode UpdateInsert(unsigned id, const void *data, size_t size, unsigned *coord)

Update or insert BLOB.

void Save(typename TDeMuxStore::ECompact compact_vectors=TDeMuxStore::eCompact)

Save storage dictionary (demux disposition).

EBDB_ErrCode Fetch(unsigned id, void **buf, size_t buf_size, CBDB_RawFile::EReallocMode allow_realloc, size_t *blob_size)

Fetch LOB record directly into the provided '*buf'.

void SetEnv(CBDB_Env &env)

Associate with the environment. Should be called before opening.

EBDB_ErrCode BlobSize(unsigned id, size_t *blob_size)

Get size of the BLOB.

void GetIdVector(TBitVector *bv) const

Get all id of all BLOBs stored.

EBDB_ErrCode GetCoordinates(unsigned id, unsigned *coords)

Find (demux) coordinates by BLOB id.

void Open(const string &storage_name, CBDB_RawFile::EOpenMode open_mode, CBDB_RawFile::EDBType db_type=CBDB_RawFile::eBtree)

Open storage (reads storage dictionary into memory)

void AssignCoordinates(unsigned id, const unsigned *coords)

Assing de-mux coordinates.

unsigned int Truncate()

Empty the database. Return number of records removed.

EBDB_ErrCode Delete(EIgnoreError on_error=eThrowOnError)

Delete record corresponding to the current key value.

void SetCondition(ECondition cond_from, ECondition cond_to=eNotSet)

Set search condition(type of interval)

EBDB_ErrCode FetchFirst()

EBDB_ErrCode Update(CBDB_File::EAfterWrite write_flag=CBDB_File::eDiscardData)

void Verify(const char *filename, const char *database, FILE *backup)

Run database verification (DB->verify)

void Open(const string &filename, EOpenMode open_mode, bool support_dirty_read=false, unsigned rec_len=0)

Open file with specified access mode.

EBDB_ErrCode Insert(EAfterWrite write_flag=eDiscardData)

Insert new record.

unsigned CountRecs(bool bFast=false)

Compute database statistic, return number of records.

void SetEnv(CBDB_Env &env)

Associate file with environment.

virtual void SetTransaction(ITransaction *trans)

Establish transaction association.

void InitMultiFetch(size_t buffer_size, EMultiFetchMode mfm=eFetchAll)

Init multi-row fetch.

EBDB_ErrCode

BDB Return codes.

EBDB_ErrCode Fetch(EFetchDirection fdir=eDefault)

Fetch record.

CBDB_ConditionHandle From

EBDB_ErrCode Delete(CBDB_File::EIgnoreError on_error=CBDB_File::eThrowOnError)

EBDB_ErrCode Fetch()

Fetches the record corresponding to the current key value.

@ eReadWriteCreate

read-write, create if it doesn't exist

@ eReadModifyUpdate

Use DB_RMW (write locking) on fetch.

virtual string GetString() const

virtual void ToString(string &str) const

void BDB_RecoverEnv(const string &path, bool fatal_recover)

Run Berkeley DB recovery in private environment.

void SetCacheSize(Uint8 cache_size, int num_caches=1)

Set cache size for the environment.

virtual void Commit()

Commit transaction.

void CleanLog()

Remove all non-active log files.

bool CheckRemove()

Try to Remove the environment, if DB_ENV::remove returns 0, but fails files ramain on disk anyway cal...

void SetLockTimeout(unsigned timeout)

Set timeout value for locks in microseconds (1 000 000 in sec)

void SetLogInMemory(bool on_off)

Configure environment for non-durable in-memory logging.

void TransactionCheckpoint()

Flush the underlying memory pools, logs and data bases.

void SetMaxLocks(unsigned locks)

Set max number of locks in the database.

void StopBackgroundWriterThread()

Stop transaction checkpoint thread.

void SetLogAutoRemove(bool on_off)

If set, Berkeley DB will automatically remove log files that are no longer needed.

void SetLogDir(const string &log_dir)

Path to directory where transaction logs are to be stored By default it is the same directory as envi...

void SetLogFileMax(unsigned int lg_max)

Set maximum size of LOG files.

void RunBackgroundWriter(TBackgroundFlags flags, unsigned thread_delay=30, int memp_trickle=0, unsigned err_max=0)

Schedule background maintenance thread which will do:

void SetTransactionTimeout(unsigned timeout)

Set timeout value for transactions in microseconds (1 000 000 in sec)

#define BDB_THROW(errcode, message)

unsigned int TEnvOpenFlags

OR-ed combination of EEnvOptions.

unsigned TBackgroundFlags

Background work flags (combination of EBackgroundWork)

void OpenWithLocks(const string &db_home)

Open environment with database locking (DB_INIT_LOCK)

void SetTransactionSync(CBDB_Transaction::ETransSync sync)

Set default syncronicity level.

bool Remove()

Non-force removal of BDB environment.

void SetCheckPointKB(unsigned kb)

If the kb parameter is non-zero, a checkpoint will be done if more than kbyte kilobytes of log data h...

void ForceRemove()

Force remove BDB environment.

void OpenErrFile(const string &file_name)

Open error reporting file for the environment.

void Open(const string &db_home, int flags)

Open environment.

void OpenWithTrans(const string &db_home, TEnvOpenFlags opt=0)

Open environment using transaction.

bool IsTransactional() const

Return TRUE if environment has been open as transactional.

void ForceTransactionCheckpoint()

Forced checkpoint.

bool IsRecovery() const

If it is DB_RUNRECOVERY error.

void SetLogBSize(unsigned lg_bsize)

Set the size of the in-memory log buffer, in bytes.

void JoinEnv(const string &db_home, TEnvOpenFlags opt=0, ETransactionDiscovery trans_test=eTestTransactions)

Join the existing environment.

@ eTransASync

Non-durable asyncronous transaction.

@ eEnvDefault

Use default from CBDB_Env.

@ eTransSync

Syncronous transaction.

@ eBackground_DeadLockDetect

@ eBackground_MempTrickle

@ eNoAssociation

No association tracking.

@ eRunRecovery

Run DB recovery first.

@ eThreaded

corresponds to DB_THREAD

@ eDeadLock_Default

Default deadlock detector.

void Set(TValue new_value) THROWS_NONE

Set atomic counter value.

TValue Add(int delta) THROWS_NONE

Atomically add value (=delta), and return new counter value.

#define LOG_POST_X(err_subcode, message)

#define ERR_POST_X(err_subcode, message)

Error posting with default error code and given error subcode.

#define ERR_POST(message)

Error posting with file, line number information but without error codes.

#define NCBI_USER_THROW(message)

Throw a quick-and-dirty runtime exception of type 'CException' with the given error message and error...

void Error(CExceptionArgs_Base &args)

void Warning(CExceptionArgs_Base &args)

virtual const char * what(void) const noexcept

Standard report (includes full backlog).

void Info(CExceptionArgs_Base &args)

TEntries GetEntries(const string &mask=kEmptyStr, TGetEntriesFlags flags=0) const

Get directory entries based on the specified "mask".

static string CreateAbsolutePath(const string &path, ERelativeToWhat rtw=eRelativeToCwd)

Get an absolute path from some, possibly relative, path.

virtual bool Exists(void) const

Check the entry existence.

Int8 GetLength(void) const

Get size of file.

virtual bool Remove(TRemoveFlags flags=eRecursive) const

Remove a directory entry.

static string MakePath(const string &dir=kEmptyStr, const string &base=kEmptyStr, const string &ext=kEmptyStr)

Assemble a path from basic components.

static string AddTrailingPathSeparator(const string &path)

Add trailing path separator, if needed.

virtual bool Exists(void) const

Check if directory "dirname" exists.

virtual ERW_Result Write(const void *buf, size_t count, size_t *bytes_written=0)

Virtual methods from IWriter.

bool Create(TCreateFlags flags=fCreate_Default) const

Create the directory using "dirname" passed in the constructor.

virtual bool Remove(TRemoveFlags flags=eRecursive) const

Delete existing directory.

const string & GetPath(void) const

Get entry path.

virtual bool Exists(void) const

Check existence of file.

void Reset(void)

Reset reference object.

bool Empty(void) const THROWS_NONE

Check if CRef is empty – not pointing to any object, which means having a null value.

static void NCBI_EntryPointImpl(TDriverInfoList &info_list, EEntryPointRequest method)

Entry point implementation.

#define NCBI_INTERFACE_VERSION(iface)

Macro to construct CVersionInfo class using interface name (relies on CInterfaceVersion class)

list< SDriverInfo > TDriverInfoList

List of driver information.

EEntryPointRequest

Actions performed by the entry point.

uint32_t Uint4

4-byte (32-bit) unsigned integer

uint64_t Uint8

8-byte (64-bit) unsigned integer

static TPid GetPid(void)

Get process identifier (pid) for the current process.

bool Set(const string &section, const string &name, const string &value, TFlags flags=0, const string &comment=kEmptyStr)

Set the configuration parameter value.

#define END_NCBI_SCOPE

End previously defined NCBI scope.

#define BEGIN_NCBI_SCOPE

Define ncbi namespace.

ERW_Result

Result codes for I/O operations.

IO_PREFIX::ofstream CNcbiOfstream

Portable alias for ofstream.

IO_PREFIX::ostream CNcbiOstream

Portable alias for ostream.

IO_PREFIX::ifstream CNcbiIfstream

Portable alias for ifstream.

IO_PREFIX::streampos CNcbiStreampos

Portable alias for streampos.

@ eRW_Eof

End of data, should be considered permanent.

@ eRW_Error

Unrecoverable error, no retry possible.

@ eRW_Success

Everything is okay, I/O completed.

static string SizetToString(size_t value, TNumToStringFlags flags=0, int base=10)

Convert size_t to string.

static string DoubleToString(double value, int precision=-1, TNumToStringFlags flags=0)

Convert double to string.

static int CompareNocase(const CTempString s1, SIZE_TYPE pos, SIZE_TYPE n, const char *s2)

Case-insensitive compare of a substring with another string.

static string IntToString(int value, TNumToStringFlags flags=0, int base=10)

Convert int to string.

static string & Replace(const string &src, const string &search, const string &replace, string &dst, SIZE_TYPE start_pos=0, SIZE_TYPE max_replace=0, SIZE_TYPE *num_replace=0)

Replace occurrences of a substring within a string.

static int Compare(const CTempString s1, SIZE_TYPE pos, SIZE_TYPE n, const char *s2, ECase use_case=eCase)

Compare of a substring with another string.

static string UIntToString(unsigned int value, TNumToStringFlags flags=0, int base=10)

Convert UInt to string.

static string ULongToString(unsigned long value, TNumToStringFlags flags=0, int base=10)

Convert unsigned long to string.

static enable_if< is_arithmetic< TNumeric >::value||is_convertible< TNumeric, Int8 >::value, string >::type NumericToString(TNumeric value, TNumToStringFlags flags=0, int base=10)

Convert numeric value to string.

virtual bool IsActive()=0

Check if monitoring is active.

virtual void Send(const char *msg, size_t length)=0

Send message.

bool Run(TRunMode flags=fRunDefault)

Run the thread.

bool TryLock(unsigned id)

Try to acquire lock of specified id.

void TakeFrom(CLockVectorGuard &lg)

Transfer lock ownership from another lock.

void Lock(void)

Acquire mutex for the current thread with no nesting checks.

void FreeUnusedMem()

Reclaim unused memory.

void Lock(unsigned id)

Acquire lock.

unsigned GetId() const

Get BLOB id.

bool IsLocked(unsigned id) const

Check if id is locked or not.

void Unlock()

Unlocks the lock.

void SetId(unsigned id)

Assign Id (no locking)

bool TryWait(unsigned int timeout_sec=0, unsigned int timeout_nsec=0)

Timed wait.

void Post(unsigned int count=1)

Increment the semaphore by "count".

void Join(void **exit_data=0)

Wait for the thread termination.

void Unlock(void)

Release mutex with no owner or nesting checks.

double Elapsed(void) const

Return time elapsed since first Start() or last Restart() call (in seconds).

string AsString(const CTimeFormat &format=kEmptyStr, TSeconds out_tz=eCurrentTimeZone) const

Transform time to string.

int GetLocalTimezone(void)

Get difference in seconds between UTC and current local time (daylight information included)

CTime GetFastLocalTime(void)

Quick and dirty getter of local time.

@ eStart

Start timer immediately after creating.

const TTreeType * FindNode(const TKeyType &key, TNodeSearchMode sflag=eImmediateAndTop) const

Search for node.

const TValue & GetValue(void) const

Return node's value.

@ eNonCompatible

major, minor does not match

operation

Bit operations.

unsigned int

A callback function used to compare two keys in a database.

Definition of all error codes used in bdb library (bdb.lib and ncbi_xcache_bdb.lib).

if(yy_accept[yy_current_state])

const string version

version string

const struct ncbi::grid::netcache::search::fields::SIZE size

const struct ncbi::grid::netcache::search::fields::KEY key

const struct ncbi::grid::netcache::search::fields::SUBKEY subkey

Defines process management classes.

Defines NCBI C++ exception handling.

Defines classes: CDirEntry, CFile, CDir, CSymLink, CMemoryFile, CFileUtil, CFileLock,...

Multi-threading – mutexes; rw-locks; semaphore.

Defines: CTimeFormat - storage class for time format.

std::istream & in(std::istream &in_, double &x_)

Int4 delta(size_t dimension_, const Int4 *score_)

Helper classes and templates to implement plugins.

static SLJIT_INLINE sljit_ins msg(sljit_gpr r, sljit_s32 d, sljit_gpr x, sljit_gpr b)

static CNamedPipeClient * client

Cache accession for internal in-memory storage.

unique_ptr< IReader > reader

BDB cache statistics (covers one elemntary unit of measurements)

BDB cache access statistics.

SCacheDescr(string x_key, int x_version, string x_subkey, int x_overflow, unsigned x_blob_id)

BLOB ID -> key, value, subkey index.


RetroSearch is an open source project built by @garambo | Open a GitHub Issue

Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo

HTML: 3.2 | Encoding: UTF-8 | Version: 0.7.4