A RetroSearch Logo

Home - News ( United States | United Kingdom | Italy | Germany ) - Football scores

Search Query:

Showing content from http://www.ncbi.nlm.nih.gov/IEB/ToolBox/CPP_DOC/doxyhtml/bmsparsevec__serial_8h_source.html below:

NCBI C++ ToolKit: include/util/bitset/bmsparsevec_serial.h Source File

1 #ifndef BMSPARSEVEC_SERIAL__H__INCLUDED__ 2 #define BMSPARSEVEC_SERIAL__H__INCLUDED__ 26 #ifndef BM__H__INCLUDED__ 29 # error missing include (bm.h or bm64.h) 41 template

<

class

BV,

typename

TOut>

44

tout << bv.count() <<

": "

;

45  typename

BV::enumerator en = bv.first();

46  for

(; en.valid(); ++en)

189 template

<

typename

SV>

340 template

<

typename

SV>

386  const unsigned char

*

buf

,

387  bool

clear_sv =

true

);

401  bool

clear_sv =

true

);

425  const unsigned char

*

buf

,

438  const unsigned char

*

buf

);

458  const unsigned char

*

buf

,

467  const unsigned char

*

buf

,

471  void load_remap

(SV& sv,

const unsigned char

* remap_buf_ptr);

549

sv_serializer.

serialize

(sv, sv_layout);

567  const unsigned char

*

buf

,

581 template

<

class

CBC>

600 template

<

class

CBC>

614  const unsigned char

*

buf

,

652 template

<

class

CBC>

658

buffer_coll.calc_stat(&

st

);

660  buf

.resize(

st

.max_serialize_mem);

663  unsigned char

* buf_ptr =

buf

.data();

669

enc.

put_8

((

unsigned char

)bo);

671  unsigned char

* mbuf1 = enc.

get_pos

();

682  size_t

addr_bv_size = bvs.

serialize

(bv, buf_ptr,

buf

.size());

683

buf_ptr += addr_bv_size;

686

enc.

put_64

(addr_bv_size);

689  size_t

coll_size = buffer_coll.size();

695  for

(

unsigned i

= 0;

i

< buffer_coll.size(); ++

i

)

698  size_t

sz = cbuf.size();

704  for

(

unsigned i

= 0;

i

< buffer_coll.size(); ++

i

)

707  size_t

sz = cbuf.size();

708

enc.

memcpy

(cbuf.buf(), sz);

715 template

<

class

CBC>

718  const unsigned char

*

buf

,

725  unsigned char

h1 = dec.

get_8

();

726  unsigned char

h2 = dec.

get_8

();

729  if

(h1 !=

'B'

&& h2 !=

'C'

)

741  const unsigned char

* bv_buf_ptr = dec.

get_pos

();

750  typename

bvector_type::size_type addr_cnt = bv.count();

751

dec.

seek

((

int

)addr_bv_size);

757  if

(coll_size != addr_cnt)

762  typedef size_t

vect_size_type;

765

buf_size_vec.

resize

(vect_size_type(coll_size));

767  for

(

unsigned i

= 0;

i

< coll_size; ++

i

)

770

buf_size_vec[

i

] = sz;

776

buf_vect.

resize

(vect_size_type(coll_size));

777  for

(

unsigned i

= 0;

i

< coll_size; ++

i

)

782

dec.

memcpy

(

b

.data(),

size_t

(sz));

793 template

<

typename

SV>

807 template

<

typename

SV>

811

bv_ref_ptr_ = bv_ref_ptr;

812

is_xor_ref_ =

bool

(bv_ref_ptr);

818 template

<

typename

SV>

822

is_xor_ref_ = is_enabled;

827 template

<

typename

SV>

833

bvs_.compute_sim_model(sim_model,

ref_vect

, params);

838 template

<

typename

SV>

842

sim_model_ptr_ = sim_model;

847 template

<

typename

SV>

851

bv_ref_.build(sv.get_bmatrix());

856 template

<

typename

SV>

860  const typename

SV::remap_matrix_type* rm = sv.get_remap_matrix();

865  size_t

rows = rmatr.rows();

866  size_t

cols = rmatr.cols();

872

remap_rlen_vect_.resize(0);

873  for

(

size_t r

= 0;

r

< rows; ++

r

)

875  const unsigned char

*

BMRESTRICT

remap_row = rmatr.row(

r

);

879

remap_rlen_vect_.push_back(

unsigned

(

cnt

));

882

rows = remap_rlen_vect_.size();

885  for

(

size_t r

= 0;

r

< rows; ++

r

)

887  unsigned

rl = remap_rlen_vect_[

r

];

888

csr_size_max += rl * 2;

891  size_t

remap_size = sv.remap_size();

893  if

(remap_size < csr_size_max)

895  const unsigned char

* matrix_buf = sv.get_remap_buffer();

901

enc.

memcpy

(matrix_buf,

size_t

(remap_size));

906

enc.

put_32

(

unsigned

(rows));

911  for

(

size_t r

= 0;

r

< rows; ++

r

)

913  unsigned

rl = remap_rlen_vect_[

r

];

918  for

(

size_t r

= 0;

r

< rows; ++

r

)

921  for

(

size_t

j = 0; j < cols; ++j)

923  unsigned char

v =

row

[j];

926

enc.

put_8

((

unsigned char

)j);

938 template

<

typename

SV>

942

bvs_.allow_stat_reset(

false

);

943

bvs_.reset_compression_stats();

948  buf

[0]=

'B'

;

buf

[1] =

'Z'

;

953  const auto

& bmatr = sv.get_bmatrix();

955

bmatr.build_plane_digest(plane_digest_bv_);

956

plane_digest_bv_.optimize();

960

bvs_.set_ref_vectors(0);

961

bvs_.serialize(plane_digest_bv_, plane_digest_buf_);

963  unsigned

planes = (unsigned)bmatr.rows();

969  typename

SV::statistics sv_stat;

970

sv.calc_stat(&sv_stat);

971

sv_stat.max_serialize_mem += plane_digest_buf_.size() + (8 * planes);

972  unsigned char

*

buf

= sv_layout.

reserve

(sv_stat.max_serialize_mem);

979  unsigned

h_size = 1 + 1 +

1000

bvs_.set_ref_vectors(bv_ref_ptr_);

1001

bvs_.set_sim_model(sim_model_ptr_);

1006

build_xor_ref_vector(sv);

1007

bvs_.set_ref_vectors(&bv_ref_);

1008  if

(bvs_.compute_sim_model(sim_model_, bv_ref_, xs_params))

1009

bvs_.set_sim_model(&sim_model_);

1017

::memset(

buf

, 0, h_size);

1018  unsigned char

* buf_ptr =

buf

+ h_size;

1020  for

(

unsigned i

= 0;

i

< planes; ++

i

)

1022  typename

SV::bvector_type_const_ptr bv = bmatr.row(

i

);

1034

idx = (unsigned)bv_ref_ptr_->find_bv(bv);

1036

idx = (unsigned)bv_ref_.find_bv(bv);

1038

bvs_.set_curr_ref_idx(idx);

1040  size_t

buf_size = (size_t)

1041

bvs_.serialize(*bv, buf_ptr, sv_stat.max_serialize_mem);

1043

sv_layout.

set_plane

(

i

, buf_ptr, buf_size);

1044

buf_ptr += buf_size;

1045  if

(sv_stat.max_serialize_mem > buf_size)

1047

sv_stat.max_serialize_mem -= buf_size;

1053

bvs_.set_ref_vectors(0);

1060  bm::encoder

enc_m(buf_ptr, sv_stat.max_serialize_mem);

1062

encode_remap_matrix(enc_m, sv);

1064

enc_m.

put_8

(

'N'

);

1065

buf_ptr += enc_m.

size

();

1071  size_t

digest_offset = size_t(buf_ptr -

buf

);

1072  auto

pd_size = plane_digest_buf_.size();

1073

::memcpy(buf_ptr, plane_digest_buf_.buf(), pd_size);

1076  bool

use_64bit =

false

;

1077

plane_off_vect_.resize(0);

1078  for

(

unsigned i

= 0;

i

< planes; ++

i

)

1080  const unsigned char

* p = sv_layout.

get_plane

(

i

);

1089

plane_off_vect_.push_back(

unsigned

(

offset

));

1092  bm::encoder

enc_o(buf_ptr, sv_stat.max_serialize_mem);

1093  if

(use_64bit || (plane_off_vect_.size() < 4))

1095

enc_o.

put_8

(

'6'

);

1098  for

(

unsigned i

= 0;

i

< planes; ++

i

)

1100  const unsigned char

* p = sv_layout.

get_plane

(

i

);

1110  BM_ASSERT

(plane_off_vect_.size() == plane_digest_bv_.count());

1111  unsigned

min_v = plane_off_vect_[0];

1112  unsigned

max_v = plane_off_vect_[plane_off_vect_.size()-1];

1114

enc_o.

put_8

(

'3'

);

1121  unsigned

(plane_off_vect_.size()-2),

1126

buf_ptr += enc_o.

size

();

1131

sv_layout.

shrink

(

size_t

(buf_ptr -

buf

));

1139  if

(sv.is_compressed())

1144

enc.

put_8

((

unsigned char

)bo);

1146  unsigned char

matr_s_ser = 1;

1152

enc.

put_8

(matr_s_ser);

1154  bm::id64_t

planes_code = planes | (1ull << 63);

1155

enc.

put_64

(planes_code);

1157

enc.

put_64

(sv.size_internal());

1165 template

<

typename

SV>

1168

temp_block_ = alloc_.alloc_bit_block();

1169

not_null_mask_bv_.set_allocator_pool(&pool_);

1170

rsc_mask_bv_.set_allocator_pool(&pool_);

1175 template

<

typename

SV>

1179

alloc_.free_bit_block(temp_block_);

1184 template

<

typename

SV>

1188

this->is_final_ = is_final;

1193 template

<

typename

SV>

1197

bv_ref_ptr_ = bv_ref_ptr;

1199

clear_xor_compression();

1204 template

<

typename

SV>

1207

op_deserial_.set_ref_vectors(0);

1208

deserial_.set_ref_vectors(0);

1214 template

<

typename

SV>

1219

op_deserial_.set_ref_vectors(bv_ref_ptr_);

1220

deserial_.set_ref_vectors(bv_ref_ptr_);

1224

op_deserial_.set_ref_vectors(&bv_ref_);

1225

deserial_.set_ref_vectors(&bv_ref_);

1231 template

<

typename

SV>

1233  const unsigned char

*

buf

,

1238

idx_range_set_ =

false

;

1239

deserialize_sv(sv,

buf

, 0, clear_sv);

1244 template

<

typename

SV>

1246  const unsigned char

*

buf

)

1250  unsigned char

matr_s_ser = 0;

1251  unsigned

planes = load_header(dec, sv, matr_s_ser);

1255

load_planes_off_table(

buf

, dec, planes);

1257

sv.get_bmatrix().allocate_rows(planes);

1259  for

(

unsigned i

= 0;

i

< planes; ++

i

)

1261  if

(!off_vect_[

i

])

1270 template

<

typename

SV>

1272  const unsigned char

*

buf

,

1278

sv.clear_all(

true

, 1);

1280

idx_range_set_ =

true

; idx_range_from_ = from; idx_range_to_ = to;

1285  unsigned char

matr_s_ser = 0;

1286  unsigned

planes = load_header(dec, sv, matr_s_ser);

1291

sv.resize_internal(

size_type

(sv_size_));

1294

load_planes_off_table(

buf

, dec, planes);

1296

setup_xor_compression();

1298

sv.get_bmatrix().allocate_rows(planes);

1301

planes = (unsigned)load_null_plane(sv,

int

(planes),

buf

, 0);

1308  bool

range_valid = sv.resolve_range(from, to, &sv_left, &sv_right);

1312

idx_range_set_ =

false

;

1317

idx_range_set_ =

true

; idx_range_from_ = sv_left; idx_range_to_ = sv_right;

1321

deserialize_planes(sv, planes,

buf

, 0);

1323

clear_xor_compression();

1330

load_remap(sv, remap_buf_ptr_);

1333

sv.sync(

true

,

true

);

1337

idx_range_set_ =

false

;

1342 template

<

typename

SV>

1344  const unsigned char

*

buf

,

1349

sv.clear_all(

true

, 1);

1354  unsigned char

matr_s_ser = 0;

1355  unsigned

planes = load_header(dec, sv, matr_s_ser);

1359

sv.resize_internal(

size_type

(sv_size_));

1362

load_planes_off_table(

buf

, dec, planes);

1364

setup_xor_compression();

1366

sv.get_bmatrix().allocate_rows(planes);

1367

planes = (unsigned)load_null_plane(sv,

int

(planes),

buf

, mask_bv);

1377

rsc_mask_bv_.clear(

true

);

1378

not_null_mask_bv_.bit_and(*bv_null, *mask_bv, bvector_type::opt_compress);

1379

rsc_compressor_.compress(rsc_mask_bv_, *bv_null, not_null_mask_bv_);

1380

mask_bv = &rsc_mask_bv_;

1386

rsc_mask_bv_.find_range(idx_range_from_, idx_range_to_);

1391

deserialize_planes(sv, planes,

buf

, mask_bv);

1395 #pragma warning( push ) 1396 #pragma warning( disable : 4127) 1398  if

(sv.max_vector_size == 1)

1402

sv.get_bmatrix().get_row(sv.sv_value_slices);

1404

sv.mark_null_idx(sv.sv_value_slices);

1407 #pragma warning( pop ) 1411

clear_xor_compression();

1418

load_remap(sv, remap_buf_ptr_);

1421

sv.sync(

true

,

true

);

1427 template

<

typename

SV>

1429  bm::decoder

& dec, SV& sv,

unsigned char

& matr_s_ser)

1433  unsigned char

h1 = dec.

get_8

();

1434  unsigned char

h2 = dec.

get_8

();

1436  BM_ASSERT

(h1 ==

'B'

&& (h2 ==

'M'

|| h2 ==

'C'

|| h2 ==

'Z'

));

1438  bool

sig2_ok = (h2 ==

'M'

|| h2 ==

'C'

|| h2 ==

'Z'

);

1439  if

(h1 !=

'B'

|| !sig2_ok)

1440

raise_invalid_header();

1441  unsigned

planes = 0;

1448  unsigned char

bv_bo = dec.

get_8

(); (void) bv_bo;

1449

planes = dec.

get_8

();

1452

matr_s_ser = dec.

get_8

();

1453

planes_code = dec.

get_64

();

1454

planes = (unsigned) planes_code;

1458  if

(matr_s_ser == 2)

1459

raise_invalid_64bit();

1464  unsigned

sv_planes = sv.stored_slices();

1465  if

(!planes || planes > sv_planes)

1466

raise_invalid_bitdepth();

1469

sv_size_ = dec.

get_64

();

1472  if

(planes_code & (1ull << 63))

1474

digest_offset_ = dec.

get_64

();

1482 template

<

typename

SV>

1486  const unsigned char

*

buf

,

1489  if

(mask_bv && !idx_range_set_)

1490

idx_range_set_ = mask_bv->find_range(idx_range_from_, idx_range_to_);

1495  for

(

int i

=

int

(planes-1);

i

>= 0; --

i

)

1497  size_t offset

= off_vect_[unsigned(

i

)];

1500  const unsigned char

* bv_buf_ptr =

buf

+

offset

;

1506

bv_ref_.add(bv,

unsigned

(

i

));

1509  typename

bvector_type::mem_pool_guard mp_g_z(pool_, *bv);

1514

deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);

1515

remap_buf_ptr_ = bv_buf_ptr + read_bytes;

1516

bv->bit_and(*mask_bv, bvector_type::opt_compress);

1521

deserial_.set_range(idx_range_from_, idx_range_to_);

1522

deserial_.deserialize(*bv, bv_buf_ptr);

1523

bv->bit_and(*mask_bv, bvector_type::opt_compress);

1532

deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);

1533

remap_buf_ptr_ = bv_buf_ptr + read_bytes;

1535

bv->keep_range(idx_range_from_, idx_range_to_);

1541

deserial_.set_range(idx_range_from_, idx_range_to_);

1542

deserial_.deserialize(*bv, bv_buf_ptr);

1543

bv->keep_range(idx_range_from_, idx_range_to_);

1548

deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);

1566

sv.set_ro_flag(

true

);

1572

deserial_.unset_range();

1578 template

<

typename

SV>

1581  const unsigned char

*

buf

,

1585  if

(!sv.is_nullable())

1588  size_t offset

= off_vect_[unsigned(

i

)];

1594  const unsigned char

* bv_buf_ptr =

buf

+

offset

;

1595  bvector_type

* bv = sv.get_bmatrix().get_row(

unsigned

(

i

));

1598

bv_ref_.add(bv,

unsigned

(

i

));

1604  size_t

read_bytes = deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);

1605

remap_buf_ptr_ = bv_buf_ptr + read_bytes;

1614  size_t

read_bytes = deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);

1615

remap_buf_ptr_ = bv_buf_ptr + read_bytes;

1617

bv->keep_range(idx_range_from_, idx_range_to_);

1622

deserial_.set_range(idx_range_from_, idx_range_to_);

1623

deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);

1624

bv->keep_range(idx_range_from_, idx_range_to_);

1625

deserial_.unset_range();

1629

deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);

1632

bv->bit_and(*mask_bv, bvector_type::opt_compress);

1650 template

<

typename

SV>

1654

off_vect_.resize(planes);

1657

plane_digest_bv_.clear(

false

);

1658  const unsigned char

* buf_ptr =

buf

+ digest_offset_;

1660

deserial_.deserialize(plane_digest_bv_, buf_ptr, temp_block_);

1661

buf_ptr += read_bytes;

1665  unsigned char

dtype = dec_o.

get_8

();

1669  for

(

unsigned i

= 0;

i

< planes; ++

i

)

1672  if

(plane_digest_bv_.test(

i

))

1683  unsigned

osize = (unsigned)plane_digest_bv_.count();

1685

off32_vect_.resize(osize);

1687  unsigned

min_v = dec_o.

get_32

();

1688  unsigned

max_v = dec_o.

get_32

();

1690

off32_vect_[0] = min_v;

1691

off32_vect_[osize-1] = max_v;

1697  for

(

unsigned i

= 0;

i

< planes; ++

i

)

1699  if

(plane_digest_bv_.test(

i

))

1701

off_vect_[

i

] = off32_vect_[k];

1716  for

(

unsigned i

= 0;

i

< planes; ++

i

)

1726 template

<

typename

SV>

1728  const unsigned char

* remap_buf_ptr)

1735  unsigned char

rh = dec_m.

get_8

();

1742  size_t

remap_size = (size_t) dec_m.

get_64

();

1743  unsigned char

* remap_buf = sv.init_remap_buffer();

1745  size_t

target_remap_size = sv.remap_size();

1746  if

(!remap_size || !remap_buf || remap_size != target_remap_size)

1748

raise_invalid_format();

1750

dec_m.

memcpy

(remap_buf, remap_size);

1757  typename

SV::remap_matrix_type* rmatr = sv.get_remap_matrix();

1760

raise_missing_remap_matrix();

1762  size_t

rows = (size_t) dec_m.

get_32

();

1763  size_t

cols = dec_m.

get_16

();

1766

raise_invalid_format();

1768

rmatr->resize(rows, cols,

false

);

1774

remap_rlen_vect_.resize(0);

1777  for

(

size_t r

= 0;

r

< rows; ++

r

)

1779  unsigned

rl = bi.

gamma

();

1780

remap_rlen_vect_.push_back(rl);

1784  for

(

size_t r

= 0;

r

< rows; ++

r

)

1787  size_t cnt

= remap_rlen_vect_[

r

];

1790

raise_invalid_format();

1792  for

(

size_t

j = 0; j <

cnt

; ++j)

1794  unsigned

idx = dec_m.

get_8

();

1795  unsigned char

v = dec_m.

get_8

();

1804

raise_invalid_format();

1809  unsigned char

end_tok = dec_m.

get_8

();

1810  if

(end_tok !=

'E'

)

1812

raise_invalid_format();

1819 template

<

typename

SV>

1823  throw

std::logic_error(

"BitMagic: Invalid serialization signature header"

);

1825

BM_THROW(BM_ERR_SERIALFORMAT);

1831 template

<

typename

SV>

1835  throw

std::logic_error(

"BitMagic: Invalid serialization target (64-bit BLOB)"

);

1837

BM_THROW(BM_ERR_SERIALFORMAT);

1843 template

<

typename

SV>

1847  throw

std::logic_error(

"BitMagic: Invalid serialization target (bit depth)"

);

1849

BM_THROW(BM_ERR_SERIALFORMAT);

1855 template

<

typename

SV>

1859  throw

std::logic_error(

"BitMagic: Invalid serialization fromat (BLOB corruption?)"

);

1861

BM_THROW(BM_ERR_SERIALFORMAT);

1867 template

<

typename

SV>

1871  throw

std::logic_error(

"BitMagic: Invalid serialization format (remap matrix)"

);

1873

BM_THROW(BM_ERR_SERIALFORMAT);

Serialization / compression of bvector<>. Set theoretical operations on compressed BLOBs.

Sparse constainer sparse_vector<> for integer types using bit-transposition transform.

Byte based reader for un-aligned bit streaming.

unsigned gamma() noexcept

decode unsigned value using Elias Gamma coding

void bic_decode_u32_cm(bm::word_t *arr, unsigned sz, bm::word_t lo, bm::word_t hi) noexcept

Binary Interpolative array decode (32-bit)

Byte based writer for un-aligned bit streaming.

void gamma(unsigned value) noexcept

Elias Gamma encode the specified value.

void bic_encode_u32_cm(const bm::word_t *arr, unsigned sz, bm::word_t lo, bm::word_t hi) noexcept

Binary Interpolative encoding (array of 32-bit ints) cm - "center-minimal".

void flush() noexcept

Flush the incomplete 32-bit accumulator word.

List of reference bit-vectors with their true index associations.

Deseriaizer for compressed collections.

bvector_type::allocator_type allocator_type

CBC::buffer_type buffer_type

CBC::bvector_type bvector_type

CBC::statistics statistics_type

CBC::container_type container_type

CBC compressed_collection_type

int deserialize(CBC &buffer_coll, const unsigned char *buf, bm::word_t *temp_block=0)

CBC::address_resolver_type address_resolver_type

Seriaizer for compressed collections.

void serialize(const CBC &buffer_coll, buffer_type &buf, bm::word_t *temp_block=0)

Serialize compressed collection into memory buffer.

CBC compressed_collection_type

CBC::bvector_type bvector_type

CBC::statistics statistics_type

CBC::address_resolver_type address_resolver_type

CBC::buffer_type buffer_type

const unsigned char * get_pos() const noexcept

Return current buffer pointer.

unsigned char get_8() noexcept

Reads character from the decoding buffer.

void memcpy(unsigned char *dst, size_t count) noexcept

read bytes from the decode buffer

void seek(int delta) noexcept

change current position

Class for decoding data from memory buffer.

bm::id64_t get_64() noexcept

Reads 64-bit word from the decoding buffer.

bm::short_t get_16() noexcept

Reads 16-bit word from the decoding buffer.

bm::word_t get_32() noexcept

Reads 32-bit word from the decoding buffer.

void put_16(bm::short_t s) noexcept

Puts short word (16 bits) into the encoding buffer.

void put_32(bm::word_t w) noexcept

Puts 32 bits word into encoding buffer.

void put_64(bm::id64_t w) noexcept

Puts 64 bits word into encoding buffer.

size_t size() const noexcept

Returns size of the current encoding stream.

void put_8(unsigned char c) noexcept

Puts one character into the encoding buffer.

void set_pos(unsigned char *buf_pos) noexcept

Set current memory stream position.

void memcpy(const unsigned char *src, size_t count) noexcept

copy bytes into target buffer or just rewind if src is NULL

unsigned char * get_pos() const noexcept

Get current memory stream position.

void resize(size_type new_size)

vector resize

Bit-vector serialization class.

void set_bookmarks(bool enable, unsigned bm_interval=256) noexcept

Add skip-markers to serialization BLOB for faster range decode at the expense of some BLOB size incre...

void gap_length_serialization(bool value) noexcept

Set GAP length serialization (serializes GAP levels of the original vector)

size_type serialize(const BV &bv, unsigned char *buf, size_t buf_size)

Bitvector serialization into memory block.

sparse vector de-serializer

sparse_vector_deserializer()

void deserialize_planes(SV &sv, unsigned planes, const unsigned char *buf, const bvector_type *mask_bv=0)

deserialize bit-vector planes

void setup_xor_compression()

setup deserializers

allocator_pool_type pool_

bm::operation_deserializer< bvector_type > op_deserial_

void deserialize(SV &sv, const unsigned char *buf, bool clear_sv=true)

bm::serializer< bvector_type >::bv_ref_vector_type bv_ref_vector_type

void deserialize(SV &sv, const unsigned char *buf, const bvector_type &mask_bv)

bm::rank_compressor< bvector_type > rsc_compressor_

void deserialize_structure(SV &sv, const unsigned char *buf)

bm::id64_t digest_offset_

void deserialize_sv(SV &sv, const unsigned char *buf, const bvector_type *mask_bv, bool clear_sv)

static void raise_invalid_bitdepth()

throw error on incorrect deserialization

static void raise_invalid_header()

throw error on incorrect deserialization

SV::bvector_type bvector_type

size_type idx_range_from_

bvector_type not_null_mask_bv_

bvector_type::allocator_type::allocator_pool_type allocator_pool_type

bvector_type::allocator_type alloc_type

static void raise_invalid_format()

throw error on incorrect deserialization

unsigned load_header(bm::decoder &dec, SV &sv, unsigned char &matr_s_ser)

Deserialize header/version and other common info.

void deserialize(SV &sv, const unsigned char *buf, size_type from, size_type to)

bm::heap_vector< size_t, alloc_type, true > off_vect_

~sparse_vector_deserializer()

bm::heap_vector< unsigned, alloc_type, true > off32_vect_

bvector_type * bvector_type_ptr

static void raise_invalid_64bit()

throw error on incorrect deserialization

bvector_type plane_digest_bv_

static void raise_missing_remap_matrix()

throw error on incorrect deserialization

sparse_vector_deserializer(const sparse_vector_deserializer &)=delete

const bvector_type * bvector_type_const_ptr

void set_xor_ref(bv_ref_vector_type *bv_ref_ptr)

Set external XOR reference vectors (data frame referenece vectors)

bv_ref_vector_type * bv_ref_ptr_

external ref bit-vect

rlen_vector_type remap_rlen_vect_

void set_finalization(bm::finalization is_final)

Set deserialization finalization to force deserialized vectors into READONLY (or READWRITE) mode.

bvector_type rsc_mask_bv_

bm::deserializer< bvector_type, bm::decoder > deserial_

const unsigned char * remap_buf_ptr_

bv_ref_vector_type bv_ref_

reference vector

sparse_vector_deserializer & operator=(const sparse_vector_deserializer &)=delete

int load_null_plane(SV &sv, int planes, const unsigned char *buf, const bvector_type *mask_bv)

load NULL bit-plane (returns new planes count)

void deserialize_range(SV &sv, const unsigned char *buf, size_type from, size_type to, bool clear_sv=true)

void load_planes_off_table(const unsigned char *buf, bm::decoder &dec, unsigned planes)

load offset table

void clear_xor_compression()

unset XOR compression vectors

void load_remap(SV &sv, const unsigned char *remap_buf_ptr)

load string remap dict

bm::finalization is_final_

bm::heap_vector< unsigned, alloc_type, true > rlen_vector_type

SV::value_type value_type

Serialize sparse vector into a memory buffer(s) structure.

sparse_vector_serializer()

bvector_type plane_digest_bv_

bv.digest of bit-planes

u32_vector_type remap_rlen_vect_

void set_bookmarks(bool enable, unsigned bm_interval=256) noexcept

Add skip-markers for faster range deserialization.

void encode_remap_matrix(bm::encoder &enc, const SV &sv)

serialize the remap matrix used for SV encoding

sparse_vector_serializer & operator=(const sparse_vector_serializer &)=delete

serializer_type::buffer buffer_type

void set_sim_model(const xor_sim_model_type *sim_model) noexcept

Attach serizalizer to a pre-computed similarity model.

bm::serializer< bvector_type >::bv_ref_vector_type bv_ref_vector_type

bool is_xor_ref() const noexcept

Returns the XOR reference compression status (enabled/disabled)

const bv_ref_vector_type * bv_ref_ptr_

bvector_type * bvector_type_ptr

void build_xor_ref_vector(const SV &sv)

SV::value_type value_type

const bvector_type * bvector_type_const_ptr

void compute_sim_model(xor_sim_model_type &sim_model, const bv_ref_vector_type &ref_vect, const bm::xor_sim_params &params)

Calculate XOR similarity model for ref_vector refernece vector must be associated before.

alloc_type::allocator_pool_type allocator_pool_type

buffer_type plane_digest_buf_

serialization buf

void set_xor_ref(bool is_enabled) noexcept

Turn ON and OFF XOR compression of sparse vectors Enables XOR reference compression for the sparse ve...

u32_vector_type plane_off_vect_

bm::serializer< bvector_type > serializer_type

SV::remap_matrix_type remap_matrix_type

void serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout)

Serialize sparse vector into a memory buffer(s) structure.

xor_sim_model_type sim_model_

void enable_xor_compression() noexcept

Enable XOR compression on vector serialization.

const xor_sim_model_type * sim_model_ptr_

bm::heap_vector< unsigned, alloc_type, true > u32_vector_type

SV::bvector_type bvector_type

bm::serializer< bvector_type > bvs_

bvector_type::allocator_type alloc_type

bv_ref_vector_type bv_ref_

bm::serializer< bvector_type >::xor_sim_model_type xor_sim_model_type

void disable_xor_compression() noexcept

Disable XOR compression on serialization.

bm::serializer< bvector_type > & get_bv_serializer() noexcept

Get access to the underlying bit-vector serializer This access can be used to fine tune compression s...

sparse_vector_serializer(const sparse_vector_serializer &)=delete

finalization

copy strategy

@ READONLY

immutable (read-only object)

size_t deserialize(BV &bv, const unsigned char *buf, bm::word_t *temp_block=0, const bm::bv_ref_vector< BV > *ref_vect=0)

Bitvector deserialization from a memory BLOB.

void sparse_vector_serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout, bm::word_t *temp_block=0)

Serialize sparse vector into a memory buffer(s) structure.

int sparse_vector_deserialize(SV &sv, const unsigned char *buf, bm::word_t *temp_block=0)

Deserialize sparse vector.

SZ count_nz(const VT *arr, SZ arr_size) noexcept

Find count of non-zero elements in the array.

ByteOrder

Byte orders recognized by the library.

void _print_bv(TOut &tout, const BV &bv)

unsigned long long int id64_t

unsigned short gap_word_t

double value_type

The numeric datatype used by the parser.

const GenericPointer< typename T::ValueType > T2 value

double r(size_t dimension_, const Int4 *score_, const double *prob_, double theta_)

static SLJIT_INLINE sljit_ins st(sljit_gpr r, sljit_s32 d, sljit_gpr x, sljit_gpr b)

std::vector< bm::id64_t > ref_vect

#define row(bind, expected)

ad-hoc conditional expressions

static ByteOrder byte_order()

layout class for serialization buffer structure

SV::bvector_type bvector_type

sizet_vector_type plane_size_

serialized plane size

serializer< bvector_type >::buffer buffer_type

void resize(size_t ssize)

Set new serialized size.

void shrink(size_t ssize)

Set new serialized size (shrink)

const unsigned char * buf() const noexcept

Return serialization buffer pointer.

const unsigned char * data() const noexcept

Return serialization buffer pointer.

unsigned char * reserve(size_t capacity)

resize capacity

void resize_slices(unsigned new_slices_size)

Resize for the target number of plains / bit-slices.

bm::heap_vector< unsigned char *, allocator_type, true > ptr_vector_type

ptr_vector_type plane_ptrs_

pointers on serialized bit-planes

size_t size() const noexcept

return current serialized size

buffer_type buf_

serialization buffer

size_t capacity() const noexcept

return serialization buffer capacity

void operator=(const sparse_vector_serial_layout &)

void set_plane(unsigned i, unsigned char *ptr, size_t buf_size) noexcept

Set plane output pointer and size.

bvector_type::allocator_type allocator_type

bm::heap_vector< size_t, allocator_type, true > sizet_vector_type

sparse_vector_serial_layout(const sparse_vector_serial_layout &)

SV::value_type value_type

sparse_vector_serial_layout() noexcept

void freemem() noexcept

free memory

const unsigned char * get_plane(unsigned i) const noexcept

Get plane pointer.

~sparse_vector_serial_layout()

Parameters for XOR similarity search.


RetroSearch is an open source project built by @garambo | Open a GitHub Issue

Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo

HTML: 3.2 | Encoding: UTF-8 | Version: 0.7.4