<
classBV,
typenameTOut>
44tout << bv.count() <<
": ";
45 typenameBV::enumerator en = bv.first();
46 for(; en.valid(); ++en)
189 template<
typenameSV>
340 template<
typenameSV>
386 const unsigned char*
buf,
387 boolclear_sv =
true);
401 boolclear_sv =
true);
425 const unsigned char*
buf,
438 const unsigned char*
buf);
458 const unsigned char*
buf,
467 const unsigned char*
buf,
471 void load_remap(SV& sv,
const unsigned char* remap_buf_ptr);
549sv_serializer.
serialize(sv, sv_layout);
567 const unsigned char*
buf,
581 template<
classCBC>
600 template<
classCBC>
614 const unsigned char*
buf,
652 template<
classCBC>
658buffer_coll.calc_stat(&
st);
660 buf.resize(
st.max_serialize_mem);
663 unsigned char* buf_ptr =
buf.data();
669enc.
put_8((
unsigned char)bo);
671 unsigned char* mbuf1 = enc.
get_pos();
682 size_taddr_bv_size = bvs.
serialize(bv, buf_ptr,
buf.size());
683buf_ptr += addr_bv_size;
686enc.
put_64(addr_bv_size);
689 size_tcoll_size = buffer_coll.size();
695 for(
unsigned i= 0;
i< buffer_coll.size(); ++
i)
698 size_tsz = cbuf.size();
704 for(
unsigned i= 0;
i< buffer_coll.size(); ++
i)
707 size_tsz = cbuf.size();
708enc.
memcpy(cbuf.buf(), sz);
715 template<
classCBC>
718 const unsigned char*
buf,
725 unsigned charh1 = dec.
get_8();
726 unsigned charh2 = dec.
get_8();
729 if(h1 !=
'B'&& h2 !=
'C')
741 const unsigned char* bv_buf_ptr = dec.
get_pos();
750 typenamebvector_type::size_type addr_cnt = bv.count();
751dec.
seek((
int)addr_bv_size);
757 if(coll_size != addr_cnt)
762 typedef size_tvect_size_type;
765buf_size_vec.
resize(vect_size_type(coll_size));
767 for(
unsigned i= 0;
i< coll_size; ++
i)
770buf_size_vec[
i] = sz;
776buf_vect.
resize(vect_size_type(coll_size));
777 for(
unsigned i= 0;
i< coll_size; ++
i)
782dec.
memcpy(
b.data(),
size_t(sz));
793 template<
typenameSV>
807 template<
typenameSV>
811bv_ref_ptr_ = bv_ref_ptr;
812is_xor_ref_ =
bool(bv_ref_ptr);
818 template<
typenameSV>
822is_xor_ref_ = is_enabled;
827 template<
typenameSV>
833bvs_.compute_sim_model(sim_model,
ref_vect, params);
838 template<
typenameSV>
842sim_model_ptr_ = sim_model;
847 template<
typenameSV>
851bv_ref_.build(sv.get_bmatrix());
856 template<
typenameSV>
860 const typenameSV::remap_matrix_type* rm = sv.get_remap_matrix();
865 size_trows = rmatr.rows();
866 size_tcols = rmatr.cols();
872remap_rlen_vect_.resize(0);
873 for(
size_t r= 0;
r< rows; ++
r)
875 const unsigned char*
BMRESTRICTremap_row = rmatr.row(
r);
879remap_rlen_vect_.push_back(
unsigned(
cnt));
882rows = remap_rlen_vect_.size();
885 for(
size_t r= 0;
r< rows; ++
r)
887 unsignedrl = remap_rlen_vect_[
r];
888csr_size_max += rl * 2;
891 size_tremap_size = sv.remap_size();
893 if(remap_size < csr_size_max)
895 const unsigned char* matrix_buf = sv.get_remap_buffer();
901enc.
memcpy(matrix_buf,
size_t(remap_size));
906enc.
put_32(
unsigned(rows));
911 for(
size_t r= 0;
r< rows; ++
r)
913 unsignedrl = remap_rlen_vect_[
r];
918 for(
size_t r= 0;
r< rows; ++
r)
921 for(
size_tj = 0; j < cols; ++j)
923 unsigned charv =
row[j];
926enc.
put_8((
unsigned char)j);
938 template<
typenameSV>
942bvs_.allow_stat_reset(
false);
943bvs_.reset_compression_stats();
948 buf[0]=
'B';
buf[1] =
'Z';
953 const auto& bmatr = sv.get_bmatrix();
955bmatr.build_plane_digest(plane_digest_bv_);
956plane_digest_bv_.optimize();
960bvs_.set_ref_vectors(0);
961bvs_.serialize(plane_digest_bv_, plane_digest_buf_);
963 unsignedplanes = (unsigned)bmatr.rows();
969 typenameSV::statistics sv_stat;
970sv.calc_stat(&sv_stat);
971sv_stat.max_serialize_mem += plane_digest_buf_.size() + (8 * planes);
972 unsigned char*
buf= sv_layout.
reserve(sv_stat.max_serialize_mem);
979 unsignedh_size = 1 + 1 +
1000bvs_.set_ref_vectors(bv_ref_ptr_);
1001bvs_.set_sim_model(sim_model_ptr_);
1006build_xor_ref_vector(sv);
1007bvs_.set_ref_vectors(&bv_ref_);
1008 if(bvs_.compute_sim_model(sim_model_, bv_ref_, xs_params))
1009bvs_.set_sim_model(&sim_model_);
1017::memset(
buf, 0, h_size);
1018 unsigned char* buf_ptr =
buf+ h_size;
1020 for(
unsigned i= 0;
i< planes; ++
i)
1022 typenameSV::bvector_type_const_ptr bv = bmatr.row(
i);
1034idx = (unsigned)bv_ref_ptr_->find_bv(bv);
1036idx = (unsigned)bv_ref_.find_bv(bv);
1038bvs_.set_curr_ref_idx(idx);
1040 size_tbuf_size = (size_t)
1041bvs_.serialize(*bv, buf_ptr, sv_stat.max_serialize_mem);
1043sv_layout.
set_plane(
i, buf_ptr, buf_size);
1044buf_ptr += buf_size;
1045 if(sv_stat.max_serialize_mem > buf_size)
1047sv_stat.max_serialize_mem -= buf_size;
1053bvs_.set_ref_vectors(0);
1060 bm::encoderenc_m(buf_ptr, sv_stat.max_serialize_mem);
1062encode_remap_matrix(enc_m, sv);
1064enc_m.
put_8(
'N');
1065buf_ptr += enc_m.
size();
1071 size_tdigest_offset = size_t(buf_ptr -
buf);
1072 autopd_size = plane_digest_buf_.size();
1073::memcpy(buf_ptr, plane_digest_buf_.buf(), pd_size);
1076 booluse_64bit =
false;
1077plane_off_vect_.resize(0);
1078 for(
unsigned i= 0;
i< planes; ++
i)
1080 const unsigned char* p = sv_layout.
get_plane(
i);
1089plane_off_vect_.push_back(
unsigned(
offset));
1092 bm::encoderenc_o(buf_ptr, sv_stat.max_serialize_mem);
1093 if(use_64bit || (plane_off_vect_.size() < 4))
1095enc_o.
put_8(
'6');
1098 for(
unsigned i= 0;
i< planes; ++
i)
1100 const unsigned char* p = sv_layout.
get_plane(
i);
1110 BM_ASSERT(plane_off_vect_.size() == plane_digest_bv_.count());
1111 unsignedmin_v = plane_off_vect_[0];
1112 unsignedmax_v = plane_off_vect_[plane_off_vect_.size()-1];
1114enc_o.
put_8(
'3');
1121 unsigned(plane_off_vect_.size()-2),
1126buf_ptr += enc_o.
size();
1131sv_layout.
shrink(
size_t(buf_ptr -
buf));
1139 if(sv.is_compressed())
1144enc.
put_8((
unsigned char)bo);
1146 unsigned charmatr_s_ser = 1;
1152enc.
put_8(matr_s_ser);
1154 bm::id64_tplanes_code = planes | (1ull << 63);
1155enc.
put_64(planes_code);
1157enc.
put_64(sv.size_internal());
1165 template<
typenameSV>
1168temp_block_ = alloc_.alloc_bit_block();
1169not_null_mask_bv_.set_allocator_pool(&pool_);
1170rsc_mask_bv_.set_allocator_pool(&pool_);
1175 template<
typenameSV>
1179alloc_.free_bit_block(temp_block_);
1184 template<
typenameSV>
1188this->is_final_ = is_final;
1193 template<
typenameSV>
1197bv_ref_ptr_ = bv_ref_ptr;
1199clear_xor_compression();
1204 template<
typenameSV>
1207op_deserial_.set_ref_vectors(0);
1208deserial_.set_ref_vectors(0);
1214 template<
typenameSV>
1219op_deserial_.set_ref_vectors(bv_ref_ptr_);
1220deserial_.set_ref_vectors(bv_ref_ptr_);
1224op_deserial_.set_ref_vectors(&bv_ref_);
1225deserial_.set_ref_vectors(&bv_ref_);
1231 template<
typenameSV>
1233 const unsigned char*
buf,
1238idx_range_set_ =
false;
1239deserialize_sv(sv,
buf, 0, clear_sv);
1244 template<
typenameSV>
1246 const unsigned char*
buf)
1250 unsigned charmatr_s_ser = 0;
1251 unsignedplanes = load_header(dec, sv, matr_s_ser);
1255load_planes_off_table(
buf, dec, planes);
1257sv.get_bmatrix().allocate_rows(planes);
1259 for(
unsigned i= 0;
i< planes; ++
i)
1261 if(!off_vect_[
i])
1270 template<
typenameSV>
1272 const unsigned char*
buf,
1278sv.clear_all(
true, 1);
1280idx_range_set_ =
true; idx_range_from_ = from; idx_range_to_ = to;
1285 unsigned charmatr_s_ser = 0;
1286 unsignedplanes = load_header(dec, sv, matr_s_ser);
1291sv.resize_internal(
size_type(sv_size_));
1294load_planes_off_table(
buf, dec, planes);
1296setup_xor_compression();
1298sv.get_bmatrix().allocate_rows(planes);
1301planes = (unsigned)load_null_plane(sv,
int(planes),
buf, 0);
1308 boolrange_valid = sv.resolve_range(from, to, &sv_left, &sv_right);
1312idx_range_set_ =
false;
1317idx_range_set_ =
true; idx_range_from_ = sv_left; idx_range_to_ = sv_right;
1321deserialize_planes(sv, planes,
buf, 0);
1323clear_xor_compression();
1330load_remap(sv, remap_buf_ptr_);
1333sv.sync(
true,
true);
1337idx_range_set_ =
false;
1342 template<
typenameSV>
1344 const unsigned char*
buf,
1349sv.clear_all(
true, 1);
1354 unsigned charmatr_s_ser = 0;
1355 unsignedplanes = load_header(dec, sv, matr_s_ser);
1359sv.resize_internal(
size_type(sv_size_));
1362load_planes_off_table(
buf, dec, planes);
1364setup_xor_compression();
1366sv.get_bmatrix().allocate_rows(planes);
1367planes = (unsigned)load_null_plane(sv,
int(planes),
buf, mask_bv);
1377rsc_mask_bv_.clear(
true);
1378not_null_mask_bv_.bit_and(*bv_null, *mask_bv, bvector_type::opt_compress);
1379rsc_compressor_.compress(rsc_mask_bv_, *bv_null, not_null_mask_bv_);
1380mask_bv = &rsc_mask_bv_;
1386rsc_mask_bv_.find_range(idx_range_from_, idx_range_to_);
1391deserialize_planes(sv, planes,
buf, mask_bv);
1395 #pragma warning( push ) 1396 #pragma warning( disable : 4127) 1398 if(sv.max_vector_size == 1)
1402sv.get_bmatrix().get_row(sv.sv_value_slices);
1404sv.mark_null_idx(sv.sv_value_slices);
1407 #pragma warning( pop ) 1411clear_xor_compression();
1418load_remap(sv, remap_buf_ptr_);
1421sv.sync(
true,
true);
1427 template<
typenameSV>
1429 bm::decoder& dec, SV& sv,
unsigned char& matr_s_ser)
1433 unsigned charh1 = dec.
get_8();
1434 unsigned charh2 = dec.
get_8();
1436 BM_ASSERT(h1 ==
'B'&& (h2 ==
'M'|| h2 ==
'C'|| h2 ==
'Z'));
1438 boolsig2_ok = (h2 ==
'M'|| h2 ==
'C'|| h2 ==
'Z');
1439 if(h1 !=
'B'|| !sig2_ok)
1440raise_invalid_header();
1441 unsignedplanes = 0;
1448 unsigned charbv_bo = dec.
get_8(); (void) bv_bo;
1449planes = dec.
get_8();
1452matr_s_ser = dec.
get_8();
1453planes_code = dec.
get_64();
1454planes = (unsigned) planes_code;
1458 if(matr_s_ser == 2)
1459raise_invalid_64bit();
1464 unsignedsv_planes = sv.stored_slices();
1465 if(!planes || planes > sv_planes)
1466raise_invalid_bitdepth();
1469sv_size_ = dec.
get_64();
1472 if(planes_code & (1ull << 63))
1474digest_offset_ = dec.
get_64();
1482 template<
typenameSV>
1486 const unsigned char*
buf,
1489 if(mask_bv && !idx_range_set_)
1490idx_range_set_ = mask_bv->find_range(idx_range_from_, idx_range_to_);
1495 for(
int i=
int(planes-1);
i>= 0; --
i)
1497 size_t offset= off_vect_[unsigned(
i)];
1500 const unsigned char* bv_buf_ptr =
buf+
offset;
1506bv_ref_.add(bv,
unsigned(
i));
1509 typenamebvector_type::mem_pool_guard mp_g_z(pool_, *bv);
1514deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1515remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1516bv->bit_and(*mask_bv, bvector_type::opt_compress);
1521deserial_.set_range(idx_range_from_, idx_range_to_);
1522deserial_.deserialize(*bv, bv_buf_ptr);
1523bv->bit_and(*mask_bv, bvector_type::opt_compress);
1532deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1533remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1535bv->keep_range(idx_range_from_, idx_range_to_);
1541deserial_.set_range(idx_range_from_, idx_range_to_);
1542deserial_.deserialize(*bv, bv_buf_ptr);
1543bv->keep_range(idx_range_from_, idx_range_to_);
1548deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1566sv.set_ro_flag(
true);
1572deserial_.unset_range();
1578 template<
typenameSV>
1581 const unsigned char*
buf,
1585 if(!sv.is_nullable())
1588 size_t offset= off_vect_[unsigned(
i)];
1594 const unsigned char* bv_buf_ptr =
buf+
offset;
1595 bvector_type* bv = sv.get_bmatrix().get_row(
unsigned(
i));
1598bv_ref_.add(bv,
unsigned(
i));
1604 size_tread_bytes = deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1605remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1614 size_tread_bytes = deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1615remap_buf_ptr_ = bv_buf_ptr + read_bytes;
1617bv->keep_range(idx_range_from_, idx_range_to_);
1622deserial_.set_range(idx_range_from_, idx_range_to_);
1623deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1624bv->keep_range(idx_range_from_, idx_range_to_);
1625deserial_.unset_range();
1629deserial_.deserialize(*bv, bv_buf_ptr, temp_block_);
1632bv->bit_and(*mask_bv, bvector_type::opt_compress);
1650 template<
typenameSV>
1654off_vect_.resize(planes);
1657plane_digest_bv_.clear(
false);
1658 const unsigned char* buf_ptr =
buf+ digest_offset_;
1660deserial_.deserialize(plane_digest_bv_, buf_ptr, temp_block_);
1661buf_ptr += read_bytes;
1665 unsigned chardtype = dec_o.
get_8();
1669 for(
unsigned i= 0;
i< planes; ++
i)
1672 if(plane_digest_bv_.test(
i))
1683 unsignedosize = (unsigned)plane_digest_bv_.count();
1685off32_vect_.resize(osize);
1687 unsignedmin_v = dec_o.
get_32();
1688 unsignedmax_v = dec_o.
get_32();
1690off32_vect_[0] = min_v;
1691off32_vect_[osize-1] = max_v;
1697 for(
unsigned i= 0;
i< planes; ++
i)
1699 if(plane_digest_bv_.test(
i))
1701off_vect_[
i] = off32_vect_[k];
1716 for(
unsigned i= 0;
i< planes; ++
i)
1726 template<
typenameSV>
1728 const unsigned char* remap_buf_ptr)
1735 unsigned charrh = dec_m.
get_8();
1742 size_tremap_size = (size_t) dec_m.
get_64();
1743 unsigned char* remap_buf = sv.init_remap_buffer();
1745 size_ttarget_remap_size = sv.remap_size();
1746 if(!remap_size || !remap_buf || remap_size != target_remap_size)
1748raise_invalid_format();
1750dec_m.
memcpy(remap_buf, remap_size);
1757 typenameSV::remap_matrix_type* rmatr = sv.get_remap_matrix();
1760raise_missing_remap_matrix();
1762 size_trows = (size_t) dec_m.
get_32();
1763 size_tcols = dec_m.
get_16();
1766raise_invalid_format();
1768rmatr->resize(rows, cols,
false);
1774remap_rlen_vect_.resize(0);
1777 for(
size_t r= 0;
r< rows; ++
r)
1779 unsignedrl = bi.
gamma();
1780remap_rlen_vect_.push_back(rl);
1784 for(
size_t r= 0;
r< rows; ++
r)
1787 size_t cnt= remap_rlen_vect_[
r];
1790raise_invalid_format();
1792 for(
size_tj = 0; j <
cnt; ++j)
1794 unsignedidx = dec_m.
get_8();
1795 unsigned charv = dec_m.
get_8();
1804raise_invalid_format();
1809 unsigned charend_tok = dec_m.
get_8();
1810 if(end_tok !=
'E')
1812raise_invalid_format();
1819 template<
typenameSV>
1823 throwstd::logic_error(
"BitMagic: Invalid serialization signature header");
1825BM_THROW(BM_ERR_SERIALFORMAT);
1831 template<
typenameSV>
1835 throwstd::logic_error(
"BitMagic: Invalid serialization target (64-bit BLOB)");
1837BM_THROW(BM_ERR_SERIALFORMAT);
1843 template<
typenameSV>
1847 throwstd::logic_error(
"BitMagic: Invalid serialization target (bit depth)");
1849BM_THROW(BM_ERR_SERIALFORMAT);
1855 template<
typenameSV>
1859 throwstd::logic_error(
"BitMagic: Invalid serialization fromat (BLOB corruption?)");
1861BM_THROW(BM_ERR_SERIALFORMAT);
1867 template<
typenameSV>
1871 throwstd::logic_error(
"BitMagic: Invalid serialization format (remap matrix)");
1873BM_THROW(BM_ERR_SERIALFORMAT);
Serialization / compression of bvector<>. Set theoretical operations on compressed BLOBs.
Sparse constainer sparse_vector<> for integer types using bit-transposition transform.
Byte based reader for un-aligned bit streaming.
unsigned gamma() noexcept
decode unsigned value using Elias Gamma coding
void bic_decode_u32_cm(bm::word_t *arr, unsigned sz, bm::word_t lo, bm::word_t hi) noexcept
Binary Interpolative array decode (32-bit)
Byte based writer for un-aligned bit streaming.
void gamma(unsigned value) noexcept
Elias Gamma encode the specified value.
void bic_encode_u32_cm(const bm::word_t *arr, unsigned sz, bm::word_t lo, bm::word_t hi) noexcept
Binary Interpolative encoding (array of 32-bit ints) cm - "center-minimal".
void flush() noexcept
Flush the incomplete 32-bit accumulator word.
List of reference bit-vectors with their true index associations.
Deseriaizer for compressed collections.
bvector_type::allocator_type allocator_type
CBC::buffer_type buffer_type
CBC::bvector_type bvector_type
CBC::statistics statistics_type
CBC::container_type container_type
CBC compressed_collection_type
int deserialize(CBC &buffer_coll, const unsigned char *buf, bm::word_t *temp_block=0)
CBC::address_resolver_type address_resolver_type
Seriaizer for compressed collections.
void serialize(const CBC &buffer_coll, buffer_type &buf, bm::word_t *temp_block=0)
Serialize compressed collection into memory buffer.
CBC compressed_collection_type
CBC::bvector_type bvector_type
CBC::statistics statistics_type
CBC::address_resolver_type address_resolver_type
CBC::buffer_type buffer_type
const unsigned char * get_pos() const noexcept
Return current buffer pointer.
unsigned char get_8() noexcept
Reads character from the decoding buffer.
void memcpy(unsigned char *dst, size_t count) noexcept
read bytes from the decode buffer
void seek(int delta) noexcept
change current position
Class for decoding data from memory buffer.
bm::id64_t get_64() noexcept
Reads 64-bit word from the decoding buffer.
bm::short_t get_16() noexcept
Reads 16-bit word from the decoding buffer.
bm::word_t get_32() noexcept
Reads 32-bit word from the decoding buffer.
void put_16(bm::short_t s) noexcept
Puts short word (16 bits) into the encoding buffer.
void put_32(bm::word_t w) noexcept
Puts 32 bits word into encoding buffer.
void put_64(bm::id64_t w) noexcept
Puts 64 bits word into encoding buffer.
size_t size() const noexcept
Returns size of the current encoding stream.
void put_8(unsigned char c) noexcept
Puts one character into the encoding buffer.
void set_pos(unsigned char *buf_pos) noexcept
Set current memory stream position.
void memcpy(const unsigned char *src, size_t count) noexcept
copy bytes into target buffer or just rewind if src is NULL
unsigned char * get_pos() const noexcept
Get current memory stream position.
void resize(size_type new_size)
vector resize
Bit-vector serialization class.
void set_bookmarks(bool enable, unsigned bm_interval=256) noexcept
Add skip-markers to serialization BLOB for faster range decode at the expense of some BLOB size incre...
void gap_length_serialization(bool value) noexcept
Set GAP length serialization (serializes GAP levels of the original vector)
size_type serialize(const BV &bv, unsigned char *buf, size_t buf_size)
Bitvector serialization into memory block.
sparse vector de-serializer
sparse_vector_deserializer()
void deserialize_planes(SV &sv, unsigned planes, const unsigned char *buf, const bvector_type *mask_bv=0)
deserialize bit-vector planes
void setup_xor_compression()
setup deserializers
allocator_pool_type pool_
bm::operation_deserializer< bvector_type > op_deserial_
void deserialize(SV &sv, const unsigned char *buf, bool clear_sv=true)
bm::serializer< bvector_type >::bv_ref_vector_type bv_ref_vector_type
void deserialize(SV &sv, const unsigned char *buf, const bvector_type &mask_bv)
bm::rank_compressor< bvector_type > rsc_compressor_
void deserialize_structure(SV &sv, const unsigned char *buf)
bm::id64_t digest_offset_
void deserialize_sv(SV &sv, const unsigned char *buf, const bvector_type *mask_bv, bool clear_sv)
static void raise_invalid_bitdepth()
throw error on incorrect deserialization
static void raise_invalid_header()
throw error on incorrect deserialization
SV::bvector_type bvector_type
size_type idx_range_from_
bvector_type not_null_mask_bv_
bvector_type::allocator_type::allocator_pool_type allocator_pool_type
bvector_type::allocator_type alloc_type
static void raise_invalid_format()
throw error on incorrect deserialization
unsigned load_header(bm::decoder &dec, SV &sv, unsigned char &matr_s_ser)
Deserialize header/version and other common info.
void deserialize(SV &sv, const unsigned char *buf, size_type from, size_type to)
bm::heap_vector< size_t, alloc_type, true > off_vect_
~sparse_vector_deserializer()
bm::heap_vector< unsigned, alloc_type, true > off32_vect_
bvector_type * bvector_type_ptr
static void raise_invalid_64bit()
throw error on incorrect deserialization
bvector_type plane_digest_bv_
static void raise_missing_remap_matrix()
throw error on incorrect deserialization
sparse_vector_deserializer(const sparse_vector_deserializer &)=delete
const bvector_type * bvector_type_const_ptr
void set_xor_ref(bv_ref_vector_type *bv_ref_ptr)
Set external XOR reference vectors (data frame referenece vectors)
bv_ref_vector_type * bv_ref_ptr_
external ref bit-vect
rlen_vector_type remap_rlen_vect_
void set_finalization(bm::finalization is_final)
Set deserialization finalization to force deserialized vectors into READONLY (or READWRITE) mode.
bvector_type rsc_mask_bv_
bm::deserializer< bvector_type, bm::decoder > deserial_
const unsigned char * remap_buf_ptr_
bv_ref_vector_type bv_ref_
reference vector
sparse_vector_deserializer & operator=(const sparse_vector_deserializer &)=delete
int load_null_plane(SV &sv, int planes, const unsigned char *buf, const bvector_type *mask_bv)
load NULL bit-plane (returns new planes count)
void deserialize_range(SV &sv, const unsigned char *buf, size_type from, size_type to, bool clear_sv=true)
void load_planes_off_table(const unsigned char *buf, bm::decoder &dec, unsigned planes)
load offset table
void clear_xor_compression()
unset XOR compression vectors
void load_remap(SV &sv, const unsigned char *remap_buf_ptr)
load string remap dict
bm::finalization is_final_
bm::heap_vector< unsigned, alloc_type, true > rlen_vector_type
SV::value_type value_type
Serialize sparse vector into a memory buffer(s) structure.
sparse_vector_serializer()
bvector_type plane_digest_bv_
bv.digest of bit-planes
u32_vector_type remap_rlen_vect_
void set_bookmarks(bool enable, unsigned bm_interval=256) noexcept
Add skip-markers for faster range deserialization.
void encode_remap_matrix(bm::encoder &enc, const SV &sv)
serialize the remap matrix used for SV encoding
sparse_vector_serializer & operator=(const sparse_vector_serializer &)=delete
serializer_type::buffer buffer_type
void set_sim_model(const xor_sim_model_type *sim_model) noexcept
Attach serizalizer to a pre-computed similarity model.
bm::serializer< bvector_type >::bv_ref_vector_type bv_ref_vector_type
bool is_xor_ref() const noexcept
Returns the XOR reference compression status (enabled/disabled)
const bv_ref_vector_type * bv_ref_ptr_
bvector_type * bvector_type_ptr
void build_xor_ref_vector(const SV &sv)
SV::value_type value_type
const bvector_type * bvector_type_const_ptr
void compute_sim_model(xor_sim_model_type &sim_model, const bv_ref_vector_type &ref_vect, const bm::xor_sim_params ¶ms)
Calculate XOR similarity model for ref_vector refernece vector must be associated before.
alloc_type::allocator_pool_type allocator_pool_type
buffer_type plane_digest_buf_
serialization buf
void set_xor_ref(bool is_enabled) noexcept
Turn ON and OFF XOR compression of sparse vectors Enables XOR reference compression for the sparse ve...
u32_vector_type plane_off_vect_
bm::serializer< bvector_type > serializer_type
SV::remap_matrix_type remap_matrix_type
void serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout)
Serialize sparse vector into a memory buffer(s) structure.
xor_sim_model_type sim_model_
void enable_xor_compression() noexcept
Enable XOR compression on vector serialization.
const xor_sim_model_type * sim_model_ptr_
bm::heap_vector< unsigned, alloc_type, true > u32_vector_type
SV::bvector_type bvector_type
bm::serializer< bvector_type > bvs_
bvector_type::allocator_type alloc_type
bv_ref_vector_type bv_ref_
bm::serializer< bvector_type >::xor_sim_model_type xor_sim_model_type
void disable_xor_compression() noexcept
Disable XOR compression on serialization.
bm::serializer< bvector_type > & get_bv_serializer() noexcept
Get access to the underlying bit-vector serializer This access can be used to fine tune compression s...
sparse_vector_serializer(const sparse_vector_serializer &)=delete
finalization
copy strategy
@ READONLY
immutable (read-only object)
size_t deserialize(BV &bv, const unsigned char *buf, bm::word_t *temp_block=0, const bm::bv_ref_vector< BV > *ref_vect=0)
Bitvector deserialization from a memory BLOB.
void sparse_vector_serialize(const SV &sv, sparse_vector_serial_layout< SV > &sv_layout, bm::word_t *temp_block=0)
Serialize sparse vector into a memory buffer(s) structure.
int sparse_vector_deserialize(SV &sv, const unsigned char *buf, bm::word_t *temp_block=0)
Deserialize sparse vector.
SZ count_nz(const VT *arr, SZ arr_size) noexcept
Find count of non-zero elements in the array.
ByteOrder
Byte orders recognized by the library.
void _print_bv(TOut &tout, const BV &bv)
unsigned long long int id64_t
unsigned short gap_word_t
double value_type
The numeric datatype used by the parser.
const GenericPointer< typename T::ValueType > T2 value
double r(size_t dimension_, const Int4 *score_, const double *prob_, double theta_)
static SLJIT_INLINE sljit_ins st(sljit_gpr r, sljit_s32 d, sljit_gpr x, sljit_gpr b)
std::vector< bm::id64_t > ref_vect
#define row(bind, expected)
ad-hoc conditional expressions
static ByteOrder byte_order()
layout class for serialization buffer structure
SV::bvector_type bvector_type
sizet_vector_type plane_size_
serialized plane size
serializer< bvector_type >::buffer buffer_type
void resize(size_t ssize)
Set new serialized size.
void shrink(size_t ssize)
Set new serialized size (shrink)
const unsigned char * buf() const noexcept
Return serialization buffer pointer.
const unsigned char * data() const noexcept
Return serialization buffer pointer.
unsigned char * reserve(size_t capacity)
resize capacity
void resize_slices(unsigned new_slices_size)
Resize for the target number of plains / bit-slices.
bm::heap_vector< unsigned char *, allocator_type, true > ptr_vector_type
ptr_vector_type plane_ptrs_
pointers on serialized bit-planes
size_t size() const noexcept
return current serialized size
buffer_type buf_
serialization buffer
size_t capacity() const noexcept
return serialization buffer capacity
void operator=(const sparse_vector_serial_layout &)
void set_plane(unsigned i, unsigned char *ptr, size_t buf_size) noexcept
Set plane output pointer and size.
bvector_type::allocator_type allocator_type
bm::heap_vector< size_t, allocator_type, true > sizet_vector_type
sparse_vector_serial_layout(const sparse_vector_serial_layout &)
SV::value_type value_type
sparse_vector_serial_layout() noexcept
void freemem() noexcept
free memory
const unsigned char * get_plane(unsigned i) const noexcept
Get plane pointer.
~sparse_vector_serial_layout()
Parameters for XOR similarity search.
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4