kSegmentSizePow2Max = 22;
68 static const size_tkSegmentSizePow2Min = 18;
69 static const doublekMaxBufferSeconds = .5;
72 #ifndef USE_RANGE_CACHE 115m_TotalReadSeconds(0),
116m_PreviousReadBytes(0),
117m_PreviousReadSeconds(0)
140 LOG_POST(
"BGZF: Total read "<<stat.first/
double(1<<20)<<
" MB" 141 " speed: "<<stat.first/(stat.second*(1<<20))<<
" MB/s" 142 " time: "<<stat.second);
150 #ifdef USE_RANGE_CACHE 160 if( page->GetFilePos() != page_pos ) {
162 if( page->GetFilePos() != page_pos ) {
166 if( !page->Contains(file_pos) ) {
168 "BGZF read @ "<<file_pos<<
" is beyond file size");
190 const Uint8add_read_bytes = 1000000;
191 const doubleadd_read_bytes_per_second = 8e6;
199add_read_bytes/add_read_bytes_per_second;
200 doubleseconds_per_byte = read_seconds/read_bytes;
202 size_tsize_pow2 = kSegmentSizePow2Max;
203 doubleseconds = (size_t(1)<<size_pow2)*seconds_per_byte;
204 while( seconds > kMaxBufferSeconds && size_pow2 > kSegmentSizePow2Min ) {
255<<
" "<<bytes<<
" bytes in "<<seconds<<
" sec" 256 " speed: "<<bytes/(seconds*(1<<20))<<
" MB/s");
266page.
m_FilePos.store(file_pos, memory_order_release);
301page =
file.GetPage(file_pos);
302 _ASSERT(page->Contains(file_pos));
303 const char* ptr = page->GetPagePtr();
305 size_toff = size_t(file_pos - page->GetFilePos());
306 size_tavail = page->GetPageSize() - off;
307 if(
len<= avail ) {
314memcpy(dst, ptr+off, avail);
320page =
file.GetPage(file_pos);
321 _ASSERT(page->Contains(file_pos));
322ptr = page->GetPagePtr();
324off = size_t(file_pos - page->GetFilePos());
325avail = page->GetPageSize() - off;
328memcpy(dst, ptr+off,
cnt);
345 return out<<
r.first <<
'-'<<
r.second;
353m_Data(new char[kMaxDataSize])
366m_TotalUncompressBytes(0),
367m_TotalUncompressSeconds(0)
377 LOG_POST(
"BGZF: Total decompressed "<<stat.first/
double(1<<20)<<
" MB" 378 " speed: "<<stat.first/(stat.second*(1<<20))<<
" MB/s" 379 " time: "<<stat.second);
390 if( block->GetFileBlockPos() != file_pos ) {
392 if( block->GetFileBlockPos() != file_pos ) {
426m_InReadBuffer(
CBGZFBlock::kMaxFileBlockSize),
494(page->GetFilePos()+page->GetPageSize() == file_pos0) ) {
505 const char* header =
507*
m_File, page, file_pos);
511 if(header[0] != 31 || header[1] !=
'\x8b'||
512header[2] != 8 || header[3] != 4) {
514 "Bad BGZF("<<file_pos0<<
") MAGIC: "<<
526 "Bad BGZF("<<file_pos0<<
") XLEN: "<<extra_size);
534 const char* more_extra =
535 s_Read(more_extra_dst, more_extra_size,
536*
m_File, page, file_pos);
537file_pos += more_extra_size;
538 if( more_extra != more_extra_dst ) {
539memcpy(more_extra_dst, more_extra, more_extra_size);
550 if( extra[0] ==
'B'&& extra[1] ==
'C'&&
551extra_data_size == 2 &&
552extra_block_size <= extra_size ) {
555 if( extra_block_size >= extra_size ) {
558extra_size -= extra_block_size;
559extra += extra_block_size;
561 if( block_size <= real_header_size +
kFooterSize) {
563 "Bad BGZF("<<file_pos0<<
") SIZE: "<<block_size);
568 const char* compressed_data =
570*
m_File, page, file_pos);
571 size_tcompressed_size = block_size - (real_header_size +
kFooterSize);
574 size_tdecompressed_size = 0;
578 const char* src = compressed_data;
579 size_tsrc_size = compressed_size;
592 if( stream.
avail_in!= src_size ||
595 "Bad BGZF("<<file_pos0<<
") compression sizes: "<<
596src_size<<
" -> "<<dst_size);
612 " @ "<<file_pos0<<
" in "<<seconds<<
" sec" 613 " speed: "<<compressed_size/(seconds*(1<<20))<<
" MB/s");
618 const char* footer = compressed_data + compressed_size;
623 "Bad BGZF("<<file_pos0<<
") DATA SIZE: "<< data_size);
630 if( decompressed_size != data_size ) {
632 "Bad BGZF("<<file_pos0<<
") decompressed data size: "<<
633decompressed_size<<
" vs "<<data_size);
640 "Bad BGZF("<<file_pos0<<
") CRC32: "<<
693 if(
count<= avail ) {
static int s_GetDebug(void)
static const EFileMode kFileMode
static char * s_Reserve(size_t size, CSimpleBufferT< char > &buffer)
static const size_t kFixedHeaderSize
static const size_t kRequiredExtraSize
static const bool kCheckBlockCRC32
static const size_t kExtraHeaderSize
static const size_t kFooterSize
static const size_t kSegmentSize
static const size_t kSegmentSizePow2
NCBI_PARAM_DECL(int, BGZF, DEBUG)
ostream & operator<<(ostream &out, const CBGZFPos &p)
static const char * s_Read(char *buffer, size_t len, CPagedFile &file, CPagedFile::TPage &page, Uint8 file_pos)
static const size_t kInitialExtraSize
static CPagedFile::TFilePos s_GetPagePos(CPagedFile::TFilePos file_pos)
NCBI_PARAM_DEF_EX(int, BGZF, DEBUG, 0, eParam_NoThread, BGZF_DEBUG)
pair< CBGZFPos, CBGZFPos > CBGZFRange
Checksum and hash calculation classes.
static const TDataSize kMaxDataSize
atomic< TFileBlockPos > m_FileBlockPos
static const TFileBlockSize kMaxFileBlockSize
TFileBlockSize m_FileBlockSize
virtual const char * GetErrCodeString(void) const override
Get error code interpreted as text.
@ eInvalidArg
invalid function argument
@ eFormatError
includes decompression errors
bool x_ReadBlock(CBGZFBlock &block, TFileBlockPos file_pos, CPagedFile::TPage &page, CSimpleBufferT< char > &buffer)
pair< Uint8, double > GetUncompressStatistics() const
CBGZFPos::TFileBlockPos TFileBlockPos
void x_AddUncompressStatistics(Uint8 bytes, double seconds)
TBlock GetBlock(TFileBlockPos file_pos, CPagedFile::TPage &page, CSimpleBufferT< char > &buffer)
Uint8 m_TotalUncompressBytes
CBGZFFile(const string &file_name)
CRef< CPagedFile > m_File
double m_TotalUncompressSeconds
CRef< TBlockCache > m_BlockCache
TByteOffset GetByteOffset() const
TFileBlockPos GetFileBlockPos() const
static CBGZFPos GetInvalid()
CBGZFBlock::TFileBlockPos GetNextBlockFilePos() const
CBGZFBlock::TDataSize GetBlockDataSize() const
void Open(CBGZFFile &file)
void Seek(CBGZFPos pos, CBGZFPos end_pos=CBGZFPos::GetInvalid())
size_t GetNextAvailableBytes()
size_t Read(char *buf, size_t count)
bool HaveBytesInBlock() const
CBGZFPos::TByteOffset m_ReadPos
CBGZFFile::TBlock m_Block
CSimpleBufferT< char > m_OutReadBuffer
CSimpleBufferT< char > m_InReadBuffer
size_t ReadAll(size_t file_pos, char *buffer, size_t buffer_size)
CFastMutex & GetValueMutex()
const range_type & get_range() const
CFastMutex & GetValueMutex()
CChecksum â Checksum calculator.
atomic< TFilePos > m_FilePos
CMemoryFileMap * m_MemFile
CSimpleBufferT< char > m_Buffer
double m_PreviousReadSeconds
TPage GetPage(TFilePos pos)
void x_AddReadStatistics(Uint8 bytes, double seconds)
pair< Uint8, double > GetReadStatistics() const
AutoPtr< CMemoryFileMap > m_MemFile
CPagedFilePage::TFilePos TFilePos
CPagedFile(const string &file_name)
Uint8 m_PreviousReadBytes
void x_ReadPage(CPagedFilePage &page, TFilePos file_pos, size_t size)
double m_TotalReadSeconds
size_t GetNextPageSizePow2() const
void SetPreviousReadStatistics(const pair< Uint8, double > &stats)
CRef< TPageCache > m_PageCache
const value_type * data() const
void resize(size_type new_size)
std::ofstream out("events_result.xml")
main entry point for tests
void reset(element_type *p=0, EOwnership ownership=eTakeOwnership)
Reset will delete the old pointer (if owned), set content to the new value, and assume the ownership ...
element_type * get(void) const
Get pointer.
element_type * get(void) const
Get pointer.
Uint4 GetChecksum(void) const
Return calculated checksum.
void AddChars(const char *str, size_t len)
Update current control sum with data provided.
#define LOG_POST(message)
This macro is deprecated and it's strongly recomended to move in all projects (except tests) to macro...
TErrCode GetErrCode(void) const
Get error code.
void Warning(CExceptionArgs_Base &args)
#define NCBI_THROW_FMT(exception_class, err_code, message)
The same as NCBI_THROW but with message processed as output to ostream.
virtual const char * GetErrCodeString(void) const
Get error code interpreted as text.
void Info(CExceptionArgs_Base &args)
void Open(const string &filename, EOpenMode open_mode, EAccessMode access_mode, EShareMode share_mode=eShare)
Open file.
void * Map(TOffsetType offset, size_t length)
Map file segment.
bool Unmap(void *ptr)
Unmap file segment.
void SetFilePos(Uint8 position) const
Set file position from beginning of the file.
size_t Read(void *buf, size_t count) const
Read file.
@ eRead
File can be read.
@ eOpen
Open an existing file, or create a new one.
@ eMMS_Shared
Changes are shared.
@ eMMP_Read
Data can be read.
void Reset(void)
Reset reference object.
#define NCBI_PARAM_TYPE(section, name)
Generate typename for a parameter from its {section, name} attributes.
@ eParam_NoThread
Do not use per-thread values.
uint32_t Uint4
4-byte (32-bit) unsigned integer
uint64_t Uint8
8-byte (64-bit) unsigned integer
position_type GetLength(void) const
position_type GetFrom(void) const
#define END_NCBI_SCOPE
End previously defined NCBI scope.
#define END_SCOPE(ns)
End the previously defined scope.
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
#define BEGIN_SCOPE(ns)
Define a new scope.
double Elapsed(void) const
Return time elapsed since first Start() or last Restart() call (in seconds).
@ eStart
Start timer immediately after creating.
static void hex(unsigned char c)
const struct ncbi::grid::netcache::search::fields::SIZE size
const GenericPointer< typename T::ValueType > T2 value
double r(size_t dimension_, const Int4 *score_, const double *prob_, double theta_)
static Uint4 MakeUint4(const char *buf)
static Uint2 MakeUint2(const char *buf)
#define inflateInit2(strm, windowBits)
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4