(
constlzo_bytep src, lzo_uint src_len,
127lzo_bytep dst, lzo_uintp dst_len,
162 returnlzo_init() == LZO_E_OK;
182param.
compress= &lzo1x_999_compress;
183param.
workmem= LZO1X_999_MEM_COMPRESS;
185param.
compress= &lzo1x_1_compress;
186param.
workmem= LZO1X_1_MEM_COMPRESS;
214 #define F_COMMENT 0x08 221 size_t* block_size = 0,
230 unsigned char*
buf= (
unsigned char*)src_buf;
235 if(header_len < kMinHeaderSize || header_len >
kMaxHeaderSize) {
238 if(header_len > src_len) {
240 ERR_COMPRESS(34,
"LZO header check failed. The length of "\
241 "input buffer is less than expected header size.");
268 while(pos < src_len &&
buf[pos++] != 0);
270 info->name.assign((
char*)
buf+start, pos-start);
276 while(pos < src_len &&
buf[pos++] != 0);
278 info->comment.assign((
char*)
buf+start, pos-start);
296 char*
buf= (
char*)src_buf;
314 if(
info&& !
info->name.empty() && buf_size > (
info->name.length() +
size) ) {
316strncpy((
char*)
buf+
size,
info->name.data(),
info->name.length());
322 if(
info&& !
info->comment.empty() && buf_size > (
info->comment.length() +
size) ) {
324strncpy((
char*)
buf+
size,
info->comment.data(),
info->comment.length());
349 file.GetTimeT(&mtime);
360 size_tdst_size = *dst_len;
363lzo_uint
n= *dst_len;
364 interrcode =
m_Param->compress((lzo_bytep)src_buf, (lzo_uint)src_len,
371 if( *dst_len + 4 > dst_size ) {
372 SetError(LZO_E_ERROR,
"Destination buffer is too small");
378crc = lzo_crc32(0,
NULL, 0);
379crc = lzo_crc32(crc, (lzo_bytep)src_buf, src_len);
397 if( *dst_len <=
offset) {
398 SetError(LZO_E_ERROR,
"Destination buffer is too small");
422 if( src_len <= 4 ) {
423 SetError(LZO_E_ERROR,
"Data block is too small to have CRC32");
429lzo_uint
n= *dst_len;
430 interrcode = lzo1x_decompress_safe((lzo_bytep)src_buf, src_len,
431(lzo_bytep)dst_buf, &
n, 0);
438lzo_uint32 crc_saved =
442crc = lzo_crc32(0,
NULL, 0);
443crc = lzo_crc32(crc, (lzo_bytep)dst_buf, *dst_len);
445 if(crc != crc_saved) {
446errcode = LZO_E_ERROR;
465 SetError(LZO_E_ERROR,
"Incorrect data block format");
477 if( block_len > src_len - 4 ) {
478 SetError(LZO_E_ERROR,
"Incomplete data block");
482block_len, dst_buf, dst_len,
flags);
483 if( errcode == LZO_E_OK) {
484*processed += block_len;
491 const void* src_buf,
size_tsrc_len,
492 void* dst_buf,
size_tdst_size,
501 if(!src_buf || !dst_buf || !dst_len) {
502 SetError(LZO_E_ERROR,
"bad argument");
511 size_tblock_size = src_len;
519 "size of the source buffer is too big, "\
520 "please use CLZOCompression::fStreamFormat flag");
538 size_tout_len = dst_size;
539 interrcode = LZO_E_OK;
543 constlzo_bytep src = (lzo_bytep)src_buf;
544lzo_bytep dst = (lzo_bytep)dst_buf;
552 size_t n=
min(src_len, block_size);
555 if( errcode != LZO_E_OK) {
567*dst_len = (
char*)dst - (
char*)dst_buf;
573(lzo_bytep)dst_buf, &out_len);
578 if( errcode != LZO_E_OK) {
587 const void* src_buf,
size_tsrc_len,
588 void* dst_buf,
size_tdst_size,
601 if(!src_buf || !dst_buf || !dst_len) {
602 SetError(LZO_E_ERROR,
"bad argument");
608 size_tout_len = dst_size;
609 interrcode = LZO_E_ERROR;
610 boolis_first_block =
true;
614 constlzo_bytep src = (lzo_bytep)src_buf;
615lzo_bytep dst = (lzo_bytep)dst_buf;
623 SetError(LZO_E_ERROR,
"LZO header missing");
626src_len -= header_len;
633 if(errcode != LZO_E_OK) {
636is_first_block =
false;
642*dst_len = (
char*)dst - (
char*)dst_buf;
646errcode = LZO_E_NOT_COMPRESSIBLE;
648 "size of the source data is too big, "\
649 "probably you forgot to specify CLZOCompression::fStreamFormat flag");
653(lzo_bytep)dst_buf, &out_len,
GetFlags());
658 if( errcode != LZO_E_OK) {
662*dst_len =
min(dst_size, src_len);
663memcpy(dst_buf, src_buf, *dst_len);
664 return(dst_size >= src_len);
686 #define ESTIMATE(block_size) (block_size + (block_size / 16) + 64 + 3) 694n_blocks = src_len / block_size;
695 if( src_len / block_size ) {
699 if( src_len % block_size ) {
731 const string& dst_file,
732 size_tfile_io_bufsize,
733 size_tcompression_in_bufsize,
734 size_tcompression_out_bufsize)
756 boolstatus = cf.
Close();
763 const string& dst_file,
764 size_tfile_io_bufsize,
765 size_tdecompression_in_bufsize,
766 size_tdecompression_out_bufsize)
787 boolstatus = cf.
Close();
795 SetError(LZO_E_ERROR,
"No dictionary support");
803 const intkErrorCount = 9;
804 static const char* kErrorDesc[kErrorCount] = {
805 "Unknown error (data is corrupted)",
808 "Input buffer is too small",
809 "Output buffer overflow",
810 "Data is corrupted",
816 if( errcode >= 0 || errcode < -kErrorCount ) {
819 returnkErrorDesc[-errcode - 1];
872 size_tcompression_in_bufsize,
873 size_tcompression_out_bufsize
878 if( !
Open(
file_name,
mode, compression_in_bufsize, compression_out_bufsize) ) {
879 const stringsmode = (
mode==
eMode_Read) ?
"reading":
"writing";
881 "[CLZOCompressionFile] Cannot open file '"+
file_name+
882 "' for "+ smode +
".");
889m_Mode(eMode_Read), m_File(0), m_Stream(0)
915 size_tcompression_in_bufsize,
size_tcompression_out_bufsize)
924compression_in_bufsize, compression_out_bufsize);
929 size_tcompression_in_bufsize,
size_tcompression_out_bufsize)
941 if( !
m_File->good() ) {
943 stringdescription =
string(
"Cannot open file '") +
file_name+
"'";
967compression_in_bufsize, compression_out_bufsize);
982compression_in_bufsize, compression_out_bufsize);
989 SetError(-1,
"Cannot create compression stream");
1003 "[CLZOCompressionFile::Read] File must be opened for reading");
1015 longnread = (long)
m_Stream->gcount();
1031 "[CLZOCompressionFile::Write] File must be opened for writing");
1042 return(
long)
len;
1152 const char* in_buf,
size_tin_len,
1153 char* out_buf,
size_tout_size,
1168 SetError(-1,
"Cannot write LZO header");
1181*in_avail = in_len -
n;
1199status =
Flush(out_buf, out_size, out_avail);
1206 char* out_buf,
size_tout_size,
1237 char* out_buf,
size_tout_size,
1280 SetError(-1,
"Cannot write LZO header");
1324 if( errcode != LZO_E_OK ) {
1375 const char* in_buf,
size_tin_len,
1376 char* out_buf,
size_tout_size,
1394*in_avail = in_len -
n;
1404 if( !header_len ) {
1406 SetError(LZO_E_ERROR,
"LZO header missing");
1438 n=
min(*in_avail, out_size);
1439memcpy(out_buf, in_buf + in_len - *in_avail,
n);
1470 return Flush(out_buf, out_size, out_avail);
1475 SetError(LZO_E_ERROR,
"Incorrect compressed block size");
1508status =
Flush(out_buf, out_size, out_avail);
1520 char* out_buf,
size_tout_size,
1547 char* out_buf,
size_tout_size,
1564 size_tin_avail = 0;
1567 size_tx_out_avail = 0;
1568status =
Process(0, 0, out_buf, out_size, &in_avail, &x_out_avail);
1572*out_avail += x_out_avail;
1580 return Flush(out_buf, out_size, out_avail);
1587 SetError(LZO_E_ERROR,
"Incomplete data block");
1620 if( errcode != LZO_E_OK ) {
CLZOCompressionFile class â.
CLZOCompressor â LZO based compressor.
CLZODecompressor â LZO based decompressor.
std::ofstream out("events_result.xml")
main entry point for tests
static const char * str(char *buf, int n)
element_type * get(void) const
Get pointer.
void reset(element_type *p=0)
Reset will delete the old pointer, set content to the new value, and assume the ownership upon the ne...
virtual void Finalize(CCompressionStream::EDirection dir=CCompressionStream::eReadWrite)
Finalize stream's compression/decompression process for read/write.
CCompressionProcessor::EStatus GetStatus(CCompressionStream::EDirection dir)
Get status of last compression/decompression stream operation.
bool GetError(CCompressionStream::EDirection dir, int &status, string &description)
Get error code and description of last compressor/decompressor stream operation.
@ eDelete
Do delete processor object.
@ eWrite
Writing into stream.
@ eRead
Reading from stream.
@ fOwnReader
Delete the reader.
@ fOwnWriter
Delete the writer.
void IncreaseProcessedSize(size_t n_bytes)
size_t m_BlockSize
Block size for (de)compression.
static size_t GetBlockSizeDefault(void)
We use 24K default block size to reduce overhead with a stream processor's methods calls,...
static size_t GetBlockSizeMin(void)
This is an artifical limit.
void SetBlockSize(size_t block_size)
Block size.
virtual CVersionInfo GetVersion(void) const
Return name and version of the compression library.
virtual EStatus Finish(char *out_buf, size_t out_size, size_t *out_avail)
Finish the compression/decompression process.
string FormatErrorMessage(string where) const
Format string with last error description.
virtual ~CLZOCompressor(void)
Destructor.
void IncreaseOutputSize(size_t n_bytes)
virtual string GetErrorDescription(void) const
virtual EStatus Finish(char *out_buf, size_t out_size, size_t *out_avail)
Finish the compression/decompression process.
void Reset(void)
Reset internal state.
static void StoreUI4(void *buf, unsigned long value)
Store 4 bytes of value in the buffer.
virtual bool DecompressFile(const string &src_file, const string &dst_file, size_t file_io_bufsize=kCompressionDefaultBufSize, size_t decompression_in_bufsize=kCompressionDefaultBufSize, size_t decompression_out_bufsize=kCompressionDefaultBufSize)
Decompress file.
virtual EStatus Flush(char *out_buf, size_t out_size, size_t *out_avail)
Flush compressed/decompressed data from the output buffer.
virtual bool CompressFile(const string &src_file, const string &dst_file, size_t file_io_bufsize=kCompressionDefaultBufSize, size_t compression_in_bufsize=kCompressionDefaultBufSize, size_t compression_out_bufsize=kCompressionDefaultBufSize)
Compress file.
static Uint2 GetUI2(const void *buf)
Read 2 bytes from buffer.
char * m_OutBegPtr
Pointer to begin of data in out buffer.
bool DecompressCache(void)
Decompress block of data in the cache buffer.
virtual bool DecompressBuffer(const void *src_buf, size_t src_len, void *dst_buf, size_t dst_size, size_t *dst_len)
Decompress data in the buffer.
void SetBusy(bool busy=true)
Set/unset compressor busy flag.
size_t compression_out
compression: recommended size for output buffer
char * m_OutBuf
Pointer to output buffer.
bool CompressCache(void)
Compress block of data in the cache buffer.
bool IsBusy(void) const
Return compressor's busy flag.
virtual EStatus Process(const char *in_buf, size_t in_len, char *out_buf, size_t out_size, size_t *in_avail, size_t *out_avail)
Compress/decompress as much data as possible, and stops when the input buffer becomes empty or the ou...
EMode m_Mode
I/O mode (read/write).
static Uint4 GetUI4(const void *buf)
Read 4 bytes from buffer.
static SRecommendedBufferSizes GetRecommendedBufferSizes(size_t round_up=0)
Get recommended buffer sizes for stream/file I/O.
virtual void SetFlags(TFlags flags)
virtual TFlags GetFlags(void) const
Get/set flags.
size_t RoundUp(size_t value, size_t precision)
char * m_OutEndPtr
Pointer to end of data in out buffer.
ELevel
Compression level.
virtual ~CLZOCompression(void)
Destructor.
virtual long Read(void *buf, size_t len)
Read data from compressed file.
CLZOCompression(ELevel level=eLevel_Default)
Constructor.
virtual bool SetDictionary(CCompressionDictionary &dict, ENcbiOwnership own=eNoOwnership)
void InitCompression(ELevel level)
Initialize compression parameters.
const streamsize kCompressionDefaultBufSize
Default compression I/O stream buffer size.
int DecompressBlockStream(const void *src_buf, size_t src_len, void *dst_buf, size_t *dst_len, TLZOFlags flags, size_t *processed)
Decompress block of data for stream format (fStreamFormat flag).
size_t m_BlockLen
Length of the compressed data in the block.
~CLZOCompressionFile(void)
Destructor.
virtual bool CompressBuffer(const void *src_buf, size_t src_len, void *dst_buf, size_t dst_size, size_t *dst_len)
Compress data in the buffer.
EDecompressMode m_DecompressMode
Decompress mode (Decompress/TransparentRead/Unknown)
virtual int GetErrorCode(void) const
Get compressor's internal status/error code and description for the last operation.
SFileInfo m_FileInfo
Compressed file info.
size_t decompression_out
decompression: recommended size for output buffer
AutoArray< char > m_WorkMem
Working memory for compressor.
virtual ELevel GetLevel(void) const
size_t decompression_in
decompression: recommended size for input buffer
virtual EStatus Process(const char *in_buf, size_t in_len, char *out_buf, size_t out_size, size_t *in_avail, size_t *out_avail)
Compress/decompress as much data as possible, and stops when the input buffer becomes empty or the ou...
virtual bool Open(const string &file_name, EMode mode, size_t compression_in_bufsize=kCompressionDefaultBufSize, size_t compression_out_bufsize=kCompressionDefaultBufSize)
Opens a compressed file for reading or writing.
string m_Cache
Buffer to cache header.
size_t GetBlockSize(void) const
virtual EStatus Flush(char *out_buf, size_t out_size, size_t *out_avail)
Flush compressed/decompressed data from the output buffer.
void ResetBuffer(size_t in_bufsize, size_t out_bufsize)
Reset internal state.
CNcbiFstream * m_File
File stream.
virtual bool x_DecompressFile(CCompressionFile &src_file, const string &dst_file, size_t file_io_bufsize=kCompressionDefaultBufSize)
Universal file decompression function.
virtual bool x_CompressFile(const string &src_file, CCompressionFile &dst_file, size_t file_io_bufsize=kCompressionDefaultBufSize)
Universal file compression function.
size_t m_InSize
Size of the input buffer.
void SetFileInfo(const SFileInfo &info)
Set information about compressed file.
static void StoreUI2(void *buf, unsigned long value)
Store 2 bytes of value in the buffer.
static bool Initialize(void)
Initialize LZO library.
#define ERR_COMPRESS(subcode, message)
Macro to report errors in compression API.
void SetError(int status, const char *description=0)
Set last action error/status code and description.
friend class CLZODecompressor
int DecompressBlock(const void *src_buf, size_t src_len, void *dst_buf, size_t *dst_len, TLZOFlags flags)
Decompress block of data.
virtual EStatus Init(void)
Initialize the internal stream state for compression/decompression.
virtual EStatus Init(void)
Initialize the internal stream state for compression/decompression.
friend class CLZOCompressor
CLZOCompression::TFlags TLZOFlags
Bitwise OR of EFlags.
static size_t GetBlockSizeMax(void)
LZO can compress/decompress data limited by its 'lzo_uint' type.
size_t compression_in
compression: recommended size for input buffer
size_t m_OutSize
Size of the output buffer.
virtual bool HaveSupport(ESupportFeature feature)
Check if compression have support for a specified feature.
EStatus
Type of the result of all basic functions.
bool m_NeedWriteHeader
TRUE if needed to write a header.
int CompressBlockStream(const void *src_buf, size_t src_len, void *dst_buf, size_t *dst_len)
Compress block of data for stream format (fStreamFormat flag).
CCompressionIOStream * m_Stream
[De]compression stream.
virtual long Write(const void *buf, size_t len)
Write data to compressed file.
int CompressBlock(const void *src_buf, size_t src_len, void *dst_buf, size_t *dst_len)
Compress block of data.
CLZOBuffer(void)
Constructor.
#define COMPRESS_HANDLE_EXCEPTIONS(subcode, message)
Macro to catch and handle exceptions (from streams in the destructor)
AutoArray< char > m_Buf
Buffer for caching (size of m_Size*2).
size_t m_HeaderLen
Length of the header.
virtual ELevel GetLevel(void) const
Get compression level.
virtual bool Close(void)
Close compressed file.
virtual size_t EstimateCompressionBufferSize(size_t src_len)
Estimate buffer size for data compression.
virtual EStatus End(int abandon=0)
Free all dynamically allocated data structures.
unique_ptr< SCompressionParam > m_Param
Compression parameters.
TLZOFlags m_HeaderFlags
Flags used for compression.
ESupportFeature
Supported features.
char * m_InBuf
Pointer to input buffer.
size_t GetProcessedSize(void)
virtual EStatus End(int abandon=0)
Free all dynamically allocated data structures.
size_t m_InLen
Length of data in the input buffer.
const char * GetLZOErrorDescription(int errcode)
Get error description for specified error code.
CLZOCompressionFile(const string &file_name, EMode mode, ELevel level=eLevel_Default, size_t compression_in_bufsize=kCompressionDefaultBufSize, size_t compression_out_bufsize=kCompressionDefaultBufSize)
Constructor.
void GetStreamError(void)
Get error code/description of last stream operation (m_Stream).
virtual ~CLZODecompressor(void)
Destructor.
@ fChecksum
Add/check (accordingly to compression or decompression) the compressed data checksum.
@ fStoreFileInfo
Store file information like file name and file modification date of the compressed file into the file...
@ fStreamFormat
Use stream compatible format for data compression.
@ fAllowEmptyData
Allow to "compress/decompress" empty data.
@ fAllowTransparentRead
Allow transparent reading data from buffer/file/stream regardless is it compressed or not.
@ eMode_Decompress
Generic decompression.
@ eMode_Unknown
Not known yet (decompress/transparent read)
@ eMode_TransparentRead
Transparent read, the data is uncompressed.
@ eStatus_Overflow
Output buffer overflow - not enough output space.
@ eStatus_Error
Error has occurred. The error code can be acquired by GetErrorCode().
@ eStatus_EndOfData
Special case of eStatus_Success.
@ eStatus_Success
Everything is fine, no errors occurred.
@ eFeature_Dictionary
Check if current compression have support for dictionaries.
@ eFeature_NoCompression
Check if current compression method have a real support for eLevel_NoCompression, so it can store dat...
@ eFeature_EstimateCompressionBufferSize
Check if current compression have implementation for the EstimateCompressionBufferSize() method.
@ eMode_Read
Reading from compressed file.
@ eMode_Write
Writing compressed data to file.
#define NCBI_THROW(exception_class, err_code, message)
Generic macro to throw an exception, given the exception class, error code and message string.
#define END_NCBI_SCOPE
End previously defined NCBI scope.
#define BEGIN_NCBI_SCOPE
Define ncbi namespace.
IO_PREFIX::fstream CNcbiFstream
Portable alias for fstream.
ENcbiOwnership
Ownership relations between objects.
unsigned int
A callback function used to compare two keys in a database.
Definition of all error codes used in util (xutil.lib).
#define ESTIMATE(block_size)
const size_t kMagicSize
Size of magic signature.
int(* TLZOCompressionFunc)(const lzo_bytep src, lzo_uint src_len, lzo_bytep dst, lzo_uintp dst_len, lzo_voidp wrkmem)
Define a pointer to LZO1X compression function.
void s_CollectFileInfo(const string &filename, CLZOCompression::SFileInfo &info)
static size_t s_WriteLZOHeader(void *src_buf, size_t buf_size, size_t block_size, CLZOCompression::TLZOFlags lzo_flags, const CLZOCompression::SFileInfo *info=0)
static size_t s_CheckLZOHeader(const void *src_buf, size_t src_len, size_t *block_size=0, CLZOCompression::TLZOFlags *lzo_flags=0, CLZOCompression::SFileInfo *info=0)
const size_t kMaxHeaderSize
Maximum header size.
#define LIMIT_SIZE_PARAM_LONG(value)
const size_t kMinHeaderSize
Minimum header size.
const char kMagic[kMagicSize]
LZO magic header (see fStreamFormat flag).
#define LIMIT_SIZE_PARAM_STREAMSIZE(value)
const struct ncbi::grid::netcache::search::fields::SIZE size
Defines classes: CDirEntry, CFile, CDir, CSymLink, CMemoryFile, CFileUtil, CFileLock,...
std::istream & in(std::istream &in_, double &x_)
Structure to get information about recommended buffer sizes for file/stream I/O to tune up a (de)comp...
Structure to keep compressed file information.
Structure to define parameters for some level of compression.
TLZOCompressionFunc compress
Pointer to compression function.
size_t workmem
Size of working memory for compressor.
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4