*** htdig-3.2.0b4-20021013/db/db.h.in Fri Jul 26 21:48:10 2002 --- htdig-3.2.0b4-20021013-zlib/db/db.h.in Thu Oct 17 16:50:33 2002 *************** *** 138,141 **** --- 138,142 ---- u_int8_t coefficient; /* Compression factor is 1<bytes; DB_ENV *dbenv = dbmfp->dbmp->dbenv; + DB_CMPR_INFO *cmpr_info = dbenv->mp_cmpr_info; int ret = 0; *************** *** 127,130 **** --- 134,144 ---- db_io->bytes = CMPR_DIVIDE(db_io->bytes); + if(memp_cmpr_zlib_level == -1) + { + memp_cmpr_zlib_level = cmpr_info->zlib_flags; + if(memp_cmpr_zlib_level == -1) + memp_cmpr_zlib_level = Z_DEFAULT_COMPRESSION; + } + /* * Page 0 is a special case. It contains the metadata information (at most 512 bytes) *************** *** 286,290 **** * We gathered all the compressed data in buffcmpr, inflate it. */ ! if((ret = (*cmpr_info->uncompress)(buffcmpr, buffcmpr_length, db_io->buf, CMPR_MULTIPLY(db_io->pagesize), cmpr_info->user_data)) != 0) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: unable to uncompress page at pgno = %ld", first_pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret); --- 300,311 ---- * We gathered all the compressed data in buffcmpr, inflate it. */ ! ! if(cmpr_info->zlib_flags != 0) ! ret = CDB___memp_cmpr_inflate(buffcmpr, buffcmpr_length, db_io->buf, CMPR_MULTIPLY(db_io->pagesize), cmpr_info->user_data); ! else ! ret = (*cmpr_info->uncompress)(buffcmpr, buffcmpr_length, db_io->buf, CMPR_MULTIPLY(db_io->pagesize), cmpr_info->user_data); ! ! if(ret != 0) ! { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: unable to uncompress page at pgno = %ld", first_pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret); *************** *** 335,339 **** u_int8_t *buffcmpr = 0; u_int8_t *buffp; ! int buffcmpr_length; u_int8_t *orig_buff = db_io->buf; DB_ENV *dbenv = dbmfp->dbmp->dbenv; --- 356,360 ---- u_int8_t *buffcmpr = 0; u_int8_t *buffp; ! unsigned int buffcmpr_length; u_int8_t *orig_buff = db_io->buf; DB_ENV *dbenv = dbmfp->dbmp->dbenv; *************** *** 343,347 **** goto err; ! if((ret = (*cmpr_info->compress)(orig_buff, CMPR_MULTIPLY(db_io->pagesize), &buffcmpr, &buffcmpr_length, cmpr_info->user_data)) != 0) { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_write: unable to compress page at pgno = %ld", db_io->pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret); --- 364,375 ---- goto err; ! ! if(cmpr_info->zlib_flags != 0) ! ret = CDB___memp_cmpr_deflate(orig_buff, CMPR_MULTIPLY(db_io->pagesize), &buffcmpr, &buffcmpr_length, cmpr_info->user_data); ! else ! ret = (*cmpr_info->compress)(orig_buff, CMPR_MULTIPLY(db_io->pagesize), &buffcmpr, &buffcmpr_length, cmpr_info->user_data); ! ! if(ret != 0) ! { CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_write: unable to compress page at pgno = %ld", db_io->pgno); ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret); *************** *** 371,376 **** /* write pages until the whole compressed data is written */ do { ! int length = buffcmpr_length - (buffp - buffcmpr); ! int copy_length = length > DB_CMPR_PAGESIZE(db_io) ? DB_CMPR_PAGESIZE(db_io) : length; /* * We handle serious compression stuff only if we need to. --- 399,404 ---- /* write pages until the whole compressed data is written */ do { ! unsigned int length = buffcmpr_length - (buffp - buffcmpr); ! unsigned int copy_length = length > DB_CMPR_PAGESIZE(db_io) ? DB_CMPR_PAGESIZE(db_io) : length; /* * We handle serious compression stuff only if we need to. *************** *** 404,408 **** cmpr.flags = DB_CMPR_INTERNAL; cmpr.next = 0; ! } while(buffp - buffcmpr < buffcmpr_length); #ifdef DEBUG_CMPR --- 432,436 ---- cmpr.flags = DB_CMPR_INTERNAL; cmpr.next = 0; ! } while((unsigned int)(buffp - buffcmpr) < buffcmpr_length); #ifdef DEBUG_CMPR *************** *** 500,507 **** } - #ifdef HAVE_LIBZ - #include "zlib.h" - #endif /* HAVE_LIBZ */ - /* * CDB___memp_cmpr_inflate -- --- 528,531 ---- *************** *** 566,569 **** --- 590,595 ---- int ret = 0; int r; + int off = 0; + int freesp = 0; z_stream c_stream; u_int8_t* outbuff; *************** *** 597,601 **** case P_IBTREE: case P_LBTREE: ! memset((char*)(inbuff + LOFFSET(pg)), '\0', P_FREESPACE(pg)); break; } --- 623,630 ---- case P_IBTREE: case P_LBTREE: ! off = LOFFSET(pg); ! freesp = P_FREESPACE(pg); ! memset((char*)(inbuff + off), 0, freesp); ! //memset((char*)(inbuff + LOFFSET(pg)), '\0', P_FREESPACE(pg)); break; } *************** *** 606,610 **** c_stream.opaque=(voidpf)0; ! if(deflateInit(&c_stream, Z_DEFAULT_COMPRESSION) != Z_OK) { ret = EIO; goto err; --- 635,641 ---- c_stream.opaque=(voidpf)0; ! //if(deflateInit(&c_stream, Z_DEFAULT_COMPRESSION) != Z_OK) ! if(deflateInit(&c_stream, memp_cmpr_zlib_level) != Z_OK) ! { ret = EIO; goto err; *** htdig-3.2.0b4-20021013/htcommon/defaults.cc Sun Sep 22 01:13:07 2002 --- htdig-3.2.0b4-20021013-zlib/htcommon/defaults.cc Fri Oct 18 14:15:52 2002 *************** *** 2540,2543 **** --- 2540,2548 ---- Zlib library is not found on the system, the default is false. \ " }, \ + { "wordlist_compress_zlib", "true", \ + "boolean", "all", "", "3.2.0b4", "Indexing:How", "wordlist_compress_zlib: true", " \ + Enables or disables the zlib compression system for the indexer. \ + wordlist_compress must be true to use this option!`\ + " }, \ { "wordlist_page_size", "0", \ "number", "all", "", "3.2.0b1", "Indexing:How", "wordlist_page_size: 8192", " \ *** htdig-3.2.0b4-20021013/htword/WordList.cc Fri Jul 26 21:48:20 2002 --- htdig-3.2.0b4-20021013-zlib/htword/WordList.cc Fri Oct 18 14:17:38 2002 *************** *** 77,81 **** if(config.Boolean("wordlist_compress") == 1) { usecompress = DB_COMPRESS; ! WordDBCompress* compressor = new WordDBCompress(); // compressor->debug = config.Value("wordlist_compress_debug"); SetCompressor(compressor); --- 77,83 ---- if(config.Boolean("wordlist_compress") == 1) { usecompress = DB_COMPRESS; ! WordDBCompress* compressor = new WordDBCompress( ! config.Boolean("wordlist_compress_zlib",0), config.Value("compression_level",0)); ! // compressor->debug = config.Value("wordlist_compress_debug"); SetCompressor(compressor); *** htdig-3.2.0b4-20021013/htword/WordDBCompress.h Fri Jul 26 21:48:20 2002 --- htdig-3.2.0b4-20021013-zlib/htword/WordDBCompress.h Fri Oct 18 14:19:28 2002 *************** *** 85,88 **** --- 85,89 ---- public: WordDBCompress(); + WordDBCompress(int, int); int Compress(const u_int8_t* inbuff, int inbuff_length, u_int8_t** outbuffp, int* outbuff_lengthp); *************** *** 97,100 **** --- 98,105 ---- private: DB_CMPR_INFO *cmprInfo; + + //ZLIB WordDBCompression Flags + int use_zlib; + int zlib_level; // DEBUGING / BENCHMARKING *** htdig-3.2.0b4-20021013/htword/WordDBCompress.cc Fri Jul 26 21:48:20 2002 --- htdig-3.2.0b4-20021013-zlib/htword/WordDBCompress.cc Fri Oct 18 14:21:24 2002 *************** *** 57,60 **** --- 57,78 ---- WordDBCompress::WordDBCompress() { + + cmprInfo = 0; + + // + // DEBUGING / BENCHMARKING + // + debug = 0; + + //zlib WordDB Compression + use_zlib = 0; + zlib_level = 0; + + } + + + WordDBCompress::WordDBCompress(int zlib, int level) + { + cmprInfo = 0; *************** *** 63,70 **** --- 81,94 ---- // debug = 0; + + //zlib WordDB Compression + use_zlib = zlib; + zlib_level = level; } + DB_CMPR_INFO* WordDBCompress::CmprInfo() { + DB_CMPR_INFO *cmpr_info = new DB_CMPR_INFO; *************** *** 74,77 **** --- 98,106 ---- cmpr_info->coefficient = 3; cmpr_info->max_npages = 9; + + if(use_zlib == 1) + cmpr_info->zlib_flags = zlib_level; + else + cmpr_info->zlib_flags = 0; cmprInfo = cmpr_info;