From 56537e46becc7491dc50f8a6696f03136b901112 Mon Sep 17 00:00:00 2001 From: Nic Limper Date: Sun, 25 Feb 2024 14:45:03 +0100 Subject: [PATCH] dirty fix for zlib compression The miniz is used for zlib compression. But because that library has some bugs related to a smal window sizes (we need 4096), I had to make some patches to the library. The miniz library happens to also be part of esp-idf. To distinguish between our version and the esp-idf version, I created a `namespace Miniz` around our version. But nevertheless, the esp-idf version is called when I'm calling Miniz::tdefl_init and Miniz::tdefl_compress . Why?? Anyway, renamed tdefl_init and tdefl_compress to tdefl_initOEPL and tdefl_compressOEPL to get it working. This is ugly. If somebody has a better solution, please let me know! --- .../lib/miniz-oepl/miniz-oepl.cpp | 24 ++++++++++++------- ESP32_AP-Flasher/lib/miniz-oepl/miniz-oepl.h | 8 +++---- ESP32_AP-Flasher/src/makeimage.cpp | 12 ++++++---- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/ESP32_AP-Flasher/lib/miniz-oepl/miniz-oepl.cpp b/ESP32_AP-Flasher/lib/miniz-oepl/miniz-oepl.cpp index 7ea4727e..e4b7fb9f 100644 --- a/ESP32_AP-Flasher/lib/miniz-oepl/miniz-oepl.cpp +++ b/ESP32_AP-Flasher/lib/miniz-oepl/miniz-oepl.cpp @@ -1,3 +1,5 @@ +#include + #include "miniz-oepl.h" #include @@ -226,7 +228,7 @@ int mz_deflateInit2(mz_streamp pStream, int level, int method, int window_bits, pStream->state = (struct mz_internal_state *)pComp; - if (tdefl_init(pComp, NULL, NULL, comp_flags) != TDEFL_STATUS_OKAY) + if (tdefl_initOEPL(pComp, NULL, NULL, comp_flags) != TDEFL_STATUS_OKAY) { mz_deflateEnd(pStream); return MZ_PARAM_ERROR; @@ -240,7 +242,7 @@ int mz_deflateReset(mz_streamp pStream) if ((!pStream) || (!pStream->state) || (!pStream->zalloc) || (!pStream->zfree)) return MZ_STREAM_ERROR; pStream->total_in = pStream->total_out = 0; - tdefl_init((tdefl_compressor *)pStream->state, NULL, NULL, ((tdefl_compressor *)pStream->state)->m_flags); + tdefl_initOEPL((tdefl_compressor *)pStream->state, NULL, NULL, ((tdefl_compressor *)pStream->state)->m_flags); return MZ_OK; } @@ -269,7 +271,7 @@ int mz_deflate(mz_streamp pStream, int flush) in_bytes = pStream->avail_in; out_bytes = pStream->avail_out; - defl_status = tdefl_compress((tdefl_compressor *)pStream->state, pStream->next_in, &in_bytes, pStream->next_out, &out_bytes, (tdefl_flush)flush); + defl_status = tdefl_compressOEPL((tdefl_compressor *)pStream->state, pStream->next_in, &in_bytes, pStream->next_out, &out_bytes, (tdefl_flush)flush); pStream->next_in += (mz_uint)in_bytes; pStream->avail_in -= (mz_uint)in_bytes; pStream->total_in += (mz_uint)in_bytes; @@ -1591,7 +1593,9 @@ static mz_bool tdefl_compress_fast(tdefl_compressor *d) if (!probe_len) cur_match_len = cur_match_dist ? TDEFL_MAX_MATCH_LEN : 0; - if ((cur_match_len < TDEFL_MIN_MATCH_LEN) || ((cur_match_len == TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 8U * 1024U))) +// fixme: hardcoded 8*1024 +// if ((cur_match_len < TDEFL_MIN_MATCH_LEN) || ((cur_match_len == TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 8U * 1024U))) + if ((cur_match_len < TDEFL_MIN_MATCH_LEN) || ((cur_match_len == TDEFL_MIN_MATCH_LEN) && (cur_match_dist >= 4U * 1024U))) { cur_match_len = 1; *pLZ_code_buf++ = (mz_uint8)first_trigram; @@ -1924,8 +1928,9 @@ static tdefl_status tdefl_flush_output_buffer(tdefl_compressor *d) return (d->m_finished && !d->m_output_flush_remaining) ? TDEFL_STATUS_DONE : TDEFL_STATUS_OKAY; } -tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush) +tdefl_status tdefl_compressOEPL(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush) { + Serial.println("tdefl_compress"); if (!d) { if (pIn_buf_size) @@ -1995,11 +2000,12 @@ tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pI tdefl_status tdefl_compress_buffer(tdefl_compressor *d, const void *pIn_buf, size_t in_buf_size, tdefl_flush flush) { MZ_ASSERT(d->m_pPut_buf_func); - return tdefl_compress(d, pIn_buf, &in_buf_size, NULL, NULL, flush); + return tdefl_compressOEPL(d, pIn_buf, &in_buf_size, NULL, NULL, flush); } -tdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags) +tdefl_status tdefl_initOEPL(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags) { + Serial.println("tdefl_init"); d->m_pPut_buf_func = pPut_buf_func; d->m_pPut_buf_user = pPut_buf_user; d->m_flags = (mz_uint)(flags); @@ -2053,7 +2059,7 @@ mz_bool tdefl_compress_mem_to_output(const void *pBuf, size_t buf_len, tdefl_put pComp = (tdefl_compressor *)MZ_MALLOC(sizeof(tdefl_compressor)); if (!pComp) return MZ_FALSE; - succeeded = (tdefl_init(pComp, pPut_buf_func, pPut_buf_user, flags) == TDEFL_STATUS_OKAY); + succeeded = (tdefl_initOEPL(pComp, pPut_buf_func, pPut_buf_user, flags) == TDEFL_STATUS_OKAY); succeeded = succeeded && (tdefl_compress_buffer(pComp, pBuf, buf_len, TDEFL_FINISH) == TDEFL_STATUS_DONE); MZ_FREE(pComp); return succeeded; @@ -2171,7 +2177,7 @@ void *tdefl_write_image_to_png_file_in_memory_ex(const void *pImage, int w, int for (z = 41; z; --z) tdefl_output_buffer_putter(&z, 1, &out_buf); /* compress image data */ - tdefl_init(pComp, tdefl_output_buffer_putter, &out_buf, s_tdefl_png_num_probes[MZ_MIN(10, level)] | TDEFL_WRITE_ZLIB_HEADER); + tdefl_initOEPL(pComp, tdefl_output_buffer_putter, &out_buf, s_tdefl_png_num_probes[MZ_MIN(10, level)] | TDEFL_WRITE_ZLIB_HEADER); for (y = 0; y < h; ++y) { tdefl_compress_buffer(pComp, &z, 1, TDEFL_NO_FLUSH); diff --git a/ESP32_AP-Flasher/lib/miniz-oepl/miniz-oepl.h b/ESP32_AP-Flasher/lib/miniz-oepl/miniz-oepl.h index 295e8be9..39326298 100644 --- a/ESP32_AP-Flasher/lib/miniz-oepl/miniz-oepl.h +++ b/ESP32_AP-Flasher/lib/miniz-oepl/miniz-oepl.h @@ -149,7 +149,7 @@ namespace Miniz { /*#define MINIZ_NO_ZLIB_APIS */ /* Define MINIZ_NO_ZLIB_COMPATIBLE_NAME to disable zlib names, to prevent conflicts against stock zlib. */ -/*#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES */ +#define MINIZ_NO_ZLIB_COMPATIBLE_NAMES /* Define MINIZ_NO_MALLOC to disable all calls to malloc, free, and realloc. Note if MINIZ_NO_MALLOC is defined then the user must always provide custom user alloc/free/realloc @@ -316,7 +316,7 @@ enum }; /* Window bits */ -#define MZ_DEFAULT_WINDOW_BITS 15 +#define MZ_DEFAULT_WINDOW_BITS 12 struct mz_internal_state; @@ -797,10 +797,10 @@ typedef struct /* pBut_buf_func: If NULL, output data will be supplied to the specified callback. In this case, the user should call the tdefl_compress_buffer() API for compression. */ /* If pBut_buf_func is NULL the user should always call the tdefl_compress() API. */ /* flags: See the above enums (TDEFL_HUFFMAN_ONLY, TDEFL_WRITE_ZLIB_HEADER, etc.) */ -MINIZ_EXPORT tdefl_status tdefl_init(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags); +MINIZ_EXPORT tdefl_status tdefl_initOEPL(tdefl_compressor *d, tdefl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags); /* Compresses a block of data, consuming as much of the specified input buffer as possible, and writing as much compressed data to the specified output buffer as possible. */ -MINIZ_EXPORT tdefl_status tdefl_compress(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush); +MINIZ_EXPORT tdefl_status tdefl_compressOEPL(tdefl_compressor *d, const void *pIn_buf, size_t *pIn_buf_size, void *pOut_buf, size_t *pOut_buf_size, tdefl_flush flush); /* tdefl_compress_buffer() is only usable when the tdefl_init() is called with a non-NULL tdefl_put_buf_func_ptr. */ /* tdefl_compress_buffer() always consumes the entire input buffer. */ diff --git a/ESP32_AP-Flasher/src/makeimage.cpp b/ESP32_AP-Flasher/src/makeimage.cpp index 16db2718..d9ee0452 100644 --- a/ESP32_AP-Flasher/src/makeimage.cpp +++ b/ESP32_AP-Flasher/src/makeimage.cpp @@ -249,7 +249,7 @@ size_t prepareHeader(uint8_t headerbuf[], uint16_t bufw, uint16_t bufh, imgParam } bool initializeCompressor(Miniz::tdefl_compressor *comp, int flags) { - return Miniz::tdefl_init(comp, NULL, NULL, flags) == Miniz::TDEFL_STATUS_OKAY; + return Miniz::tdefl_initOEPL(comp, NULL, NULL, flags) == Miniz::TDEFL_STATUS_OKAY; } size_t compressAndWrite(Miniz::tdefl_compressor *comp, const void *inbuf, size_t inbytes, void *zlibbuf, size_t outsize, size_t totalbytes, File &f_out, Miniz::tdefl_flush flush) { @@ -257,7 +257,7 @@ size_t compressAndWrite(Miniz::tdefl_compressor *comp, const void *inbuf, size_t size_t outbytes_compressed = outsize; uint32_t t = millis(); - tdefl_compress(comp, inbuf, &inbytes_compressed, zlibbuf, &outbytes_compressed, flush); + Miniz::tdefl_compressOEPL(comp, inbuf, &inbytes_compressed, zlibbuf, &outbytes_compressed, flush); Serial.printf("zlib: compressed %d into %d bytes in %d ms\n", inbytes_compressed, outbytes_compressed, millis()-t); f_out.write((const uint8_t *)zlibbuf, outbytes_compressed); @@ -266,7 +266,8 @@ size_t compressAndWrite(Miniz::tdefl_compressor *comp, const void *inbuf, size_t void rewriteHeader(File &f_out) { // https://www.rfc-editor.org/rfc/rfc1950 - const uint8_t cmf = 0x48; + const uint8_t cmf = 0x48; // 4096 + // const uint8_t cmf = 0x58; // 8192 uint8_t flg, flevel = 3; uint16_t header = cmf << 8 | (flevel << 6); header += 31 - (header % 31); @@ -333,6 +334,7 @@ void spr2buffer(TFT_eSprite &spr, String &fileout, imgParam &imageParams) { f_out.write(reinterpret_cast(&totalbytes), sizeof(uint32_t)); + // 768 = compression level 9, 1500 = unofficial level 10 if (comp == NULL || zlibbuf == NULL || totalbytes == 0 || !initializeCompressor(comp, Miniz::TDEFL_WRITE_ZLIB_HEADER | 1500)) { Serial.println("Failed to initialize compressor or allocate memory for zlib"); if (zlibbuf != NULL) free(zlibbuf); @@ -348,10 +350,10 @@ void spr2buffer(TFT_eSprite &spr, String &fileout, imgParam &imageParams) { compressAndWrite(comp, buffer, buffer_size, zlibbuf, buffer_size, buffer_size, f_out, Miniz::TDEFL_FINISH); } - rewriteHeader(f_out); - free(zlibbuf); free(comp); + + rewriteHeader(f_out); } else { f_out.write(buffer, buffer_size); if (imageParams.hasRed && imageParams.bpp > 1) {