blob: 07ddfefd6db9be5f450148765b0a99a7d4e9ddb3 [file] [log] [blame]
Yann Collet32fb4072017-08-18 16:52:05 -07001/*
Elliott Hughes44aba642023-09-12 20:18:59 +00002 * Copyright (c) Meta Platforms, Inc. and affiliates.
Yann Collet4ded9e52016-08-30 10:04:33 -07003 * All rights reserved.
4 *
Yann Collet32fb4072017-08-18 16:52:05 -07005 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
Yann Collet3128e032017-09-08 00:09:23 -07008 * You may select, at your option, one of the above-listed licenses.
Yann Collet4ded9e52016-08-30 10:04:33 -07009 */
Yann Collet4856a002015-01-24 01:58:16 +010010
Yann Collet4856a002015-01-24 01:58:16 +010011
Yann Collet0d9ce042016-03-19 13:21:08 +010012/*-************************************
Yann Collet4856a002015-01-24 01:58:16 +010013* Compiler specific
14**************************************/
15#ifdef _MSC_VER /* Visual Studio */
Yann Colleta5ffe3d2017-05-12 16:29:19 -070016# define _CRT_SECURE_NO_WARNINGS /* fgets */
17# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
Yann Collet4856a002015-01-24 01:58:16 +010019#endif
20
Yann Collet4856a002015-01-24 01:58:16 +010021
Yann Collet0d9ce042016-03-19 13:21:08 +010022/*-************************************
Yann Collet4856a002015-01-24 01:58:16 +010023* Includes
24**************************************/
Yann Collet6fa05a22016-07-20 14:58:49 +020025#include <stdlib.h> /* free */
26#include <stdio.h> /* fgets, sscanf */
Yann Collet6fa05a22016-07-20 14:58:49 +020027#include <string.h> /* strcmp */
Elliott Hughes44aba642023-09-12 20:18:59 +000028#include <time.h> /* time(), time_t */
29#undef NDEBUG /* always enable assert() */
Yann Colletd88c6712017-12-19 10:16:09 +010030#include <assert.h>
Yann Colleta5ffe3d2017-05-12 16:29:19 -070031#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
W. Felix Handtedacbcd22020-05-01 12:24:51 -040032#include "debug.h" /* DEBUG_STATIC_ASSERT */
Nick Terrellb9faaa12018-07-30 12:57:11 -070033#include "fse.h"
sen40def702021-05-13 14:41:21 -040034#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
Yann Collet6fa05a22016-07-20 14:58:49 +020035#include "zstd.h" /* ZSTD_VERSION_STRING */
Yann Collete7a41a52016-12-05 16:21:06 -080036#include "zstd_errors.h" /* ZSTD_getErrorCode */
Nick Terrella8b4fe02017-01-02 18:45:19 -080037#define ZDICT_STATIC_LINKING_ONLY
Yann Collet6fa05a22016-07-20 14:58:49 +020038#include "zdict.h" /* ZDICT_trainFromBuffer */
Yann Collet353c5d22015-10-21 14:39:26 +010039#include "mem.h"
Yann Collet20bd2462020-05-11 19:29:36 -070040#include "datagen.h" /* RDG_genBuffer */
Yann Colleta146ee02018-03-11 05:21:53 -070041#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
Yann Collet6fa05a22016-07-20 14:58:49 +020042#include "xxhash.h" /* XXH64 */
Nick Terrell9a2f6f42017-11-29 19:11:12 -080043#include "util.h"
Yann Collet59a71162019-04-10 12:37:03 -070044#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
Yann Collet20bd2462020-05-11 19:29:36 -070045/* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
Yann Collet7fce9a42021-09-08 14:05:57 -070046#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
send8d6e482021-05-07 11:13:44 -040047#include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */
Yann Collet4856a002015-01-24 01:58:16 +010048
49
Yann Collet0d9ce042016-03-19 13:21:08 +010050/*-************************************
51* Constants
Yann Collet4856a002015-01-24 01:58:16 +010052**************************************/
Yann Collet4856a002015-01-24 01:58:16 +010053#define GB *(1U<<30)
54
Yann Colletededcfc2018-12-21 16:19:44 -080055static const int FUZ_compressibility_default = 50;
56static const int nbTestsDefault = 30000;
Yann Collet4856a002015-01-24 01:58:16 +010057
58
Yann Collet0d9ce042016-03-19 13:21:08 +010059/*-************************************
Yann Collet4856a002015-01-24 01:58:16 +010060* Display Macros
61**************************************/
Yann Collet4b525af2018-02-02 15:58:13 -080062#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
Yann Collet4856a002015-01-24 01:58:16 +010063#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
64static U32 g_displayLevel = 2;
65
Nick Terrell9a2f6f42017-11-29 19:11:12 -080066static const U64 g_refreshRate = SEC_TO_MICRO / 6;
67static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
Yann Collet553cf6a2015-12-04 17:25:26 +010068
Yann Collet4baecdf2019-05-28 13:15:48 -070069#define DISPLAYUPDATE(l, ...) \
70 if (g_displayLevel>=l) { \
71 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
72 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
73 if (g_displayLevel>=4) fflush(stderr); } \
74 }
Yann Collet4856a002015-01-24 01:58:16 +010075
Yann Colletff795582018-01-10 20:33:45 -080076
Yann Collet31769ce2018-06-22 17:58:21 -070077/*-*******************************************************
78* Compile time test
79*********************************************************/
Nick Terrell9a2f6f42017-11-29 19:11:12 -080080#undef MIN
81#undef MAX
Yann Collet4baecdf2019-05-28 13:15:48 -070082/* Declaring the function, to avoid -Wmissing-prototype */
Nick Terrellf2d6db42018-09-27 15:13:43 -070083void FUZ_bug976(void);
Yann Collet2103a622018-01-11 04:49:19 -080084void FUZ_bug976(void)
Yann Colletff795582018-01-10 20:33:45 -080085{ /* these constants shall not depend on MIN() macro */
Elliott Hughes44aba642023-09-12 20:18:59 +000086 DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
87 DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
Yann Colletff795582018-01-10 20:33:45 -080088}
89
Yann Collet31769ce2018-06-22 17:58:21 -070090
Yann Colletff795582018-01-10 20:33:45 -080091/*-*******************************************************
92* Internal functions
93*********************************************************/
Yann Collete93add02016-02-15 17:44:14 +010094#define MIN(a,b) ((a)<(b)?(a):(b))
Sean Purcellf5e50512017-03-15 15:04:54 -070095#define MAX(a,b) ((a)>(b)?(a):(b))
Yann Collet110cc142015-11-19 12:02:28 +010096
Yann Colletc0a9bf32016-05-30 01:56:08 +020097#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
Yann Colletededcfc2018-12-21 16:19:44 -080098static U32 FUZ_rand(U32* src)
Yann Collet4856a002015-01-24 01:58:16 +010099{
Yann Collet0d9ce042016-03-19 13:21:08 +0100100 static const U32 prime1 = 2654435761U;
101 static const U32 prime2 = 2246822519U;
Yann Collet4856a002015-01-24 01:58:16 +0100102 U32 rand32 = *src;
103 rand32 *= prime1;
104 rand32 += prime2;
105 rand32 = FUZ_rotl32(rand32, 13);
106 *src = rand32;
107 return rand32 >> 5;
108}
109
Yann Colletededcfc2018-12-21 16:19:44 -0800110static U32 FUZ_highbit32(U32 v32)
Yann Collet4856a002015-01-24 01:58:16 +0100111{
112 unsigned nbBits = 0;
113 if (v32==0) return 0;
Yann Colletc0a9bf32016-05-30 01:56:08 +0200114 while (v32) v32 >>= 1, nbBits++;
Yann Collet4856a002015-01-24 01:58:16 +0100115 return nbBits;
116}
Yann Collet4856a002015-01-24 01:58:16 +0100117
118
Yann Colletcb327632016-08-23 00:30:31 +0200119/*=============================================
Yann Collet574e7532017-12-19 11:24:14 +0100120* Test macros
121=============================================*/
Elliott Hughes44aba642023-09-12 20:18:59 +0000122#define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
123
124#define CHECK_Z(f) { \
125 size_t const err = f; \
126 if (ZSTD_isError(err)) { \
127 DISPLAY("Error => %s : %s ", \
128 #f, ZSTD_getErrorName(err)); \
129 exit(1); \
Yann Collet574e7532017-12-19 11:24:14 +0100130} }
131
Elliott Hughes44aba642023-09-12 20:18:59 +0000132#define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
Yann Collet810a9ca2019-08-01 16:59:22 +0200133#define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
Yann Collet810a9ca2019-08-01 16:59:22 +0200134#define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
Yann Collet574e7532017-12-19 11:24:14 +0100135
Nick Terrell6efce7c2019-02-19 11:07:52 -0800136#define CHECK_OP(op, lhs, rhs) { \
137 if (!((lhs) op (rhs))) { \
138 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
Elliott Hughes44aba642023-09-12 20:18:59 +0000139 exit(1); \
Nick Terrell280a2362018-04-12 11:50:12 -0700140 } \
141}
Nick Terrell6efce7c2019-02-19 11:07:52 -0800142#define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
143#define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
Nick Terrell280a2362018-04-12 11:50:12 -0700144
Yann Collet574e7532017-12-19 11:24:14 +0100145
146/*=============================================
Yann Colletf9524cf2017-07-10 13:48:41 -0700147* Memory Tests
148=============================================*/
149#if defined(__APPLE__) && defined(__MACH__)
150
151#include <malloc/malloc.h> /* malloc_size */
152
153typedef struct {
154 unsigned long long totalMalloc;
Yann Collet670b1fc2017-07-10 16:30:55 -0700155 size_t currentMalloc;
Yann Colletf9524cf2017-07-10 13:48:41 -0700156 size_t peakMalloc;
157 unsigned nbMalloc;
158 unsigned nbFree;
159} mallocCounter_t;
160
Yann Collet670b1fc2017-07-10 16:30:55 -0700161static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };
Yann Colletf9524cf2017-07-10 13:48:41 -0700162
163static void* FUZ_mallocDebug(void* counter, size_t size)
164{
165 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
166 void* const ptr = malloc(size);
167 if (ptr==NULL) return NULL;
Yann Colletef0ff7f2017-07-11 08:54:29 -0700168 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
Yann Colletededcfc2018-12-21 16:19:44 -0800169 (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */
Yann Colletf9524cf2017-07-10 13:48:41 -0700170 mcPtr->totalMalloc += size;
Yann Collet670b1fc2017-07-10 16:30:55 -0700171 mcPtr->currentMalloc += size;
172 if (mcPtr->currentMalloc > mcPtr->peakMalloc)
173 mcPtr->peakMalloc = mcPtr->currentMalloc;
Yann Colletf9524cf2017-07-10 13:48:41 -0700174 mcPtr->nbMalloc += 1;
175 return ptr;
176}
177
178static void FUZ_freeDebug(void* counter, void* address)
179{
180 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
Yann Colletededcfc2018-12-21 16:19:44 -0800181 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10));
Yann Colletf9524cf2017-07-10 13:48:41 -0700182 mcPtr->nbFree += 1;
Yann Collet670b1fc2017-07-10 16:30:55 -0700183 mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */
184 free(address);
Yann Colletf9524cf2017-07-10 13:48:41 -0700185}
186
187static void FUZ_displayMallocStats(mallocCounter_t count)
188{
Yann Collet1ca12882017-07-19 16:01:16 -0700189 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n",
Yann Colletededcfc2018-12-21 16:19:44 -0800190 (unsigned)(count.peakMalloc >> 10),
Yann Colletf9524cf2017-07-10 13:48:41 -0700191 count.nbMalloc,
Yann Colletededcfc2018-12-21 16:19:44 -0800192 (unsigned)(count.totalMalloc >> 10));
Yann Colletf9524cf2017-07-10 13:48:41 -0700193}
194
Yann Collet3692c312018-08-14 16:56:07 -0700195static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
196 void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
Yann Colletf9524cf2017-07-10 13:48:41 -0700197{
Yann Colletf9524cf2017-07-10 13:48:41 -0700198 /* test only played in verbose mode, as they are long */
199 if (g_displayLevel<3) return 0;
200
201 /* Create compressible noise */
202 if (!inBuffer || !outBuffer) {
203 DISPLAY("Not enough memory, aborting\n");
204 exit(1);
205 }
206 RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
207
208 /* simple compression tests */
Yann Collet1ca12882017-07-19 16:01:16 -0700209 if (part <= 1)
Yann Colletf9524cf2017-07-10 13:48:41 -0700210 { int compressionLevel;
Yann Colletee3423d2017-07-10 14:09:16 -0700211 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
Yann Collet3510efb2017-07-10 14:21:40 -0700212 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
213 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
Yann Colletf9524cf2017-07-10 13:48:41 -0700214 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
Yann Collet670b1fc2017-07-10 16:30:55 -0700215 CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
Yann Colletf9524cf2017-07-10 13:48:41 -0700216 ZSTD_freeCCtx(cctx);
217 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
218 FUZ_displayMallocStats(malcount);
Yann Colletf9524cf2017-07-10 13:48:41 -0700219 } }
220
221 /* streaming compression tests */
Yann Collet1ca12882017-07-19 16:01:16 -0700222 if (part <= 2)
Yann Colletf9524cf2017-07-10 13:48:41 -0700223 { int compressionLevel;
Yann Colletee3423d2017-07-10 14:09:16 -0700224 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
Yann Collet3510efb2017-07-10 14:21:40 -0700225 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
226 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
Yann Colletf9524cf2017-07-10 13:48:41 -0700227 ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
228 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
229 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
Yann Collet670b1fc2017-07-10 16:30:55 -0700230 CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
231 CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
232 CHECK_Z( ZSTD_endStream(cstream, &out) );
Yann Colletf9524cf2017-07-10 13:48:41 -0700233 ZSTD_freeCStream(cstream);
234 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
235 FUZ_displayMallocStats(malcount);
Yann Colletf9524cf2017-07-10 13:48:41 -0700236 } }
237
Yann Colletf9524cf2017-07-10 13:48:41 -0700238 /* advanced MT API test */
Yann Collet1ca12882017-07-19 16:01:16 -0700239 if (part <= 3)
Yann Collet98692c22019-08-01 15:58:17 +0200240 { int nbThreads;
Yann Colletee3423d2017-07-10 14:09:16 -0700241 for (nbThreads=1; nbThreads<=4; nbThreads++) {
242 int compressionLevel;
Yann Colletee3423d2017-07-10 14:09:16 -0700243 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
Yann Collet3510efb2017-07-10 14:21:40 -0700244 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
245 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
Yann Colletee3423d2017-07-10 14:09:16 -0700246 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
Yann Collet3583d192018-12-05 17:26:02 -0800247 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
248 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
Yann Collet6ced8f72018-12-03 14:22:38 -0800249 CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
Yann Colletee3423d2017-07-10 14:09:16 -0700250 ZSTD_freeCCtx(cctx);
Yann Collet98692c22019-08-01 15:58:17 +0200251 DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
Yann Colletee3423d2017-07-10 14:09:16 -0700252 nbThreads, compressionLevel);
253 FUZ_displayMallocStats(malcount);
Yann Colletee3423d2017-07-10 14:09:16 -0700254 } } }
Yann Colletf9524cf2017-07-10 13:48:41 -0700255
256 /* advanced MT streaming API test */
Yann Collet1ca12882017-07-19 16:01:16 -0700257 if (part <= 4)
Yann Collet4baecdf2019-05-28 13:15:48 -0700258 { int nbThreads;
Yann Colletee3423d2017-07-10 14:09:16 -0700259 for (nbThreads=1; nbThreads<=4; nbThreads++) {
260 int compressionLevel;
Yann Colletee3423d2017-07-10 14:09:16 -0700261 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
Yann Collet3510efb2017-07-10 14:21:40 -0700262 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
263 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
Yann Colletee3423d2017-07-10 14:09:16 -0700264 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
265 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
266 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
Yann Collet3583d192018-12-05 17:26:02 -0800267 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
268 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
Yann Colletd8e215c2018-11-30 11:16:26 -0800269 CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
270 while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
Yann Colletee3423d2017-07-10 14:09:16 -0700271 ZSTD_freeCCtx(cctx);
Yann Collet4baecdf2019-05-28 13:15:48 -0700272 DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ",
Yann Colletee3423d2017-07-10 14:09:16 -0700273 nbThreads, compressionLevel);
274 FUZ_displayMallocStats(malcount);
Yann Colletee3423d2017-07-10 14:09:16 -0700275 } } }
Yann Colletf9524cf2017-07-10 13:48:41 -0700276
277 return 0;
278}
279
Yann Collet3692c312018-08-14 16:56:07 -0700280static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
281{
282 size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
283 size_t const outSize = ZSTD_compressBound(inSize);
284 void* const inBuffer = malloc(inSize);
285 void* const outBuffer = malloc(outSize);
286 int result;
287
288 /* Create compressible noise */
289 if (!inBuffer || !outBuffer) {
290 DISPLAY("Not enough memory, aborting \n");
291 exit(1);
292 }
293
294 result = FUZ_mallocTests_internal(seed, compressibility, part,
295 inBuffer, inSize, outBuffer, outSize);
296
297 free(inBuffer);
298 free(outBuffer);
299 return result;
300}
301
Yann Colletf9524cf2017-07-10 13:48:41 -0700302#else
303
Yann Collet5e6c5202017-07-20 15:11:56 -0700304static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
Yann Colletf9524cf2017-07-10 13:48:41 -0700305{
Yann Collet38ba7002017-07-20 18:39:04 -0700306 (void)seed; (void)compressibility; (void)part;
Yann Colletf9524cf2017-07-10 13:48:41 -0700307 return 0;
308}
309
310#endif
311
senhuang429102f302020-11-02 11:30:31 -0500312static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
313 BYTE* src, size_t size, ZSTD_sequenceFormat_e format)
Bimba Shresthabb274722019-09-26 15:38:31 -0700314{
315 size_t i;
Bimba Shrestha36528b92019-10-03 09:26:51 -0700316 size_t j;
senhuang42dc448562020-10-27 12:28:46 -0400317 for(i = 0; i < seqsSize; ++i) {
318 assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size);
319 assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size);
senhuang429102f302020-11-02 11:30:31 -0500320 if (format == ZSTD_sf_noBlockDelimiters) {
321 assert(seqs[i].matchLength != 0 || seqs[i].offset != 0);
322 }
Bimba Shresthabb274722019-09-26 15:38:31 -0700323
324 memcpy(dst, src, seqs[i].litLength);
325 dst += seqs[i].litLength;
326 src += seqs[i].litLength;
327 size -= seqs[i].litLength;
328
senhuang42dc448562020-10-27 12:28:46 -0400329 if (seqs[i].offset != 0) {
330 for (j = 0; j < seqs[i].matchLength; ++j)
331 dst[j] = dst[j - seqs[i].offset];
332 dst += seqs[i].matchLength;
333 src += seqs[i].matchLength;
334 size -= seqs[i].matchLength;
335 }
Bimba Shresthabb274722019-09-26 15:38:31 -0700336 }
senhuang429102f302020-11-02 11:30:31 -0500337 if (format == ZSTD_sf_noBlockDelimiters) {
338 memcpy(dst, src, size);
339 }
Bimba Shresthabb274722019-09-26 15:38:31 -0700340}
341
send8d6e482021-05-07 11:13:44 -0400342#ifdef ZSTD_MULTITHREAD
Elliott Hughes44aba642023-09-12 20:18:59 +0000343
send8d6e482021-05-07 11:13:44 -0400344typedef struct {
345 ZSTD_CCtx* cctx;
346 ZSTD_threadPool* pool;
senb35c2502021-05-12 12:56:57 -0400347 void* CNBuffer;
send8d6e482021-05-07 11:13:44 -0400348 size_t CNBuffSize;
senb35c2502021-05-12 12:56:57 -0400349 void* compressedBuffer;
send8d6e482021-05-07 11:13:44 -0400350 size_t compressedBufferSize;
senb35c2502021-05-12 12:56:57 -0400351 void* decodedBuffer;
send8d6e482021-05-07 11:13:44 -0400352 int err;
353} threadPoolTests_compressionJob_payload;
354
355static void* threadPoolTests_compressionJob(void* payload) {
356 threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload;
357 size_t cSize;
358 if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1;
359 cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize);
360 if (ZSTD_isError(cSize)) args->err = 1;
361 if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1;
362 return payload;
363}
364
365static int threadPoolTests(void) {
366 int testResult = 0;
367 size_t err;
368
369 size_t const CNBuffSize = 5 MB;
370 void* const CNBuffer = malloc(CNBuffSize);
371 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
372 void* const compressedBuffer = malloc(compressedBufferSize);
373 void* const decodedBuffer = malloc(CNBuffSize);
374
375 size_t const kPoolNumThreads = 8;
376
377 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0);
378
379 DISPLAYLEVEL(3, "thread pool test : threadPool re-use roundtrips: ");
380 {
381 ZSTD_CCtx* cctx = ZSTD_createCCtx();
382 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
383
384 size_t nbThreads = 1;
385 for (; nbThreads <= kPoolNumThreads; ++nbThreads) {
386 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
387 ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads);
388 err = ZSTD_CCtx_refThreadPool(cctx, pool);
389 if (ZSTD_isError(err)) {
390 DISPLAYLEVEL(3, "refThreadPool error!\n");
391 ZSTD_freeCCtx(cctx);
392 goto _output_error;
393 }
394 err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
395 if (ZSTD_isError(err)) {
396 DISPLAYLEVEL(3, "Compression error!\n");
397 ZSTD_freeCCtx(cctx);
398 goto _output_error;
399 }
400 err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err);
401 if (ZSTD_isError(err)) {
402 DISPLAYLEVEL(3, "Decompression error!\n");
403 ZSTD_freeCCtx(cctx);
404 goto _output_error;
405 }
406 }
407
408 ZSTD_freeCCtx(cctx);
409 ZSTD_freeThreadPool(pool);
410 }
411 DISPLAYLEVEL(3, "OK \n");
412
413 DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: ");
414 {
415 void* const decodedBuffer2 = malloc(CNBuffSize);
416 void* const compressedBuffer2 = malloc(compressedBufferSize);
417 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
418 ZSTD_CCtx* cctx1 = ZSTD_createCCtx();
419 ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
420
421 ZSTD_pthread_t t1;
422 ZSTD_pthread_t t2;
423 threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize,
424 compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */};
425 threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize,
426 compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */};
427
428 ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2);
429 ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2);
430 ZSTD_CCtx_refThreadPool(cctx1, pool);
431 ZSTD_CCtx_refThreadPool(cctx2, pool);
432
433 ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
434 ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
Elliott Hughes44aba642023-09-12 20:18:59 +0000435 ZSTD_pthread_join(t1);
436 ZSTD_pthread_join(t2);
send8d6e482021-05-07 11:13:44 -0400437
438 assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
439 free(decodedBuffer2);
440 free(compressedBuffer2);
441
442 ZSTD_freeThreadPool(pool);
443 ZSTD_freeCCtx(cctx1);
444 ZSTD_freeCCtx(cctx2);
445
446 if (p1.err || p2.err) goto _output_error;
447 }
448 DISPLAYLEVEL(3, "OK \n");
449
450_end:
451 free(CNBuffer);
452 free(compressedBuffer);
453 free(decodedBuffer);
454 return testResult;
455
456_output_error:
457 testResult = 1;
458 DISPLAY("Error detected in Unit tests ! \n");
459 goto _end;
460}
461#endif /* ZSTD_MULTITHREAD */
462
Yann Colletf9524cf2017-07-10 13:48:41 -0700463/*=============================================
464* Unit tests
Yann Colletcb327632016-08-23 00:30:31 +0200465=============================================*/
466
Elliott Hughes44aba642023-09-12 20:18:59 +0000467static void test_compressBound(unsigned tnb)
468{
469 DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
470
471 /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
472 * for a large range of known valid values */
473 DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
474 { int s;
475 for (s=0; s<30; s++) {
476 size_t const w = (size_t)1 << s;
477 CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
478 } }
479
480 /* Ensure error if srcSize too big */
481 { size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
482 CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
483 CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
484 }
485
486 DISPLAYLEVEL(3, "OK \n");
487}
488
489static void test_decompressBound(unsigned tnb)
490{
491 DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
492
493 /* Simple compression, with size : should provide size; */
494 { const char example[] = "abcd";
495 char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
496 size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
497 CHECK_Z(cSize);
498 CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
499 }
500
501 /* Simple small compression without size : should provide 1 block size */
502 { char cBuffer[ZSTD_COMPRESSBOUND(0)];
503 ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
504 ZSTD_inBuffer in = { NULL, 0, 0 };
505 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
506 assert(cctx);
507 CHECK_Z( ZSTD_initCStream(cctx, 0) );
508 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
509 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
510 CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
511 ZSTD_freeCCtx(cctx);
512 }
513
514 /* Attempt to overflow 32-bit intermediate multiplication result
515 * This requires dBound >= 4 GB, aka 2^32.
516 * This requires 2^32 / 2^17 = 2^15 blocks
517 * => create 2^15 blocks (can be empty, or just 1 byte). */
518 { const char input[] = "a";
519 size_t const nbBlocks = (1 << 15) + 1;
520 size_t blockNb;
521 size_t const outCapacity = 1 << 18; /* large margin */
522 char* const outBuffer = malloc (outCapacity);
523 ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
524 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
525 assert(cctx);
526 assert(outBuffer);
527 CHECK_Z( ZSTD_initCStream(cctx, 0) );
528 for (blockNb=0; blockNb<nbBlocks; blockNb++) {
529 ZSTD_inBuffer in = { input, sizeof(input), 0 };
530 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
531 CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
532 }
533 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
534 CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000LLU /* 4 GB */ );
535 ZSTD_freeCCtx(cctx);
536 free(outBuffer);
537 }
538
539 DISPLAYLEVEL(3, "OK \n");
540}
541
542static void test_setCParams(unsigned tnb)
543{
544 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
545 ZSTD_compressionParameters cparams;
546 assert(cctx);
547
548 DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
549
550 /* valid cparams */
551 cparams = ZSTD_getCParams(1, 0, 0);
552 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
553
554 /* invalid cparams (must fail) */
555 cparams.windowLog = 99;
556 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
557
558 free(cctx);
559 DISPLAYLEVEL(3, "OK \n");
560}
561
Yann Collet37f47e52019-08-02 17:34:53 +0200562static int basicUnitTests(U32 const seed, double compressibility)
Yann Collet4856a002015-01-24 01:58:16 +0100563{
Yann Collet30009522016-05-30 16:17:33 +0200564 size_t const CNBuffSize = 5 MB;
Yann Colletd2858e92016-05-30 15:10:09 +0200565 void* const CNBuffer = malloc(CNBuffSize);
Yann Colletf9524cf2017-07-10 13:48:41 -0700566 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
567 void* const compressedBuffer = malloc(compressedBufferSize);
Yann Colletd2858e92016-05-30 15:10:09 +0200568 void* const decodedBuffer = malloc(CNBuffSize);
Yann Colletc0a9bf32016-05-30 01:56:08 +0200569 int testResult = 0;
Yann Colletededcfc2018-12-21 16:19:44 -0800570 unsigned testNb=0;
Yann Colletc0a9bf32016-05-30 01:56:08 +0200571 size_t cSize;
Yann Collet4856a002015-01-24 01:58:16 +0100572
Yann Colletc0a9bf32016-05-30 01:56:08 +0200573 /* Create compressible noise */
Yann Colletd1d210f2016-03-19 12:12:07 +0100574 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
Yann Collet94f998b2015-07-04 23:10:40 -0800575 DISPLAY("Not enough memory, aborting\n");
576 testResult = 1;
577 goto _end;
578 }
Yann Colletd2858e92016-05-30 15:10:09 +0200579 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
Yann Collet4856a002015-01-24 01:58:16 +0100580
Yann Colletd5d9bc32015-08-23 23:13:49 +0100581 /* Basic tests */
Yann Colletededcfc2018-12-21 16:19:44 -0800582 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++);
Yann Collet9a69ec42016-08-01 16:25:58 +0200583 { const char* errorString = ZSTD_getErrorName(0);
Yann Collet64482c22017-12-29 17:04:37 +0100584 DISPLAYLEVEL(3, "OK : %s \n", errorString);
Yann Collet9a69ec42016-08-01 16:25:58 +0200585 }
586
Yann Colletededcfc2018-12-21 16:19:44 -0800587 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++);
Yann Collet9a69ec42016-08-01 16:25:58 +0200588 { const char* errorString = ZSTD_getErrorName(499);
Yann Collet64482c22017-12-29 17:04:37 +0100589 DISPLAYLEVEL(3, "OK : %s \n", errorString);
Yann Collet9a69ec42016-08-01 16:25:58 +0200590 }
591
Yann Colletededcfc2018-12-21 16:19:44 -0800592 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++);
Yann Colleta54c86c2018-09-20 16:17:49 -0700593 { int const mcl = ZSTD_minCLevel();
594 DISPLAYLEVEL(3, "%i (OK) \n", mcl);
595 }
Yann Collet0ef68032017-03-29 16:58:57 -0700596
Sen Huange3987442021-03-24 10:43:48 -0700597 DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++);
598 { int const defaultCLevel = ZSTD_defaultCLevel();
599 if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error;
600 DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel);
601 }
602
Bimba Shrestha6a4258a2020-02-05 16:55:00 -0800603 DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
604 { unsigned const vn = ZSTD_versionNumber();
605 DISPLAYLEVEL(3, "%u (OK) \n", vn);
606 }
607
Elliott Hughes44aba642023-09-12 20:18:59 +0000608 test_compressBound(testNb++);
609
610 test_decompressBound(testNb++);
611
612 test_setCParams(testNb++);
613
Nick Terrelld5c688e2020-10-01 13:12:23 -0700614 DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
615 {
616 ZSTD_compressionParameters params;
617 memset(&params, 0, sizeof(params));
618 params.windowLog = 10;
619 params.hashLog = 19;
620 params.chainLog = 19;
621 params = ZSTD_adjustCParams(params, 1000, 100000);
622 if (params.hashLog != 18) goto _output_error;
623 if (params.chainLog != 17) goto _output_error;
624 }
625 DISPLAYLEVEL(3, "OK \n");
626
Yann Colletededcfc2018-12-21 16:19:44 -0800627 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
Yann Colleta54c86c2018-09-20 16:17:49 -0700628 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Yann Collete32fb0c2017-07-10 12:29:57 -0700629 if (cctx==NULL) goto _output_error;
Yann Collet810a9ca2019-08-01 16:59:22 +0200630 CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
Yann Colletf9524cf2017-07-10 13:48:41 -0700631 compressedBuffer, compressedBufferSize,
Yann Collet810a9ca2019-08-01 16:59:22 +0200632 CNBuffer, CNBuffSize, 1) );
Yann Colletededcfc2018-12-21 16:19:44 -0800633 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Collete32fb0c2017-07-10 12:29:57 -0700634
Yann Collet64482c22017-12-29 17:04:37 +0100635 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
Yann Collete32fb0c2017-07-10 12:29:57 -0700636 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
Yann Colletededcfc2018-12-21 16:19:44 -0800637 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize);
Yann Collete32fb0c2017-07-10 12:29:57 -0700638 }
639 ZSTD_freeCCtx(cctx);
640 }
Yann Collet4856a002015-01-24 01:58:16 +0100641
Nick Terrellaaea4ef2018-12-12 15:26:35 -0800642 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++);
643 {
644 char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
645 size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8);
646 if (!ZSTD_isError(size)) goto _output_error;
647 }
648 DISPLAYLEVEL(3, "OK \n");
649
Yann Collet64482c22017-12-29 17:04:37 +0100650 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
Yann Collet0ef68032017-03-29 16:58:57 -0700651 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
652 if (rSize != CNBuffSize) goto _output_error;
653 }
Yann Collet64482c22017-12-29 17:04:37 +0100654 DISPLAYLEVEL(3, "OK \n");
Yann Collet0ef68032017-03-29 16:58:57 -0700655
Bimba Shrestha6a4258a2020-02-05 16:55:00 -0800656 DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++);
657 { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
658 if (rSize != CNBuffSize) goto _output_error;
659 }
660 DISPLAYLEVEL(3, "OK \n");
661
Yann Collet64482c22017-12-29 17:04:37 +0100662 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
Sean Purcell4e709712017-02-07 13:50:09 -0800663 { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
Yann Colletf323bf72016-07-07 13:14:21 +0200664 if (rSize != CNBuffSize) goto _output_error;
665 }
Yann Collet64482c22017-12-29 17:04:37 +0100666 DISPLAYLEVEL(3, "OK \n");
Yann Colletf323bf72016-07-07 13:14:21 +0200667
shakeelrao820af1e2019-02-28 00:42:49 -0800668 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
669 {
shakeelrao95dfd482019-03-01 23:11:15 -0800670 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
671 if (bound != CNBuffSize) goto _output_error;
shakeelrao820af1e2019-02-28 00:42:49 -0800672 }
673 DISPLAYLEVEL(3, "OK \n");
674
shakeelrao18d3a972019-03-13 01:43:40 -0700675 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
676 {
677 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
678 if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
679 }
680 DISPLAYLEVEL(3, "OK \n");
681
Yann Colletededcfc2018-12-21 16:19:44 -0800682 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
Yann Colletc991cc12016-07-28 00:55:43 +0200683 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
684 if (r != CNBuffSize) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +0100685 DISPLAYLEVEL(3, "OK \n");
Yann Collet4856a002015-01-24 01:58:16 +0100686
Elliott Hughes44aba642023-09-12 20:18:59 +0000687 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
688 {
689 ZSTD_DCtx* dctx = ZSTD_createDCtx();
690 size_t r;
691 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
692 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
693 if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
694 ZSTD_freeDCtx(dctx);
695 }
696 DISPLAYLEVEL(3, "OK \n");
697
Yann Collet64482c22017-12-29 17:04:37 +0100698 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
Yann Colletc0a9bf32016-05-30 01:56:08 +0200699 { size_t u;
Yann Colletd2858e92016-05-30 15:10:09 +0200700 for (u=0; u<CNBuffSize; u++) {
Ed Masteb81d7cc2019-08-15 21:17:06 -0400701 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
Yann Colletc0a9bf32016-05-30 01:56:08 +0200702 } }
Yann Collet64482c22017-12-29 17:04:37 +0100703 DISPLAYLEVEL(3, "OK \n");
Yann Collet4856a002015-01-24 01:58:16 +0100704
Yann Colletc91a0852020-09-14 10:56:08 -0700705 DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++);
706 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
707 ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 };
708 ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 };
709 if (cctx==NULL) goto _output_error;
Yann Colletdec1a782020-09-14 11:46:23 -0700710 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */
711 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */
Yann Colletc91a0852020-09-14 10:56:08 -0700712 ZSTD_freeCCtx(cctx);
713 }
714 DISPLAYLEVEL(3, "OK \n");
715
Bimba Shrestha6a4258a2020-02-05 16:55:00 -0800716 DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++);
717 {
718 ZSTD_parameters params = ZSTD_getParams(3, 0, 0);
719 assert(!ZSTD_checkCParams(params.cParams));
720 }
721 DISPLAYLEVEL(3, "OK \n");
722
723 DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++);
724 {
725 ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
726 assert(dctx != NULL);
727 assert(ZSTD_sizeof_DCtx(dctx) != 0);
728 ZSTD_freeDCtx(dctx);
729 }
730 DISPLAYLEVEL(3, "OK \n");
731
732 DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++);
733 {
W. Felix Handtedacbcd22020-05-01 12:24:51 -0400734 /* %p takes a void*. In ISO C, it's illegal to cast a function pointer
735 * to a data pointer. (Although in POSIX you're required to be allowed
736 * to do it...) So we have to fall back to our trusty friend memcpy. */
737 unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) =
738 ZSTD_getDictID_fromDDict;
739 ZSTD_DStream* (* const funcptr_createDStream)(
740 ZSTD_customMem customMem) = ZSTD_createDStream_advanced;
741 void (* const funcptr_copyDCtx)(
742 ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx;
743 ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) =
744 ZSTD_nextInputType;
745 const void *voidptr_getDictID;
746 const void *voidptr_createDStream;
747 const void *voidptr_copyDCtx;
748 const void *voidptr_nextInputType;
749 DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID));
W. Felix Handte2cf72d52020-05-01 16:48:21 -0400750 memcpy(
751 (void*)&voidptr_getDictID,
752 (const void*)&funcptr_getDictID,
753 sizeof(void*));
754 memcpy(
755 (void*)&voidptr_createDStream,
756 (const void*)&funcptr_createDStream,
757 sizeof(void*));
758 memcpy(
759 (void*)&voidptr_copyDCtx,
760 (const void*)&funcptr_copyDCtx,
761 sizeof(void*));
762 memcpy(
763 (void*)&voidptr_nextInputType,
764 (const void*)&funcptr_nextInputType,
765 sizeof(void*));
W. Felix Handtedacbcd22020-05-01 12:24:51 -0400766 DISPLAYLEVEL(3, "%p ", voidptr_getDictID);
767 DISPLAYLEVEL(3, "%p ", voidptr_createDStream);
768 DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx);
769 DISPLAYLEVEL(3, "%p ", voidptr_nextInputType);
Bimba Shrestha6a4258a2020-02-05 16:55:00 -0800770 }
771 DISPLAYLEVEL(3, ": OK \n");
772
Yann Collet64482c22017-12-29 17:04:37 +0100773 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
Yann Colletaec945f2018-12-04 15:35:37 -0800774 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
Yann Collet2fb8d1a2018-12-04 15:54:01 -0800775 { size_t const r = ZSTD_decompress_usingDict(dctx,
776 decodedBuffer, CNBuffSize,
777 compressedBuffer, cSize,
778 NULL, 0);
779 if (r != CNBuffSize) goto _output_error;
780 }
Yann Colletaec945f2018-12-04 15:35:37 -0800781 ZSTD_freeDCtx(dctx);
782 }
Yann Collet64482c22017-12-29 17:04:37 +0100783 DISPLAYLEVEL(3, "OK \n");
Yann Colletc0b17312017-02-28 01:02:46 -0800784
Yann Collet64482c22017-12-29 17:04:37 +0100785 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
Yann Colletaec945f2018-12-04 15:35:37 -0800786 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
Yann Collet2fb8d1a2018-12-04 15:54:01 -0800787 { size_t const r = ZSTD_decompress_usingDDict(dctx,
788 decodedBuffer, CNBuffSize,
789 compressedBuffer, cSize,
790 NULL);
791 if (r != CNBuffSize) goto _output_error;
792 }
Yann Colletaec945f2018-12-04 15:35:37 -0800793 ZSTD_freeDCtx(dctx);
794 }
Yann Collet64482c22017-12-29 17:04:37 +0100795 DISPLAYLEVEL(3, "OK \n");
Yann Colletc0b17312017-02-28 01:02:46 -0800796
Yann Collet64482c22017-12-29 17:04:37 +0100797 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
Yann Colletd2858e92016-05-30 15:10:09 +0200798 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
Yann Colletc0a9bf32016-05-30 01:56:08 +0200799 if (!ZSTD_isError(r)) goto _output_error;
800 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +0100801 DISPLAYLEVEL(3, "OK \n");
Yann Collet4856a002015-01-24 01:58:16 +0100802
Yann Collet64482c22017-12-29 17:04:37 +0100803 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
Yann Colletd2858e92016-05-30 15:10:09 +0200804 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
Yann Colletc0a9bf32016-05-30 01:56:08 +0200805 if (!ZSTD_isError(r)) goto _output_error;
806 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +0100807 DISPLAYLEVEL(3, "OK \n");
Yann Collet4856a002015-01-24 01:58:16 +0100808
Yann Colletd59cf022018-05-14 15:32:28 -0700809 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++);
810 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize);
811 if (!ZSTD_isError(r)) goto _output_error;
812 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
813 DISPLAYLEVEL(3, "OK \n");
814
Nick Terrell5717bd32020-05-01 16:35:35 -0700815 DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++);
816 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize);
817 if (!ZSTD_isError(r)) goto _output_error;
818 if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; }
819 DISPLAYLEVEL(3, "OK \n");
820
senhuang4220eb0952020-08-22 13:26:33 -0400821 DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++);
822 { /* create compressed buffer with checksumming enabled */
823 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
senhuang4244c54a32020-08-24 16:14:19 -0400824 if (!cctx) {
825 DISPLAY("Not enough memory, aborting\n");
826 testResult = 1;
827 goto _end;
828 }
senhuang4220eb0952020-08-22 13:26:33 -0400829 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
830 CHECK_VAR(cSize, ZSTD_compress2(cctx,
831 compressedBuffer, compressedBufferSize,
832 CNBuffer, CNBuffSize) );
833 ZSTD_freeCCtx(cctx);
834 }
835 { /* copy the compressed buffer and corrupt the checksum */
senhuang42ffaa0df2020-08-22 16:58:41 -0400836 size_t r;
senhuang42ffaa0df2020-08-22 16:58:41 -0400837 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
senhuang4244c54a32020-08-24 16:14:19 -0400838 if (!dctx) {
senhuang4220eb0952020-08-22 13:26:33 -0400839 DISPLAY("Not enough memory, aborting\n");
840 testResult = 1;
841 goto _end;
842 }
843
senhuang4244c54a32020-08-24 16:14:19 -0400844 ((char*)compressedBuffer)[cSize-1] += 1;
845 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
senhuang4220eb0952020-08-22 13:26:33 -0400846 if (!ZSTD_isError(r)) goto _output_error;
847 if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error;
W. Felix Handte2cc2b402020-09-10 11:32:16 -0400848
senhuang42a0305602020-08-24 17:28:00 -0400849 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum));
senhuang4244c54a32020-08-24 16:14:19 -0400850 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
851 if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */
852 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
senhuang4220eb0952020-08-22 13:26:33 -0400853 if (ZSTD_isError(r)) goto _output_error;
854
855 ZSTD_freeDCtx(dctx);
senhuang4220eb0952020-08-22 13:26:33 -0400856 }
857 DISPLAYLEVEL(3, "OK \n");
858
859
shakeelrao3da3dc22019-03-01 21:27:30 -0800860 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
861 { /* create compressed buffer with content size missing */
Yann Collet98692c22019-08-01 15:58:17 +0200862 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
shakeelrao3da3dc22019-03-01 21:27:30 -0800863 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
Yann Collet810a9ca2019-08-01 16:59:22 +0200864 CHECK_VAR(cSize, ZSTD_compress2(cctx,
shakeelrao3da3dc22019-03-01 21:27:30 -0800865 compressedBuffer, compressedBufferSize,
Yann Collet810a9ca2019-08-01 16:59:22 +0200866 CNBuffer, CNBuffSize) );
shakeelrao3da3dc22019-03-01 21:27:30 -0800867 ZSTD_freeCCtx(cctx);
868 }
869 { /* ensure frame content size is missing */
870 ZSTD_frameHeader zfh;
shakeelrao95dfd482019-03-01 23:11:15 -0800871 size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
872 if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
shakeelrao3da3dc22019-03-01 21:27:30 -0800873 }
874 { /* ensure CNBuffSize <= decompressBound */
shakeelrao95dfd482019-03-01 23:11:15 -0800875 unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
shakeelrao3da3dc22019-03-01 21:27:30 -0800876 if (CNBuffSize > bound) goto _output_error;
877 }
878 DISPLAYLEVEL(3, "OK \n");
879
Bimba Shrestha31e76f12020-04-04 08:49:24 -0700880 DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++);
881 {
882 size_t const largeFrameSrcSize = 200;
883 size_t const smallFrameSrcSize = 10;
884 size_t const nbFrames = 256;
885
886 size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0;
887 int sizeReduced = 0;
888
889 BYTE* const dst = (BYTE*)compressedBuffer;
890 ZSTD_DCtx* dctx = ZSTD_createDCtx();
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500891
Bimba Shrestha31e76f12020-04-04 08:49:24 -0700892 /* create a large frame and then a bunch of small frames */
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500893 size_t srcSize = ZSTD_compress((void*)dst,
Bimba Shrestha31e76f12020-04-04 08:49:24 -0700894 compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500895 for (i = 0; i < nbFrames; i++)
896 srcSize += ZSTD_compress((void*)(dst + srcSize),
897 compressedBufferSize - srcSize, CNBuffer,
Bimba Shrestha31e76f12020-04-04 08:49:24 -0700898 smallFrameSrcSize, 3);
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500899
Bimba Shrestha31e76f12020-04-04 08:49:24 -0700900 /* decompressStream and make sure that dctx size was reduced at least once */
901 while (consumed < srcSize) {
902 ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
903 ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0};
904 ZSTD_decompressStream(dctx, &out, &in);
905 consumed += in.pos;
906 produced += out.pos;
907
908 /* success! size was reduced from the previous frame */
909 if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx))
910 sizeReduced = 1;
911
912 prevDCtxSize = ZSTD_sizeof_DCtx(dctx);
913 }
914
915 assert(sizeReduced);
916
917 ZSTD_freeDCtx(dctx);
918 }
919 DISPLAYLEVEL(3, "OK \n");
920
Nick Terrell77365492021-02-12 18:52:08 -0800921 {
922 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
923 ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1);
924 ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
925 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
926
927 DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++);
928 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1));
929 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
930 DISPLAYLEVEL(3, "OK \n");
931
932 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++);
933 CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1));
934 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
935 DISPLAYLEVEL(3, "OK \n");
936
937 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++);
938 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict));
939 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
940 DISPLAYLEVEL(3, "OK \n");
941
942 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++);
943 CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params));
944 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
945 DISPLAYLEVEL(3, "OK \n");
946
947 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++);
948 CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams));
949 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
950 DISPLAYLEVEL(3, "OK \n");
951
952 ZSTD_freeCDict(cdict);
953 ZSTD_freeCCtx(cctx);
954 }
955
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500956 DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
957 {
958 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
959
960 size_t const size = (1U << 10);
961 size_t const dstCapacity = ZSTD_compressBound(size);
962 void* dict = (void*)malloc(size);
963 void* src = (void*)malloc(size);
964 void* dst = (void*)malloc(dstCapacity);
965
966 RDG_genBuffer(dict, size, 0.5, 0.5, seed);
967 RDG_genBuffer(src, size, 0.5, 0.5, seed);
968
Elliott Hughes44aba642023-09-12 20:18:59 +0000969 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500970 assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
971
972 ZSTD_freeCCtx(cctx);
973 free(dict);
974 free(src);
975 free(dst);
976 }
977 DISPLAYLEVEL(3, "OK \n");
978
979 DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
980 {
981 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Nick Terrell87dbd6d2020-05-18 12:56:34 -0700982 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500983 void* dict = (void*)malloc(CNBuffSize);
Nick Terrell87dbd6d2020-05-18 12:56:34 -0700984 int nbWorkers;
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500985
Nick Terrell87dbd6d2020-05-18 12:56:34 -0700986 for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) {
987 RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
988 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500989
Nick Terrell651d3d72020-05-19 16:14:14 -0700990 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
Nick Terrell87dbd6d2020-05-18 12:56:34 -0700991 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
992 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
Elliott Hughes44aba642023-09-12 20:18:59 +0000993 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
Nick Terrell87dbd6d2020-05-18 12:56:34 -0700994 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
995 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
996 CHECK_Z(cSize);
997 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize));
998 }
Bimba Shrestha5b0a4522020-04-17 15:58:53 -0500999
1000 ZSTD_freeCCtx(cctx);
Nick Terrell87dbd6d2020-05-18 12:56:34 -07001001 ZSTD_freeDCtx(dctx);
Bimba Shrestha5b0a4522020-04-17 15:58:53 -05001002 free(dict);
1003 }
1004 DISPLAYLEVEL(3, "OK \n");
1005
Nick Terrell172b4b62021-05-05 12:18:47 -07001006 DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++);
1007 {
1008 size_t const testSize = 1024;
1009 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1010 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1011 char* dict = (char*)malloc(2 * testSize);
1012 int ldmEnabled, level;
1013
1014 RDG_genBuffer(dict, testSize, 0.5, 0.5, seed);
1015 RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
1016 memcpy(dict + testSize, CNBuffer, testSize);
1017 for (level = 1; level <= 5; ++level) {
Elliott Hughes44aba642023-09-12 20:18:59 +00001018 for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
Nick Terrell172b4b62021-05-05 12:18:47 -07001019 size_t cSize0;
1020 XXH64_hash_t compressedChecksum0;
1021
1022 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1023 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level));
1024 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled));
1025 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1));
1026
1027 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1028 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize);
1029 CHECK_Z(cSize);
1030 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize));
1031
1032 cSize0 = cSize;
1033 compressedChecksum0 = XXH64(compressedBuffer, cSize, 0);
1034
1035 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1036 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize);
1037 CHECK_Z(cSize);
1038
1039 if (cSize != cSize0) goto _output_error;
1040 if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error;
1041 }
1042 }
1043
1044 ZSTD_freeCCtx(cctx);
1045 ZSTD_freeDCtx(dctx);
1046 free(dict);
1047 }
1048 DISPLAYLEVEL(3, "OK \n");
1049
senhuang42169fc072020-10-27 16:59:43 -04001050 DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
1051 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1052 ZSTD_DCtx* dctx = ZSTD_createDCtx();
senhuang427198ebb2020-10-28 17:49:54 -04001053 size_t const srcSize = 300 KB;
1054 size_t const flushSize = 128 KB + 5;
1055 size_t const dstSize = ZSTD_compressBound(srcSize);
1056 char* src = (char*)CNBuffer;
1057 char* dst = (char*)compressedBuffer;
1058
1059 ZSTD_outBuffer out = { dst, dstSize, 0 };
1060 ZSTD_inBuffer in = { src, flushSize, 0 };
1061
senhuang4260a52c22020-10-28 16:22:22 -04001062 if (!cctx || !dctx) {
1063 DISPLAY("Not enough memory, aborting\n");
1064 testResult = 1;
1065 goto _end;
1066 }
senhuang42169fc072020-10-27 16:59:43 -04001067
1068 RDG_genBuffer(src, srcSize, 0.5, 0.5, seed);
1069 /* Force an LDM to exist that crosses block boundary into uncompressible block */
1070 memcpy(src + 125 KB, src, 3 KB + 5);
1071
1072 /* Enable MT, LDM, and opt parser */
1073 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
Elliott Hughes44aba642023-09-12 20:18:59 +00001074 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
senhuang42169fc072020-10-27 16:59:43 -04001075 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1077
1078 /* Flushes a block of 128 KB and block of 5 bytes */
1079 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1080
1081 /* Compress the rest */
1082 in.size = 300 KB;
1083 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1084
1085 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos));
1086
1087 ZSTD_freeCCtx(cctx);
1088 ZSTD_freeDCtx(dctx);
1089 }
1090 DISPLAYLEVEL(3, "OK \n");
1091
Nick Terrell7b317b42020-05-18 15:11:02 -07001092 DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++);
1093 {
1094 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1095 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1096 void* dict = (void*)malloc(CNBuffSize);
1097 size_t const kWindowLog = 10;
1098 size_t const kWindowSize = (size_t)1 << kWindowLog;
1099 size_t const dictSize = kWindowSize * 10;
1100 size_t const srcSize1 = kWindowSize / 2;
1101 size_t const srcSize2 = kWindowSize * 10;
Nick Terrell7b317b42020-05-18 15:11:02 -07001102
1103 if (CNBuffSize < dictSize) goto _output_error;
1104
1105 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
1106 RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed);
1107
1108 /* Enable checksum to verify round trip. */
1109 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1110 /* Disable content size to skip single-pass decompression. */
1111 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
1112 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
Elliott Hughes44aba642023-09-12 20:18:59 +00001113 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
Nick Terrell7b317b42020-05-18 15:11:02 -07001114 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
1115 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
Nick Terrell651d3d72020-05-19 16:14:14 -07001116 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
1117 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3));
Nick Terrell7b317b42020-05-18 15:11:02 -07001118
Nick Terrell651d3d72020-05-19 16:14:14 -07001119 /* Round trip once with a dictionary. */
1120 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1121 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
1122 CHECK_Z(cSize);
1123 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1124 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
1125 /* Streaming decompression to catch out of bounds offsets. */
1126 {
1127 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1128 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1129 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1130 CHECK_Z(dSize);
1131 if (dSize != 0) goto _output_error;
1132 }
1133
1134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
1135 /* Round trip once with a dictionary. */
1136 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1137 {
1138 ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
1139 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1140 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1141 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1142 cSize = out.pos;
1143 }
1144 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1145 {
1146 ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
1147 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1148 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1149 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1150 cSize = out.pos;
1151 }
1152 /* Streaming decompression to catch out of bounds offsets. */
1153 {
1154 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1155 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1156 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1157 CHECK_Z(dSize);
1158 if (dSize != 0) goto _output_error;
Nick Terrell7b317b42020-05-18 15:11:02 -07001159 }
1160
1161 ZSTD_freeCCtx(cctx);
1162 ZSTD_freeDCtx(dctx);
1163 free(dict);
1164 }
1165 DISPLAYLEVEL(3, "OK \n");
1166
Bimba Shrestha5b0a4522020-04-17 15:58:53 -05001167 /* Note: this test takes 0.5 seconds to run */
1168 DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
1169 {
1170 /* test a big buffer so that ldm can take effect */
1171 size_t const size = 100 MB;
1172 int const windowLog = 27;
1173 size_t const dstSize = ZSTD_compressBound(size);
1174
1175 void* dict = (void*)malloc(size);
1176 void* src = (void*)malloc(size);
1177 void* dst = (void*)malloc(dstSize);
1178 void* recon = (void*)malloc(size);
1179
1180 size_t refPrefixCompressedSize = 0;
Elliott Hughes44aba642023-09-12 20:18:59 +00001181 size_t refPrefixLdmCompressedSize = 0;
Bimba Shrestha5b0a4522020-04-17 15:58:53 -05001182 size_t reconSize = 0;
1183
1184 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1185 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1186
1187 /* make dict and src the same uncompressible data */
1188 RDG_genBuffer(src, size, 0, 0, seed);
1189 memcpy(dict, src, size);
1190 assert(!memcmp(dict, src, size));
1191
1192 /* set level 1 and windowLog to cover src */
1193 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1));
1194 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog));
1195
1196 /* compress on level 1 using just refPrefix and no ldm */
1197 ZSTD_CCtx_refPrefix(cctx, dict, size);
1198 refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1199 assert(!ZSTD_isError(refPrefixCompressedSize));
1200
1201 /* test round trip just refPrefix */
1202 ZSTD_DCtx_refPrefix(dctx, dict, size);
1203 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize);
1204 assert(!ZSTD_isError(reconSize));
1205 assert(reconSize == size);
1206 assert(!memcmp(recon, src, size));
1207
1208 /* compress on level 1 using refPrefix and ldm */
1209 ZSTD_CCtx_refPrefix(cctx, dict, size);;
Elliott Hughes44aba642023-09-12 20:18:59 +00001210 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
1211 refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1212 assert(!ZSTD_isError(refPrefixLdmCompressedSize));
Bimba Shrestha5b0a4522020-04-17 15:58:53 -05001213
1214 /* test round trip refPrefix + ldm*/
1215 ZSTD_DCtx_refPrefix(dctx, dict, size);
Elliott Hughes44aba642023-09-12 20:18:59 +00001216 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize);
Bimba Shrestha5b0a4522020-04-17 15:58:53 -05001217 assert(!ZSTD_isError(reconSize));
1218 assert(reconSize == size);
1219 assert(!memcmp(recon, src, size));
1220
1221 /* make sure that refPrefixCompressedSize is significantly greater */
Elliott Hughes44aba642023-09-12 20:18:59 +00001222 assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize);
1223 /* make sure the ldm compressed size is less than 1% of original */
1224 assert((double)refPrefixLdmCompressedSize / (double)size < 0.01);
Bimba Shrestha5b0a4522020-04-17 15:58:53 -05001225
1226 ZSTD_freeDCtx(dctx);
1227 ZSTD_freeCCtx(cctx);
1228 free(recon);
1229 free(dict);
1230 free(src);
1231 free(dst);
1232 }
1233 DISPLAYLEVEL(3, "OK \n");
1234
Elliott Hughes44aba642023-09-12 20:18:59 +00001235 DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++);
1236 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX);
1237 CHECK_Z(cSize);
1238 CHECK_LT(CNBuffSize, cSize);
1239 {
1240 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1241 size_t const outputSize = (CNBuffSize + margin);
1242 char* output = malloc(outputSize);
1243 char* input = output + outputSize - cSize;
1244 CHECK_LT(cSize, CNBuffSize + margin);
1245 CHECK(output != NULL);
1246 CHECK_Z(margin);
1247 CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX));
1248 memcpy(input, compressedBuffer, cSize);
1249
1250 {
1251 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1252 CHECK_Z(dSize);
1253 CHECK_EQ(dSize, CNBuffSize);
1254 }
1255 CHECK(!memcmp(output, CNBuffer, CNBuffSize));
1256 free(output);
1257 }
1258 DISPLAYLEVEL(3, "OK \n");
1259
1260 DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++);
1261 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1262 CHECK_Z(cSize);
1263 {
1264 size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1265 CHECK_Z(cSize2);
1266 cSize += cSize2;
1267 }
1268 {
1269 size_t const srcSize = (CNBuffSize / 3) * 2;
1270 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1271 size_t const outputSize = (CNBuffSize + margin);
1272 char* output = malloc(outputSize);
1273 char* input = output + outputSize - cSize;
1274 CHECK_LT(cSize, CNBuffSize + margin);
1275 CHECK(output != NULL);
1276 CHECK_Z(margin);
1277 memcpy(input, compressedBuffer, cSize);
1278
1279 {
1280 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1281 CHECK_Z(dSize);
1282 CHECK_EQ(dSize, srcSize);
1283 }
1284 CHECK(!memcmp(output, CNBuffer, srcSize));
1285 free(output);
1286 }
1287 DISPLAYLEVEL(3, "OK \n");
1288
1289 DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++);
1290 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1291 size_t const srcSize = 256 * 1024;
1292 U32 const compressibleLenU32 = 32 * 1024 / 4;
1293 U32 const blockSizeU32 = 128 * 1024 / 4;
1294 U32 const litLenU32 = 64 * 1024 / 4;
1295 U32* data = (U32*)malloc(srcSize);
1296 size_t dSize;
1297
1298 if (data == NULL || cctx == NULL) goto _output_error;
1299
1300 /* Generate data without any matches */
1301 RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U);
1302 /* Generate 32K of compressible data */
1303 RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe);
1304
1305 /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */
1306 data[compressibleLenU32 + 0] = 0xFFFFFFFF;
1307 data[compressibleLenU32 + 1] = 0xEEEEEEEE;
1308 data[compressibleLenU32 + 4] = 0xFFFFFFFF;
1309 data[compressibleLenU32 + 5] = 0xEEEEEEEE;
1310
1311 /* Add a match of offset=16, length=8 at idx=64K + 64.
1312 * This generates a sequence with llen=64K, and repeat code 1.
1313 * The block splitter thought this was ll0, and corrupted the
1314 * repeat offset history.
1315 */
1316 data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD;
1317 data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC;
1318 data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD;
1319 data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC;
1320
1321 /* Add a match of offset=16, length=8 at idx=128K + 16.
1322 * This should generate a sequence with repeat code = 1.
1323 * But the block splitters mistake caused zstd to generate
1324 * repeat code = 2, corrupting the data.
1325 */
1326 data[blockSizeU32] = 0xBBBBBBBB;
1327 data[blockSizeU32 + 1] = 0xAAAAAAAA;
1328 data[blockSizeU32 + 4] = 0xBBBBBBBB;
1329 data[blockSizeU32 + 5] = 0xAAAAAAAA;
1330
1331 /* Generate a golden file from this data in case datagen changes and
1332 * doesn't generate the exact same data. We will also test this golden file.
1333 */
1334 if (0) {
1335 FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb");
1336 fwrite(data, 1, srcSize, f);
1337 fclose(f);
1338 }
1339
1340 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1341 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7));
1342 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable));
1343
1344 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize);
1345 CHECK_Z(cSize);
1346 dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1347 CHECK_Z(dSize);
1348 CHECK_EQ(dSize, srcSize);
1349 CHECK(!memcmp(decodedBuffer, data, srcSize));
1350
1351 free(data);
1352 ZSTD_freeCCtx(cctx);
1353 }
1354 DISPLAYLEVEL(3, "OK \n");
1355
Bimba Shrestha31e76f12020-04-04 08:49:24 -07001356 DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
Bimba Shresthae3cd2782019-12-13 15:31:29 -08001357 {
Bimba Shrestha56415ef2019-12-17 17:16:51 -08001358 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1359 const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;
Bimba Shresthae3cd2782019-12-13 15:31:29 -08001360 size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize);
1361
1362 /* This is the number of bytes to stream before ending. This value
1363 * was obtained by trial and error :/. */
1364
1365 const size_t streamCompressThreshold = 161792;
1366 const size_t streamCompressDelta = 1024;
1367
Bimba Shresthab1f53b12020-01-03 16:53:51 -08001368 /* The first 1/5 of the buffer is compressible and the last 4/5 is
Bimba Shrestha56415ef2019-12-17 17:16:51 -08001369 * uncompressible. This is an approximation of the type of data
Bimba Shresthae3cd2782019-12-13 15:31:29 -08001370 * the fuzzer generated to catch this bug. Streams like this were making
1371 * zstd generate noCompress superblocks (which are larger than the src
1372 * they come from). Do this enough times, and we'll run out of room
1373 * and throw a dstSize_tooSmall error. */
1374
Bimba Shresthab1f53b12020-01-03 16:53:51 -08001375 const size_t compressiblePartSize = srcSize/5;
Bimba Shrestha56415ef2019-12-17 17:16:51 -08001376 const size_t uncompressiblePartSize = srcSize-compressiblePartSize;
1377 RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed);
1378 RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed);
Bimba Shresthae3cd2782019-12-13 15:31:29 -08001379
1380 /* Setting target block size so that superblock is used */
1381
Bimba Shrestha56415ef2019-12-17 17:16:51 -08001382 assert(cctx != NULL);
Bimba Shresthae3cd2782019-12-13 15:31:29 -08001383 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81);
1384
Yann Colletc91a0852020-09-14 10:56:08 -07001385 { size_t read;
1386 for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) {
1387 ZSTD_inBuffer in = {src, streamCompressDelta, 0};
1388 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1389 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1390 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1391 src += streamCompressDelta; srcSize -= streamCompressDelta;
1392 dst += out.pos; dstCapacity -= out.pos;
1393 } }
Bimba Shresthae3cd2782019-12-13 15:31:29 -08001394
1395 /* This is trying to catch a dstSize_tooSmall error */
1396
Yann Colletc91a0852020-09-14 10:56:08 -07001397 { ZSTD_inBuffer in = {src, srcSize, 0};
1398 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1399 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1400 }
Nick Terrelle103d7b2020-05-01 16:11:47 -07001401 ZSTD_freeCCtx(cctx);
1402 }
1403 DISPLAYLEVEL(3, "OK \n");
1404
1405 DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++);
1406 /* Generate the same data 20 times over */
Yann Colletc91a0852020-09-14 10:56:08 -07001407 { size_t const avgChunkSize = CNBuffSize / 20;
Nick Terrelle103d7b2020-05-01 16:11:47 -07001408 size_t b;
1409 for (b = 0; b < CNBuffSize; b += avgChunkSize) {
1410 size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize);
1411 RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed);
Yann Colletc91a0852020-09-14 10:56:08 -07001412 } }
1413 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Nick Terrelle103d7b2020-05-01 16:11:47 -07001414 size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1415 size_t const allowedExpansion = (CNBuffSize * 3 / 1000);
1416 size_t superCSize;
1417 CHECK_Z(normalCSize);
1418 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
1419 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000);
1420 superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1421 CHECK_Z(superCSize);
1422 if (superCSize > normalCSize + allowedExpansion) {
1423 DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion);
1424 goto _output_error;
1425 }
Bimba Shresthae3cd2782019-12-13 15:31:29 -08001426 ZSTD_freeCCtx(cctx);
1427 }
1428 DISPLAYLEVEL(3, "OK \n");
1429
1430 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed);
Bimba Shrestha707a12c2019-11-22 17:25:36 -08001431 DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++)
Yann Colletc91a0852020-09-14 10:56:08 -07001432 /* This tests whether or not we leave enough room for the checksum at the end
1433 * of the dst buffer. The bug that motivated this test was found by the
1434 * stream_round_trip fuzzer but this crashes for the same reason and is
1435 * far more compact than re-creating the stream_round_trip fuzzer's code path */
1436 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Bimba Shrestha707a12c2019-11-22 17:25:36 -08001437 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64);
1438 assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278)));
1439 ZSTD_freeCCtx(cctx);
1440 }
1441 DISPLAYLEVEL(3, "OK \n");
Bimba Shresthae3cd2782019-12-13 15:31:29 -08001442
Nick Terrell659e9f02019-11-20 18:21:51 -08001443 DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
1444 { int level = -1;
Elliott Hughes44aba642023-09-12 20:18:59 +00001445 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Nick Terrell659e9f02019-11-20 18:21:51 -08001446 if (!cctx) goto _output_error;
1447 for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
1448 CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
1449 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
1450 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
1451 }
1452 ZSTD_freeCCtx(cctx);
1453 }
1454 DISPLAYLEVEL(3, "OK \n");
Bimba Shrestha707a12c2019-11-22 17:25:36 -08001455
Yann Collet64482c22017-12-29 17:04:37 +01001456 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
Yann Collet806a5c82018-10-24 16:34:35 -07001457 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Nick Terrell86b81342017-10-25 17:10:57 -07001458 size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
1459 if (ZSTD_isError(r)) goto _output_error;
1460 if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
1461 ZSTD_freeCCtx(cctx);
Yann Collet806a5c82018-10-24 16:34:35 -07001462 cSize = r;
1463 }
1464 DISPLAYLEVEL(3, "OK \n");
1465
1466 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++);
1467 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize);
1468 if (ZSTD_isError(r)) goto _output_error;
1469 if (r != 0) goto _output_error;
1470 }
1471 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1472 ZSTD_outBuffer output;
1473 if (cctx==NULL) goto _output_error;
1474 output.dst = compressedBuffer;
1475 output.size = compressedBufferSize;
1476 output.pos = 0;
1477 CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */
1478 CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */
1479 CHECK_Z( ZSTD_endStream(cctx, &output) );
1480 ZSTD_freeCCtx(cctx);
1481 /* single scan decompression */
1482 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos);
1483 if (ZSTD_isError(r)) goto _output_error;
1484 if (r != 0) goto _output_error;
1485 }
1486 /* streaming decompression */
1487 { ZSTD_DCtx* const dstream = ZSTD_createDStream();
1488 ZSTD_inBuffer dinput;
1489 ZSTD_outBuffer doutput;
1490 size_t ipos;
1491 if (dstream==NULL) goto _output_error;
1492 dinput.src = compressedBuffer;
1493 dinput.size = 0;
1494 dinput.pos = 0;
1495 doutput.dst = NULL;
1496 doutput.size = 0;
1497 doutput.pos = 0;
1498 CHECK_Z ( ZSTD_initDStream(dstream) );
1499 for (ipos=1; ipos<=output.pos; ipos++) {
1500 dinput.size = ipos;
1501 CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) );
1502 }
1503 if (doutput.pos != 0) goto _output_error;
1504 ZSTD_freeDStream(dstream);
1505 }
Nick Terrell86b81342017-10-25 17:10:57 -07001506 }
Yann Collet64482c22017-12-29 17:04:37 +01001507 DISPLAYLEVEL(3, "OK \n");
Yann Collet0ef68032017-03-29 16:58:57 -07001508
Yann Collet64482c22017-12-29 17:04:37 +01001509 DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++);
Yann Colletd88c6712017-12-19 10:16:09 +01001510 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1511 ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1512 assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
1513 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
1514 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
1515
1516 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1517 { size_t const inSize = 2* 128 KB;
1518 size_t const outSize = ZSTD_compressBound(inSize);
1519 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
1520 /* will fail if blockSize is not resized */
1521 }
1522 ZSTD_freeCCtx(cctx);
1523 }
Yann Collet64482c22017-12-29 17:04:37 +01001524 DISPLAYLEVEL(3, "OK \n");
Yann Colletd88c6712017-12-19 10:16:09 +01001525
cyan49739597b432018-07-17 18:52:57 +02001526 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++);
Yann Collet78c4ea42018-12-19 14:10:27 -08001527 { size_t const sampleSize = 30;
1528 int i;
cyan49739597b432018-07-17 18:52:57 +02001529 for (i=0; i<20; i++)
cyan49737d1bc9c2018-07-18 16:10:23 +02001530 ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */
cyan49739597b432018-07-17 18:52:57 +02001531 memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
1532 for (i=1; i<=19; i++) {
1533 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1534 size_t size1, size2;
1535 DISPLAYLEVEL(5, "l%i ", i);
Yann Collet78c4ea42018-12-19 14:10:27 -08001536 size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
cyan49739597b432018-07-17 18:52:57 +02001537 CHECK_Z(size1);
Yann Collet78c4ea42018-12-19 14:10:27 -08001538
1539 size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
cyan49739597b432018-07-17 18:52:57 +02001540 CHECK_Z(size2);
1541 CHECK_EQ(size1, size2);
1542
Yann Collet78c4ea42018-12-19 14:10:27 -08001543 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) );
1544 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize);
1545 CHECK_Z(size2);
1546 CHECK_EQ(size1, size2);
1547
1548 size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */
1549 CHECK_Z(size2);
1550 CHECK_EQ(size1, size2);
1551
1552 { ZSTD_inBuffer inb;
1553 ZSTD_outBuffer outb;
1554 inb.src = CNBuffer;
1555 inb.pos = 0;
1556 inb.size = sampleSize;
1557 outb.dst = compressedBuffer;
1558 outb.pos = 0;
1559 outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */
1560 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );
1561 assert(inb.pos == inb.size);
1562 CHECK_EQ(size1, outb.pos);
1563 }
1564
cyan49739597b432018-07-17 18:52:57 +02001565 ZSTD_freeCCtx(cctx);
1566 }
1567 }
1568 DISPLAYLEVEL(3, "OK \n");
1569
Yann Collet2898afa2018-12-19 16:54:15 -08001570 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++);
1571 { size_t const sampleSize = 1024;
1572 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1573 ZSTD_inBuffer inb;
1574 ZSTD_outBuffer outb;
1575 inb.src = CNBuffer;
1576 inb.pos = 0;
1577 inb.size = 0;
1578 outb.dst = compressedBuffer;
1579 outb.pos = 0;
1580 outb.size = compressedBufferSize;
1581 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) );
1582
1583 inb.size = sampleSize; /* start with something, so that context is already used */
1584 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1585 assert(inb.pos == inb.size);
1586 outb.pos = 0; /* cancel output */
1587
1588 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) );
1589 inb.size = 4; /* too small size : compression will be skipped */
1590 inb.pos = 0;
1591 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1592 assert(inb.pos == inb.size);
1593
1594 inb.size += 5; /* too small size : compression will be skipped */
1595 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1596 assert(inb.pos == inb.size);
1597
1598 inb.size += 11; /* small enough to attempt compression */
1599 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1600 assert(inb.pos == inb.size);
1601
1602 assert(inb.pos < sampleSize);
1603 inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */
1604 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1605 assert(inb.pos == inb.size);
1606 ZSTD_freeCCtx(cctx);
1607 }
1608 DISPLAYLEVEL(3, "OK \n");
1609
Nick Terrell280a2362018-04-12 11:50:12 -07001610 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
1611 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1612 ZSTD_outBuffer out = {NULL, 0, 0};
1613 ZSTD_inBuffer in = {NULL, 0, 0};
Yann Colletd4d4e102018-11-21 15:37:26 -08001614 int value;
Nick Terrell280a2362018-04-12 11:50:12 -07001615
Yann Collet3583d192018-12-05 17:26:02 -08001616 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001617 CHECK_EQ(value, 3);
Yann Collet3583d192018-12-05 17:26:02 -08001618 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001619 CHECK_EQ(value, 0);
Yann Collet3583d192018-12-05 17:26:02 -08001620 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
1621 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001622 CHECK_EQ(value, 3);
Yann Collet3583d192018-12-05 17:26:02 -08001623 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001624 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
Yann Collet3583d192018-12-05 17:26:02 -08001625 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
1626 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001627 CHECK_EQ(value, 7);
Yann Collet3583d192018-12-05 17:26:02 -08001628 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001629 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1630 /* Start a compression job */
Yann Colletd8e215c2018-11-30 11:16:26 -08001631 ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
Yann Collet3583d192018-12-05 17:26:02 -08001632 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001633 CHECK_EQ(value, 7);
Yann Collet3583d192018-12-05 17:26:02 -08001634 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001635 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1636 /* Reset the CCtx */
Yann Collet5c686392018-11-15 16:12:39 -08001637 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
Yann Collet3583d192018-12-05 17:26:02 -08001638 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001639 CHECK_EQ(value, 7);
Yann Collet3583d192018-12-05 17:26:02 -08001640 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
Nick Terrell280a2362018-04-12 11:50:12 -07001641 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
Nick Terrell9f76eeb2018-04-12 16:54:07 -07001642 /* Reset the parameters */
Yann Collet5c686392018-11-15 16:12:39 -08001643 ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
Yann Collet3583d192018-12-05 17:26:02 -08001644 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
Nick Terrell9f76eeb2018-04-12 16:54:07 -07001645 CHECK_EQ(value, 3);
Yann Collet3583d192018-12-05 17:26:02 -08001646 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
Nick Terrell9f76eeb2018-04-12 16:54:07 -07001647 CHECK_EQ(value, 0);
Nick Terrell280a2362018-04-12 11:50:12 -07001648
1649 ZSTD_freeCCtx(cctx);
1650 }
1651 DISPLAYLEVEL(3, "OK \n");
1652
Elliott Hughes44aba642023-09-12 20:18:59 +00001653 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++);
1654 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1655 int value;
1656 ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0);
1657 cparams.strategy = -1;
1658 /* Set invalid cParams == no change. */
1659 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
1660
1661 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1662 CHECK_EQ(value, 0);
1663 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1664 CHECK_EQ(value, 0);
1665 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1666 CHECK_EQ(value, 0);
1667 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1668 CHECK_EQ(value, 0);
1669 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1670 CHECK_EQ(value, 0);
1671 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1672 CHECK_EQ(value, 0);
1673 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1674 CHECK_EQ(value, 0);
1675
1676 cparams = ZSTD_getCParams(12, 0, 0);
1677 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
1678
1679 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1680 CHECK_EQ(value, (int)cparams.windowLog);
1681 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1682 CHECK_EQ(value, (int)cparams.chainLog);
1683 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1684 CHECK_EQ(value, (int)cparams.hashLog);
1685 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1686 CHECK_EQ(value, (int)cparams.searchLog);
1687 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1688 CHECK_EQ(value, (int)cparams.minMatch);
1689 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1690 CHECK_EQ(value, (int)cparams.targetLength);
1691 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1692 CHECK_EQ(value, (int)cparams.strategy);
1693
1694 ZSTD_freeCCtx(cctx);
1695 }
1696
1697 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++);
1698 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1699 int value;
1700 ZSTD_frameParameters fparams = {0, 1, 1};
1701
1702 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1703 CHECK_EQ(value, 1);
1704 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1705 CHECK_EQ(value, 0);
1706 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1707 CHECK_EQ(value, 1);
1708
1709 CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams));
1710
1711 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1712 CHECK_EQ(value, fparams.contentSizeFlag);
1713 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1714 CHECK_EQ(value, fparams.checksumFlag);
1715 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1716 CHECK_EQ(value, !fparams.noDictIDFlag);
1717
1718 ZSTD_freeCCtx(cctx);
1719 }
1720
1721 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCarams() : ", testNb++);
1722 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1723 int value;
1724 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
1725 params.cParams.strategy = -1;
1726 /* Set invalid params == no change. */
1727 CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params)));
1728
1729 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1730 CHECK_EQ(value, 0);
1731 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1732 CHECK_EQ(value, 0);
1733 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1734 CHECK_EQ(value, 0);
1735 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1736 CHECK_EQ(value, 0);
1737 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1738 CHECK_EQ(value, 0);
1739 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1740 CHECK_EQ(value, 0);
1741 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1742 CHECK_EQ(value, 0);
1743 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1744 CHECK_EQ(value, 1);
1745 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1746 CHECK_EQ(value, 0);
1747 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1748 CHECK_EQ(value, 1);
1749
1750 params = ZSTD_getParams(12, 0, 0);
1751 params.fParams.contentSizeFlag = 0;
1752 params.fParams.checksumFlag = 1;
1753 params.fParams.noDictIDFlag = 1;
1754 CHECK_Z(ZSTD_CCtx_setParams(cctx, params));
1755
1756 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1757 CHECK_EQ(value, (int)params.cParams.windowLog);
1758 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1759 CHECK_EQ(value, (int)params.cParams.chainLog);
1760 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1761 CHECK_EQ(value, (int)params.cParams.hashLog);
1762 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1763 CHECK_EQ(value, (int)params.cParams.searchLog);
1764 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1765 CHECK_EQ(value, (int)params.cParams.minMatch);
1766 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1767 CHECK_EQ(value, (int)params.cParams.targetLength);
1768 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1769 CHECK_EQ(value, (int)params.cParams.strategy);
1770 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1771 CHECK_EQ(value, params.fParams.contentSizeFlag);
1772 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1773 CHECK_EQ(value, params.fParams.checksumFlag);
1774 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1775 CHECK_EQ(value, !params.fParams.noDictIDFlag);
1776
1777 ZSTD_freeCCtx(cctx);
1778 }
1779
senhuang42df470e12020-10-19 10:52:41 -04001780 DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
1781 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1782 ZSTD_outBuffer out = {NULL, 0, 0};
1783 ZSTD_inBuffer in = {NULL, 0, 0};
1784 int value;
1785
1786 /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt),
1787 * we should not modify the actual parameter specified by the user within the CCtx
1788 */
1789 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27));
1790 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt));
1791
1792 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1793 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value));
1794 CHECK_EQ(value, 0);
1795
1796 ZSTD_freeCCtx(cctx);
1797 }
1798 DISPLAYLEVEL(3, "OK \n");
1799
Yann Colletf7392f32018-06-05 14:53:28 -07001800 /* this test is really too long, and should be made faster */
Yann Collet3d523c72018-06-05 11:23:18 -07001801 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
Nick Terrell33fb9662018-03-06 13:07:28 -08001802 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Yann Colletaec945f2018-12-04 15:35:37 -08001803 ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1804 size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
Yann Collet3d523c72018-06-05 11:23:18 -07001805 size_t cnb;
1806 assert(cctx != NULL);
Nick Terrell33fb9662018-03-06 13:07:28 -08001807 params.fParams.contentSizeFlag = 0;
1808 params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
Yann Collet3d523c72018-06-05 11:23:18 -07001809 for (cnb = 0; cnb < nbCompressions; ++cnb) {
1810 DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions);
Nick Terrell33fb9662018-03-06 13:07:28 -08001811 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1812 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
1813 }
1814 ZSTD_freeCCtx(cctx);
1815 }
1816 DISPLAYLEVEL(3, "OK \n");
1817
Yann Colletf7392f32018-06-05 14:53:28 -07001818 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
1819 { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
1820 assert(largeCCtx != NULL);
1821 CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
1822 CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
1823 { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
1824 { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
1825 assert(smallCCtx != NULL);
1826 CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1827 { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
1828 DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
1829 largeCCtxSize>>10, smallCCtxSize>>10);
1830 assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
1831 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
1832 }
1833 ZSTD_freeCCtx(smallCCtx);
1834 }
1835 { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
1836 * currently defined as 128x, but could be adjusted in the future.
1837 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
Yann Colletededcfc2018-12-21 16:19:44 -08001838 unsigned u;
Yann Colletf7392f32018-06-05 14:53:28 -07001839 for (u=0; u<maxNbAttempts; u++) {
Yann Colletf1ea3832018-06-06 15:04:12 -07001840 CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
Yann Colletf7392f32018-06-05 14:53:28 -07001841 if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
1842 }
1843 DISPLAYLEVEL(5, "size down after %u attempts : ", u);
1844 if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
1845 }
1846 }
1847 ZSTD_freeCCtx(largeCCtx);
1848 }
1849 DISPLAYLEVEL(3, "OK \n");
1850
Yann Colletc7fe2622017-05-23 13:16:00 -07001851 /* Static CCtx tests */
Yann Collet608f1bf2020-05-11 18:16:38 -07001852#define STATIC_CCTX_LEVEL 4
Yann Colletdd026ca2020-05-09 11:30:45 -07001853 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1854 { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
1855 void* const staticCCtxBuffer = malloc(staticCStreamSize);
Yann Collet0fdc71c2017-05-24 17:41:41 -07001856 size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
1857 void* const staticDCtxBuffer = malloc(staticDCtxSize);
Yann Colletdd026ca2020-05-09 11:30:45 -07001858 DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize);
Yann Collet0fdc71c2017-05-24 17:41:41 -07001859 if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
1860 free(staticCCtxBuffer);
1861 free(staticDCtxBuffer);
Yann Colletc7fe2622017-05-23 13:16:00 -07001862 DISPLAY("Not enough memory, aborting\n");
1863 testResult = 1;
1864 goto _end;
1865 }
Yann Collet608f1bf2020-05-11 18:16:38 -07001866 { size_t const smallInSize = 32 KB;
Yann Collet91ad0122020-05-11 18:50:10 -07001867 ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0);
1868 size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small);
Yann Collet608f1bf2020-05-11 18:16:38 -07001869 size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL);
Yann Collet91ad0122020-05-11 18:50:10 -07001870 ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize);
Yann Colletdd026ca2020-05-09 11:30:45 -07001871 ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
Yann Collet91ad0122020-05-11 18:50:10 -07001872 DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize);
1873 DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize);
Yann Collet608f1bf2020-05-11 18:16:38 -07001874 if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
Yann Collet64482c22017-12-29 17:04:37 +01001875 DISPLAYLEVEL(3, "OK \n");
Yann Collet91ad0122020-05-11 18:50:10 -07001876
1877 DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++);
1878 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1879 compressedBuffer, compressedBufferSize,
1880 CNBuffer, smallInSize, STATIC_CCTX_LEVEL) );
1881 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1882 (unsigned)cSize, (double)cSize/smallInSize*100);
1883
1884 DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++);
1885 { size_t const r = ZSTD_compressCCtx(staticCCtx,
1886 compressedBuffer, compressedBufferSize,
1887 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL);
1888 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error;
1889 }
1890 DISPLAYLEVEL(3, "OK \n");
1891
1892 DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
1893 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
Nick Terrell8bf699a2021-09-17 11:42:08 -07001894 DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx);
Yann Collete0017152020-05-11 20:35:47 -07001895 if (staticCCtx == NULL) goto _output_error;
Yann Collet91ad0122020-05-11 18:50:10 -07001896 DISPLAYLEVEL(3, "OK \n");
1897
1898 DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++);
Yann Colletdd026ca2020-05-09 11:30:45 -07001899 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1900 compressedBuffer, compressedBufferSize,
1901 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1902 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1903 (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Collet91ad0122020-05-11 18:50:10 -07001904
Yann Collet608f1bf2020-05-11 18:16:38 -07001905 DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++);
1906 { int nbc;
Yann Collete0017152020-05-11 20:35:47 -07001907 assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */
Yann Collet608f1bf2020-05-11 18:16:38 -07001908 assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3);
1909 for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) {
1910 CHECK_Z(ZSTD_compressCCtx(staticCCtx,
1911 compressedBuffer, compressedBufferSize,
1912 (char*)CNBuffer + nbc, smallInSize,
1913 STATIC_CCTX_LEVEL) );
1914 } }
1915 DISPLAYLEVEL(3, "OK \n")
Yann Colletdd026ca2020-05-09 11:30:45 -07001916
Yann Collet64482c22017-12-29 17:04:37 +01001917 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
Yann Colletdd026ca2020-05-09 11:30:45 -07001918 CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) );
Yann Collet64482c22017-12-29 17:04:37 +01001919 DISPLAYLEVEL(3, "OK \n");
Yann Colletc7fe2622017-05-23 13:16:00 -07001920
Yann Colletdd026ca2020-05-09 11:30:45 -07001921 DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++);
Yann Collet810a9ca2019-08-01 16:59:22 +02001922 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1923 compressedBuffer, compressedBufferSize,
1924 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
Yann Collet64482c22017-12-29 17:04:37 +01001925 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
Yann Colletededcfc2018-12-21 16:19:44 -08001926 (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Colletc7fe2622017-05-23 13:16:00 -07001927
Yann Collet64482c22017-12-29 17:04:37 +01001928 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
Yann Collet0fdc71c2017-05-24 17:41:41 -07001929 { size_t const r = ZSTD_decompressDCtx(staticDCtx,
1930 decodedBuffer, CNBuffSize,
1931 compressedBuffer, cSize);
Yann Colletc7fe2622017-05-23 13:16:00 -07001932 if (r != CNBuffSize) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01001933 DISPLAYLEVEL(3, "OK \n");
Yann Colletc7fe2622017-05-23 13:16:00 -07001934
Yann Collet64482c22017-12-29 17:04:37 +01001935 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
Yann Colletdd026ca2020-05-09 11:30:45 -07001936 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
Yann Collet64482c22017-12-29 17:04:37 +01001937 DISPLAYLEVEL(3, "OK \n");
Yann Collet0fdc71c2017-05-24 17:41:41 -07001938
Yann Collet64482c22017-12-29 17:04:37 +01001939 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
Yann Colletc7fe2622017-05-23 13:16:00 -07001940 { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
1941 if (!ZSTD_isError(r)) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01001942 DISPLAYLEVEL(3, "OK \n");
Yann Colletc7fe2622017-05-23 13:16:00 -07001943
Yann Collet64482c22017-12-29 17:04:37 +01001944 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
Elliott Hughes44aba642023-09-12 20:18:59 +00001945 CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
Yann Collet64482c22017-12-29 17:04:37 +01001946 DISPLAYLEVEL(3, "OK \n");
Yann Colletc7fe2622017-05-23 13:16:00 -07001947
Yann Colletdd026ca2020-05-09 11:30:45 -07001948 DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
1949 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1950 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1951 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1952 size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */
1953 if (!ZSTD_isError(r)) goto _output_error;
1954 }
1955 DISPLAYLEVEL(3, "OK \n");
1956
1957 DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++);
1958 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize);
Yann Collet91ad0122020-05-11 18:50:10 -07001959 assert(staticCCtx != NULL);
Yann Colletdd026ca2020-05-09 11:30:45 -07001960 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1961 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1962 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1963 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1964 }
1965 DISPLAYLEVEL(3, "OK \n");
1966
1967 DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1);
1968 CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */
1969 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1970 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1971 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1972 }
Yann Collet64482c22017-12-29 17:04:37 +01001973 DISPLAYLEVEL(3, "OK \n");
Yann Colletc7fe2622017-05-23 13:16:00 -07001974
Yann Collet98692c22019-08-01 15:58:17 +02001975 DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++);
Yann Colletc7fe2622017-05-23 13:16:00 -07001976 { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
1977 if (!ZSTD_isError(r)) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01001978 DISPLAYLEVEL(3, "OK \n");
Yann Collet0fdc71c2017-05-24 17:41:41 -07001979
Yann Colletdd026ca2020-05-09 11:30:45 -07001980 DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++);
1981 CHECK_Z( ZSTD_initDStream(staticDCtx) );
Yann Collet0fdc71c2017-05-24 17:41:41 -07001982 { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
1983 ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
1984 size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
1985 if (!ZSTD_isError(r)) goto _output_error;
1986 }
Yann Collet64482c22017-12-29 17:04:37 +01001987 DISPLAYLEVEL(3, "OK \n");
Elliott Hughes44aba642023-09-12 20:18:59 +00001988
1989 DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
1990 {
1991 // Test ZSTD_estimateCCtxSize_usingCCtxParams
1992 {
1993 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
1994 size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
1995 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
1996 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1997 compressedBuffer, compressedBufferSize,
1998 CNBuffer, CNBuffSize, 3));
1999
2000 {
2001 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2002 decodedBuffer, CNBuffSize,
2003 compressedBuffer, cSize);
2004 if (r != CNBuffSize) goto _output_error;
2005 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2006 }
2007 ZSTD_freeCCtxParams(params);
2008 }
2009
2010 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2011 {
2012 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2013 size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2014 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2015 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2016 compressedBuffer, compressedBufferSize,
2017 CNBuffer, CNBuffSize, 3) );
2018
2019 {
2020 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2021 decodedBuffer, CNBuffSize,
2022 compressedBuffer, cSize);
2023 if (r != CNBuffSize) goto _output_error;
2024 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2025 }
2026 ZSTD_freeCCtxParams(params);
2027 }
2028 }
2029 DISPLAYLEVEL(3, "OK \n");
2030
2031 DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
2032 {
2033 // Test ZSTD_estimateCCtxSize_usingCCtxParams
2034 {
2035 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2036 size_t cctxSizeDefault;
2037 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2038 cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2039 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2040 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2041 compressedBuffer, compressedBufferSize,
2042 CNBuffer, CNBuffSize, 3) );
2043
2044 {
2045 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2046 decodedBuffer, CNBuffSize,
2047 compressedBuffer, cSize);
2048 if (r != CNBuffSize) goto _output_error;
2049 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2050 }
2051 ZSTD_freeCCtxParams(params);
2052 }
2053
2054 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2055 {
2056 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2057 size_t cctxSizeDefault;
2058 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2059 cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2060 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2061 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2062 compressedBuffer, compressedBufferSize,
2063 CNBuffer, CNBuffSize, 3) );
2064
2065 {
2066 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2067 decodedBuffer, CNBuffSize,
2068 compressedBuffer, cSize);
2069 if (r != CNBuffSize) goto _output_error;
2070 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2071 }
2072 ZSTD_freeCCtxParams(params);
2073 }
2074 }
2075 DISPLAYLEVEL(3, "OK \n");
Yann Colletc7fe2622017-05-23 13:16:00 -07002076 }
2077 free(staticCCtxBuffer);
Yann Collet0fdc71c2017-05-24 17:41:41 -07002078 free(staticDCtxBuffer);
Yann Colletc7fe2622017-05-23 13:16:00 -07002079 }
2080
Yann Colletdd026ca2020-05-09 11:30:45 -07002081 DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++);
Nick Terrelld7def452018-12-18 14:24:49 -08002082 { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1);
2083 size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1);
2084 size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1);
2085 size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1);
2086
2087 if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error;
2088 if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error;
2089 }
2090 DISPLAYLEVEL(3, "OK \n");
2091
Yann Colletc7fe2622017-05-23 13:16:00 -07002092
Yann Collet1e1e26f2017-03-29 17:09:59 -07002093 /* ZSTDMT simple MT compression test */
Yann Collet64482c22017-12-29 17:04:37 +01002094 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
Nick Terrellc51a9e72020-10-01 18:47:54 -07002095 { ZSTD_CCtx* const mtctx = ZSTD_createCCtx();
Yann Collet0ef68032017-03-29 16:58:57 -07002096 if (mtctx==NULL) {
Yann Collet5cf1b242019-08-02 14:21:39 +02002097 DISPLAY("mtctx : not enough memory, aborting \n");
Yann Collet0ef68032017-03-29 16:58:57 -07002098 testResult = 1;
2099 goto _end;
2100 }
Elliott Hughes44aba642023-09-12 20:18:59 +00002101 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
2102 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
Yann Collet64482c22017-12-29 17:04:37 +01002103 DISPLAYLEVEL(3, "OK \n");
Yann Collet0ef68032017-03-29 16:58:57 -07002104
Yann Colletededcfc2018-12-21 16:19:44 -08002105 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
Nick Terrellc51a9e72020-10-01 18:47:54 -07002106 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
Yann Colletf9524cf2017-07-10 13:48:41 -07002107 compressedBuffer, compressedBufferSize,
Nick Terrellc51a9e72020-10-01 18:47:54 -07002108 CNBuffer, CNBuffSize) );
Yann Colletededcfc2018-12-21 16:19:44 -08002109 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Collet0ef68032017-03-29 16:58:57 -07002110
Yann Collet64482c22017-12-29 17:04:37 +01002111 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
Yann Collet0ef68032017-03-29 16:58:57 -07002112 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2113 if (rSize != CNBuffSize) {
Yann Colletededcfc2018-12-21 16:19:44 -08002114 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize);
Yann Collet0ef68032017-03-29 16:58:57 -07002115 goto _output_error;
2116 } }
Yann Collet64482c22017-12-29 17:04:37 +01002117 DISPLAYLEVEL(3, "OK \n");
Yann Collet0ef68032017-03-29 16:58:57 -07002118
Yann Colletededcfc2018-12-21 16:19:44 -08002119 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
Yann Collet0ef68032017-03-29 16:58:57 -07002120 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2121 if (r != CNBuffSize) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01002122 DISPLAYLEVEL(3, "OK \n");
Yann Collet0ef68032017-03-29 16:58:57 -07002123
Yann Collet64482c22017-12-29 17:04:37 +01002124 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
Yann Collet0ef68032017-03-29 16:58:57 -07002125 { size_t u;
2126 for (u=0; u<CNBuffSize; u++) {
Ed Masteb81d7cc2019-08-15 21:17:06 -04002127 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
Yann Collet0ef68032017-03-29 16:58:57 -07002128 } }
Yann Collet64482c22017-12-29 17:04:37 +01002129 DISPLAYLEVEL(3, "OK \n");
Yann Collet0ef68032017-03-29 16:58:57 -07002130
Yann Collet64482c22017-12-29 17:04:37 +01002131 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00002132 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
2133 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
2134 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
Nick Terrellc51a9e72020-10-01 18:47:54 -07002135 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2136 compressedBuffer, compressedBufferSize,
2137 CNBuffer, CNBuffSize) );
Yann Colletededcfc2018-12-21 16:19:44 -08002138 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Collet052a95f2017-07-11 17:18:26 -07002139
Yann Colletededcfc2018-12-21 16:19:44 -08002140 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
Yann Collet052a95f2017-07-11 17:18:26 -07002141 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2142 if (r != CNBuffSize) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01002143 DISPLAYLEVEL(3, "OK \n");
Yann Collet052a95f2017-07-11 17:18:26 -07002144
Nick Terrellc51a9e72020-10-01 18:47:54 -07002145 ZSTD_freeCCtx(mtctx);
Yann Collet0ef68032017-03-29 16:58:57 -07002146 }
2147
Nick Terrell8c474f92019-11-06 17:41:29 -08002148 DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++);
2149 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2150 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2151 char out[32];
2152 if (cctx == NULL || dctx == NULL) goto _output_error;
Elliott Hughes44aba642023-09-12 20:18:59 +00002153 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
Nick Terrell8c474f92019-11-06 17:41:29 -08002154 CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
2155 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
2156
Elliott Hughes44aba642023-09-12 20:18:59 +00002157 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
Nick Terrell8c474f92019-11-06 17:41:29 -08002158 { char const* outPtr = out;
2159 ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
2160 ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
2161 size_t dSize;
2162 CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) );
2163 if (dSize != 0) goto _output_error;
2164 }
2165
2166 ZSTD_freeDCtx(dctx);
2167 ZSTD_freeCCtx(cctx);
2168 }
2169
senhuang42e2bb2152020-12-29 15:56:13 -05002170 DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
2171 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
Elliott Hughes44aba642023-09-12 20:18:59 +00002172 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
senhuang42e2bb2152020-12-29 15:56:13 -05002173 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
Elliott Hughes44aba642023-09-12 20:18:59 +00002174 CHECK_Z(cSize);
senhuang42e2bb2152020-12-29 15:56:13 -05002175 ZSTD_freeCCtx(cctx);
2176 }
2177 DISPLAYLEVEL(3, "OK \n");
2178
Nick Terrell6efce7c2019-02-19 11:07:52 -08002179 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
2180 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2181 size_t cSize1, cSize2;
Elliott Hughes44aba642023-09-12 20:18:59 +00002182 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2183 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
Nick Terrell6efce7c2019-02-19 11:07:52 -08002184 cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
Elliott Hughes44aba642023-09-12 20:18:59 +00002185 CHECK_Z(cSize1);
2186 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
Nick Terrell6efce7c2019-02-19 11:07:52 -08002187 cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
Elliott Hughes44aba642023-09-12 20:18:59 +00002188 CHECK_Z(cSize2);
Nick Terrell6efce7c2019-02-19 11:07:52 -08002189 CHECK_LT(cSize1, cSize2);
2190 ZSTD_freeCCtx(cctx);
2191 }
2192 DISPLAYLEVEL(3, "OK \n");
2193
Nick Terrell48a64272019-04-09 16:24:17 -07002194 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++)
2195 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2196 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
2197 * ZSTDMT is forced to not take the shortcut.
2198 */
Elliott Hughes44aba642023-09-12 20:18:59 +00002199 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2200 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
2201 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
2202 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
Nick Terrell48a64272019-04-09 16:24:17 -07002203 ZSTD_freeCCtx(cctx);
2204 }
2205 DISPLAYLEVEL(3, "OK \n");
2206
Nick Terrell6efce7c2019-02-19 11:07:52 -08002207 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
2208 { ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
Yann Collet8f86c292021-05-04 10:54:34 -07002209 int const jobSize = 512 KB;
Nick Terrell6efce7c2019-02-19 11:07:52 -08002210 int value;
2211 /* Check that the overlap log and job size are unset. */
Elliott Hughes44aba642023-09-12 20:18:59 +00002212 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
Nick Terrell6efce7c2019-02-19 11:07:52 -08002213 CHECK_EQ(value, 0);
Elliott Hughes44aba642023-09-12 20:18:59 +00002214 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
Nick Terrell6efce7c2019-02-19 11:07:52 -08002215 CHECK_EQ(value, 0);
2216 /* Set and check the overlap log and job size. */
Elliott Hughes44aba642023-09-12 20:18:59 +00002217 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
2218 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
2219 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
Nick Terrell6efce7c2019-02-19 11:07:52 -08002220 CHECK_EQ(value, 5);
Elliott Hughes44aba642023-09-12 20:18:59 +00002221 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
Yann Collet8f86c292021-05-04 10:54:34 -07002222 CHECK_EQ(value, jobSize);
Josh Sorefa880ca22019-04-12 14:18:11 -04002223 /* Set the number of workers and check the overlap log and job size. */
Elliott Hughes44aba642023-09-12 20:18:59 +00002224 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
2225 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
Nick Terrell6efce7c2019-02-19 11:07:52 -08002226 CHECK_EQ(value, 5);
Elliott Hughes44aba642023-09-12 20:18:59 +00002227 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
Yann Collet8f86c292021-05-04 10:54:34 -07002228 CHECK_EQ(value, jobSize);
Nick Terrell6efce7c2019-02-19 11:07:52 -08002229 ZSTD_freeCCtxParams(params);
Nick Terrell6efce7c2019-02-19 11:07:52 -08002230 }
2231 DISPLAYLEVEL(3, "OK \n");
Yann Collet0ef68032017-03-29 16:58:57 -07002232
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002233 /* Simple API multiframe test */
Yann Collet64482c22017-12-29 17:04:37 +01002234 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002235 { size_t off = 0;
2236 int i;
2237 int const segs = 4;
2238 /* only use the first half so we don't push against size limit of compressedBuffer */
2239 size_t const segSize = (CNBuffSize / 2) / segs;
senhuang42339d8ba2020-12-21 11:33:27 -05002240
senhuang42339d8ba2020-12-21 11:33:27 -05002241 const U32 skipLen = 129 KB;
Yann Collet7f8be042020-12-28 14:07:31 -08002242 char* const skipBuff = (char*)malloc(skipLen);
2243 assert(skipBuff != NULL);
senhuang42339d8ba2020-12-21 11:33:27 -05002244 memset(skipBuff, 0, skipLen);
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002245 for (i = 0; i < segs; i++) {
Yann Collet810a9ca2019-08-01 16:59:22 +02002246 CHECK_NEWV(r, ZSTD_compress(
2247 (BYTE*)compressedBuffer + off, CNBuffSize - off,
2248 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
2249 5) );
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002250 off += r;
2251 if (i == segs/2) {
2252 /* insert skippable frame */
Yann Collet7f8be042020-12-28 14:07:31 -08002253 size_t const skippableSize =
Yann Colletff2f8882020-12-29 11:44:37 -08002254 ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize,
Yann Collet7f8be042020-12-28 14:07:31 -08002255 skipBuff, skipLen, seed % 15);
senhuang42339d8ba2020-12-21 11:33:27 -05002256 CHECK_Z(skippableSize);
2257 off += skippableSize;
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002258 }
2259 }
2260 cSize = off;
senhuang42339d8ba2020-12-21 11:33:27 -05002261 free(skipBuff);
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002262 }
Yann Collet64482c22017-12-29 17:04:37 +01002263 DISPLAYLEVEL(3, "OK \n");
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002264
Yann Collet64482c22017-12-29 17:04:37 +01002265 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++);
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002266 { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
2267 if (r != CNBuffSize / 2) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01002268 DISPLAYLEVEL(3, "OK \n");
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002269
shakeelrao820af1e2019-02-28 00:42:49 -08002270 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
shakeelrao95dfd482019-03-01 23:11:15 -08002271 { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize);
2272 if (bound != CNBuffSize / 2) goto _output_error; }
shakeelrao820af1e2019-02-28 00:42:49 -08002273 DISPLAYLEVEL(3, "OK \n");
2274
Yann Collet64482c22017-12-29 17:04:37 +01002275 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
Yann Collet810a9ca2019-08-01 16:59:22 +02002276 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002277 if (r != CNBuffSize / 2) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01002278 DISPLAYLEVEL(3, "OK \n");
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002279
Yann Collet64482c22017-12-29 17:04:37 +01002280 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002281 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
Yann Collet64482c22017-12-29 17:04:37 +01002282 DISPLAYLEVEL(3, "OK \n");
Sean Purcellba2ad9f2017-02-07 16:16:55 -08002283
Binh Vo9d9f7682021-06-11 12:11:58 -04002284 /* Simple API skippable frame test */
2285 DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++);
Yann Colletf0fc8cb2021-09-03 13:44:07 -07002286 { U32 i;
Binh Vo9d9f7682021-06-11 12:11:58 -04002287 unsigned readMagic;
2288 unsigned long long receivedSize;
2289 size_t skippableSize;
2290 const U32 skipLen = 129 KB;
2291 char* const skipBuff = (char*)malloc(skipLen);
2292 assert(skipBuff != NULL);
2293 for (i = 0; i < skipLen; i++)
Yann Colletf0fc8cb2021-09-03 13:44:07 -07002294 skipBuff[i] = (char) ((seed + i) % 256);
2295 skippableSize = ZSTD_writeSkippableFrame(
2296 compressedBuffer, compressedBufferSize,
2297 skipBuff, skipLen, seed % 15);
Binh Vo9d9f7682021-06-11 12:11:58 -04002298 CHECK_Z(skippableSize);
2299 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2300 receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize);
2301 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2302 CHECK_EQ(seed % 15, readMagic);
2303 if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error;
2304
2305 free(skipBuff);
2306 }
2307 DISPLAYLEVEL(3, "OK \n");
2308
2309 DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++);
2310 {
2311 unsigned readMagic;
2312 unsigned long long receivedSize;
2313 size_t skippableSize;
Yann Colletf0fc8cb2021-09-03 13:44:07 -07002314 skippableSize = ZSTD_writeSkippableFrame(
2315 compressedBuffer, compressedBufferSize,
2316 CNBuffer, 0, seed % 15);
Binh Vo9d9f7682021-06-11 12:11:58 -04002317 CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize);
2318 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2319 receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize);
2320 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2321 CHECK_EQ(seed % 15, readMagic);
2322 }
2323 DISPLAYLEVEL(3, "OK \n");
2324
Yann Collet389648c2016-04-12 19:13:08 +02002325 /* Dictionary and CCtx Duplication tests */
Yann Colletc0a9bf32016-05-30 01:56:08 +02002326 { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
2327 ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
Yann Colletaec945f2018-12-04 15:35:37 -08002328 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
Yann Colletc0a9bf32016-05-30 01:56:08 +02002329 static const size_t dictSize = 551;
Yann Colletaec945f2018-12-04 15:35:37 -08002330 assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
Yann Collet60096272016-01-08 17:27:50 +01002331
Yann Collet64482c22017-12-29 17:04:37 +01002332 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
Yann Collet97b378a2016-09-21 17:20:19 +02002333 { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
Yann Colletc0a9bf32016-05-30 01:56:08 +02002334 if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
Yann Collet64482c22017-12-29 17:04:37 +01002335 DISPLAYLEVEL(3, "OK \n");
Yann Collet887e7da2016-04-11 20:12:27 +02002336
Yann Collet64482c22017-12-29 17:04:37 +01002337 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00002338 CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
2339 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
Yann Collet64482c22017-12-29 17:04:37 +01002340 DISPLAYLEVEL(3, "OK \n");
Yann Collet60096272016-01-08 17:27:50 +01002341
Yann Collet64482c22017-12-29 17:04:37 +01002342 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
Yann Collet60096272016-01-08 17:27:50 +01002343 cSize = 0;
Yann Collet810a9ca2019-08-01 16:59:22 +02002344 CHECKPLUS(r, ZSTD_compressEnd(ctxOrig,
2345 compressedBuffer, compressedBufferSize,
2346 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
Yann Colletc0a9bf32016-05-30 01:56:08 +02002347 cSize += r);
Yann Colletededcfc2018-12-21 16:19:44 -08002348 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Collet60096272016-01-08 17:27:50 +01002349
Yann Collet64482c22017-12-29 17:04:37 +01002350 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
Yann Colletc0a9bf32016-05-30 01:56:08 +02002351 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
Yann Colletd2858e92016-05-30 15:10:09 +02002352 decodedBuffer, CNBuffSize,
Yann Colletc0a9bf32016-05-30 01:56:08 +02002353 compressedBuffer, cSize,
2354 CNBuffer, dictSize),
Yann Colletd2858e92016-05-30 15:10:09 +02002355 if (r != CNBuffSize - dictSize) goto _output_error);
Yann Collet64482c22017-12-29 17:04:37 +01002356 DISPLAYLEVEL(3, "OK \n");
Yann Collet60096272016-01-08 17:27:50 +01002357
Yann Collet64482c22017-12-29 17:04:37 +01002358 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
Yann Collet389648c2016-04-12 19:13:08 +02002359 { size_t const cSizeOrig = cSize;
2360 cSize = 0;
Yann Collet810a9ca2019-08-01 16:59:22 +02002361 CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated,
2362 compressedBuffer, compressedBufferSize,
2363 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
Yann Colletc0a9bf32016-05-30 01:56:08 +02002364 cSize += r);
Yann Collet227cc392016-07-15 11:27:09 +02002365 if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
Yann Collet389648c2016-04-12 19:13:08 +02002366 }
Yann Colletededcfc2018-12-21 16:19:44 -08002367 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Collet60096272016-01-08 17:27:50 +01002368
Yann Collet64482c22017-12-29 17:04:37 +01002369 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
Yann Colletc0a9bf32016-05-30 01:56:08 +02002370 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
Yann Colletd2858e92016-05-30 15:10:09 +02002371 decodedBuffer, CNBuffSize,
Yann Collet60096272016-01-08 17:27:50 +01002372 compressedBuffer, cSize,
Yann Colletc0a9bf32016-05-30 01:56:08 +02002373 CNBuffer, dictSize),
Yann Colletd2858e92016-05-30 15:10:09 +02002374 if (r != CNBuffSize - dictSize) goto _output_error);
Yann Collet64482c22017-12-29 17:04:37 +01002375 DISPLAYLEVEL(3, "OK \n");
Yann Collet541dc7c2016-04-12 18:00:20 +02002376
Yann Collet64482c22017-12-29 17:04:37 +01002377 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++);
Yann Collet57827f92017-05-25 15:44:06 -07002378 { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
Yann Collet8dff9562017-02-25 10:11:15 -08002379 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2380 if (r != CNBuffSize - dictSize) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08002381 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict));
Yann Collet8dff9562017-02-25 10:11:15 -08002382 ZSTD_freeDDict(ddict);
2383 }
2384
Yann Collet64482c22017-12-29 17:04:37 +01002385 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
Stella Lauc88fb922017-08-29 11:55:02 -07002386 { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
Yann Collet810a9ca2019-08-01 16:59:22 +02002387 void* const ddictBuffer = malloc(ddictBufferSize);
Yann Collet57827f92017-05-25 15:44:06 -07002388 if (ddictBuffer == NULL) goto _output_error;
Yann Collet6873fec2018-03-20 15:13:14 -07002389 { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
Yann Collet57827f92017-05-25 15:44:06 -07002390 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2391 if (r != CNBuffSize - dictSize) goto _output_error;
2392 }
Yann Colletff8f83b2017-06-20 12:17:32 -07002393 free(ddictBuffer);
Yann Colletededcfc2018-12-21 16:19:44 -08002394 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize);
Yann Collet57827f92017-05-25 15:44:06 -07002395 }
2396
Yann Collet64482c22017-12-29 17:04:37 +01002397 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
Yann Colletd2858e92016-05-30 15:10:09 +02002398 { size_t const testSize = CNBuffSize / 3;
Elliott Hughes44aba642023-09-12 20:18:59 +00002399 CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
2400 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
Yann Colletc0a9bf32016-05-30 01:56:08 +02002401
Yann Collet810a9ca2019-08-01 16:59:22 +02002402 CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
2403 (const char*)CNBuffer + dictSize, testSize) );
Yann Collet990449b2017-07-07 15:21:35 -07002404 { ZSTD_frameHeader zfh;
2405 if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
2406 if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
Yann Collet541dc7c2016-04-12 18:00:20 +02002407 } }
Yann Collet64482c22017-12-29 17:04:37 +01002408 DISPLAYLEVEL(3, "OK \n");
Yann Collet541dc7c2016-04-12 18:00:20 +02002409
Yann Collet7fce9a42021-09-08 14:05:57 -07002410 /* Note : these tests should be replaced by proper regression tests,
2411 * but existing ones do not focus on small data + dictionary + all levels.
2412 */
Yann Colletd1927f02019-08-02 15:31:00 +02002413 if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
2414 size_t const flatdictSize = 22 KB;
Yann Collet810a9ca2019-08-01 16:59:22 +02002415 size_t const contentSize = 9 KB;
2416 const void* const dict = (const char*)CNBuffer;
Yann Collet21152922019-08-01 17:12:26 +02002417 const void* const contentStart = (const char*)dict + flatdictSize;
Nick Terrell46944232020-11-02 17:52:29 -08002418 /* These upper bounds are generally within a few bytes of the compressed size */
Sen Huang2ff5c7b2021-06-07 00:44:23 -07002419 size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
2420 3770, 3770, 3770, 3750, 3750,
Yann Collet7fce9a42021-09-08 14:05:57 -07002421 3742, 3675, 3674, 3665, 3664,
2422 3663, 3662, 3661, 3660, 3660,
Sen Huang2ff5c7b2021-06-07 00:44:23 -07002423 3660, 3660, 3660 };
W. Felix Handte450fca92021-12-13 17:29:32 -05002424 size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940,
Elliott Hughes44aba642023-09-12 20:18:59 +00002425 2950, 2950, 2925, 2900, 2892,
Yann Collet7fce9a42021-09-08 14:05:57 -07002426 2910, 2910, 2910, 2780, 2775,
2427 2765, 2760, 2755, 2754, 2753,
2428 2753, 2753, 2753 };
Yann Collet810a9ca2019-08-01 16:59:22 +02002429 int l = 1;
2430 int const maxLevel = ZSTD_maxCLevel();
Sen Huang2ff5c7b2021-06-07 00:44:23 -07002431 /* clevels with strategies that support rowhash on small inputs */
2432 int rowLevel = 4;
2433 int const rowLevelEnd = 8;
Yann Colletd1927f02019-08-02 15:31:00 +02002434
2435 DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
Yann Colleta9e43b32021-12-20 11:43:14 -08002436 assert(maxLevel == 22);
Yann Colletd1927f02019-08-02 15:31:00 +02002437 RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
Yann Colletfb3522a2021-12-08 09:17:06 -08002438 DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n",
2439 (unsigned long long)XXH64(contentStart, contentSize, 0),
2440 (unsigned long long)XXH64(dict, flatdictSize, 0));
Yann Colletd1927f02019-08-02 15:31:00 +02002441
Yann Collet810a9ca2019-08-01 16:59:22 +02002442 for ( ; l <= maxLevel; l++) {
2443 size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize,
2444 contentStart, contentSize, l);
2445 if (nodict_cSize > target_nodict_cSize[l]) {
2446 DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n",
2447 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2448 goto _output_error;
2449 }
Yann Colletd1927f02019-08-02 15:31:00 +02002450 DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n",
Yann Collet810a9ca2019-08-01 16:59:22 +02002451 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2452 }
2453 for ( l=1 ; l <= maxLevel; l++) {
2454 size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig,
2455 compressedBuffer, compressedBufferSize,
2456 contentStart, contentSize,
2457 dict, flatdictSize,
2458 l);
2459 if (wdict_cSize > target_wdict_cSize[l]) {
2460 DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n",
2461 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2462 goto _output_error;
2463 }
Yann Colletd1927f02019-08-02 15:31:00 +02002464 DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
Yann Collet810a9ca2019-08-01 16:59:22 +02002465 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2466 }
Sen Huang2ff5c7b2021-06-07 00:44:23 -07002467 /* Compression with ZSTD_compress2 and row match finder force enabled.
2468 * Give some slack for force-enabled row matchfinder since we're on a small input (9KB)
2469 */
2470 for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5;
2471 for (l=1 ; l <= maxLevel; l++) {
2472 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2473 size_t nodict_cSize;
2474 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l);
senhuang42b5c35d72021-09-20 09:04:07 -04002475 ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable);
Sen Huang2ff5c7b2021-06-07 00:44:23 -07002476 nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize,
2477 contentStart, contentSize);
2478 if (nodict_cSize > target_nodict_cSize[l]) {
2479 DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n",
2480 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2481 ZSTD_freeCCtx(cctx);
2482 goto _output_error;
2483 }
2484 DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n",
2485 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2486 ZSTD_freeCCtx(cctx);
2487 }
Nick Terrell46944232020-11-02 17:52:29 -08002488 /* Dict compression with DMS */
2489 for ( l=1 ; l <= maxLevel; l++) {
2490 size_t wdict_cSize;
2491 CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) );
2492 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) );
2493 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) );
2494 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) );
Elliott Hughes44aba642023-09-12 20:18:59 +00002495 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) );
Nick Terrell46944232020-11-02 17:52:29 -08002496 wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize);
2497 if (wdict_cSize > target_wdict_cSize[l]) {
2498 DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n",
2499 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2500 goto _output_error;
2501 }
2502 DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n",
2503 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2504 }
Yann Collet810a9ca2019-08-01 16:59:22 +02002505
Yann Colletd1927f02019-08-02 15:31:00 +02002506 DISPLAYLEVEL(4, "compression efficiency tests OK \n");
2507 }
Yann Collet810a9ca2019-08-01 16:59:22 +02002508
Yann Collet541dc7c2016-04-12 18:00:20 +02002509 ZSTD_freeCCtx(ctxOrig);
2510 ZSTD_freeCCtx(ctxDuplicated);
Yann Colletaec945f2018-12-04 15:35:37 -08002511 ZSTD_freeDCtx(dctx);
Yann Collet60096272016-01-08 17:27:50 +01002512 }
2513
Yann Collet30009522016-05-30 16:17:33 +02002514 /* Dictionary and dictBuilder tests */
2515 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Yann Collete8093dd2018-01-11 11:16:32 -08002516 size_t const dictBufferCapacity = 16 KB;
Yann Collet4baecdf2019-05-28 13:15:48 -07002517 void* const dictBuffer = malloc(dictBufferCapacity);
Yann Collet30009522016-05-30 16:17:33 +02002518 size_t const totalSampleSize = 1 MB;
2519 size_t const sampleUnitSize = 8 KB;
Yann Colletb81cbba2016-05-30 22:29:45 +02002520 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
Yann Collet30009522016-05-30 16:17:33 +02002521 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
Yann Collete8093dd2018-01-11 11:16:32 -08002522 size_t dictSize;
Yann Colletf586bdf2016-12-06 06:11:46 +01002523 U32 dictID;
Sen Huang97b7f712019-11-04 14:33:52 -05002524 size_t dictHeaderSize;
senhuang4222b7bff2020-12-28 16:43:04 -05002525 size_t dictBufferFixedSize = 144;
2526 unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
2527 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2528 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
2529 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
2530 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2531 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2532 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
2533 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
2534 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
2535 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
2536 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
2537 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69};
Yann Collet30009522016-05-30 16:17:33 +02002538
2539 if (dictBuffer==NULL || samplesSizes==NULL) {
2540 free(dictBuffer);
2541 free(samplesSizes);
2542 goto _output_error;
2543 }
2544
Yann Colletcacf47c2018-01-11 13:25:08 -08002545 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++);
Yann Collet218e9fe2018-01-11 09:42:38 -08002546 assert(compressedBufferSize >= totalSampleSize);
Yann Collete8093dd2018-01-11 11:16:32 -08002547 { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; }
Yann Collet218e9fe2018-01-11 09:42:38 -08002548 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
Yann Collete8093dd2018-01-11 11:16:32 -08002549 { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2550 decodedBuffer, samplesSizes, nbSamples);
2551 if (ZDICT_isError(sDictSize)) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08002552 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize);
Yann Collete8093dd2018-01-11 11:16:32 -08002553 }
Yann Collet218e9fe2018-01-11 09:42:38 -08002554
Yann Collet64482c22017-12-29 17:04:37 +01002555 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++);
Yann Collet30009522016-05-30 16:17:33 +02002556 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
Yann Collete8093dd2018-01-11 11:16:32 -08002557 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
Yann Collet30009522016-05-30 16:17:33 +02002558 CNBuffer, samplesSizes, nbSamples);
2559 if (ZDICT_isError(dictSize)) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08002560 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
Yann Collet30009522016-05-30 16:17:33 +02002561
Nick Terrell103d1ee2018-11-08 10:58:51 -08002562 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++);
2563 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2564 { ZDICT_cover_params_t coverParams;
2565 memset(&coverParams, 0, sizeof(coverParams));
2566 coverParams.steps = 8;
2567 coverParams.nbThreads = 4;
2568 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2569 dictBuffer, dictBufferCapacity,
Yann Colletaec945f2018-12-04 15:35:37 -08002570 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
Nick Terrell103d1ee2018-11-08 10:58:51 -08002571 &coverParams);
2572 if (ZDICT_isError(dictSize)) goto _output_error;
2573 }
Yann Colletededcfc2018-12-21 16:19:44 -08002574 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
Nick Terrell103d1ee2018-11-08 10:58:51 -08002575
Tyler-Tranc55d2e72019-06-27 16:26:57 -07002576 DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++);
2577 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2578 { ZDICT_cover_params_t coverParams;
2579 memset(&coverParams, 0, sizeof(coverParams));
2580 coverParams.steps = 8;
2581 coverParams.nbThreads = 4;
2582 coverParams.shrinkDict = 1;
2583 coverParams.shrinkDictMaxRegression = 1;
2584 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2585 dictBuffer, dictBufferCapacity,
2586 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2587 &coverParams);
2588 if (ZDICT_isError(dictSize)) goto _output_error;
2589 }
2590 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2591
Nick Terrell103d1ee2018-11-08 10:58:51 -08002592 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++);
2593 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2594 { ZDICT_fastCover_params_t fastCoverParams;
2595 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2596 fastCoverParams.steps = 8;
2597 fastCoverParams.nbThreads = 4;
2598 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2599 dictBuffer, dictBufferCapacity,
2600 CNBuffer, samplesSizes, nbSamples,
2601 &fastCoverParams);
2602 if (ZDICT_isError(dictSize)) goto _output_error;
2603 }
Yann Colletededcfc2018-12-21 16:19:44 -08002604 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
Nick Terrell103d1ee2018-11-08 10:58:51 -08002605
Tyler-Tranc55d2e72019-06-27 16:26:57 -07002606 DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++);
2607 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2608 { ZDICT_fastCover_params_t fastCoverParams;
2609 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2610 fastCoverParams.steps = 8;
2611 fastCoverParams.nbThreads = 4;
2612 fastCoverParams.shrinkDict = 1;
2613 fastCoverParams.shrinkDictMaxRegression = 1;
2614 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2615 dictBuffer, dictBufferCapacity,
2616 CNBuffer, samplesSizes, nbSamples,
2617 &fastCoverParams);
2618 if (ZDICT_isError(dictSize)) goto _output_error;
2619 }
2620 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2621
Yann Collet64482c22017-12-29 17:04:37 +01002622 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
Yann Colletf586bdf2016-12-06 06:11:46 +01002623 dictID = ZDICT_getDictID(dictBuffer, dictSize);
2624 if (dictID==0) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08002625 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
Yann Colletda3fbcb2016-08-19 14:23:58 +02002626
Sen Huange1edc552019-11-03 17:44:28 -05002627 DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++);
2628 dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize);
2629 if (dictHeaderSize==0) goto _output_error;
2630 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2631
2632 DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
senhuang4222b7bff2020-12-28 16:43:04 -05002633 { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize);
Sen Huange1edc552019-11-03 17:44:28 -05002634 if (dictHeaderSize != 115) goto _output_error;
2635 }
2636 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2637
Yann Collet64482c22017-12-29 17:04:37 +01002638 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
Yann Colletf9524cf2017-07-10 13:48:41 -07002639 cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
Yann Collet30009522016-05-30 16:17:33 +02002640 CNBuffer, CNBuffSize,
2641 dictBuffer, dictSize, 4);
2642 if (ZSTD_isError(cSize)) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08002643 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Collet30009522016-05-30 16:17:33 +02002644
Yann Collet64482c22017-12-29 17:04:37 +01002645 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++);
Yann Colletf586bdf2016-12-06 06:11:46 +01002646 { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
2647 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
Yann Collete7a41a52016-12-05 16:21:06 -08002648 }
Yann Collet64482c22017-12-29 17:04:37 +01002649 DISPLAYLEVEL(3, "OK \n");
Yann Collete7a41a52016-12-05 16:21:06 -08002650
Yann Collet64482c22017-12-29 17:04:37 +01002651 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
Yann Colletf586bdf2016-12-06 06:11:46 +01002652 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2653 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
Yann Collete7a41a52016-12-05 16:21:06 -08002654 }
Yann Collet64482c22017-12-29 17:04:37 +01002655 DISPLAYLEVEL(3, "OK \n");
Yann Collete7a41a52016-12-05 16:21:06 -08002656
Yann Collet64482c22017-12-29 17:04:37 +01002657 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
Yann Colletaec945f2018-12-04 15:35:37 -08002658 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2659 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2660 decodedBuffer, CNBuffSize,
2661 compressedBuffer, cSize,
2662 dictBuffer, dictSize),
2663 if (r != CNBuffSize) goto _output_error);
2664 ZSTD_freeDCtx(dctx);
2665 }
Yann Collet64482c22017-12-29 17:04:37 +01002666 DISPLAYLEVEL(3, "OK \n");
Yann Collet30009522016-05-30 16:17:33 +02002667
Yann Collet64482c22017-12-29 17:04:37 +01002668 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
Yann Colleta1d67042017-05-08 17:51:49 -07002669 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
Stella Lauc88fb922017-08-29 11:55:02 -07002670 size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef);
Yann Colletededcfc2018-12-21 16:19:44 -08002671 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize);
Yann Colleta1d67042017-05-08 17:51:49 -07002672 }
2673
Yann Collet64482c22017-12-29 17:04:37 +01002674 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
Yann Collet2f734272017-04-27 14:39:39 -07002675 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
Yann Collet25989e32017-05-25 15:07:37 -07002676 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
Yann Collet6873fec2018-03-20 15:13:14 -07002677 ZSTD_dlm_byRef, ZSTD_dct_auto,
Yann Collet7bd1a292017-06-21 11:50:33 -07002678 cParams, ZSTD_defaultCMem);
Yann Collet4baecdf2019-05-28 13:15:48 -07002679 assert(cdict != NULL);
Yann Colletededcfc2018-12-21 16:19:44 -08002680 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict));
Luke Pitteac309c2020-11-04 11:37:37 +00002681 assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict));
Yann Colletf9524cf2017-07-10 13:48:41 -07002682 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
Nick Terrell39a6cc52017-04-03 21:00:44 -07002683 CNBuffer, CNBuffSize, cdict);
Yann Collet2f734272017-04-27 14:39:39 -07002684 ZSTD_freeCDict(cdict);
2685 if (ZSTD_isError(cSize)) goto _output_error;
Nick Terrell39a6cc52017-04-03 21:00:44 -07002686 }
Yann Colletededcfc2018-12-21 16:19:44 -08002687 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Nick Terrell39a6cc52017-04-03 21:00:44 -07002688
Yann Collet64482c22017-12-29 17:04:37 +01002689 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
Nick Terrell39a6cc52017-04-03 21:00:44 -07002690 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2691 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2692 }
Yann Collet64482c22017-12-29 17:04:37 +01002693 DISPLAYLEVEL(3, "OK \n");
Nick Terrell39a6cc52017-04-03 21:00:44 -07002694
Yann Collet64482c22017-12-29 17:04:37 +01002695 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
Yann Colletaec945f2018-12-04 15:35:37 -08002696 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2697 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2698 decodedBuffer, CNBuffSize,
2699 compressedBuffer, cSize,
2700 dictBuffer, dictSize),
2701 if (r != CNBuffSize) goto _output_error);
2702 ZSTD_freeDCtx(dctx);
2703 }
Yann Collet64482c22017-12-29 17:04:37 +01002704 DISPLAYLEVEL(3, "OK \n");
Nick Terrell39a6cc52017-04-03 21:00:44 -07002705
Yann Collet64482c22017-12-29 17:04:37 +01002706 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
Nick Terrell9c8c69e2018-01-31 11:12:54 -08002707 { int const maxLevel = ZSTD_maxCLevel();
2708 int level;
2709 for (level = 1; level <= maxLevel; ++level) {
2710 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize);
2711 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
2712 void* const cdictBuffer = malloc(cdictSize);
2713 if (cdictBuffer==NULL) goto _output_error;
2714 { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict(
2715 cdictBuffer, cdictSize,
2716 dictBuffer, dictSize,
Yann Collet6873fec2018-03-20 15:13:14 -07002717 ZSTD_dlm_byCopy, ZSTD_dct_auto,
Nick Terrell9c8c69e2018-01-31 11:12:54 -08002718 cParams);
2719 if (cdict == NULL) {
2720 DISPLAY("ZSTD_initStaticCDict failed ");
2721 goto _output_error;
2722 }
2723 cSize = ZSTD_compress_usingCDict(cctx,
2724 compressedBuffer, compressedBufferSize,
2725 CNBuffer, MIN(10 KB, CNBuffSize), cdict);
2726 if (ZSTD_isError(cSize)) {
2727 DISPLAY("ZSTD_compress_usingCDict failed ");
2728 goto _output_error;
2729 } }
2730 free(cdictBuffer);
2731 } }
Yann Colletededcfc2018-12-21 16:19:44 -08002732 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Colletcdf7e822017-05-25 18:05:49 -07002733
Yann Collet64482c22017-12-29 17:04:37 +01002734 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
Yann Collet2f734272017-04-27 14:39:39 -07002735 { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
2736 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
Yann Collet6873fec2018-03-20 15:13:14 -07002737 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
Yann Collet4baecdf2019-05-28 13:15:48 -07002738 assert(cdict != NULL);
2739 cSize = ZSTD_compress_usingCDict_advanced(cctx,
2740 compressedBuffer, compressedBufferSize,
2741 CNBuffer, CNBuffSize,
2742 cdict, fParams);
Yann Collet2f734272017-04-27 14:39:39 -07002743 ZSTD_freeCDict(cdict);
2744 if (ZSTD_isError(cSize)) goto _output_error;
2745 }
Yann Colletededcfc2018-12-21 16:19:44 -08002746 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Collet2f734272017-04-27 14:39:39 -07002747
Yann Collet64482c22017-12-29 17:04:37 +01002748 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++);
Yann Collet2f734272017-04-27 14:39:39 -07002749 { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2750 if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
2751 }
Yann Collet64482c22017-12-29 17:04:37 +01002752 DISPLAYLEVEL(3, "OK (unknown)\n");
Yann Collet2f734272017-04-27 14:39:39 -07002753
Yann Collet64482c22017-12-29 17:04:37 +01002754 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
Yann Collet4baecdf2019-05-28 13:15:48 -07002755 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2756 assert(dctx != NULL);
Yann Colletaec945f2018-12-04 15:35:37 -08002757 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2758 decodedBuffer, CNBuffSize,
2759 compressedBuffer, cSize,
2760 dictBuffer, dictSize),
2761 if (r != CNBuffSize) goto _output_error);
2762 ZSTD_freeDCtx(dctx);
2763 }
Yann Collet64482c22017-12-29 17:04:37 +01002764 DISPLAYLEVEL(3, "OK \n");
Yann Collet2f734272017-04-27 14:39:39 -07002765
Yann Collet64482c22017-12-29 17:04:37 +01002766 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
Yann Collet6c6e1752016-06-27 15:28:45 +02002767 { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
2768 p.fParams.noDictIDFlag = 1;
Yann Colletf9524cf2017-07-10 13:48:41 -07002769 cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
Yann Collet30009522016-05-30 16:17:33 +02002770 CNBuffer, CNBuffSize,
2771 dictBuffer, dictSize, p);
2772 if (ZSTD_isError(cSize)) goto _output_error;
2773 }
Yann Colletededcfc2018-12-21 16:19:44 -08002774 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
Yann Collet30009522016-05-30 16:17:33 +02002775
Yann Collet64482c22017-12-29 17:04:37 +01002776 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
Yann Colletaec945f2018-12-04 15:35:37 -08002777 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2778 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2779 decodedBuffer, CNBuffSize,
2780 compressedBuffer, cSize,
2781 dictBuffer, dictSize),
2782 if (r != CNBuffSize) goto _output_error);
2783 ZSTD_freeDCtx(dctx);
2784 }
Yann Collet64482c22017-12-29 17:04:37 +01002785 DISPLAYLEVEL(3, "OK \n");
Yann Collet30009522016-05-30 16:17:33 +02002786
Yann Collet64482c22017-12-29 17:04:37 +01002787 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
Yann Collet2fb8d1a2018-12-04 15:54:01 -08002788 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2789 assert(dctx != NULL);
2790 { const size_t ret = ZSTD_decompress_usingDict(
Yann Colletaec945f2018-12-04 15:35:37 -08002791 dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
2792 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
Yann Collet2fb8d1a2018-12-04 15:54:01 -08002793 if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
2794 goto _output_error;
2795 }
Yann Colletaec945f2018-12-04 15:35:37 -08002796 ZSTD_freeDCtx(dctx);
Sean Purcell834ab502017-01-11 17:31:06 -08002797 }
Yann Collet64482c22017-12-29 17:04:37 +01002798 DISPLAYLEVEL(3, "OK \n");
Sean Purcell834ab502017-01-11 17:31:06 -08002799
Elliott Hughes44aba642023-09-12 20:18:59 +00002800 DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++);
2801 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2802 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2803 ZSTD_frameParameters const fParams = { 0, 1, 0 };
2804 size_t cBlockSize;
2805 cSize = 0;
2806 CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN));
2807 cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000);
2808 CHECK_Z(cBlockSize);
2809 cSize += cBlockSize;
2810 cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000);
2811 CHECK_Z(cBlockSize);
2812 cSize += cBlockSize;
2813
2814 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
2815
2816 ZSTD_freeCDict(cdict);
2817 ZSTD_freeDCtx(dctx);
2818 }
2819 DISPLAYLEVEL(3, "OK \n");
2820
Yann Collet63239662019-10-16 16:14:04 -07002821 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
Yann Collet7bd1a292017-06-21 11:50:33 -07002822 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
Yann Collet6873fec2018-03-20 15:13:14 -07002823 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
Yann Collet7bd1a292017-06-21 11:50:33 -07002824 if (cdict==NULL) goto _output_error;
2825 ZSTD_freeCDict(cdict);
2826 }
Yann Collet64482c22017-12-29 17:04:37 +01002827 DISPLAYLEVEL(3, "OK \n");
Yann Collet7bd1a292017-06-21 11:50:33 -07002828
Yann Collet63239662019-10-16 16:14:04 -07002829 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
Yann Collet7bd1a292017-06-21 11:50:33 -07002830 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
Yann Collet6873fec2018-03-20 15:13:14 -07002831 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
Yann Collet7bd1a292017-06-21 11:50:33 -07002832 if (cdict!=NULL) goto _output_error;
2833 ZSTD_freeCDict(cdict);
2834 }
Yann Collet64482c22017-12-29 17:04:37 +01002835 DISPLAYLEVEL(3, "OK \n");
Yann Collet7bd1a292017-06-21 11:50:33 -07002836
Nick Terrell7083f792020-10-05 15:17:44 -07002837 { char* rawDictBuffer = (char*)malloc(dictSize);
2838 assert(rawDictBuffer);
2839 memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2);
2840 memset(rawDictBuffer + dictSize - 2, 0, 2);
2841 MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY);
Stella Laua6e20e12017-08-29 18:36:18 -07002842
Nick Terrell7083f792020-10-05 15:17:44 -07002843 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
2844 {
2845 size_t ret;
2846 /* Either operation is allowed to fail, but one must fail. */
2847 ret = ZSTD_CCtx_loadDictionary_advanced(
2848 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
2849 if (!ZSTD_isError(ret)) {
2850 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2851 if (!ZSTD_isError(ret)) goto _output_error;
2852 }
2853 }
2854 DISPLAYLEVEL(3, "OK \n");
2855
2856 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
2857 {
2858 size_t ret;
2859 ret = ZSTD_CCtx_loadDictionary_advanced(
2860 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
2861 if (ZSTD_isError(ret)) goto _output_error;
2862 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2863 if (ZSTD_isError(ret)) goto _output_error;
2864 }
2865 DISPLAYLEVEL(3, "OK \n");
2866
2867 DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++);
2868 { size_t const srcSize = MIN(CNBuffSize, 100);
2869 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2870 /* Force the dictionary to be reloaded in raw content mode */
2871 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad));
2872 CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent));
2873 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
2874 CHECK_Z(cSize);
2875 }
2876 DISPLAYLEVEL(3, "OK \n");
2877
2878 free(rawDictBuffer);
Stella Laua6e20e12017-08-29 18:36:18 -07002879 }
Stella Laua6e20e12017-08-29 18:36:18 -07002880
Nick Terrell787b7692019-03-13 15:23:24 -07002881 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
2882 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
Nick Terrell7083f792020-10-05 15:17:44 -07002883 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
Nick Terrell787b7692019-03-13 15:23:24 -07002884 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2885 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2886 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2887 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2888 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2889 ZSTD_freeCDict(cdict);
2890 }
2891 DISPLAYLEVEL(3, "OK \n");
2892
Nick Terrell6b053b92019-03-21 15:17:41 -07002893 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++);
2894 {
2895 size_t size1, size2;
2896 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2897 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2898 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2899 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2900 if (ZSTD_isError(size1)) goto _output_error;
2901
2902 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2903 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2904 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2905 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2906 if (ZSTD_isError(size2)) goto _output_error;
2907
2908 if (size1 != size2) goto _output_error;
2909 }
2910 DISPLAYLEVEL(3, "OK \n");
2911
2912 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++);
2913 {
2914 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2915 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2916 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2917 }
2918 DISPLAYLEVEL(3, "OK \n");
2919
2920 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++);
2921 {
2922 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2923 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2924 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2925 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2926 ZSTD_freeCDict(cdict);
2927 }
2928 DISPLAYLEVEL(3, "OK \n");
2929
2930 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++);
2931 {
2932 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2933 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2934 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2935 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2936 ZSTD_freeCDict(cdict);
2937 }
2938 DISPLAYLEVEL(3, "OK \n");
2939
2940 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++);
2941 {
2942 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2943 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2944 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2945 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2946 ZSTD_freeCDict(cdict);
2947 }
2948 DISPLAYLEVEL(3, "OK \n");
2949
2950 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++);
2951 {
2952 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2953 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2954 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2955 }
2956 DISPLAYLEVEL(3, "OK \n");
2957
2958 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++);
2959 {
2960 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2961 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2962 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2963 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2964 ZSTD_freeCDict(cdict);
2965 }
2966 DISPLAYLEVEL(3, "OK \n");
2967
2968 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++);
2969 {
2970 size_t size1, size2;
2971 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2972 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2973 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2974 if (ZSTD_isError(size1)) goto _output_error;
2975
2976 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
2977 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2978 if (ZSTD_isError(size2)) goto _output_error;
2979
2980 if (size1 != size2) goto _output_error;
2981 }
2982 DISPLAYLEVEL(3, "OK \n");
2983
2984 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++);
2985 {
2986 size_t size1, size2;
2987 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2988 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2989 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2990 if (ZSTD_isError(size1)) goto _output_error;
2991
2992 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2993 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2994 if (ZSTD_isError(size2)) goto _output_error;
2995
2996 if (size1 == size2) goto _output_error;
2997 }
2998 DISPLAYLEVEL(3, "OK \n");
2999
Nick Terrell824aaa62019-04-09 17:59:27 -07003000 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3001 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) );
3002 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3003 CHECK_Z(cSize);
3004 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++);
3005 {
3006 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3007 size_t ret;
3008 /* We should fail to decompress without a dictionary. */
3009 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3010 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3011 if (!ZSTD_isError(ret)) goto _output_error;
3012 /* We should succeed to decompress with the dictionary. */
3013 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3014 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) );
3015 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
Dimitris Apostolouebbd6752021-11-13 10:04:04 +02003016 /* The dictionary should persist across calls. */
Nick Terrell824aaa62019-04-09 17:59:27 -07003017 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3018 /* When we reset the context the dictionary is cleared. */
3019 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3020 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3021 if (!ZSTD_isError(ret)) goto _output_error;
3022 ZSTD_freeDCtx(dctx);
3023 }
3024 DISPLAYLEVEL(3, "OK \n");
3025
Nick Terrell50b9c412019-04-10 12:34:21 -07003026 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
3027 {
3028 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3029 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
3030 size_t ret;
3031 /* We should succeed to decompress with the ddict. */
3032 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3033 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
3034 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
Dimitris Apostolouebbd6752021-11-13 10:04:04 +02003035 /* The ddict should persist across calls. */
Nick Terrell50b9c412019-04-10 12:34:21 -07003036 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3037 /* When we reset the context the ddict is cleared. */
3038 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3039 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3040 if (!ZSTD_isError(ret)) goto _output_error;
3041 ZSTD_freeDCtx(dctx);
3042 ZSTD_freeDDict(ddict);
3043 }
3044 DISPLAYLEVEL(3, "OK \n");
3045
3046 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
3047 {
3048 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3049 size_t ret;
3050 /* We should succeed to decompress with the prefix. */
3051 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3052 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
3053 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3054 /* The prefix should be cleared after the first compression. */
3055 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3056 if (!ZSTD_isError(ret)) goto _output_error;
3057 ZSTD_freeDCtx(dctx);
3058 }
3059 DISPLAYLEVEL(3, "OK \n");
3060
Elliott Hughes44aba642023-09-12 20:18:59 +00003061 DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3062 {
3063 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3064 ZSTD_customMem customMem = {NULL, NULL, NULL};
3065 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3066 ZSTD_CDict* cdict;
3067 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast));
3068 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3069 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3070 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3071 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3072 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3073 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3074 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3075 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3077 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3078 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3079 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3080 CHECK_Z(cSize);
3081 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3082 ZSTD_freeCDict(cdict);
3083 ZSTD_freeDCtx(dctx);
3084 ZSTD_freeCCtxParams(cctxParams);
3085 }
3086 DISPLAYLEVEL(3, "OK \n");
3087
3088 DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3089 {
3090 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3091 ZSTD_customMem customMem = {NULL, NULL, NULL};
3092 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3093 ZSTD_CDict* cdict;
3094 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast));
3095 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3096 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3097 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3098 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3099 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3100 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3101 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3102 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3103 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3104 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3105 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3106 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3107 CHECK_Z(cSize);
3108 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3109 ZSTD_freeCDict(cdict);
3110 ZSTD_freeDCtx(dctx);
3111 ZSTD_freeCCtxParams(cctxParams);
3112 }
3113 DISPLAYLEVEL(3, "OK \n");
3114
3115 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++);
3116 if (MEM_64bits()) {
3117 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3118 ZSTD_customMem customMem = {NULL, NULL, NULL};
3119 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3120 ZSTD_CDict* cdict;
3121 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy));
3122 /* Force enable row based match finder, and disable dedicated dict search. */
3123 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
3124 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0));
3125 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4));
3126 /* Set windowLog to 29 so hash/chain logs don't get sized down */
3127 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29));
3128 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29));
3129 /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */
3130 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29));
3131 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3132 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3133 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3135 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3136 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3137 CHECK_Z(cSize);
3138 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3139 ZSTD_freeCDict(cdict);
3140 ZSTD_freeDCtx(dctx);
3141 ZSTD_freeCCtxParams(cctxParams);
3142 }
3143 DISPLAYLEVEL(3, "OK \n");
3144
Yann Collet64482c22017-12-29 17:04:37 +01003145 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
Nick Terrell22727a72017-12-12 17:37:06 -08003146 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3147 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
3148 CNBuffer, samplesSizes, nbSamples);
3149 if (ZDICT_isError(dictSize)) goto _output_error;
3150 /* Set all the repcodes to non-default */
3151 {
3152 BYTE* dictPtr = (BYTE*)dictBuffer;
3153 BYTE* dictLimit = dictPtr + dictSize - 12;
3154 /* Find the repcodes */
3155 while (dictPtr < dictLimit &&
3156 (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
3157 MEM_readLE32(dictPtr + 8) != 8)) {
3158 ++dictPtr;
3159 }
3160 if (dictPtr >= dictLimit) goto _output_error;
3161 MEM_writeLE32(dictPtr + 0, 10);
3162 MEM_writeLE32(dictPtr + 4, 10);
3163 MEM_writeLE32(dictPtr + 8, 10);
3164 /* Set the last 8 bytes to 'x' */
3165 memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
3166 }
3167 /* The optimal parser checks all the repcodes.
3168 * Make sure at least one is a match >= targetLength so that it is
3169 * immediately chosen. This will make sure that the compressor and
3170 * decompressor agree on at least one of the repcodes.
3171 */
3172 { size_t dSize;
3173 BYTE data[1024];
Yann Colletaec945f2018-12-04 15:35:37 -08003174 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
Nick Terrell22727a72017-12-12 17:37:06 -08003175 ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
3176 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
Yann Collet6873fec2018-03-20 15:13:14 -07003177 ZSTD_dlm_byRef, ZSTD_dct_auto,
Nick Terrell22727a72017-12-12 17:37:06 -08003178 cParams, ZSTD_defaultCMem);
Yann Colletaec945f2018-12-04 15:35:37 -08003179 assert(dctx != NULL); assert(cdict != NULL);
Nick Terrell22727a72017-12-12 17:37:06 -08003180 memset(data, 'x', sizeof(data));
3181 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
3182 data, sizeof(data), cdict);
3183 ZSTD_freeCDict(cdict);
3184 if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
3185 dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
3186 if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
3187 if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
Yann Colletaec945f2018-12-04 15:35:37 -08003188 ZSTD_freeDCtx(dctx);
Nick Terrell22727a72017-12-12 17:37:06 -08003189 }
Yann Collet64482c22017-12-29 17:04:37 +01003190 DISPLAYLEVEL(3, "OK \n");
Nick Terrell22727a72017-12-12 17:37:06 -08003191
senhuang4222b7bff2020-12-28 16:43:04 -05003192 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++);
3193 {
3194 const size_t numDicts = 128;
3195 const size_t numFrames = 4;
3196 size_t i;
3197 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3198 ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts);
3199 ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts);
3200 U32 dictIDSeed = seed;
3201 /* Create new compressed buffer that will hold frames with differing dictIDs */
3202 char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */
3203
3204 ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize);
senhuang4222b7bff2020-12-28 16:43:04 -05003205 /* Create a bunch of DDicts with random dict IDs */
3206 for (i = 0; i < numDicts; ++i) {
3207 U32 currDictID = FUZ_rand(&dictIDSeed);
3208 MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID);
3209 ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize);
3210 cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3);
3211 if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) {
3212 goto _output_error;
3213 }
3214 }
3215 /* Compress a few frames using random CDicts */
3216 {
3217 size_t off = 0;
3218 /* only use the first half so we don't push against size limit of compressedBuffer */
3219 size_t const segSize = (CNBuffSize / 2) / numFrames;
3220 for (i = 0; i < numFrames; i++) {
3221 size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts;
3222 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3223 { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx,
3224 (BYTE*)compressedBuffer + off, CNBuffSize - off,
3225 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
3226 cdictTable[dictIdx]));
3227 off += r;
3228 }
3229 }
3230 cSize = off;
3231 }
3232
3233 /* We should succeed to decompression even though different dicts were used on different frames */
3234 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3235 ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts);
3236 /* Reference every single ddict we made */
3237 for (i = 0; i < numDicts; ++i) {
3238 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i]));
3239 }
3240 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
senhuang4217222652021-01-07 12:07:35 -05003241 /* Streaming decompression should also work */
3242 {
3243 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
3244 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
3245 while (in.pos < in.size) {
3246 CHECK_Z(ZSTD_decompressStream(dctx, &out, &in));
3247 }
3248 }
senhuang4222b7bff2020-12-28 16:43:04 -05003249 ZSTD_freeDCtx(dctx);
3250 for (i = 0; i < numDicts; ++i) {
3251 ZSTD_freeCDict(cdictTable[i]);
3252 ZSTD_freeDDict(ddictTable[i]);
3253 }
3254 free(dictBufferMulti);
3255 free(ddictTable);
3256 free(cdictTable);
3257 }
3258 DISPLAYLEVEL(3, "OK \n");
3259
Yann Collet30009522016-05-30 16:17:33 +02003260 ZSTD_freeCCtx(cctx);
Yann Collet30009522016-05-30 16:17:33 +02003261 free(dictBuffer);
3262 free(samplesSizes);
3263 }
3264
Nick Terrella8b4fe02017-01-02 18:45:19 -08003265 /* COVER dictionary builder tests */
3266 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Nick Terrella8b4fe02017-01-02 18:45:19 -08003267 size_t dictSize = 16 KB;
3268 size_t optDictSize = dictSize;
3269 void* dictBuffer = malloc(dictSize);
3270 size_t const totalSampleSize = 1 MB;
3271 size_t const sampleUnitSize = 8 KB;
3272 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3273 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
Yann Collet37f47e52019-08-02 17:34:53 +02003274 U32 seed32 = seed;
Nick Terrell5b7fd7c2017-06-26 21:07:14 -07003275 ZDICT_cover_params_t params;
Nick Terrella8b4fe02017-01-02 18:45:19 -08003276 U32 dictID;
3277
3278 if (dictBuffer==NULL || samplesSizes==NULL) {
3279 free(dictBuffer);
3280 free(samplesSizes);
3281 goto _output_error;
3282 }
3283
Yann Collet64482c22017-12-29 17:04:37 +01003284 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003285 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3286 memset(&params, 0, sizeof(params));
Yann Collet37f47e52019-08-02 17:34:53 +02003287 params.d = 1 + (FUZ_rand(&seed32) % 16);
3288 params.k = params.d + (FUZ_rand(&seed32) % 256);
Nick Terrell5b7fd7c2017-06-26 21:07:14 -07003289 dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize,
3290 CNBuffer, samplesSizes, nbSamples,
3291 params);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003292 if (ZDICT_isError(dictSize)) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08003293 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003294
Yann Collet64482c22017-12-29 17:04:37 +01003295 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003296 dictID = ZDICT_getDictID(dictBuffer, dictSize);
3297 if (dictID==0) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08003298 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003299
Yann Collet64482c22017-12-29 17:04:37 +01003300 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003301 memset(&params, 0, sizeof(params));
3302 params.steps = 4;
Nick Terrell5b7fd7c2017-06-26 21:07:14 -07003303 optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize,
3304 CNBuffer, samplesSizes,
3305 nbSamples / 4, &params);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003306 if (ZDICT_isError(optDictSize)) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08003307 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003308
Yann Collet64482c22017-12-29 17:04:37 +01003309 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003310 dictID = ZDICT_getDictID(dictBuffer, optDictSize);
3311 if (dictID==0) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08003312 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003313
3314 ZSTD_freeCCtx(cctx);
Nick Terrella8b4fe02017-01-02 18:45:19 -08003315 free(dictBuffer);
3316 free(samplesSizes);
3317 }
3318
Yann Collet4856a002015-01-24 01:58:16 +01003319 /* Decompression defense tests */
Yann Collet64482c22017-12-29 17:04:37 +01003320 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++);
Yann Collet20d5e032017-04-11 18:34:02 -07003321 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
Yann Colletc0a9bf32016-05-30 01:56:08 +02003322 if (!ZSTD_isError(r)) goto _output_error;
Yann Collet20d5e032017-04-11 18:34:02 -07003323 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01003324 DISPLAYLEVEL(3, "OK \n");
Yann Collet4856a002015-01-24 01:58:16 +01003325
Yann Collet64482c22017-12-29 17:04:37 +01003326 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++);
Yann Collet4856a002015-01-24 01:58:16 +01003327 ((char*)(CNBuffer))[0] = 1;
Yann Colletd2858e92016-05-30 15:10:09 +02003328 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
Yann Collet33341de2016-05-29 23:09:51 +02003329 if (!ZSTD_isError(r)) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01003330 DISPLAYLEVEL(3, "OK \n");
Yann Collet4856a002015-01-24 01:58:16 +01003331
Yann Collet20d5e032017-04-11 18:34:02 -07003332 /* content size verification test */
Yann Collet64482c22017-12-29 17:04:37 +01003333 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++);
Yann Collet20d5e032017-04-11 18:34:02 -07003334 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3335 size_t const srcSize = 5000;
3336 size_t const wrongSrcSize = (srcSize + 1000);
3337 ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
3338 params.fParams.contentSizeFlag = 1;
Elliott Hughes44aba642023-09-12 20:18:59 +00003339 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
Yann Collet20d5e032017-04-11 18:34:02 -07003340 { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
3341 if (!ZSTD_isError(result)) goto _output_error;
3342 if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
Yann Collet64482c22017-12-29 17:04:37 +01003343 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result));
Yann Colletf913cbe2017-04-13 22:46:41 -07003344 }
3345 ZSTD_freeCCtx(cctx);
3346 }
Yann Collet20d5e032017-04-11 18:34:02 -07003347
Yann Collet43a56972018-06-07 14:46:55 -07003348 /* negative compression level test : ensure simple API and advanced API produce same result */
3349 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++);
Yann Collet8537bfd2018-06-07 15:12:13 -07003350 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3351 size_t const srcSize = CNBuffSize / 5;
3352 int const compressionLevel = -1;
3353
3354 assert(cctx != NULL);
sen698f2612021-05-06 17:59:32 -04003355 { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize,
3356 CNBuffer, srcSize, compressionLevel);
Yann Collet8537bfd2018-06-07 15:12:13 -07003357 if (ZSTD_isError(cSize_1pass)) goto _output_error;
3358
Elliott Hughes44aba642023-09-12 20:18:59 +00003359 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
Yann Collet6ced8f72018-12-03 14:22:38 -08003360 { size_t const compressionResult = ZSTD_compress2(cctx,
3361 compressedBuffer, compressedBufferSize,
3362 CNBuffer, srcSize);
3363 DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
Yann Collet43a56972018-06-07 14:46:55 -07003364 if (ZSTD_isError(compressionResult)) goto _output_error;
Yann Collet6ced8f72018-12-03 14:22:38 -08003365 if (compressionResult != cSize_1pass) goto _output_error;
Yann Collet8537bfd2018-06-07 15:12:13 -07003366 } }
3367 ZSTD_freeCCtx(cctx);
Yann Collet43a56972018-06-07 14:46:55 -07003368 }
3369 DISPLAYLEVEL(3, "OK \n");
3370
Yann Collete5ddfae2018-03-19 16:02:51 -07003371 /* parameters order test */
3372 { size_t const inputSize = CNBuffSize / 2;
3373 U64 xxh64;
3374
Yann Collet43a56972018-06-07 14:46:55 -07003375 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Yann Collete5ddfae2018-03-19 16:02:51 -07003376 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
Yann Collet43a56972018-06-07 14:46:55 -07003377 assert(cctx != NULL);
Elliott Hughes44aba642023-09-12 20:18:59 +00003378 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3379 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3380 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
Yann Collet6ced8f72018-12-03 14:22:38 -08003381 { size_t const compressedSize = ZSTD_compress2(cctx,
3382 compressedBuffer, ZSTD_compressBound(inputSize),
3383 CNBuffer, inputSize);
Elliott Hughes44aba642023-09-12 20:18:59 +00003384 CHECK_Z(compressedSize);
Yann Collet6ced8f72018-12-03 14:22:38 -08003385 cSize = compressedSize;
3386 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
Yann Collete5ddfae2018-03-19 16:02:51 -07003387 }
Yann Colletededcfc2018-12-21 16:19:44 -08003388 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
Yann Collete5ddfae2018-03-19 16:02:51 -07003389 ZSTD_freeCCtx(cctx);
3390 }
3391
3392 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
3393 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00003394 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3395 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3396 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
Yann Collet6ced8f72018-12-03 14:22:38 -08003397 { size_t const result = ZSTD_compress2(cctx,
3398 compressedBuffer, ZSTD_compressBound(inputSize),
3399 CNBuffer, inputSize);
Elliott Hughes44aba642023-09-12 20:18:59 +00003400 CHECK_Z(result);
Yann Collet6ced8f72018-12-03 14:22:38 -08003401 if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
3402 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
Yann Colletededcfc2018-12-21 16:19:44 -08003403 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
Yann Collete5ddfae2018-03-19 16:02:51 -07003404 }
3405 ZSTD_freeCCtx(cctx);
3406 }
3407 }
3408
Yann Colletaec945f2018-12-04 15:35:37 -08003409 /* advanced parameters for decompression */
3410 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3411 assert(dctx != NULL);
3412
3413 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
3414 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
Elliott Hughes44aba642023-09-12 20:18:59 +00003415 CHECK_Z(bounds.error);
Yann Colletaec945f2018-12-04 15:35:37 -08003416 }
3417 DISPLAYLEVEL(3, "OK \n");
3418
3419 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
3420 { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
3421 if (!ZSTD_isError(sr)) goto _output_error;
3422 }
3423 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
3424 if (!ZSTD_isError(bounds.error)) goto _output_error;
3425 }
3426 DISPLAYLEVEL(3, "OK \n");
3427
3428 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
3429 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
3430 if (!ZSTD_isError(sr)) goto _output_error;
3431 }
3432 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
3433 if (!ZSTD_isError(sr)) goto _output_error;
3434 }
3435 DISPLAYLEVEL(3, "OK \n");
3436
3437 ZSTD_freeDCtx(dctx);
3438 }
3439
3440
Yann Collet62568c92017-09-25 14:26:26 -07003441 /* custom formats tests */
3442 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Yann Colletaec945f2018-12-04 15:35:37 -08003443 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
Yann Colletdf4e9bb2017-09-26 14:31:06 -07003444 size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
Yann Colletaec945f2018-12-04 15:35:37 -08003445 assert(dctx != NULL); assert(cctx != NULL);
Yann Collet62568c92017-09-25 14:26:26 -07003446
3447 /* basic block compression */
Yann Collet64482c22017-12-29 17:04:37 +01003448 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00003449 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
Yann Collet62568c92017-09-25 14:26:26 -07003450 { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
3451 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
Yann Colletd8e215c2018-11-30 11:16:26 -08003452 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
Yann Collet62568c92017-09-25 14:26:26 -07003453 if (result != 0) goto _output_error;
3454 if (in.pos != in.size) goto _output_error;
3455 cSize = out.pos;
3456 }
Yann Colletededcfc2018-12-21 16:19:44 -08003457 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
Yann Collet62568c92017-09-25 14:26:26 -07003458
Yann Collet64482c22017-12-29 17:04:37 +01003459 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++);
Yann Collet62568c92017-09-25 14:26:26 -07003460 { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3461 if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
Yann Collet64482c22017-12-29 17:04:37 +01003462 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult));
Yann Collet62568c92017-09-25 14:26:26 -07003463 }
3464
Yann Collete6e848b2018-03-29 17:51:08 -06003465 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
Yann Collet5c686392018-11-15 16:12:39 -08003466 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
Elliott Hughes44aba642023-09-12 20:18:59 +00003467 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
Yann Collete6e848b2018-03-29 17:51:08 -06003468 { ZSTD_frameHeader zfh;
3469 size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
3470 if (zfhrt != 0) goto _output_error;
3471 }
Yann Collet3e042d52018-12-04 17:30:58 -08003472 /* one shot */
3473 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3474 if (result != inputSize) goto _output_error;
3475 DISPLAYLEVEL(3, "one-shot OK, ");
3476 }
3477 /* streaming */
Yann Collet62568c92017-09-25 14:26:26 -07003478 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3479 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
Yann Collet34e146f2018-12-04 10:28:36 -08003480 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
Yann Collet62568c92017-09-25 14:26:26 -07003481 if (result != 0) goto _output_error;
3482 if (in.pos != in.size) goto _output_error;
3483 if (out.pos != inputSize) goto _output_error;
Yann Colletededcfc2018-12-21 16:19:44 -08003484 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
Yann Collet62568c92017-09-25 14:26:26 -07003485 }
3486
Nick Terrellb1ec94e2019-10-21 19:42:14 -07003487 /* basic block compression */
3488 DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00003489 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
Nick Terrellb1ec94e2019-10-21 19:42:14 -07003490 { ZSTD_inBuffer in = { CNBuffer, 0, 0 };
3491 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
3492 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3493 if (result != 0) goto _output_error;
3494 if (in.pos != in.size) goto _output_error;
3495 cSize = out.pos;
3496 }
3497 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
3498
3499 DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
3500 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
Elliott Hughes44aba642023-09-12 20:18:59 +00003501 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
Nick Terrellb1ec94e2019-10-21 19:42:14 -07003502 /* one shot */
3503 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3504 if (result != 0) goto _output_error;
3505 DISPLAYLEVEL(3, "one-shot OK, ");
3506 }
3507 /* streaming */
3508 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3509 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3510 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3511 if (result != 0) goto _output_error;
3512 if (in.pos != in.size) goto _output_error;
3513 if (out.pos != 0) goto _output_error;
3514 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3515 }
3516
Yann Collet62568c92017-09-25 14:26:26 -07003517 ZSTD_freeCCtx(cctx);
Yann Colletaec945f2018-12-04 15:35:37 -08003518 ZSTD_freeDCtx(dctx);
Yann Collet62568c92017-09-25 14:26:26 -07003519 }
3520
Nick Terrell2e7d1742020-09-29 16:25:03 -07003521 DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++);
3522 {
3523 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3524 /* Attempt to future proof this to new parameters. */
3525 int const maxParam = 2000;
3526 int param;
3527 if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error;
3528 for (param = 0; param < maxParam; ++param) {
3529 ZSTD_dParameter dParam = (ZSTD_dParameter)param;
3530 ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam);
3531 int value1;
3532 int value2;
3533 int check;
3534 if (ZSTD_isError(bounds.error))
3535 continue;
Elliott Hughes44aba642023-09-12 20:18:59 +00003536 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
Nick Terrell2e7d1742020-09-29 16:25:03 -07003537 value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
Elliott Hughes44aba642023-09-12 20:18:59 +00003538 CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
3539 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
Nick Terrell2e7d1742020-09-29 16:25:03 -07003540 if (check != value2) goto _output_error;
Elliott Hughes44aba642023-09-12 20:18:59 +00003541 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
3542 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
Nick Terrell2e7d1742020-09-29 16:25:03 -07003543 if (check != value1) goto _output_error;
3544 }
3545 ZSTD_freeDCtx(dctx);
3546 }
3547 DISPLAYLEVEL(3, "OK \n");
3548
Yann Colletbf42c8e2016-01-09 01:08:23 +01003549 /* block API tests */
Yann Colletd1d210f2016-03-19 12:12:07 +01003550 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
Yann Colletaec945f2018-12-04 15:35:37 -08003551 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
Yann Colletd3d2db52016-07-15 12:20:26 +02003552 static const size_t dictSize = 65 KB;
3553 static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
Yann Colletd4f4e582016-06-27 01:31:35 +02003554 size_t cSize2;
Yann Colletaec945f2018-12-04 15:35:37 -08003555 assert(cctx != NULL); assert(dctx != NULL);
Yann Colletbf42c8e2016-01-09 01:08:23 +01003556
3557 /* basic block compression */
Yann Collet64482c22017-12-29 17:04:37 +01003558 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00003559 CHECK_Z( ZSTD_compressBegin(cctx, 5) );
3560 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
Yann Colletefe84962019-08-02 19:31:19 +02003561 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
Yann Collet64482c22017-12-29 17:04:37 +01003562 DISPLAYLEVEL(3, "OK \n");
Yann Colletbf42c8e2016-01-09 01:08:23 +01003563
Yann Collet64482c22017-12-29 17:04:37 +01003564 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00003565 CHECK_Z( ZSTD_decompressBegin(dctx) );
Yann Collet810a9ca2019-08-01 16:59:22 +02003566 { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
Yann Colletd2858e92016-05-30 15:10:09 +02003567 if (r != blockSize) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01003568 DISPLAYLEVEL(3, "OK \n");
Yann Colletbf42c8e2016-01-09 01:08:23 +01003569
Yann Colletf98c69d2018-09-26 14:24:28 -07003570 /* very long stream of block compression */
3571 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00003572 CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
3573 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
Yann Colletf98c69d2018-09-26 14:24:28 -07003574 { U64 const toCompress = 5000000000ULL; /* > 4 GB */
3575 U64 compressed = 0;
3576 while (compressed < toCompress) {
3577 size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
Yann Colletaec945f2018-12-04 15:35:37 -08003578 assert(blockCSize != 0);
3579 if (ZSTD_isError(blockCSize)) goto _output_error;
Yann Colletf98c69d2018-09-26 14:24:28 -07003580 compressed += blockCSize;
Yann Colletefe84962019-08-02 19:31:19 +02003581 } }
Yann Colletf98c69d2018-09-26 14:24:28 -07003582 DISPLAYLEVEL(3, "OK \n");
3583
Yann Colletb0125102016-01-09 02:00:10 +01003584 /* dictionary block compression */
Yann Collet64482c22017-12-29 17:04:37 +01003585 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00003586 CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
Yann Colletefe84962019-08-02 19:31:19 +02003587 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
Yann Collet0b0b83e2019-08-03 16:43:34 +02003588 RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
3589 { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
3590 assert(r == 0); /* non-compressible block */ }
3591 memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */
Yann Colletefe84962019-08-02 19:31:19 +02003592 CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
3593 (char*)CNBuffer+dictSize+2*blockSize, blockSize));
Yann Collet64482c22017-12-29 17:04:37 +01003594 DISPLAYLEVEL(3, "OK \n");
Yann Colletb0125102016-01-09 02:00:10 +01003595
Yann Collet64482c22017-12-29 17:04:37 +01003596 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
Elliott Hughes44aba642023-09-12 20:18:59 +00003597 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
Yann Colletefe84962019-08-02 19:31:19 +02003598 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
Yann Collet387e20d2019-08-02 18:02:54 +02003599 if (r != blockSize) {
Yann Colletefe84962019-08-02 19:31:19 +02003600 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
Yann Collet387e20d2019-08-02 18:02:54 +02003601 goto _output_error;
3602 } }
Yann Colletefe84962019-08-02 19:31:19 +02003603 memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize);
Yann Colletaa2628d2016-07-07 15:28:41 +02003604 ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
Yann Colletefe84962019-08-02 19:31:19 +02003605 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
Yann Collet387e20d2019-08-02 18:02:54 +02003606 if (r != blockSize) {
Yann Colletefe84962019-08-02 19:31:19 +02003607 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
Yann Collet387e20d2019-08-02 18:02:54 +02003608 goto _output_error;
3609 } }
Yann Collet0b0b83e2019-08-03 16:43:34 +02003610 assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */
Yann Collet64482c22017-12-29 17:04:37 +01003611 DISPLAYLEVEL(3, "OK \n");
Yann Colletb0125102016-01-09 02:00:10 +01003612
Yann Colletad4524d2018-05-07 12:54:13 -07003613 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
3614 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
3615 if (cdict==NULL) goto _output_error;
Elliott Hughes44aba642023-09-12 20:18:59 +00003616 CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
3617 CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
Yann Colletad4524d2018-05-07 12:54:13 -07003618 ZSTD_freeCDict(cdict);
3619 }
3620 DISPLAYLEVEL(3, "OK \n");
3621
Yann Colletbf42c8e2016-01-09 01:08:23 +01003622 ZSTD_freeCCtx(cctx);
Yann Colletaec945f2018-12-04 15:35:37 -08003623 ZSTD_freeDCtx(dctx);
Yann Colletbf42c8e2016-01-09 01:08:23 +01003624 }
3625
Yann Collet213089c2015-06-18 07:43:16 -08003626 /* long rle test */
Yann Colletd1d210f2016-03-19 12:12:07 +01003627 { size_t sampleSize = 0;
Sen Huangbc3e2152019-11-18 16:39:16 -05003628 size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
Yann Collet64482c22017-12-29 17:04:37 +01003629 DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
Yann Collet213089c2015-06-18 07:43:16 -08003630 memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
3631 sampleSize += 256 KB - 1;
Sen Huang75c34682019-11-15 12:26:48 -05003632 memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
Yann Collet213089c2015-06-18 07:43:16 -08003633 sampleSize += 96 KB;
Yann Collet5be2dd22015-11-11 13:43:58 +01003634 cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
Sen Huangbc3e2152019-11-18 16:39:16 -05003635 if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
Yann Collet810a9ca2019-08-01 16:59:22 +02003636 { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
Yann Colletc0a9bf32016-05-30 01:56:08 +02003637 if (regenSize!=sampleSize) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01003638 DISPLAYLEVEL(3, "OK \n");
Yann Collet213089c2015-06-18 07:43:16 -08003639 }
3640
senhuang4251abd582020-11-06 10:53:22 -05003641 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++);
Bimba Shrestha1f93be02019-09-16 13:35:45 -07003642 {
senhuang429102f302020-11-02 11:30:31 -05003643 size_t srcSize = 150 KB;
Bimba Shresthabb274722019-09-26 15:38:31 -07003644 BYTE* src = (BYTE*)CNBuffer;
3645 BYTE* decoded = (BYTE*)compressedBuffer;
3646
Bimba Shrestha1f93be02019-09-16 13:35:45 -07003647 ZSTD_CCtx* cctx = ZSTD_createCCtx();
Bimba Shrestha91daee52019-09-26 16:21:57 -07003648 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
Bimba Shrestha36528b92019-10-03 09:26:51 -07003649 size_t seqsSize;
Bimba Shresthabb274722019-09-26 15:38:31 -07003650
Bimba Shrestha91daee52019-09-26 16:21:57 -07003651 if (seqs == NULL) goto _output_error;
Bimba Shresthabe0bebd2019-09-23 15:08:18 -07003652 assert(cctx != NULL);
senhuang429d936d62020-11-13 09:41:44 -05003653 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
Bimba Shresthabb274722019-09-26 15:38:31 -07003654 /* Populate src with random data */
senhuang421a8af0d2020-11-12 11:09:01 -05003655 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
Bimba Shresthabb274722019-09-26 15:38:31 -07003656
senhuang429102f302020-11-02 11:30:31 -05003657 /* Test with block delimiters roundtrip */
senhuang427d1dea02020-11-06 10:56:56 -05003658 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
senhuang42d4d03462020-11-02 11:32:56 -05003659 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters);
senhuang429102f302020-11-02 11:30:31 -05003660 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
Bimba Shresthabb274722019-09-26 15:38:31 -07003661
senhuang429102f302020-11-02 11:30:31 -05003662 /* Test no block delimiters roundtrip */
senhuang427d1dea02020-11-06 10:56:56 -05003663 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
senhuang429102f302020-11-02 11:30:31 -05003664 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters);
Bimba Shresthab63a1e72019-09-27 07:20:20 -07003665 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
Bimba Shresthabb274722019-09-26 15:38:31 -07003666
Bimba Shrestha1f93be02019-09-16 13:35:45 -07003667 ZSTD_freeCCtx(cctx);
Bimba Shresthabb274722019-09-26 15:38:31 -07003668 free(seqs);
Bimba Shrestha1f93be02019-09-16 13:35:45 -07003669 }
senhuang42dc448562020-10-27 12:28:46 -04003670 DISPLAYLEVEL(3, "OK \n");
Yann Collet7f8be042020-12-28 14:07:31 -08003671
senhuang422bbdddf2020-11-03 18:53:44 -05003672 DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
3673 {
Elliott Hughes44aba642023-09-12 20:18:59 +00003674 const size_t srcSize = 500 KB;
3675 const BYTE* const src = (BYTE*)CNBuffer;
3676 BYTE* const dst = (BYTE*)compressedBuffer;
3677 const size_t dstCapacity = ZSTD_compressBound(srcSize);
3678 const size_t decompressSize = srcSize;
3679 char* const decompressBuffer = (char*)malloc(decompressSize);
senhuang422bbdddf2020-11-03 18:53:44 -05003680 size_t compressedSize;
senhuang422bbdddf2020-11-03 18:53:44 -05003681
Elliott Hughes44aba642023-09-12 20:18:59 +00003682 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3683 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3684 size_t nbSeqs;
senhuang422bbdddf2020-11-03 18:53:44 -05003685
3686 if (seqs == NULL) goto _output_error;
3687 assert(cctx != NULL);
3688
3689 /* Populate src with random data */
3690 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
3691
Elliott Hughes44aba642023-09-12 20:18:59 +00003692 /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */
3693 nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
senhuang423d266152020-11-06 13:24:43 -05003694 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
senhuang4255b90ef2020-11-16 10:47:26 -05003695 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
Elliott Hughes44aba642023-09-12 20:18:59 +00003696 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize);
senhuang422bbdddf2020-11-03 18:53:44 -05003697 if (ZSTD_isError(compressedSize)) {
3698 DISPLAY("Error in sequence compression with block delims\n");
3699 goto _output_error;
3700 }
Elliott Hughes44aba642023-09-12 20:18:59 +00003701 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3702 if (ZSTD_isError(dSize)) {
3703 DISPLAY("Error in sequence compression roundtrip with block delims\n");
3704 goto _output_error;
3705 } }
senhuang422bbdddf2020-11-03 18:53:44 -05003706 assert(!memcmp(decompressBuffer, src, srcSize));
3707
Elliott Hughes44aba642023-09-12 20:18:59 +00003708 /* Roundtrip Test with no block delimiters */
3709 { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs);
3710 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3711 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
3712 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize);
3713 }
senhuang422bbdddf2020-11-03 18:53:44 -05003714 if (ZSTD_isError(compressedSize)) {
3715 DISPLAY("Error in sequence compression with no block delims\n");
3716 goto _output_error;
3717 }
Elliott Hughes44aba642023-09-12 20:18:59 +00003718 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3719 if (ZSTD_isError(dSize)) {
3720 DISPLAY("Error in sequence compression roundtrip with no block delims\n");
3721 goto _output_error;
3722 } }
senhuang422bbdddf2020-11-03 18:53:44 -05003723 assert(!memcmp(decompressBuffer, src, srcSize));
3724
3725 ZSTD_freeCCtx(cctx);
3726 free(decompressBuffer);
3727 free(seqs);
3728 }
3729 DISPLAYLEVEL(3, "OK \n");
Bimba Shrestha5f8b0f62019-08-30 09:18:44 -07003730
bimbashresthae5704bb2019-08-28 08:32:34 -07003731 /* Multiple blocks of zeros test */
3732 #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
3733 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3734 memset(CNBuffer, 0, LONGZEROSLENGTH);
3735 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) );
3736 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100);
3737
3738 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3739 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) );
3740 if (r != LONGZEROSLENGTH) goto _output_error; }
3741 DISPLAYLEVEL(3, "OK \n");
3742
Yann Colletc0a9bf32016-05-30 01:56:08 +02003743 /* All zeroes test (test bug #137) */
Yann Collet4ba85342016-03-07 20:01:45 +01003744 #define ZEROESLENGTH 100
Yann Collet64482c22017-12-29 17:04:37 +01003745 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
Yann Collet4ba85342016-03-07 20:01:45 +01003746 memset(CNBuffer, 0, ZEROESLENGTH);
Yann Collet810a9ca2019-08-01 16:59:22 +02003747 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
Yann Colletededcfc2018-12-21 16:19:44 -08003748 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100);
Yann Collet4ba85342016-03-07 20:01:45 +01003749
Yann Collet64482c22017-12-29 17:04:37 +01003750 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
Yann Collet810a9ca2019-08-01 16:59:22 +02003751 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
Yann Colletc0a9bf32016-05-30 01:56:08 +02003752 if (r != ZEROESLENGTH) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01003753 DISPLAYLEVEL(3, "OK \n");
Yann Collet4ba85342016-03-07 20:01:45 +01003754
3755 /* nbSeq limit test */
Yann Colletd1d210f2016-03-19 12:12:07 +01003756 #define _3BYTESTESTLENGTH 131000
3757 #define NB3BYTESSEQLOG 9
3758 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
3759 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
Yann Collet0d9ce042016-03-19 13:21:08 +01003760 /* creates a buffer full of 3-bytes sequences */
Yann Colletd1d210f2016-03-19 12:12:07 +01003761 { BYTE _3BytesSeqs[NB3BYTESSEQ][3];
Yann Collet0d9ce042016-03-19 13:21:08 +01003762 U32 rSeed = 1;
Yann Collet4ba85342016-03-07 20:01:45 +01003763
Yann Collet0d9ce042016-03-19 13:21:08 +01003764 /* create batch of 3-bytes sequences */
Yann Colletf323bf72016-07-07 13:14:21 +02003765 { int i;
3766 for (i=0; i < NB3BYTESSEQ; i++) {
3767 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
3768 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
3769 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
3770 } }
Yann Collet4ba85342016-03-07 20:01:45 +01003771
Yann Collet0d9ce042016-03-19 13:21:08 +01003772 /* randomly fills CNBuffer with prepared 3-bytes sequences */
Yann Colletf323bf72016-07-07 13:14:21 +02003773 { int i;
3774 for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
3775 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
3776 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
3777 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
3778 ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
3779 } } }
Nick Terrell5a4e6c92018-08-28 13:20:37 -07003780 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++);
3781 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3782 size_t const maxNbSeq = _3BYTESTESTLENGTH / 3;
3783 size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
3784 size_t nbSeq = 1;
3785 while (nbSeq <= maxNbSeq) {
Elliott Hughes44aba642023-09-12 20:18:59 +00003786 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
Nick Terrell5a4e6c92018-08-28 13:20:37 -07003787 /* Check every sequence for the first 100, then skip more rapidly. */
3788 if (nbSeq < 100) {
3789 ++nbSeq;
3790 } else {
3791 nbSeq += (nbSeq >> 2);
3792 }
3793 }
3794 ZSTD_freeCCtx(cctx);
3795 }
3796 DISPLAYLEVEL(3, "OK \n");
3797
Yann Collet64482c22017-12-29 17:04:37 +01003798 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
Yann Collet810a9ca2019-08-01 16:59:22 +02003799 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
3800 CNBuffer, _3BYTESTESTLENGTH, 19) );
Yann Colletededcfc2018-12-21 16:19:44 -08003801 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
Yann Collet4ba85342016-03-07 20:01:45 +01003802
Yann Collet64482c22017-12-29 17:04:37 +01003803 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
Yann Collet810a9ca2019-08-01 16:59:22 +02003804 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
Yann Colletc0a9bf32016-05-30 01:56:08 +02003805 if (r != _3BYTESTESTLENGTH) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01003806 DISPLAYLEVEL(3, "OK \n");
Yann Collet4ba85342016-03-07 20:01:45 +01003807
Nick Terrell5a4e6c92018-08-28 13:20:37 -07003808
Nick Terrelle984d012018-08-28 13:42:01 -07003809 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++);
Nick Terrellcc152232017-07-18 11:21:19 -07003810 RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
Nick Terrell5a4e6c92018-08-28 13:20:37 -07003811 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3812 size_t const bound = ZSTD_compressBound(CNBuffSize);
3813 size_t size = 1;
3814 while (size <= CNBuffSize) {
Elliott Hughes44aba642023-09-12 20:18:59 +00003815 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
Nick Terrell5a4e6c92018-08-28 13:20:37 -07003816 /* Check every size for the first 100, then skip more rapidly. */
3817 if (size < 100) {
3818 ++size;
3819 } else {
3820 size += (size >> 2);
3821 }
3822 }
3823 ZSTD_freeCCtx(cctx);
3824 }
3825 DISPLAYLEVEL(3, "OK \n");
3826
3827 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
Nick Terrellcc152232017-07-18 11:21:19 -07003828 { /* Train a dictionary on low characters */
3829 size_t dictSize = 16 KB;
3830 void* const dictBuffer = malloc(dictSize);
3831 size_t const totalSampleSize = 1 MB;
3832 size_t const sampleUnitSize = 8 KB;
3833 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3834 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3835 if (!dictBuffer || !samplesSizes) goto _output_error;
3836 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3837 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
3838 if (ZDICT_isError(dictSize)) goto _output_error;
3839 /* Reverse the characters to make the dictionary ill suited */
3840 { U32 u;
3841 for (u = 0; u < CNBuffSize; ++u) {
3842 ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
Elliott Hughes44aba642023-09-12 20:18:59 +00003843 } }
Nick Terrellcc152232017-07-18 11:21:19 -07003844 { /* Compress the data */
3845 size_t const inputSize = 500;
3846 size_t const outputSize = ZSTD_compressBound(inputSize);
3847 void* const outputBuffer = malloc(outputSize);
3848 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3849 if (!outputBuffer || !cctx) goto _output_error;
Elliott Hughes44aba642023-09-12 20:18:59 +00003850 CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
Nick Terrellcc152232017-07-18 11:21:19 -07003851 free(outputBuffer);
3852 ZSTD_freeCCtx(cctx);
3853 }
3854
3855 free(dictBuffer);
3856 free(samplesSizes);
3857 }
Yann Collet64482c22017-12-29 17:04:37 +01003858 DISPLAYLEVEL(3, "OK \n");
Nick Terrellcc152232017-07-18 11:21:19 -07003859
3860
Sean Purcell9050e192017-02-22 12:12:32 -08003861 /* findFrameCompressedSize on skippable frames */
Yann Collet64482c22017-12-29 17:04:37 +01003862 DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++);
Sean Purcell9050e192017-02-22 12:12:32 -08003863 { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
3864 size_t const frameSrcSize = 13;
3865 if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
Yann Collet64482c22017-12-29 17:04:37 +01003866 DISPLAYLEVEL(3, "OK \n");
Sean Purcell9050e192017-02-22 12:12:32 -08003867
Sean Purcelle0b32652017-02-08 15:31:47 -08003868 /* error string tests */
Yann Collet64482c22017-12-29 17:04:37 +01003869 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
Sean Purcelle0b32652017-02-08 15:31:47 -08003870 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
3871 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
3872 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
3873 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
3874 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
3875 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
Yann Collet64482c22017-12-29 17:04:37 +01003876 DISPLAYLEVEL(3, "OK \n");
Sean Purcelle0b32652017-02-08 15:31:47 -08003877
Yann Collet9b184352018-02-13 10:09:01 -08003878 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
Nick Terrell4b7c4e52018-01-11 16:45:16 -08003879 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
3880 {
3881 size_t const size = MIN(128 KB, CNBuffSize);
3882 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3883 ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
3884 ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
3885 ZSTD_frameHeader lgHeader;
3886 ZSTD_frameHeader smHeader;
3887
3888 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
3889 CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
3890 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
3891 CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
3892
3893 if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
3894
3895 ZSTD_freeCDict(smCDict);
3896 ZSTD_freeCDict(lgCDict);
3897 ZSTD_freeCCtx(cctx);
3898 }
Yann Collet9b184352018-02-13 10:09:01 -08003899 DISPLAYLEVEL(3, "OK \n");
Nick Terrell4b7c4e52018-01-11 16:45:16 -08003900
Nick Terrellb9faaa12018-07-30 12:57:11 -07003901 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++);
3902 {
3903 short norm[32];
3904 unsigned count[32];
3905 unsigned const tableLog = 5;
3906 size_t const nbSeq = 32;
3907 unsigned const maxSymbolValue = 31;
3908 size_t i;
3909
3910 for (i = 0; i < 32; ++i)
3911 count[i] = 1;
3912 /* Calling FSE_normalizeCount() on a uniform distribution should not
3913 * cause a division by zero.
3914 */
Nick Terrell575731b2020-08-18 15:26:54 -07003915 FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1);
Nick Terrellb9faaa12018-07-30 12:57:11 -07003916 }
3917 DISPLAYLEVEL(3, "OK \n");
senhuang4299b5e7b2021-09-22 11:27:56 -04003918
3919 DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++);
3920 {
3921 size_t const outBufSize = 9;
3922 short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18};
3923 unsigned const tableLog = 5;
3924 unsigned const maxSymbolValue = 10;
3925 BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE));
3926
Yann Colletfb3522a2021-12-08 09:17:06 -08003927 /* Ensure that this write doesn't write out of bounds, and that
senhuang4299b5e7b2021-09-22 11:27:56 -04003928 * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1.
3929 */
3930 FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog);
3931 free(outBuf);
3932 }
3933 DISPLAYLEVEL(3, "OK \n");
3934
Elliott Hughes44aba642023-09-12 20:18:59 +00003935 DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
3936 {
3937 U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
3938 U32 rand32 = FUZ_rand(&seed_copy);
3939 U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
3940 U32 lowbit_only_32 = 1;
3941 U64 lowbit_only_64 = 1;
3942 U32 highbit_only_32 = (U32)1 << 31;
3943 U64 highbit_only_64 = (U64)1 << 63;
3944 U32 i;
3945 if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
3946 if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
3947
3948 /* Test ZSTD_countTrailingZeros32 */
3949 CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
3950 CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u);
3951 CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32));
3952
3953 /* Test ZSTD_countLeadingZeros32 */
3954 CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u);
3955 CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u);
3956 CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32));
3957
3958 /* Test ZSTD_countTrailingZeros64 */
3959 CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u);
3960 CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u);
3961
3962 /* Test ZSTD_countLeadingZeros64 */
3963 CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u);
3964 CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u);
3965
3966 /* Test ZSTD_highbit32 */
3967 CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u);
3968 CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u);
3969
3970 /* Test ZSTD_NbCommonBytes */
3971 if (MEM_isLittleEndian()) {
3972 if (MEM_64bits()) {
3973 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
3974 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
3975 } else {
3976 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
3977 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
3978 }
3979 } else {
3980 if (MEM_64bits()) {
3981 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u);
3982 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u);
3983 } else {
3984 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u);
3985 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u);
3986 }
3987 }
3988
3989 /* Test MEM_ intrinsics */
3990 CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32));
3991 CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64));
3992
3993 /* Test fallbacks vs intrinsics on a range of small integers */
3994 for (i=1; i <= 1000; i++) {
3995 CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i));
3996 CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i));
3997 CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i));
3998 CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i));
3999 }
4000 }
4001 DISPLAYLEVEL(3, "OK \n");
4002
Bimba Shresthaf7a74092020-04-21 22:26:48 -07004003#ifdef ZSTD_MULTITHREAD
Yann Colletdd026ca2020-05-09 11:30:45 -07004004 DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++);
Yann Colletc91a0852020-09-14 10:56:08 -07004005 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4006 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
Bimba Shresthaf7a74092020-04-21 22:26:48 -07004007 size_t const dstSize = ZSTD_compressBound(srcSize);
4008 void* const src = CNBuffer;
4009 void* const dst = compressedBuffer;
4010 void* dict = (void*)malloc(srcSize);
4011
4012 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4013 RDG_genBuffer(dict, srcSize, compressibility, 0., seed);
4014
4015 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4016 memset(dict, 0, sizeof(U32));
4017
4018 /* something more than 1 */
4019 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4020 /* lie and claim this is a full dict */
4021 CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict));
4022
Yann Colletc91a0852020-09-14 10:56:08 -07004023 { ZSTD_outBuffer out = {dst, dstSize, 0};
Bimba Shresthaf7a74092020-04-21 22:26:48 -07004024 ZSTD_inBuffer in = {src, srcSize, 0};
Bimba Shresthaf7a74092020-04-21 22:26:48 -07004025 /* should fail because its not a full dict like we said it was */
4026 assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)));
4027 }
4028
4029 ZSTD_freeCCtx(cctx);
4030 free(dict);
4031 }
4032 DISPLAYLEVEL(3, "OK \n");
Nick Terrellbf0591e2020-05-14 12:06:55 -07004033
4034 DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++);
Yann Colletc91a0852020-09-14 10:56:08 -07004035 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4036 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
Nick Terrellbf0591e2020-05-14 12:06:55 -07004037 size_t const dictSize = 10;
4038 size_t const dstSize = ZSTD_compressBound(srcSize);
4039 void* const src = CNBuffer;
4040 void* const dst = compressedBuffer;
4041 void* dict = (void*)malloc(dictSize);
4042
4043 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4044 RDG_genBuffer(dict, dictSize, compressibility, 0., seed);
4045
4046 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4047 memset(dict, 0, sizeof(U32));
4048
4049 /* Enable MT, LDM, and use refPrefix() for a small dict */
4050 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
Elliott Hughes44aba642023-09-12 20:18:59 +00004051 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
Nick Terrellbf0591e2020-05-14 12:06:55 -07004052 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
4053
4054 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
4055
4056 ZSTD_freeCCtx(cctx);
4057 free(dict);
4058 }
4059 DISPLAYLEVEL(3, "OK \n");
Nick Terrell58476bc2021-01-04 15:51:23 -08004060
4061 DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++);
4062 {
4063 ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0);
4064 ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0);
4065 ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400);
4066 ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000);
4067 ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000);
4068
4069 assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict)));
4070 assert(!memcmp(&medium, &mediumDict, sizeof(medium)));
4071 assert(!memcmp(&large, &largeDict, sizeof(large)));
4072 }
4073 DISPLAYLEVEL(3, "OK \n");
4074
4075 DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++);
4076 {
4077 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0);
4078 ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400);
4079 ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400);
4080
4081 assert(smallSrcAndDict.windowLog == 10);
4082 assert(!memcmp(&cParams, &smallDict, sizeof(cParams)));
4083 }
4084 DISPLAYLEVEL(3, "OK \n");
4085
Sen Huangdff4a0e2021-03-19 16:56:58 -07004086 DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
4087 {
4088 int level = 1;
4089 size_t prevSize = 0;
4090 for (; level < ZSTD_maxCLevel(); ++level) {
4091 size_t const currSize = ZSTD_estimateCCtxSize(level);
4092 if (prevSize > currSize) {
4093 DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
4094 prevSize, level-1, currSize, level);
4095 goto _output_error;
4096 }
4097 prevSize = currSize;
4098 }
4099 }
4100 DISPLAYLEVEL(3, "OK \n");
4101
4102 DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
Sen Huangb9dd8212021-03-16 08:22:13 -07004103 {
4104 size_t const kSizeIncrement = 2 KB;
4105 int level = -3;
4106
4107 for (; level <= ZSTD_maxCLevel(); ++level) {
4108 size_t dictSize = 0;
4109 for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
4110 size_t srcSize = 2 KB;
Sen Huangb9dd8212021-03-16 08:22:13 -07004111 for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
4112 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
Sen Huangdff4a0e2021-03-19 16:56:58 -07004113 size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
4114 size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
4115 if (cctxSizeUsingLevel < cctxSizeUsingCParams
4116 || ZSTD_isError(cctxSizeUsingCParams)
4117 || ZSTD_isError(cctxSizeUsingLevel)) {
4118 DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
4119 level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
Sen Huangb9dd8212021-03-16 08:22:13 -07004120 goto _output_error;
Yann Collet7fce9a42021-09-08 14:05:57 -07004121 } } } } }
Sen Huangb9dd8212021-03-16 08:22:13 -07004122 DISPLAYLEVEL(3, "OK \n");
4123
send8d6e482021-05-07 11:13:44 -04004124 DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++)
4125 {
4126 int const threadPoolTestResult = threadPoolTests();
4127 if (threadPoolTestResult) {
4128 goto _output_error;
4129 }
4130 }
4131 DISPLAYLEVEL(3, "thread pool tests OK \n");
4132
4133#endif /* ZSTD_MULTITHREAD */
Nick Terrellb9faaa12018-07-30 12:57:11 -07004134
W. Felix Handte9398acb2020-09-17 12:57:39 -04004135_end:
4136 free(CNBuffer);
4137 free(compressedBuffer);
4138 free(decodedBuffer);
4139 return testResult;
4140
4141_output_error:
4142 testResult = 1;
4143 DISPLAY("Error detected in Unit tests ! \n");
4144 goto _end;
4145}
4146
4147static int longUnitTests(U32 const seed, double compressibility)
4148{
4149 size_t const CNBuffSize = 5 MB;
4150 void* const CNBuffer = malloc(CNBuffSize);
4151 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
4152 void* const compressedBuffer = malloc(compressedBufferSize);
4153 void* const decodedBuffer = malloc(CNBuffSize);
4154 int testResult = 0;
4155 unsigned testNb=0;
4156 size_t cSize;
4157
4158 /* Create compressible noise */
4159 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
4160 DISPLAY("Not enough memory, aborting\n");
4161 testResult = 1;
4162 goto _end;
4163 }
4164 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
4165
Yann Collet608f1bf2020-05-11 18:16:38 -07004166 /* note : this test is rather long, it would be great to find a way to speed up its execution */
W. Felix Handte9398acb2020-09-17 12:57:39 -04004167 DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++);
Yann Colletc91a0852020-09-14 10:56:08 -07004168 { int cLevel;
W. Felix Handteff67c622019-09-11 13:59:09 -04004169 size_t approxIndex = 0;
4170 size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004171
W. Felix Handtef7d9b362019-09-20 14:11:29 -04004172 /* Provision enough space in a static context so that we can do all
4173 * this without ever reallocating, which would reset the indices. */
4174 size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22);
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004175 void* const staticCCtxBuffer = malloc(staticCCtxSize);
Yann Colletdd026ca2020-05-09 11:30:45 -07004176 ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004177
4178 /* bump the indices so the following compressions happen at high
4179 * indices. */
Yann Colletdd026ca2020-05-09 11:30:45 -07004180 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004181 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4182 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4183 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4184 while (approxIndex <= (maxIndex / 4) * 3) {
4185 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4186 approxIndex += in.pos;
Elliott Hughes44aba642023-09-12 20:18:59 +00004187 CHECK_Z(in.pos == in.size);
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004188 in.pos = 0;
4189 out.pos = 0;
4190 }
4191 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4192 }
4193
4194 /* spew a bunch of stuff into the table area */
4195 for (cLevel = 1; cLevel <= 22; cLevel++) {
Yann Colletdd026ca2020-05-09 11:30:45 -07004196 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004197 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4198 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4199 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4200 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4201 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4202 approxIndex += in.pos;
4203 }
4204
4205 /* now crank the indices so we overflow */
Yann Colletdd026ca2020-05-09 11:30:45 -07004206 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004207 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4208 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4209 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4210 while (approxIndex <= maxIndex) {
4211 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4212 approxIndex += in.pos;
Elliott Hughes44aba642023-09-12 20:18:59 +00004213 CHECK_Z(in.pos == in.size);
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004214 in.pos = 0;
4215 out.pos = 0;
4216 }
4217 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4218 }
4219
4220 /* do a bunch of compressions again in low indices and ensure we don't
4221 * hit untracked invalid indices */
4222 for (cLevel = 1; cLevel <= 22; cLevel++) {
Yann Colletdd026ca2020-05-09 11:30:45 -07004223 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004224 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4225 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4226 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4227 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4228 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4229 approxIndex += in.pos;
4230 }
4231
W. Felix Handte194c5422019-09-11 14:25:30 -04004232 free(staticCCtxBuffer);
W. Felix Handteed4c2c62019-09-11 13:17:19 -04004233 }
4234 DISPLAYLEVEL(3, "OK \n");
4235
senhuang4281a2c022020-10-19 15:27:40 -04004236 DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++);
Yann Collet7fce9a42021-09-08 14:05:57 -07004237 { size_t cSizeLdm;
senhuang4281a2c022020-10-19 15:27:40 -04004238 size_t cSizeNoLdm;
4239 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4240
4241 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed);
4242
4243 /* Enable checksum to verify round trip. */
4244 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
Elliott Hughes44aba642023-09-12 20:18:59 +00004245 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
senhuang4281a2c022020-10-19 15:27:40 -04004246 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4247
4248 /* Round trip once with ldm. */
4249 cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4250 CHECK_Z(cSizeLdm);
4251 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm));
4252
4253 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4254 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
Elliott Hughes44aba642023-09-12 20:18:59 +00004255 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
senhuang4281a2c022020-10-19 15:27:40 -04004256 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4257
4258 /* Round trip once without ldm. */
4259 cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4260 CHECK_Z(cSizeNoLdm);
4261 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm));
4262
4263 if (cSizeLdm > cSizeNoLdm) {
4264 DISPLAY("Using long mode should not cause regressions for btopt+\n");
4265 testResult = 1;
4266 goto _end;
4267 }
4268
4269 ZSTD_freeCCtx(cctx);
4270 }
4271 DISPLAYLEVEL(3, "OK \n");
4272
W. Felix Handte9398acb2020-09-17 12:57:39 -04004273 DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++);
Yann Colletc91a0852020-09-14 10:56:08 -07004274 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004275 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
W. Felix Handte2cc2b402020-09-10 11:32:16 -04004276 size_t dictSize = CNBuffSize;
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004277 void* dict = (void*)malloc(dictSize);
4278 ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams();
4279 ZSTD_dictAttachPref_e const attachPrefs[] = {
4280 ZSTD_dictDefaultAttach,
4281 ZSTD_dictForceAttach,
4282 ZSTD_dictForceCopy,
4283 ZSTD_dictForceLoad,
W. Felix Handte2cc2b402020-09-10 11:32:16 -04004284 ZSTD_dictDefaultAttach,
4285 ZSTD_dictForceAttach,
4286 ZSTD_dictForceCopy,
4287 ZSTD_dictForceLoad
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004288 };
W. Felix Handte2cc2b402020-09-10 11:32:16 -04004289 int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1};
4290 int cLevel;
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004291 int i;
4292
4293 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
4294 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
4295
Elliott Hughes44aba642023-09-12 20:18:59 +00004296 CHECK_Z(cctx_params != NULL);
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004297
W. Felix Handte6d3f8162020-09-10 22:29:19 -04004298 for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
W. Felix Handted6246d42020-09-10 23:35:42 -04004299 DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
W. Felix Handte6d3f8162020-09-10 22:29:19 -04004300 for (cLevel = 4; cLevel < 13; cLevel++) {
W. Felix Handte2cc2b402020-09-10 11:32:16 -04004301 for (i = 0; i < 8; ++i) {
4302 ZSTD_dictAttachPref_e const attachPref = attachPrefs[i];
4303 int const enableDDS = enableDedicatedDictSearch[i];
4304 ZSTD_CDict* cdict;
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004305
W. Felix Handted6246d42020-09-10 23:35:42 -04004306 DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i);
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004307
W. Felix Handte2cc2b402020-09-10 11:32:16 -04004308 ZSTD_CCtxParams_init(cctx_params, cLevel);
4309 CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS));
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004310
W. Felix Handte2cc2b402020-09-10 11:32:16 -04004311 cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem);
4312 CHECK(cdict != NULL);
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004313
W. Felix Handte2cc2b402020-09-10 11:32:16 -04004314 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
Yann Colletf0fc8cb2021-09-03 13:44:07 -07004315 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref));
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004316
W. Felix Handte2cc2b402020-09-10 11:32:16 -04004317 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4318 CHECK_Z(cSize);
4319 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004320
W. Felix Handteb6df3fd2020-09-10 19:19:39 -04004321 DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize);
W. Felix Handte2cc2b402020-09-10 11:32:16 -04004322
4323 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
4324 ZSTD_freeCDict(cdict);
Yann Colletc91a0852020-09-14 10:56:08 -07004325 } } }
W. Felix Handte2cf6cfc2020-08-11 15:29:12 -04004326
4327 ZSTD_freeCCtx(cctx);
4328 ZSTD_freeDCtx(dctx);
4329 ZSTD_freeCCtxParams(cctx_params);
4330 free(dict);
4331 }
4332 DISPLAYLEVEL(3, "OK \n");
4333
Yann Collet4856a002015-01-24 01:58:16 +01004334_end:
4335 free(CNBuffer);
4336 free(compressedBuffer);
4337 free(decodedBuffer);
4338 return testResult;
Yann Collet4856a002015-01-24 01:58:16 +01004339}
4340
4341
4342static size_t findDiff(const void* buf1, const void* buf2, size_t max)
4343{
Yann Collet213089c2015-06-18 07:43:16 -08004344 const BYTE* b1 = (const BYTE*)buf1;
4345 const BYTE* b2 = (const BYTE*)buf2;
Yann Colletc0a9bf32016-05-30 01:56:08 +02004346 size_t u;
4347 for (u=0; u<max; u++) {
4348 if (b1[u] != b2[u]) break;
Yann Collet4856a002015-01-24 01:58:16 +01004349 }
Yann Colletc0a9bf32016-05-30 01:56:08 +02004350 return u;
Yann Collet4856a002015-01-24 01:58:16 +01004351}
4352
Yann Collet1fce6e02016-04-08 20:26:33 +02004353
Yann Collet73213452017-04-27 14:19:34 -07004354static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
4355{
4356 ZSTD_parameters params;
4357 params.cParams = cParams;
4358 params.fParams = fParams;
4359 return params;
4360}
4361
Yann Collet1fce6e02016-04-08 20:26:33 +02004362static size_t FUZ_rLogLength(U32* seed, U32 logLength)
4363{
4364 size_t const lengthMask = ((size_t)1 << logLength) - 1;
4365 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
4366}
4367
4368static size_t FUZ_randomLength(U32* seed, U32 maxLog)
4369{
4370 U32 const logLength = FUZ_rand(seed) % maxLog;
4371 return FUZ_rLogLength(seed, logLength);
4372}
4373
Yann Colletc0a9bf32016-05-30 01:56:08 +02004374#undef CHECK
Yann Collet7bd1a292017-06-21 11:50:33 -07004375#define CHECK(cond, ...) { \
4376 if (cond) { \
4377 DISPLAY("Error => "); \
4378 DISPLAY(__VA_ARGS__); \
Yann Colletededcfc2018-12-21 16:19:44 -08004379 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
Yann Collet7bd1a292017-06-21 11:50:33 -07004380 goto _output_error; \
4381} }
4382
Yann Collet670b1fc2017-07-10 16:30:55 -07004383#undef CHECK_Z
Yann Collet7bd1a292017-06-21 11:50:33 -07004384#define CHECK_Z(f) { \
4385 size_t const err = f; \
4386 if (ZSTD_isError(err)) { \
4387 DISPLAY("Error => %s : %s ", \
4388 #f, ZSTD_getErrorName(err)); \
Yann Colletededcfc2018-12-21 16:19:44 -08004389 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
Yann Collet7bd1a292017-06-21 11:50:33 -07004390 goto _output_error; \
4391} }
4392
Yann Collet4856a002015-01-24 01:58:16 +01004393
Yann Colletededcfc2018-12-21 16:19:44 -08004394static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
Yann Collet4856a002015-01-24 01:58:16 +01004395{
Yann Collet1fce6e02016-04-08 20:26:33 +02004396 static const U32 maxSrcLog = 23;
4397 static const U32 maxSampleLog = 22;
Yann Colletc0a9bf32016-05-30 01:56:08 +02004398 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
4399 size_t const dstBufferSize = (size_t)1<<maxSampleLog;
4400 size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
Yann Colletd5d9bc32015-08-23 23:13:49 +01004401 BYTE* cNoiseBuffer[5];
Yann Colletc0a9bf32016-05-30 01:56:08 +02004402 BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
4403 BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
4404 BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
Yann Colletd2858e92016-05-30 15:10:09 +02004405 ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
4406 ZSTD_CCtx* const ctx = ZSTD_createCCtx();
4407 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
Yann Collet30009522016-05-30 16:17:33 +02004408 U32 result = 0;
Yann Colletededcfc2018-12-21 16:19:44 -08004409 unsigned testNb = 0;
Yann Collet3692c312018-08-14 16:56:07 -07004410 U32 coreSeed = seed;
Nick Terrell9a2f6f42017-11-29 19:11:12 -08004411 UTIL_time_t const startClock = UTIL_getTime();
4412 U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
Sean Purcell7ebf2de2017-03-20 11:25:00 -07004413 int const cLevelLimiter = bigTests ? 3 : 2;
Yann Collet4856a002015-01-24 01:58:16 +01004414
4415 /* allocation */
Yann Colletd5d9bc32015-08-23 23:13:49 +01004416 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
4417 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
4418 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
4419 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
4420 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
Yann Collete47c4e52015-12-05 09:23:53 +01004421 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
Yann Colletecd651b2016-01-07 15:35:18 +01004422 || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
Yann Colletd5d9bc32015-08-23 23:13:49 +01004423 "Not enough memory, fuzzer tests cancelled");
Yann Collet4856a002015-01-24 01:58:16 +01004424
Yann Colletd5d9bc32015-08-23 23:13:49 +01004425 /* Create initial samples */
4426 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
4427 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
4428 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
4429 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
4430 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
Yann Collet4856a002015-01-24 01:58:16 +01004431
4432 /* catch up testNb */
Yann Collet546c9b12016-03-19 12:47:52 +01004433 for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
Yann Collet4856a002015-01-24 01:58:16 +01004434
Yann Collet546c9b12016-03-19 12:47:52 +01004435 /* main test loop */
Nick Terrell9a2f6f42017-11-29 19:11:12 -08004436 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
Yann Collet3692c312018-08-14 16:56:07 -07004437 BYTE* srcBuffer; /* jumping pointer */
4438 U32 lseed;
Yann Colletc0a9bf32016-05-30 01:56:08 +02004439 size_t sampleSize, maxTestSize, totalTestSize;
4440 size_t cSize, totalCSize, totalGenSize;
Yann Collet1fce6e02016-04-08 20:26:33 +02004441 U64 crcOrig;
Yann Collet110cc142015-11-19 12:02:28 +01004442 BYTE* sampleBuffer;
Yann Collet4bfe4152015-12-06 13:18:37 +01004443 const BYTE* dict;
4444 size_t dictSize;
Yann Collet4856a002015-01-24 01:58:16 +01004445
Yann Collet546c9b12016-03-19 12:47:52 +01004446 /* notification */
4447 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
Yann Collet4c0b44f2016-11-01 11:13:22 -07004448 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
Yann Collet1c2ddba2015-12-04 17:45:35 +01004449
Yann Collet4856a002015-01-24 01:58:16 +01004450 FUZ_rand(&coreSeed);
Yann Collet0d9ce042016-03-19 13:21:08 +01004451 { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
Yann Collet1fce6e02016-04-08 20:26:33 +02004452
4453 /* srcBuffer selection [0-4] */
4454 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
4455 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
4456 else {
4457 buffNb >>= 3;
4458 if (buffNb & 7) {
4459 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
4460 buffNb = tnb[buffNb >> 3];
4461 } else {
4462 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
4463 buffNb = tnb[buffNb >> 3];
4464 } }
4465 srcBuffer = cNoiseBuffer[buffNb];
4466 }
4467
4468 /* select src segment */
Yann Colletd2858e92016-05-30 15:10:09 +02004469 sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
Yann Collet4856a002015-01-24 01:58:16 +01004470
Yann Collet110cc142015-11-19 12:02:28 +01004471 /* create sample buffer (to catch read error with valgrind & sanitizers) */
4472 sampleBuffer = (BYTE*)malloc(sampleSize);
Yann Colletd2858e92016-05-30 15:10:09 +02004473 CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
Yann Colletc0a9bf32016-05-30 01:56:08 +02004474 { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
4475 memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
Yann Collet110cc142015-11-19 12:02:28 +01004476 crcOrig = XXH64(sampleBuffer, sampleSize, 0);
4477
Yann Collet1fce6e02016-04-08 20:26:33 +02004478 /* compression tests */
Yann Colletdd026ca2020-05-09 11:30:45 -07004479 { int const cLevelPositive = (int)
Yann Collet20d5e032017-04-11 18:34:02 -07004480 ( FUZ_rand(&lseed) %
Yann Colletdd026ca2020-05-09 11:30:45 -07004481 ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) )
Yann Collet02f64ef2017-12-29 19:08:51 +01004482 + 1;
Yann Colleta146ee02018-03-11 05:21:53 -07004483 int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ?
4484 - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */
4485 cLevelPositive;
Yann Collet02f64ef2017-12-29 19:08:51 +01004486 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel);
Yann Collet1fce6e02016-04-08 20:26:33 +02004487 cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
Yann Colleta7737f62016-09-06 09:44:59 +02004488 CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
Yann Collet4856a002015-01-24 01:58:16 +01004489
Yann Collet1fce6e02016-04-08 20:26:33 +02004490 /* compression failure test : too small dest buffer */
Yann Collet3e5cdf12018-11-05 17:50:30 -08004491 assert(cSize > 3);
4492 { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;
Yann Collet1fce6e02016-04-08 20:26:33 +02004493 const size_t tooSmallSize = cSize - missing;
Yann Colletededcfc2018-12-21 16:19:44 -08004494 const unsigned endMark = 0x4DC2B1A9;
4495 memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark));
Yann Collet3e5cdf12018-11-05 17:50:30 -08004496 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
4497 testNb, (unsigned)tooSmallSize, (unsigned)missing);
Yann Collet1fce6e02016-04-08 20:26:33 +02004498 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
Nick Terrell1302f8d2020-08-15 12:32:57 -07004499 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
Yann Colletededcfc2018-12-21 16:19:44 -08004500 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck));
Yann Collet3e5cdf12018-11-05 17:50:30 -08004501 CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); }
Yann Colletd2858e92016-05-30 15:10:09 +02004502 } }
Yann Collet1fce6e02016-04-08 20:26:33 +02004503
Yann Collet990449b2017-07-07 15:21:35 -07004504 /* frame header decompression test */
4505 { ZSTD_frameHeader zfh;
4506 CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
4507 CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
4508 }
4509
Yann Colletf323bf72016-07-07 13:14:21 +02004510 /* Decompressed size test */
Sean Purcell4e709712017-02-07 13:50:09 -08004511 { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
Yann Colletf323bf72016-07-07 13:14:21 +02004512 CHECK(rSize != sampleSize, "decompressed size incorrect");
4513 }
4514
Yann Collet546c9b12016-03-19 12:47:52 +01004515 /* successful decompression test */
Yann Collet64482c22017-12-29 17:04:37 +01004516 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb);
Yann Collet1fce6e02016-04-08 20:26:33 +02004517 { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
Yann Colletc0a9bf32016-05-30 01:56:08 +02004518 size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
Yann Colletededcfc2018-12-21 16:19:44 -08004519 CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize);
Yann Collet1fce6e02016-04-08 20:26:33 +02004520 { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
Yann Colletededcfc2018-12-21 16:19:44 -08004521 CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize);
Yann Collet1fce6e02016-04-08 20:26:33 +02004522 } }
Yann Collet110cc142015-11-19 12:02:28 +01004523
4524 free(sampleBuffer); /* no longer useful after this point */
Yann Colletf3cb79b2015-08-20 00:02:43 +01004525
4526 /* truncated src decompression test */
Yann Collet64482c22017-12-29 17:04:37 +01004527 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb);
Yann Collet1fce6e02016-04-08 20:26:33 +02004528 { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4529 size_t const tooSmallSize = cSize - missing;
Yann Colletd2858e92016-05-30 15:10:09 +02004530 void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
Yann Colletf3cb79b2015-08-20 00:02:43 +01004531 CHECK(cBufferTooSmall == NULL, "not enough memory !");
Yann Colletd5d9bc32015-08-23 23:13:49 +01004532 memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
Yann Collet1fce6e02016-04-08 20:26:33 +02004533 { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
4534 CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
Yann Colletf3cb79b2015-08-20 00:02:43 +01004535 free(cBufferTooSmall);
4536 }
4537
4538 /* too small dst decompression test */
Yann Collet64482c22017-12-29 17:04:37 +01004539 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb);
Yann Colletd1d210f2016-03-19 12:12:07 +01004540 if (sampleSize > 3) {
Yann Colletea63bb72016-04-08 15:25:32 +02004541 size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4542 size_t const tooSmallSize = sampleSize - missing;
Yann Colletf3cb79b2015-08-20 00:02:43 +01004543 static const BYTE token = 0xA9;
4544 dstBuffer[tooSmallSize] = token;
Yann Collet1fce6e02016-04-08 20:26:33 +02004545 { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
Nick Terrell1302f8d2020-08-15 12:32:57 -07004546 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
Yann Colletf3cb79b2015-08-20 00:02:43 +01004547 CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
4548 }
Yann Collet997f9ee2015-08-21 02:44:20 +01004549
4550 /* noisy src decompression test */
Yann Colletd1d210f2016-03-19 12:12:07 +01004551 if (cSize > 6) {
4552 /* insert noise into src */
4553 { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
4554 size_t pos = 4; /* preserve magic number (too easy to detect) */
4555 for (;;) {
4556 /* keep some original src */
4557 { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
4558 size_t const mask = (1<<nbBits) - 1;
4559 size_t const skipLength = FUZ_rand(&lseed) & mask;
4560 pos += skipLength;
4561 }
conor420e88f6e2018-01-11 11:42:58 +10004562 if (pos >= cSize) break;
Yann Colletd1d210f2016-03-19 12:12:07 +01004563 /* add noise */
Yann Collet33341de2016-05-29 23:09:51 +02004564 { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
4565 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
4566 size_t const mask = (1<<nbBits) - 1;
4567 size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
4568 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
4569 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
Yann Colletd1d210f2016-03-19 12:12:07 +01004570 memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
4571 pos += noiseLength;
4572 } } }
Yann Collet997f9ee2015-08-21 02:44:20 +01004573
4574 /* decompress noisy source */
Yann Collet64482c22017-12-29 17:04:37 +01004575 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb);
Yann Colletd1d210f2016-03-19 12:12:07 +01004576 { U32 const endMark = 0xA9B1C3D6;
Yann Collet997f9ee2015-08-21 02:44:20 +01004577 memcpy(dstBuffer+sampleSize, &endMark, 4);
Yann Collet1fce6e02016-04-08 20:26:33 +02004578 { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
4579 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
4580 CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
Yann Colletededcfc2018-12-21 16:19:44 -08004581 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize);
Yann Collet1fce6e02016-04-08 20:26:33 +02004582 }
4583 { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
4584 CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
4585 } } } /* noisy src decompression test */
Yann Collet417890c2015-12-04 17:16:37 +01004586
Yann Collet64482c22017-12-29 17:04:37 +01004587 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
4588 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
Yann Collet1fce6e02016-04-08 20:26:33 +02004589 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
Sean Purcellf5e50512017-03-15 15:04:54 -07004590 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
Elliott Hughes44aba642023-09-12 20:18:59 +00004591 int const cLevel = (int)(FUZ_rand(&lseed) %
4592 ((U32)ZSTD_maxCLevel() -
4593 (MAX(testLog, dictLog) / (U32)cLevelLimiter))) +
Sean Purcell7ebf2de2017-03-20 11:25:00 -07004594 1;
Yann Collet1fce6e02016-04-08 20:26:33 +02004595 maxTestSize = FUZ_rLogLength(&lseed, testLog);
4596 if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
4597
Sean Purcellf5e50512017-03-15 15:04:54 -07004598 dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
Yann Colletc0a9bf32016-05-30 01:56:08 +02004599 dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
Yann Collet1fce6e02016-04-08 20:26:33 +02004600
Yann Collet64482c22017-12-29 17:04:37 +01004601 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
Yann Colletededcfc2018-12-21 16:19:44 -08004602 testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize);
Yann Collet64482c22017-12-29 17:04:37 +01004603
Yann Colletf2a3b6e2016-05-31 18:13:56 +02004604 if (FUZ_rand(&lseed) & 0xF) {
Yann Collet7bd1a292017-06-21 11:50:33 -07004605 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
Yann Collet33341de2016-05-29 23:09:51 +02004606 } else {
Yann Collet64482c22017-12-29 17:04:37 +01004607 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
Yann Colletf4bd8572017-04-27 11:31:55 -07004608 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
Yann Colletf2a3b6e2016-05-31 18:13:56 +02004609 !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
4610 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
Yann Collet73213452017-04-27 14:19:34 -07004611 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
Yann Collet7bd1a292017-06-21 11:50:33 -07004612 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
Yann Collet33341de2016-05-29 23:09:51 +02004613 }
Yann Collet7bd1a292017-06-21 11:50:33 -07004614 CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
4615 }
Yann Colletf4bd8572017-04-27 11:31:55 -07004616
Yann Colletd2858e92016-05-30 15:10:09 +02004617 { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
4618 U32 n;
Yann Colletf4bd8572017-04-27 11:31:55 -07004619 XXH64_state_t xxhState;
4620 XXH64_reset(&xxhState, 0);
Yann Colletd2858e92016-05-30 15:10:09 +02004621 for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
4622 size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
4623 size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
Yann Collet417890c2015-12-04 17:16:37 +01004624
Yann Colletd2858e92016-05-30 15:10:09 +02004625 if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
4626 if (totalTestSize+segmentSize > maxTestSize) break;
Yann Collet417890c2015-12-04 17:16:37 +01004627
Yann Colletd2858e92016-05-30 15:10:09 +02004628 { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
4629 CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
4630 cSize += compressResult;
4631 }
4632 XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
4633 memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
4634 totalTestSize += segmentSize;
Yann Colletf4bd8572017-04-27 11:31:55 -07004635 }
Yann Colletd2858e92016-05-30 15:10:09 +02004636
Yann Colletf4bd8572017-04-27 11:31:55 -07004637 { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
4638 CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
4639 cSize += flushResult;
4640 }
4641 crcOrig = XXH64_digest(&xxhState);
Yann Collet1fce6e02016-04-08 20:26:33 +02004642 }
Yann Collete47c4e52015-12-05 09:23:53 +01004643
4644 /* streaming decompression test */
Yann Collet64482c22017-12-29 17:04:37 +01004645 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
Yann Collet058ed2a2017-09-09 01:03:29 -07004646 /* ensure memory requirement is good enough (should always be true) */
4647 { ZSTD_frameHeader zfh;
Yann Colletb83d1e72018-11-13 16:56:32 -08004648 CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
Yann Collet058ed2a2017-09-09 01:03:29 -07004649 "ZSTD_getFrameHeader(): error retrieving frame information");
4650 { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
4651 CHECK_Z(roundBuffSize);
4652 CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN),
4653 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)",
Yann Colletededcfc2018-12-21 16:19:44 -08004654 (unsigned)roundBuffSize, (unsigned)totalTestSize );
Yann Collet058ed2a2017-09-09 01:03:29 -07004655 } }
Yann Collet33341de2016-05-29 23:09:51 +02004656 if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
Yann Collet7bd1a292017-06-21 11:50:33 -07004657 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
Yann Collete47c4e52015-12-05 09:23:53 +01004658 totalCSize = 0;
4659 totalGenSize = 0;
Yann Colletd1d210f2016-03-19 12:12:07 +01004660 while (totalCSize < cSize) {
Yann Collet1fce6e02016-04-08 20:26:33 +02004661 size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
4662 size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
Yann Collet45dc3562016-07-12 09:47:31 +02004663 CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize));
Yann Collete47c4e52015-12-05 09:23:53 +01004664 totalGenSize += genSize;
4665 totalCSize += inSize;
4666 }
4667 CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
Yann Colletc0a9bf32016-05-30 01:56:08 +02004668 CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
Yann Collete47c4e52015-12-05 09:23:53 +01004669 CHECK (totalCSize != cSize, "compressed data should be fully read")
Yann Collet1fce6e02016-04-08 20:26:33 +02004670 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
Yann Collet3692c312018-08-14 16:56:07 -07004671 CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
Yann Colletededcfc2018-12-21 16:19:44 -08004672 (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize);
Yann Collet3692c312018-08-14 16:56:07 -07004673 }
Yann Collet1fce6e02016-04-08 20:26:33 +02004674 } /* for ( ; (testNb <= nbTests) */
Yann Collet1c2ddba2015-12-04 17:45:35 +01004675 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
Yann Collet4856a002015-01-24 01:58:16 +01004676
4677_cleanup:
Yann Colletecd651b2016-01-07 15:35:18 +01004678 ZSTD_freeCCtx(refCtx);
Yann Collet2f648e52015-10-29 18:23:38 +01004679 ZSTD_freeCCtx(ctx);
Yann Collete47c4e52015-12-05 09:23:53 +01004680 ZSTD_freeDCtx(dctx);
Yann Colletd5d9bc32015-08-23 23:13:49 +01004681 free(cNoiseBuffer[0]);
4682 free(cNoiseBuffer[1]);
4683 free(cNoiseBuffer[2]);
4684 free(cNoiseBuffer[3]);
4685 free(cNoiseBuffer[4]);
Yann Collet4856a002015-01-24 01:58:16 +01004686 free(cBuffer);
4687 free(dstBuffer);
Yann Collete47c4e52015-12-05 09:23:53 +01004688 free(mirrorBuffer);
Elliott Hughes44aba642023-09-12 20:18:59 +00004689 return (int)result;
Yann Collet4856a002015-01-24 01:58:16 +01004690
4691_output_error:
4692 result = 1;
4693 goto _cleanup;
4694}
4695
4696
Yann Colletd1d210f2016-03-19 12:12:07 +01004697/*_*******************************************************
Yann Collet4856a002015-01-24 01:58:16 +01004698* Command line
4699*********************************************************/
Yann Colletc6915422017-04-27 16:24:53 -07004700static int FUZ_usage(const char* programName)
Yann Collet4856a002015-01-24 01:58:16 +01004701{
4702 DISPLAY( "Usage :\n");
4703 DISPLAY( " %s [args]\n", programName);
4704 DISPLAY( "\n");
4705 DISPLAY( "Arguments :\n");
Taylor Braun-Jones3cbc3d32020-03-23 17:46:56 -04004706 DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault);
4707 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
Yann Collet4856a002015-01-24 01:58:16 +01004708 DISPLAY( " -s# : Select seed (default:prompt user)\n");
4709 DISPLAY( " -t# : Select starting test number (default:0)\n");
Yann Colletededcfc2018-12-21 16:19:44 -08004710 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default);
Yann Collet4856a002015-01-24 01:58:16 +01004711 DISPLAY( " -v : verbose\n");
Yann Collete9853b22015-08-07 19:07:32 +01004712 DISPLAY( " -p : pause at the end\n");
Yann Collet4856a002015-01-24 01:58:16 +01004713 DISPLAY( " -h : display help and exit\n");
4714 return 0;
4715}
4716
Yann Colletc6915422017-04-27 16:24:53 -07004717/*! readU32FromChar() :
4718 @return : unsigned integer value read from input in `char` format
4719 allows and interprets K, KB, KiB, M, MB and MiB suffix.
4720 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
4721 Note : function result can overflow if digit string > MAX_UINT */
4722static unsigned readU32FromChar(const char** stringPtr)
4723{
4724 unsigned result = 0;
4725 while ((**stringPtr >='0') && (**stringPtr <='9'))
Elliott Hughes44aba642023-09-12 20:18:59 +00004726 result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ;
Yann Colletc6915422017-04-27 16:24:53 -07004727 if ((**stringPtr=='K') || (**stringPtr=='M')) {
4728 result <<= 10;
4729 if (**stringPtr=='M') result <<= 10;
4730 (*stringPtr)++ ;
4731 if (**stringPtr=='i') (*stringPtr)++;
4732 if (**stringPtr=='B') (*stringPtr)++;
4733 }
4734 return result;
4735}
Yann Collet4856a002015-01-24 01:58:16 +01004736
Yann Collet1ca12882017-07-19 16:01:16 -07004737/** longCommandWArg() :
4738 * check if *stringPtr is the same as longCommand.
4739 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
4740 * @return 0 and doesn't modify *stringPtr otherwise.
4741 */
Yann Colleted38b642019-05-29 15:26:06 -07004742static int longCommandWArg(const char** stringPtr, const char* longCommand)
Yann Collet1ca12882017-07-19 16:01:16 -07004743{
4744 size_t const comSize = strlen(longCommand);
4745 int const result = !strncmp(*stringPtr, longCommand, comSize);
4746 if (result) *stringPtr += comSize;
4747 return result;
4748}
4749
Yann Colletd1d210f2016-03-19 12:12:07 +01004750int main(int argc, const char** argv)
Yann Collet4856a002015-01-24 01:58:16 +01004751{
Yann Colletc6915422017-04-27 16:24:53 -07004752 U32 seed = 0;
4753 int seedset = 0;
Yann Collet4856a002015-01-24 01:58:16 +01004754 int argNb;
4755 int nbTests = nbTestsDefault;
4756 int testNb = 0;
Yann Colletededcfc2018-12-21 16:19:44 -08004757 int proba = FUZ_compressibility_default;
W. Felix Handte9398acb2020-09-17 12:57:39 -04004758 double probfloat;
Yann Colletc6915422017-04-27 16:24:53 -07004759 int result = 0;
Yann Collet4856a002015-01-24 01:58:16 +01004760 U32 mainPause = 0;
Yann Collet0d9ce042016-03-19 13:21:08 +01004761 U32 maxDuration = 0;
Sean Purcell7ebf2de2017-03-20 11:25:00 -07004762 int bigTests = 1;
W. Felix Handte9398acb2020-09-17 12:57:39 -04004763 int longTests = 0;
Yann Colletf9524cf2017-07-10 13:48:41 -07004764 U32 memTestsOnly = 0;
Yann Colletc6915422017-04-27 16:24:53 -07004765 const char* const programName = argv[0];
Yann Collet4856a002015-01-24 01:58:16 +01004766
4767 /* Check command line */
Yann Colletd1d210f2016-03-19 12:12:07 +01004768 for (argNb=1; argNb<argc; argNb++) {
4769 const char* argument = argv[argNb];
Yann Collet4856a002015-01-24 01:58:16 +01004770 if(!argument) continue; /* Protection if argument empty */
4771
4772 /* Handle commands. Aggregated commands are allowed */
Yann Colletd1d210f2016-03-19 12:12:07 +01004773 if (argument[0]=='-') {
Sean Purcell7ebf2de2017-03-20 11:25:00 -07004774
Yann Collet1ca12882017-07-19 16:01:16 -07004775 if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; }
4776
Yann Colletf9524cf2017-07-10 13:48:41 -07004777 if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
Sean Purcell7ebf2de2017-03-20 11:25:00 -07004778 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
W. Felix Handte9398acb2020-09-17 12:57:39 -04004779 if (!strcmp(argument, "--long-tests")) { longTests=1; continue; }
4780 if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; }
Sean Purcell7ebf2de2017-03-20 11:25:00 -07004781
Yann Collet4856a002015-01-24 01:58:16 +01004782 argument++;
Yann Colletd1d210f2016-03-19 12:12:07 +01004783 while (*argument!=0) {
Yann Collet4856a002015-01-24 01:58:16 +01004784 switch(*argument)
4785 {
4786 case 'h':
4787 return FUZ_usage(programName);
Yann Colletc6915422017-04-27 16:24:53 -07004788
Yann Collet4856a002015-01-24 01:58:16 +01004789 case 'v':
4790 argument++;
Yann Collet64482c22017-12-29 17:04:37 +01004791 g_displayLevel++;
Yann Collet4856a002015-01-24 01:58:16 +01004792 break;
Yann Colletc6915422017-04-27 16:24:53 -07004793
Yann Collet4856a002015-01-24 01:58:16 +01004794 case 'q':
4795 argument++;
4796 g_displayLevel--;
4797 break;
Yann Colletc6915422017-04-27 16:24:53 -07004798
Yann Collet4856a002015-01-24 01:58:16 +01004799 case 'p': /* pause at the end */
4800 argument++;
4801 mainPause = 1;
4802 break;
4803
4804 case 'i':
Yann Colletc6915422017-04-27 16:24:53 -07004805 argument++; maxDuration = 0;
Yann Colleted38b642019-05-29 15:26:06 -07004806 nbTests = (int)readU32FromChar(&argument);
Yann Collet4856a002015-01-24 01:58:16 +01004807 break;
4808
Yann Collet553cf6a2015-12-04 17:25:26 +01004809 case 'T':
4810 argument++;
Yann Colletc6915422017-04-27 16:24:53 -07004811 nbTests = 0;
4812 maxDuration = readU32FromChar(&argument);
4813 if (*argument=='s') argument++; /* seconds */
4814 if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
Yann Collet553cf6a2015-12-04 17:25:26 +01004815 if (*argument=='n') argument++;
Yann Collet553cf6a2015-12-04 17:25:26 +01004816 break;
4817
Yann Collet4856a002015-01-24 01:58:16 +01004818 case 's':
4819 argument++;
Yann Colletc6915422017-04-27 16:24:53 -07004820 seedset = 1;
4821 seed = readU32FromChar(&argument);
Yann Collet4856a002015-01-24 01:58:16 +01004822 break;
4823
4824 case 't':
4825 argument++;
Yann Colleted38b642019-05-29 15:26:06 -07004826 testNb = (int)readU32FromChar(&argument);
Yann Collet4856a002015-01-24 01:58:16 +01004827 break;
4828
4829 case 'P': /* compressibility % */
4830 argument++;
Yann Colleted38b642019-05-29 15:26:06 -07004831 proba = (int)readU32FromChar(&argument);
Yann Colletc6915422017-04-27 16:24:53 -07004832 if (proba>100) proba = 100;
Yann Collet4856a002015-01-24 01:58:16 +01004833 break;
4834
4835 default:
Yann Colletc6915422017-04-27 16:24:53 -07004836 return (FUZ_usage(programName), 1);
Yann Colletd1d210f2016-03-19 12:12:07 +01004837 } } } } /* for (argNb=1; argNb<argc; argNb++) */
Yann Collet4856a002015-01-24 01:58:16 +01004838
4839 /* Get Seed */
Yann Collet45f84ab2016-05-20 12:34:40 +02004840 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
Yann Collet4856a002015-01-24 01:58:16 +01004841
Yann Collet3f01c882016-06-16 13:38:10 +02004842 if (!seedset) {
4843 time_t const t = time(NULL);
4844 U32 const h = XXH32(&t, sizeof(t), 1);
4845 seed = h % 10000;
4846 }
4847
Yann Colletededcfc2018-12-21 16:19:44 -08004848 DISPLAY("Seed = %u\n", (unsigned)seed);
4849 if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba);
Yann Collet4856a002015-01-24 01:58:16 +01004850
W. Felix Handte9398acb2020-09-17 12:57:39 -04004851 probfloat = ((double)proba) / 100;
4852
Yann Colletf9524cf2017-07-10 13:48:41 -07004853 if (memTestsOnly) {
Yann Collet670b1fc2017-07-10 16:30:55 -07004854 g_displayLevel = MAX(3, g_displayLevel);
W. Felix Handte9398acb2020-09-17 12:57:39 -04004855 return FUZ_mallocTests(seed, probfloat, memTestsOnly);
Yann Colletf9524cf2017-07-10 13:48:41 -07004856 }
4857
Yann Collet803c05e2016-06-16 11:32:57 +02004858 if (nbTests < testNb) nbTests = testNb;
4859
W. Felix Handte9398acb2020-09-17 12:57:39 -04004860 if (testNb==0) {
4861 result = basicUnitTests(0, probfloat); /* constant seed for predictability */
4862
4863 if (!result && longTests) {
4864 result = longUnitTests(0, probfloat);
4865 }
4866 }
Yann Collet4856a002015-01-24 01:58:16 +01004867 if (!result)
Elliott Hughes44aba642023-09-12 20:18:59 +00004868 result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests);
Yann Colletd1d210f2016-03-19 12:12:07 +01004869 if (mainPause) {
Yann Colletb5e06dc2015-07-04 23:20:56 -08004870 int unused;
Yann Collet4856a002015-01-24 01:58:16 +01004871 DISPLAY("Press Enter \n");
Yann Colletb5e06dc2015-07-04 23:20:56 -08004872 unused = getchar();
4873 (void)unused;
Yann Collet4856a002015-01-24 01:58:16 +01004874 }
4875 return result;
4876}