mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
575 lines
26 KiB
575 lines
26 KiB
/* |
|
* FSE : Finite State Entropy codec |
|
* Public Prototypes declaration |
|
* Copyright (C) 2013-2016, Yann Collet. |
|
* |
|
* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) |
|
* |
|
* Redistribution and use in source and binary forms, with or without |
|
* modification, are permitted provided that the following conditions are |
|
* met: |
|
* |
|
* * Redistributions of source code must retain the above copyright |
|
* notice, this list of conditions and the following disclaimer. |
|
* * Redistributions in binary form must reproduce the above |
|
* copyright notice, this list of conditions and the following disclaimer |
|
* in the documentation and/or other materials provided with the |
|
* distribution. |
|
* |
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
* |
|
* This program is free software; you can redistribute it and/or modify it under |
|
* the terms of the GNU General Public License version 2 as published by the |
|
* Free Software Foundation. This program is dual-licensed; you may select |
|
* either version 2 of the GNU General Public License ("GPL") or BSD license |
|
* ("BSD"). |
|
* |
|
* You can contact the author at : |
|
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy |
|
*/ |
|
#ifndef FSE_H |
|
#define FSE_H |
|
|
|
/*-***************************************** |
|
* Dependencies |
|
******************************************/ |
|
#include <linux/types.h> /* size_t, ptrdiff_t */ |
|
|
|
/*-***************************************** |
|
* FSE_PUBLIC_API : control library symbols visibility |
|
******************************************/ |
|
#define FSE_PUBLIC_API |
|
|
|
/*------ Version ------*/ |
|
#define FSE_VERSION_MAJOR 0 |
|
#define FSE_VERSION_MINOR 9 |
|
#define FSE_VERSION_RELEASE 0 |
|
|
|
#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE |
|
#define FSE_QUOTE(str) #str |
|
#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) |
|
#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) |
|
|
|
#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR * 100 * 100 + FSE_VERSION_MINOR * 100 + FSE_VERSION_RELEASE) |
|
FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ |
|
|
|
/*-***************************************** |
|
* Tool functions |
|
******************************************/ |
|
FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ |
|
|
|
/* Error Management */ |
|
FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ |
|
|
|
/*-***************************************** |
|
* FSE detailed API |
|
******************************************/ |
|
/*! |
|
FSE_compress() does the following: |
|
1. count symbol occurrence from source[] into table count[] |
|
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) |
|
3. save normalized counters to memory buffer using writeNCount() |
|
4. build encoding table 'CTable' from normalized counters |
|
5. encode the data stream using encoding table 'CTable' |
|
|
|
FSE_decompress() does the following: |
|
1. read normalized counters with readNCount() |
|
2. build decoding table 'DTable' from normalized counters |
|
3. decode the data stream using decoding table 'DTable' |
|
|
|
The following API allows targeting specific sub-functions for advanced tasks. |
|
For example, it's possible to compress several blocks using the same 'CTable', |
|
or to save and provide normalized distribution using external method. |
|
*/ |
|
|
|
/* *** COMPRESSION *** */ |
|
/*! FSE_optimalTableLog(): |
|
dynamically downsize 'tableLog' when conditions are met. |
|
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. |
|
@return : recommended tableLog (necessarily <= 'maxTableLog') */ |
|
FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); |
|
|
|
/*! FSE_normalizeCount(): |
|
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) |
|
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). |
|
@return : tableLog, |
|
or an errorCode, which can be tested using FSE_isError() */ |
|
FSE_PUBLIC_API size_t FSE_normalizeCount(short *normalizedCounter, unsigned tableLog, const unsigned *count, size_t srcSize, unsigned maxSymbolValue); |
|
|
|
/*! FSE_NCountWriteBound(): |
|
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. |
|
Typically useful for allocation purpose. */ |
|
FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); |
|
|
|
/*! FSE_writeNCount(): |
|
Compactly save 'normalizedCounter' into 'buffer'. |
|
@return : size of the compressed table, |
|
or an errorCode, which can be tested using FSE_isError(). */ |
|
FSE_PUBLIC_API size_t FSE_writeNCount(void *buffer, size_t bufferSize, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); |
|
|
|
/*! Constructor and Destructor of FSE_CTable. |
|
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ |
|
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ |
|
|
|
/*! FSE_compress_usingCTable(): |
|
Compress `src` using `ct` into `dst` which must be already allocated. |
|
@return : size of compressed data (<= `dstCapacity`), |
|
or 0 if compressed data could not fit into `dst`, |
|
or an errorCode, which can be tested using FSE_isError() */ |
|
FSE_PUBLIC_API size_t FSE_compress_usingCTable(void *dst, size_t dstCapacity, const void *src, size_t srcSize, const FSE_CTable *ct); |
|
|
|
/*! |
|
Tutorial : |
|
---------- |
|
The first step is to count all symbols. FSE_count() does this job very fast. |
|
Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. |
|
'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] |
|
maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) |
|
FSE_count() will return the number of occurrence of the most frequent symbol. |
|
This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. |
|
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). |
|
|
|
The next step is to normalize the frequencies. |
|
FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. |
|
It also guarantees a minimum of 1 to any Symbol with frequency >= 1. |
|
You can use 'tableLog'==0 to mean "use default tableLog value". |
|
If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), |
|
which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). |
|
|
|
The result of FSE_normalizeCount() will be saved into a table, |
|
called 'normalizedCounter', which is a table of signed short. |
|
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. |
|
The return value is tableLog if everything proceeded as expected. |
|
It is 0 if there is a single symbol within distribution. |
|
If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). |
|
|
|
'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). |
|
'buffer' must be already allocated. |
|
For guaranteed success, buffer size must be at least FSE_headerBound(). |
|
The result of the function is the number of bytes written into 'buffer'. |
|
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). |
|
|
|
'normalizedCounter' can then be used to create the compression table 'CTable'. |
|
The space required by 'CTable' must be already allocated, using FSE_createCTable(). |
|
You can then use FSE_buildCTable() to fill 'CTable'. |
|
If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). |
|
|
|
'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). |
|
Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' |
|
The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. |
|
If it returns '0', compressed data could not fit into 'dst'. |
|
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). |
|
*/ |
|
|
|
/* *** DECOMPRESSION *** */ |
|
|
|
/*! FSE_readNCount(): |
|
Read compactly saved 'normalizedCounter' from 'rBuffer'. |
|
@return : size read from 'rBuffer', |
|
or an errorCode, which can be tested using FSE_isError(). |
|
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ |
|
FSE_PUBLIC_API size_t FSE_readNCount(short *normalizedCounter, unsigned *maxSymbolValuePtr, unsigned *tableLogPtr, const void *rBuffer, size_t rBuffSize); |
|
|
|
/*! Constructor and Destructor of FSE_DTable. |
|
Note that its size depends on 'tableLog' */ |
|
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ |
|
|
|
/*! FSE_buildDTable(): |
|
Builds 'dt', which must be already allocated, using FSE_createDTable(). |
|
return : 0, or an errorCode, which can be tested using FSE_isError() */ |
|
FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize); |
|
|
|
/*! FSE_decompress_usingDTable(): |
|
Decompress compressed source `cSrc` of size `cSrcSize` using `dt` |
|
into `dst` which must be already allocated. |
|
@return : size of regenerated data (necessarily <= `dstCapacity`), |
|
or an errorCode, which can be tested using FSE_isError() */ |
|
FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt); |
|
|
|
/*! |
|
Tutorial : |
|
---------- |
|
(Note : these functions only decompress FSE-compressed blocks. |
|
If block is uncompressed, use memcpy() instead |
|
If block is a single repeated byte, use memset() instead ) |
|
|
|
The first step is to obtain the normalized frequencies of symbols. |
|
This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). |
|
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. |
|
In practice, that means it's necessary to know 'maxSymbolValue' beforehand, |
|
or size the table to handle worst case situations (typically 256). |
|
FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. |
|
The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. |
|
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. |
|
If there is an error, the function will return an error code, which can be tested using FSE_isError(). |
|
|
|
The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. |
|
This is performed by the function FSE_buildDTable(). |
|
The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). |
|
If there is an error, the function will return an error code, which can be tested using FSE_isError(). |
|
|
|
`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). |
|
`cSrcSize` must be strictly correct, otherwise decompression will fail. |
|
FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). |
|
If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) |
|
*/ |
|
|
|
/* *** Dependency *** */ |
|
#include "bitstream.h" |
|
|
|
/* ***************************************** |
|
* Static allocation |
|
*******************************************/ |
|
/* FSE buffer bounds */ |
|
#define FSE_NCOUNTBOUND 512 |
|
#define FSE_BLOCKBOUND(size) (size + (size >> 7)) |
|
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ |
|
|
|
/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ |
|
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1 << (maxTableLog - 1)) + ((maxSymbolValue + 1) * 2)) |
|
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1 << maxTableLog)) |
|
|
|
/* ***************************************** |
|
* FSE advanced API |
|
*******************************************/ |
|
/* FSE_count_wksp() : |
|
* Same as FSE_count(), but using an externally provided scratch buffer. |
|
* `workSpace` size must be table of >= `1024` unsigned |
|
*/ |
|
size_t FSE_count_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *source, size_t sourceSize, unsigned *workSpace); |
|
|
|
/* FSE_countFast_wksp() : |
|
* Same as FSE_countFast(), but using an externally provided scratch buffer. |
|
* `workSpace` must be a table of minimum `1024` unsigned |
|
*/ |
|
size_t FSE_countFast_wksp(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize, unsigned *workSpace); |
|
|
|
/*! FSE_count_simple |
|
* Same as FSE_countFast(), but does not use any additional memory (not even on stack). |
|
* This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`). |
|
*/ |
|
size_t FSE_count_simple(unsigned *count, unsigned *maxSymbolValuePtr, const void *src, size_t srcSize); |
|
|
|
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); |
|
/**< same as FSE_optimalTableLog(), which used `minus==2` */ |
|
|
|
size_t FSE_buildCTable_raw(FSE_CTable *ct, unsigned nbBits); |
|
/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ |
|
|
|
size_t FSE_buildCTable_rle(FSE_CTable *ct, unsigned char symbolValue); |
|
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ |
|
|
|
/* FSE_buildCTable_wksp() : |
|
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). |
|
* `wkspSize` must be >= `(1<<tableLog)`. |
|
*/ |
|
size_t FSE_buildCTable_wksp(FSE_CTable *ct, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workSpace, size_t wkspSize); |
|
|
|
size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits); |
|
/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ |
|
|
|
size_t FSE_buildDTable_rle(FSE_DTable *dt, unsigned char symbolValue); |
|
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ |
|
|
|
size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize); |
|
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DTABLE_SIZE_U32(maxLog)` */ |
|
|
|
/* ***************************************** |
|
* FSE symbol compression API |
|
*******************************************/ |
|
/*! |
|
This API consists of small unitary functions, which highly benefit from being inlined. |
|
Hence their body are included in next section. |
|
*/ |
|
typedef struct { |
|
ptrdiff_t value; |
|
const void *stateTable; |
|
const void *symbolTT; |
|
unsigned stateLog; |
|
} FSE_CState_t; |
|
|
|
static void FSE_initCState(FSE_CState_t *CStatePtr, const FSE_CTable *ct); |
|
|
|
static void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *CStatePtr, unsigned symbol); |
|
|
|
static void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *CStatePtr); |
|
|
|
/**< |
|
These functions are inner components of FSE_compress_usingCTable(). |
|
They allow the creation of custom streams, mixing multiple tables and bit sources. |
|
|
|
A key property to keep in mind is that encoding and decoding are done **in reverse direction**. |
|
So the first symbol you will encode is the last you will decode, like a LIFO stack. |
|
|
|
You will need a few variables to track your CStream. They are : |
|
|
|
FSE_CTable ct; // Provided by FSE_buildCTable() |
|
BIT_CStream_t bitStream; // bitStream tracking structure |
|
FSE_CState_t state; // State tracking structure (can have several) |
|
|
|
|
|
The first thing to do is to init bitStream and state. |
|
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); |
|
FSE_initCState(&state, ct); |
|
|
|
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); |
|
You can then encode your input data, byte after byte. |
|
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. |
|
Remember decoding will be done in reverse direction. |
|
FSE_encodeByte(&bitStream, &state, symbol); |
|
|
|
At any time, you can also add any bit sequence. |
|
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders |
|
BIT_addBits(&bitStream, bitField, nbBits); |
|
|
|
The above methods don't commit data to memory, they just store it into local register, for speed. |
|
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). |
|
Writing data to memory is a manual operation, performed by the flushBits function. |
|
BIT_flushBits(&bitStream); |
|
|
|
Your last FSE encoding operation shall be to flush your last state value(s). |
|
FSE_flushState(&bitStream, &state); |
|
|
|
Finally, you must close the bitStream. |
|
The function returns the size of CStream in bytes. |
|
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) |
|
If there is an error, it returns an errorCode (which can be tested using FSE_isError()). |
|
size_t size = BIT_closeCStream(&bitStream); |
|
*/ |
|
|
|
/* ***************************************** |
|
* FSE symbol decompression API |
|
*******************************************/ |
|
typedef struct { |
|
size_t state; |
|
const void *table; /* precise table may vary, depending on U16 */ |
|
} FSE_DState_t; |
|
|
|
static void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt); |
|
|
|
static unsigned char FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD); |
|
|
|
static unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr); |
|
|
|
/**< |
|
Let's now decompose FSE_decompress_usingDTable() into its unitary components. |
|
You will decode FSE-encoded symbols from the bitStream, |
|
and also any other bitFields you put in, **in reverse order**. |
|
|
|
You will need a few variables to track your bitStream. They are : |
|
|
|
BIT_DStream_t DStream; // Stream context |
|
FSE_DState_t DState; // State context. Multiple ones are possible |
|
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() |
|
|
|
The first thing to do is to init the bitStream. |
|
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); |
|
|
|
You should then retrieve your initial state(s) |
|
(in reverse flushing order if you have several ones) : |
|
errorCode = FSE_initDState(&DState, &DStream, DTablePtr); |
|
|
|
You can then decode your data, symbol after symbol. |
|
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. |
|
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). |
|
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); |
|
|
|
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) |
|
Note : maximum allowed nbBits is 25, for 32-bits compatibility |
|
size_t bitField = BIT_readBits(&DStream, nbBits); |
|
|
|
All above operations only read from local register (which size depends on size_t). |
|
Refueling the register from memory is manually performed by the reload method. |
|
endSignal = FSE_reloadDStream(&DStream); |
|
|
|
BIT_reloadDStream() result tells if there is still some more data to read from DStream. |
|
BIT_DStream_unfinished : there is still some data left into the DStream. |
|
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. |
|
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. |
|
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. |
|
|
|
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, |
|
to properly detect the exact end of stream. |
|
After each decoded symbol, check if DStream is fully consumed using this simple test : |
|
BIT_reloadDStream(&DStream) >= BIT_DStream_completed |
|
|
|
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. |
|
Checking if DStream has reached its end is performed by : |
|
BIT_endOfDStream(&DStream); |
|
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. |
|
FSE_endOfDState(&DState); |
|
*/ |
|
|
|
/* ***************************************** |
|
* FSE unsafe API |
|
*******************************************/ |
|
static unsigned char FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD); |
|
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ |
|
|
|
/* ***************************************** |
|
* Implementation of inlined functions |
|
*******************************************/ |
|
typedef struct { |
|
int deltaFindState; |
|
U32 deltaNbBits; |
|
} FSE_symbolCompressionTransform; /* total 8 bytes */ |
|
|
|
ZSTD_STATIC void FSE_initCState(FSE_CState_t *statePtr, const FSE_CTable *ct) |
|
{ |
|
const void *ptr = ct; |
|
const U16 *u16ptr = (const U16 *)ptr; |
|
const U32 tableLog = ZSTD_read16(ptr); |
|
statePtr->value = (ptrdiff_t)1 << tableLog; |
|
statePtr->stateTable = u16ptr + 2; |
|
statePtr->symbolTT = ((const U32 *)ct + 1 + (tableLog ? (1 << (tableLog - 1)) : 1)); |
|
statePtr->stateLog = tableLog; |
|
} |
|
|
|
/*! FSE_initCState2() : |
|
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) |
|
* uses the smallest state value possible, saving the cost of this symbol */ |
|
ZSTD_STATIC void FSE_initCState2(FSE_CState_t *statePtr, const FSE_CTable *ct, U32 symbol) |
|
{ |
|
FSE_initCState(statePtr, ct); |
|
{ |
|
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol]; |
|
const U16 *stateTable = (const U16 *)(statePtr->stateTable); |
|
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1 << 15)) >> 16); |
|
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; |
|
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; |
|
} |
|
} |
|
|
|
ZSTD_STATIC void FSE_encodeSymbol(BIT_CStream_t *bitC, FSE_CState_t *statePtr, U32 symbol) |
|
{ |
|
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform *)(statePtr->symbolTT))[symbol]; |
|
const U16 *const stateTable = (const U16 *)(statePtr->stateTable); |
|
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); |
|
BIT_addBits(bitC, statePtr->value, nbBitsOut); |
|
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; |
|
} |
|
|
|
ZSTD_STATIC void FSE_flushCState(BIT_CStream_t *bitC, const FSE_CState_t *statePtr) |
|
{ |
|
BIT_addBits(bitC, statePtr->value, statePtr->stateLog); |
|
BIT_flushBits(bitC); |
|
} |
|
|
|
/* ====== Decompression ====== */ |
|
|
|
typedef struct { |
|
U16 tableLog; |
|
U16 fastMode; |
|
} FSE_DTableHeader; /* sizeof U32 */ |
|
|
|
typedef struct { |
|
unsigned short newState; |
|
unsigned char symbol; |
|
unsigned char nbBits; |
|
} FSE_decode_t; /* size == U32 */ |
|
|
|
ZSTD_STATIC void FSE_initDState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD, const FSE_DTable *dt) |
|
{ |
|
const void *ptr = dt; |
|
const FSE_DTableHeader *const DTableH = (const FSE_DTableHeader *)ptr; |
|
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); |
|
BIT_reloadDStream(bitD); |
|
DStatePtr->table = dt + 1; |
|
} |
|
|
|
ZSTD_STATIC BYTE FSE_peekSymbol(const FSE_DState_t *DStatePtr) |
|
{ |
|
FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; |
|
return DInfo.symbol; |
|
} |
|
|
|
ZSTD_STATIC void FSE_updateState(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) |
|
{ |
|
FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; |
|
U32 const nbBits = DInfo.nbBits; |
|
size_t const lowBits = BIT_readBits(bitD, nbBits); |
|
DStatePtr->state = DInfo.newState + lowBits; |
|
} |
|
|
|
ZSTD_STATIC BYTE FSE_decodeSymbol(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) |
|
{ |
|
FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; |
|
U32 const nbBits = DInfo.nbBits; |
|
BYTE const symbol = DInfo.symbol; |
|
size_t const lowBits = BIT_readBits(bitD, nbBits); |
|
|
|
DStatePtr->state = DInfo.newState + lowBits; |
|
return symbol; |
|
} |
|
|
|
/*! FSE_decodeSymbolFast() : |
|
unsafe, only works if no symbol has a probability > 50% */ |
|
ZSTD_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t *DStatePtr, BIT_DStream_t *bitD) |
|
{ |
|
FSE_decode_t const DInfo = ((const FSE_decode_t *)(DStatePtr->table))[DStatePtr->state]; |
|
U32 const nbBits = DInfo.nbBits; |
|
BYTE const symbol = DInfo.symbol; |
|
size_t const lowBits = BIT_readBitsFast(bitD, nbBits); |
|
|
|
DStatePtr->state = DInfo.newState + lowBits; |
|
return symbol; |
|
} |
|
|
|
ZSTD_STATIC unsigned FSE_endOfDState(const FSE_DState_t *DStatePtr) { return DStatePtr->state == 0; } |
|
|
|
/* ************************************************************** |
|
* Tuning parameters |
|
****************************************************************/ |
|
/*!MEMORY_USAGE : |
|
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) |
|
* Increasing memory usage improves compression ratio |
|
* Reduced memory usage can improve speed, due to cache effect |
|
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ |
|
#ifndef FSE_MAX_MEMORY_USAGE |
|
#define FSE_MAX_MEMORY_USAGE 14 |
|
#endif |
|
#ifndef FSE_DEFAULT_MEMORY_USAGE |
|
#define FSE_DEFAULT_MEMORY_USAGE 13 |
|
#endif |
|
|
|
/*!FSE_MAX_SYMBOL_VALUE : |
|
* Maximum symbol value authorized. |
|
* Required for proper stack allocation */ |
|
#ifndef FSE_MAX_SYMBOL_VALUE |
|
#define FSE_MAX_SYMBOL_VALUE 255 |
|
#endif |
|
|
|
/* ************************************************************** |
|
* template functions type & suffix |
|
****************************************************************/ |
|
#define FSE_FUNCTION_TYPE BYTE |
|
#define FSE_FUNCTION_EXTENSION |
|
#define FSE_DECODE_TYPE FSE_decode_t |
|
|
|
/* *************************************************************** |
|
* Constants |
|
*****************************************************************/ |
|
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE - 2) |
|
#define FSE_MAX_TABLESIZE (1U << FSE_MAX_TABLELOG) |
|
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE - 1) |
|
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE - 2) |
|
#define FSE_MIN_TABLELOG 5 |
|
|
|
#define FSE_TABLELOG_ABSOLUTE_MAX 15 |
|
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX |
|
#error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" |
|
#endif |
|
|
|
#define FSE_TABLESTEP(tableSize) ((tableSize >> 1) + (tableSize >> 3) + 3) |
|
|
|
#endif /* FSE_H */
|
|
|