Add support for Huffman compressed GPMF streams

yoshiyama · Jan 11, 2019 · 3b9424a · 3b9424a
1 parent 10c282b
commit 3b9424a
Show file tree

Hide file tree

Showing 4 changed files with 851 additions and 117 deletions.
diff --git a/GPMF_bitstream.h b/GPMF_bitstream.h
@@ -0,0 +1,153 @@
+/*! @file GPMF_bitstream.h
+*
+*  @brief GPMF Parser library include
+* 
+*  Some GPMF streams may contain compressed data, this is useful for high frequency 
+*  sensor data that is highly correlated like IMU data.  The compression is Huffman 
+*  coding of the delta between samples, with addition codewords for runs of zeros, 
+*  and optional quantization. The compression scheme is similar to the Huffman coding 
+*  in JPEG. As it intended for lossless compression (with quantize set to 1) it can 
+*  only comrpess/decompress integer based streams.  
+*
+*  @version 1.2.0
+*
+*  (C) Copyright 2017 GoPro Inc (http://gopro.com/).
+*
+*  Licensed under either:
+*  - Apache License, Version 2.0, http://www.apache.org/licenses/LICENSE-2.0
+*  - MIT license, http://opensource.org/licenses/MIT
+*  at your option.
+*
+*  Unless required by applicable law or agreed to in writing, software
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*
+*/
+
+#ifndef _GPMF_BITSTREAM_H
+#define _GPMF_BITSTREAM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+typedef struct rlv {	// Codebook entries for arbitrary runs
+	uint16_t size;		// Size of code word in bits
+	uint16_t bits;		// Code word bits right justified
+	uint16_t count;		// Run length for zeros
+	int16_t  value;		// Value for difference
+} RLV;
+
+typedef const struct {
+	int length;			// Number of entries in the code book
+	RLV entries[39];
+} RLVTABLE;
+
+#define BITSTREAM_WORD_TYPE			uint16_t		// use 16-bit buffer for compression
+#define BITSTREAM_WORD_SIZE			16				// use 16-bit buffer for compression
+#define BITSTREAM_ERROR_OVERFLOW	1
+
+#define BITMASK(n)		_bitmask[n]
+#define _BITMASK(n)		((((BITSTREAM_WORD_TYPE )1 << (n))) - 1)
+
+static const BITSTREAM_WORD_TYPE  _bitmask[] =
+{
+	_BITMASK(0),  _BITMASK(1),  _BITMASK(2),  _BITMASK(3),
+	_BITMASK(4),  _BITMASK(5),  _BITMASK(6),  _BITMASK(7),
+	_BITMASK(8),  _BITMASK(9),  _BITMASK(10), _BITMASK(11),
+	_BITMASK(12), _BITMASK(13), _BITMASK(14), _BITMASK(15),
+	0xFFFF
+};
+
+
+
+typedef struct bitstream
+{
+	int32_t error;				// Error parsing the bitstream
+	int32_t bitsFree;			// Number of bits available in the current word
+	uint8_t *lpCurrentWord;		// Pointer to next word in block
+	int32_t wordsUsed;			// Number of words used in the block
+	int32_t dwBlockLength;		// Number of entries in the block
+	uint16_t wBuffer;			// Current word bit buffer
+	uint16_t bits_per_src_word;	// Bitused in the source word. e.g. 's' = 16-bits
+} BITSTREAM;
+
+
+static RLVTABLE enchuftable = {
+	39,
+	{
+	  { 1, 0b0,   1,  0 },	// m0
+	  { 2, 0b10,   1,  1 },	// m1
+	  { 4, 0b1100,   1,  2 },	// m2
+	  { 5, 0b11011,   1,  3 },	// m3
+	  { 5, 0b11101,   1,  4 },	// m4
+	  { 6, 0b110100,   1,  5 },	// m5
+	  { 6, 0b110101,   1,  6 },	// m6
+	  { 6, 0b111110,   1,  7 },	// m7
+	  { 7, 0b1110000,   1,  8 },	// m8
+	  { 7, 0b1110011,   1,  9 },	// m9
+	  { 7, 0b1111000,   1, 10 },	// m10
+	  { 7, 0b1111001,   1, 11 },	// m11
+	  { 7, 0b1111011,   1, 12 },	// m12
+	  { 8, 0b11100100,   1, 13 },	// m13
+	  { 8, 0b11100101,   1, 14 },	// m14
+	  { 8, 0b11110100,   1, 15 },	// m15
+	  { 9, 0b111000101,   1, 16 },	// m16
+	  { 9, 0b111000110,   1, 17 },	// m17
+	  { 9, 0b111101010,   1, 18 },	// m18
+	  { 10, 0b1110001000,   1, 19 },	// m19
+	  { 10, 0b1110001110,   1, 20 },	// m20
+	  { 10, 0b1111010110,   1, 21 },	// m21
+	  { 10, 0b1111111100,   1, 22 },	// m22
+	  { 11, 0b11100010010,   1, 23 },	// m23
+	  { 11, 0b11100011111,   1, 24 },	// m24
+	  { 11, 0b11110101110,   1, 25 },	// m25
+	  { 12, 0b111000100111,   1, 26 },	// m26
+	  { 12, 0b111000111101,   1, 27 },	// m27
+	  { 12, 0b111101011111,   1, 28 },	// m28
+	  { 13, 0b1110001001101,   1, 29 },	// m29
+	  { 13, 0b1110001111001,   1, 30 },	// m30
+	  { 13, 0b1111010111101,   1, 31 },	// m31
+	  { 14, 0b11100010011000,   1, 32 },	// m32
+	  { 14, 0b11100011110000,   1, 33 },	// m33
+	  { 14, 0b11110101111000,   1, 34 },	// m34
+	  { 14, 0b11110101111001,   1, 35 },	// m35
+	  { 15, 0b111000100110010,   1, 36 },	// m36
+	  { 15, 0b111000100110011,   1, 37 },	// m37
+	  { 15, 0b111000111100011,   1, 38 },	// m38
+	}		  
+};			  
+
+
+
+static RLVTABLE enczerorunstable = {
+	4,
+	{
+		{ 7, 0b1111110,  16,  0 },		// z16
+		{ 8, 0b11111110,  32,  0 },		// z32
+		{ 9, 0b111111111,  64,  0 },	// z64
+		{ 10,0b1111111101, 128,  0 },	// z128
+	}
+};	
+
+#define HUFF_ESC_CODE_ENTRY		0	
+#define HUFF_END_CODE_ENTRY		1	
+static RLVTABLE enccontrolcodestable = {
+	2,
+	{
+		{ 16, 0b1110001111000100, 0, 0 },	// escape code for direct data <ESC><data>Continue
+		{ 16, 0b1110001111000101, 0, 0 },	// end code.  Ends each compressed stream
+	}
+};
+
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/GPMF_common.h b/GPMF_common.h
@@ -0,0 +1,130 @@
+/*! @file GPMF_parser.h
+ * 
+ *  @brief GPMF Parser library include
+ * 
+ *  @version 1.1.0
+ * 
+ *  (C) Copyright 2017 GoPro Inc (http://gopro.com/).
+ *
+ *  Licensed under either:
+ *  - Apache License, Version 2.0, http://www.apache.org/licenses/LICENSE-2.0  
+ *  - MIT license, http://opensource.org/licenses/MIT
+ *  at your option.
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * 
+ */
+
+#ifndef _GPMF_COMMON_H
+#define _GPMF_COMMON_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum GPMF_ERROR
+{
+	GPMF_OK = 0,
+	GPMF_ERROR_MEMORY,
+	GPMF_ERROR_BAD_STRUCTURE,
+	GPMF_ERROR_BUFFER_END,
+	GPMF_ERROR_FIND,
+	GPMF_ERROR_LAST,
+	GPMF_ERROR_TYPE_NOT_SUPPORTED,
+	GPMF_ERROR_SCALE_NOT_SUPPORTED,
+	GPMF_ERROR_SCALE_COUNT,
+	GPMF_ERROR_RESERVED
+} GPMF_ERROR;
+
+#define GPMF_ERR	uint32_t
+
+typedef enum
+{
+	GPMF_TYPE_STRING_ASCII = 'c', //single byte 'c' style character string
+	GPMF_TYPE_SIGNED_BYTE = 'b',//single byte signed number
+	GPMF_TYPE_UNSIGNED_BYTE = 'B', //single byte unsigned number
+	GPMF_TYPE_SIGNED_SHORT = 's',//16-bit integer
+	GPMF_TYPE_UNSIGNED_SHORT = 'S',//16-bit integer
+	GPMF_TYPE_FLOAT = 'f', //32-bit single precision float (IEEE 754)
+	GPMF_TYPE_FOURCC = 'F', //32-bit four character tag 
+	GPMF_TYPE_SIGNED_LONG = 'l',//32-bit integer
+	GPMF_TYPE_UNSIGNED_LONG = 'L', //32-bit integer
+	GPMF_TYPE_Q15_16_FIXED_POINT = 'q', // Q number Q15.16 - 16-bit signed integer (A) with 16-bit fixed point (B) for A.B value (range -32768.0 to 32767.99998). 
+	GPMF_TYPE_Q31_32_FIXED_POINT = 'Q', // Q number Q31.32 - 32-bit signed integer (A) with 32-bit fixed point (B) for A.B value. 
+	GPMF_TYPE_SIGNED_64BIT_INT = 'j', //64 bit signed long
+	GPMF_TYPE_UNSIGNED_64BIT_INT = 'J', //64 bit unsigned long	
+	GPMF_TYPE_DOUBLE = 'd', //64 bit double precision float (IEEE 754)
+	GPMF_TYPE_UTC_DATE_TIME = 'U', //128-bit ASCII Date + UTC Time format yymmddhhmmss.sss - 16 bytes ASCII (years 20xx covered)
+	GPMF_TYPE_GUID = 'G', //128-bit ID (like UUID)
+
+	GPMF_TYPE_COMPLEX = '?', //for sample with complex data structures, base size in bytes.  Data is either opaque, or the stream has a TYPE structure field for the sample.
+	GPMF_TYPE_COMPRESSED = '#', //Huffman compression STRM payloads.  4-CC <type><size><rpt> <data ...> is compressed as 4-CC '#'<new size/rpt> <type><size><rpt> <compressed data ...>
+
+	GPMF_TYPE_NEST = 0, // used to nest more GPMF formatted metadata 
+
+} GPMF_SampleType;
+
+
+
+#define MAKEID(a,b,c,d)			(((d&0xff)<<24)|((c&0xff)<<16)|((b&0xff)<<8)|(a&0xff))
+#define STR2FOURCC(s)			((s[0]<<0)|(s[1]<<8)|(s[2]<<16)|(s[3]<<24))
+
+#define BYTESWAP64(a)			(((a&0xff)<<56)|((a&0xff00)<<40)|((a&0xff0000)<<24)|((a&0xff000000)<<8) | ((a>>56)&0xff)|((a>>40)&0xff00)|((a>>24)&0xff0000)|((a>>8)&0xff000000) )
+#define BYTESWAP32(a)			(((a&0xff)<<24)|((a&0xff00)<<8)|((a>>8)&0xff00)|((a>>24)&0xff))
+#define BYTESWAP16(a)			((((a)>>8)&0xff)|(((a)<<8)&0xff00))
+#define BYTESWAPin32(a,s)		((s>=4)?(((a&0xff)<<24)|((a&0xff00)<<8)|((a>>8)&0xff00)|((a>>24)&0xff)):((s==2)?(((a>>8)&0xff)|((a<<8)&0xff00)|((a>>8)&0xff0000)|((a<<8)&0xff000000)):(a)))
+#define NOSWAP8(a)				(a)
+
+#define GPMF_SAMPLES(a)			(((a>>24) & 0xff)|(((a>>16)&0xff)<<8))
+#define GPMF_SAMPLE_SIZE(a)		(((a)>>8)&0xff)
+#define GPMF_SAMPLE_TYPE(a)		(a&0xff)
+#define GPMF_MAKE_TYPE_SIZE_COUNT(t,s,c)		((t)&0xff)|(((s)&0xff)<<8)|(((c)&0xff)<<24)|(((c)&0xff00)<<8)
+#define GPMF_DATA_SIZE(a)		((GPMF_SAMPLE_SIZE(a)*GPMF_SAMPLES(a)+3)&~0x3)
+#define GPMF_DATA_PACKEDSIZE(a)	((GPMF_SAMPLE_SIZE(a)*GPMF_SAMPLES(a)))
+#define GPMF_VALID_FOURCC(a)	(((((a>>24)&0xff)>='a'&&((a>>24)&0xff)<='z') || (((a>>24)&0xff)>='A'&&((a>>24)&0xff)<='Z') || (((a>>24)&0xff)>='0'&&((a>>24)&0xff)<='9') || (((a>>24)&0xff)==' ') ) && \
+								( (((a>>16)&0xff)>='a'&&((a>>24)&0xff)<='z') || (((a>>16)&0xff)>='A'&&((a>>16)&0xff)<='Z') || (((a>>16)&0xff)>='0'&&((a>>16)&0xff)<='9') || (((a>>16)&0xff)==' ') ) && \
+								( (((a>>8)&0xff)>='a'&&((a>>24)&0xff)<='z') || (((a>>8)&0xff)>='A'&&((a>>8)&0xff)<='Z') || (((a>>8)&0xff)>='0'&&((a>>8)&0xff)<='9') || (((a>>8)&0xff)==' ') ) && \
+								( (((a>>0)&0xff)>='a'&&((a>>24)&0xff)<='z') || (((a>>0)&0xff)>='A'&&((a>>0)&0xff)<='Z') || (((a>>0)&0xff)>='0'&&((a>>0)&0xff)<='9') || (((a>>0)&0xff)==' ') )) 
+#define GPMF_KEY_TYPE(a)		(a&0xff)
+
+#define PRINTF_4CC(k)			((k) >> 0) & 0xff, ((k) >> 8) & 0xff, ((k) >> 16) & 0xff, ((k) >> 24) & 0xff
+
+
+typedef enum GPMFKey // TAG in all caps are GoPro preserved (are defined by GoPro, but can be used by others.)
+{
+	// Internal Metadata structure and formatting tags
+	GPMF_KEY_DEVICE =			MAKEID('D','E','V','C'),//DEVC - nested device data to speed the parsing of multiple devices in post 
+	GPMF_KEY_DEVICE_ID =		MAKEID('D','V','I','D'),//DVID - unique id per stream for a metadata source (in camera or external input) (single 4 byte int)
+	GPMF_KEY_DEVICE_NAME =		MAKEID('D','V','N','M'),//DVNM - human readable device type/name (char string)
+	GPMF_KEY_STREAM =			MAKEID('S','T','R','M'),//STRM - nested channel/stream of telemetry data
+	GPMF_KEY_STREAM_NAME =		MAKEID('S','T','N','M'),//STNM - human readable telemetry/metadata stream type/name (char string)
+	GPMF_KEY_SI_UNITS =			MAKEID('S','I','U','N'),//SIUN - Display string for metadata units where inputs are in SI units "uT","rad/s","km/s","m/s","mm/s" etc.
+	GPMF_KEY_UNITS =			MAKEID('U','N','I','T'),//UNIT - Freedform display string for metadata units (char sting like "RPM", "MPH", "km/h", etc)
+	GPMF_KEY_SCALE =			MAKEID('S','C','A','L'),//SCAL - divisor for input data to scale to the correct units.
+	GPMF_KEY_TYPE =				MAKEID('T','Y','P','E'),//TYPE - Type define for complex data structures
+	GPMF_KEY_TOTAL_SAMPLES =	MAKEID('T','S','M','P'),//TOTL - Total Sample Count including the current payload 	
+	GPMF_KEY_TICK =				MAKEID('T','I','C','K'),//TICK - Beginning of data timing (arrival) in milliseconds. 
+	GPMF_KEY_TOCK =				MAKEID('T','O','C','K'),//TOCK - End of data timing (arrival)  in milliseconds. 
+	GPMF_KEY_TIME_STAMP =		MAKEID('S','T','M','P'),//STMP - Time stamp at the source in microseconds for the first sample. 
+	GPMF_KEY_TIME_STAMPS =		MAKEID('S','T','P','S'),//STPS - Stream of all the timestamps delivered. 
+	GPMF_KEY_PREFORMATTED =		MAKEID('P','F','R','M'),//PFRM - GPMF data
+	GPMF_KEY_TEMPERATURE_C =	MAKEID('T','M','P','C'),//TMPC - Temperature in Celsius
+	GPMF_KEY_EMPTY_PAYLOADS =	MAKEID('E','M','P','T'),//EMPT - Payloads that are empty since the device start (e.g. BLE disconnect.)
+	GPMF_KEY_VERSION =			MAKEID('V','E','R','S'),//VERS - version of the metadata stream (debugging)
+	GPMF_KEY_FREESPACE =		MAKEID('F','R','E','E'),//FREE - n bytes reserved for more metadata added to an existing stream
+	GPMF_KEY_REMARK =			MAKEID('R','M','R','K'),//RMRK - adding comments to the bitstream (debugging)
+
+	GPMF_KEY_END = 0//(null)
+} GPMFKey;
+
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif