From cf27b8d592db321e5ab3c1db0bf8caa36d8bc79e Mon Sep 17 00:00:00 2001 From: oikomi Date: Sun, 10 Jan 2016 21:58:47 -0500 Subject: [PATCH] add supper_block parse --- store/tools/supper_block_parse/compile.sh | 3 + store/tools/supper_block_parse/const.go | 25 ++ .../tools/supper_block_parse/supper_block.go | 218 ++++++++++++++++++ .../supper_block_parse/supper_block_parse.go | 48 ++++ store/tools/supper_block_parse/util.go | 110 +++++++++ 5 files changed, 404 insertions(+) create mode 100755 store/tools/supper_block_parse/compile.sh create mode 100755 store/tools/supper_block_parse/const.go create mode 100755 store/tools/supper_block_parse/supper_block.go create mode 100755 store/tools/supper_block_parse/supper_block_parse.go create mode 100755 store/tools/supper_block_parse/util.go diff --git a/store/tools/supper_block_parse/compile.sh b/store/tools/supper_block_parse/compile.sh new file mode 100755 index 0000000..905ad2d --- /dev/null +++ b/store/tools/supper_block_parse/compile.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +go build -o supper_block_parse \ No newline at end of file diff --git a/store/tools/supper_block_parse/const.go b/store/tools/supper_block_parse/const.go new file mode 100755 index 0000000..b8753ac --- /dev/null +++ b/store/tools/supper_block_parse/const.go @@ -0,0 +1,25 @@ +package main + +const ( + SUPPER_BLOCK_MAGIC_SIZE = 4 + SUPPER_BLOCK_VER_SIZE = 1 + SUPPER_BLOCK_PADDING_SIZE = 3 + + NEEDLE_HEADER_MAGIC_SIZE = 4 + NEEDLE_COOKIE_SIZE = 4 + NEEDLE_KEY_SIZE = 8 + NEEDLE_FLAG_SIZE = 1 + NEEDLE_DATA_SIZE = 4 + NEEDLE_FOOTER_MAGIC_SIZE = 4 + NEEDLE_CHECKSUM_SIZE = 4 + +) + +const ( + SUPPER_BLOCK_MAGIC_STR = "[0xAB 0xCD 0xEF 0x00]" + SUPPER_BLOCK_VER_STR = "[0x01]" + SUPPER_BLOCK_PADDING_STR = "[0x00 0x00 0x00]" + + NEEDLE_HEADER_MAGIC_STR = "[0x12 0x34 0x56 0x78]" + NEEDLE_FOOTER_MAGIC_STR = "[0x87 0x65 0x43 0x21]" +) diff --git a/store/tools/supper_block_parse/supper_block.go b/store/tools/supper_block_parse/supper_block.go new file mode 100755 index 0000000..93423b4 --- /dev/null +++ b/store/tools/supper_block_parse/supper_block.go @@ -0,0 +1,218 @@ +package main + +import ( + "os" + "io" + "fmt" + "bytes" + "bufio" +) + +var ( + MAGIC = []byte{0xab, 0xcd, 0xef, 0x00} + VER = []byte{byte(1)} + PADDING = bytes.Repeat([]byte{byte(0)}, 3) + + NEEDLE_HEADER_MAGIC = []byte{0x12, 0x34, 0x56, 0x78} + NEEDLE_FOOTER_MAGIC = []byte{0x87, 0x65, 0x43, 0x21} +) + +type NeedleData struct { + NeedleHeaderMagic string `json:"needle_header_magic"` + NeedleCookie int32 `json:"needle_cookie"` + NeedleKey int64 `json:"needle_key"` + NeedleFlag int64 `json:"needle_flag"` + NeedleDataSize int32 `json:"needle_data_size"` + NeedleFooterMagic string `json:"needle_footer_magic"` + NeedleCheckSum int32 `json:"needle_checksum"` +} + +type SuperBlockHeader struct { + Magic string `json:"magic"` + Ver string `json:"ver"` + Padding string `json:"padding"` +} + +type SuperBlock struct { + fd *os.File + r *bufio.Reader + offset int64 + + SuperBlockHeader *SuperBlockHeader `json:"supper_block_header"` + + NeedleNum int64 `json:"needle_num"` + + NeedleDetail []*NeedleData `json:"needle_detail"` +} + + +func NewSuperBlock() *SuperBlock { + return &SuperBlock { + SuperBlockHeader : new(SuperBlockHeader), + NeedleDetail : make([]*NeedleData, 0), + } +} + +func (s *SuperBlock)parseSupperBlockHeader() (err error) { + var ( + n int + supperBlockMagicBuf []byte = make([]byte, SUPPER_BLOCK_MAGIC_SIZE) + supperBlockVerBuf []byte = make([]byte, SUPPER_BLOCK_VER_SIZE) + supperBlockPaddingBuf []byte = make([]byte, SUPPER_BLOCK_PADDING_SIZE) + ) + + n, err = s.fd.Read(supperBlockMagicBuf) + if err != nil { + return + } + s.offset += int64(n) + + if !bytes.Equal(supperBlockMagicBuf, MAGIC) { + fmt.Println("Magic not match") + return + } else { + s.SuperBlockHeader.Magic = SUPPER_BLOCK_MAGIC_STR + } + + n, err = s.fd.Read(supperBlockVerBuf) + if err != nil { + return + } + s.offset += int64(n) + + if !bytes.Equal(supperBlockVerBuf, VER) { + fmt.Println("ver not match") + return + } else { + s.SuperBlockHeader.Ver = SUPPER_BLOCK_VER_STR + } + + n, err = s.fd.Read(supperBlockPaddingBuf) + if err != nil { + return + } + s.offset += int64(n) + + if !bytes.Equal(supperBlockPaddingBuf, PADDING) { + fmt.Println("ver not match") + return + } else { + s.SuperBlockHeader.Padding = SUPPER_BLOCK_PADDING_STR + } + + return +} + +func (s *SuperBlock)parseNeedles() (err error) { + var ( + n int + needleHeaderMagicBuf []byte = make([]byte, NEEDLE_HEADER_MAGIC_SIZE) + needleCookieBuf []byte = make([]byte, NEEDLE_COOKIE_SIZE) + needleKeyBuf []byte = make([]byte, NEEDLE_KEY_SIZE) + needleFlagBuf []byte = make([]byte, NEEDLE_FLAG_SIZE) + needleDataBuf []byte = make([]byte, NEEDLE_DATA_SIZE) + needleFooterMagicBuf []byte = make([]byte, NEEDLE_FOOTER_MAGIC_SIZE) + needleCheckSumBuf []byte = make([]byte, NEEDLE_CHECKSUM_SIZE) + ) + + for { + needle := new(NeedleData) + + n, err = s.fd.Read(needleHeaderMagicBuf) + if err != nil || err == io.EOF { + return + } + s.offset += int64(n) + + if !bytes.Equal(needleHeaderMagicBuf, NEEDLE_HEADER_MAGIC) { + fmt.Println("header magic not match") + break + } else { + needle.NeedleHeaderMagic = NEEDLE_HEADER_MAGIC_STR + } + + n, err = s.fd.Read(needleCookieBuf) + if err != nil || err == io.EOF { + return + } else { + needle.NeedleCookie = Byte4ToInt32(needleCookieBuf, BigEndian) + } + s.offset += int64(n) + + n, err = s.fd.Read(needleKeyBuf) + if err != nil || err == io.EOF { + return + } else { + needle.NeedleKey = Byte8ToInt64(needleKeyBuf, BigEndian) + } + s.offset += int64(n) + + n, err = s.fd.Read(needleFlagBuf) + if err != nil || err == io.EOF { + return + } else { + needle.NeedleFlag = Byte2Int(needleFlagBuf) + } + s.offset += int64(n) + + n, err = s.fd.Read(needleDataBuf) + if err != nil || err == io.EOF { + return + } else { + needle.NeedleDataSize = Byte4ToInt32(needleDataBuf, BigEndian) + } + s.offset += int64(n) + s.offset += (int64)(needle.NeedleDataSize) + s.fd.Seek(s.offset, 0) + + n, err = s.fd.Read(needleFooterMagicBuf) + if err != nil || err == io.EOF { + return + } + if !bytes.Equal(needleFooterMagicBuf, NEEDLE_FOOTER_MAGIC) { + fmt.Println("footer magic not match") + break + } else { + needle.NeedleFooterMagic = NEEDLE_FOOTER_MAGIC_STR + } + s.offset += int64(n) + + n, err = s.fd.Read(needleCheckSumBuf) + if err != nil || err == io.EOF { + return + } else { + needle.NeedleCheckSum = Byte4ToInt32(needleCheckSumBuf, BigEndian) + } + s.offset += int64(n) + + if s.offset % 8 > 0 { + paddingBuf := make([]byte, 8 - s.offset % 8) + + n, err = s.fd.Read(paddingBuf) + if err != nil || err == io.EOF { + return + } + s.offset += int64(n) + s.fd.Seek(s.offset, 0) + } + + s.NeedleDetail = append(s.NeedleDetail, needle) + s.NeedleNum ++ + } + + return nil +} + +func (s *SuperBlock)doParse() (err error){ + err = s.parseSupperBlockHeader() + if err != nil { + return + } + err = s.parseNeedles() + if err != nil { + return + } + + return +} + diff --git a/store/tools/supper_block_parse/supper_block_parse.go b/store/tools/supper_block_parse/supper_block_parse.go new file mode 100755 index 0000000..c4f043f --- /dev/null +++ b/store/tools/supper_block_parse/supper_block_parse.go @@ -0,0 +1,48 @@ +package main + +import ( + "os" + "fmt" + "flag" + "encoding/json" +) + +var ( + supperBlockFile string +) + +func init() { + flag.StringVar(&supperBlockFile, "i", "", "input a supper_block file") +} + +func help() { + fmt.Println("supper_block_parse -i supper_block_file") +} + +func main() { + var ( + err error + ) + + flag.Parse() + + if supperBlockFile == "" { + help() + } + + supperBlock := NewSuperBlock() + + supperBlock.fd, err = os.Open(supperBlockFile) + if err != nil { + return + } + + err = supperBlock.doParse() + res, err := json.Marshal(supperBlock) + if err != nil { + return + } + fmt.Println(string(res)) + + supperBlock.fd.Close() +} \ No newline at end of file diff --git a/store/tools/supper_block_parse/util.go b/store/tools/supper_block_parse/util.go new file mode 100755 index 0000000..8244334 --- /dev/null +++ b/store/tools/supper_block_parse/util.go @@ -0,0 +1,110 @@ +package main + +import ( + "encoding/binary" + "encoding/hex" + //"fmt" + "strconv" +) + +const ( + BigEndian = 0 +) + +func Uint32ToBytes(i uint32) []byte { + var buf = make([]byte, 4) + binary.BigEndian.PutUint32(buf, i) + return buf +} + +func Byte8ToInt64(data []byte, endian int) int64 { + var i int64 + if 0 == endian { + i = int64(int64(data[7]) + int64(data[6])<<8 + int64(data[5])<<16 + int64(data[4])<<24 + + int64(data[3])<<32 + int64(data[2])<<40 + int64(data[1])<<48 + int64(data[0])<<56) + } + + if 1 == endian { + i = int64(int64(data[0]) + int64(data[1])<<8 + int64(data[2])<<16 + int64(data[3])<<24 + + int64(data[4])<<32 + int64(data[5])<<40 + int64(data[6])<<48 + int64(data[7])<<56) + } + + return i +} + +func Byte4ToInt32(data []byte, endian int) int32 { + var i int32 + if 0 == endian { + i = int32(int32(data[3]) + int32(data[2])<<8 + int32(data[1])<<16 + int32(data[0])<<24) + } + + if 1 == endian { + i = int32(int32(data[0]) + int32(data[1])<<8 + int32(data[2])<<16 + int32(data[3])<<24) + } + + return i +} + +func Byte32Uint32(data []byte, endian int) uint32 { + var i uint32 + if 0 == endian { + i = uint32(uint32(data[2]) + uint32(data[1])<<8 + uint32(data[0])<<16) + } + + if 1 == endian { + i = uint32(uint32(data[0]) + uint32(data[1])<<8 + uint32(data[2])<<16) + } + + return i +} + +func Byte22Uint16(data []byte, endian int) uint16 { + var i uint16 + if 0 == endian { + i = uint16(uint16(data[1]) + uint16(data[0])<<8) + } + + if 1 == endian { + i = uint16(uint16(data[0]) + uint16(data[1])<<8) + } + + return i +} + + + +func Byte2Int(b []byte) int64 { + + num := hex.EncodeToString(b) + + i, err := strconv.ParseInt(num, 16, 64) + if err != nil { + panic(err) + } + //fmt.Println(i) + + return i +} + + +func ToHex(ten int) (hex []int, length int) { + m := 0 + + hex = make([]int, 0) + length = 0; + + for { + m = ten / 16 + ten = ten % 16 + + if(m == 0) { + hex = append(hex, ten) + length++ + break + } + + hex = append(hex, m) + length++; + } + return +} \ No newline at end of file