From 032b4bed7f138a928fffde6ff528f76a15ac5987 Mon Sep 17 00:00:00 2001 From: GCHQ 77703 Date: Mon, 27 Aug 2018 01:17:06 +0100 Subject: [PATCH 1/5] Add Length Value Decoder Operatoin --- src/core/config/Categories.json | 3 +- src/core/lib/LengthValue.mjs | 71 ++++++++++++++++++++ src/core/operations/LengthValueDecoder.mjs | 78 ++++++++++++++++++++++ 3 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 src/core/lib/LengthValue.mjs create mode 100644 src/core/operations/LengthValueDecoder.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 66663f4a7f..b87d0ddb75 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -53,7 +53,8 @@ "To MessagePack", "From MessagePack", "To Braille", - "From Braille" + "From Braille", + "From Length Value" ] }, { diff --git a/src/core/lib/LengthValue.mjs b/src/core/lib/LengthValue.mjs new file mode 100644 index 0000000000..3ec0c5e571 --- /dev/null +++ b/src/core/lib/LengthValue.mjs @@ -0,0 +1,71 @@ +/** + * @author gchq77703 [] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +const defaults = { + location: 0, + bytesInLength: 1, + basicEncodingRules: false +}; + +/** + * Length Value library + */ +export default class LengthValue { + + /** + * LengthValue constructor + */ + constructor(input, options) { + this.input = input; + Object.assign(this, defaults, options); + } + + /** + * @returns {Number} + */ + getLength() { + if (this.basicEncodingRules) { + const bit = this.input[this.location]; + if (bit & 0x80) { + this.bytesInLength = bit & ~0x80; + } else { + this.location++; + return bit & ~0x80; + } + } + + let length = 0; + + for (let i = 0; i < this.bytesInLength; i++) { + length += this.input[this.location] * Math.pow(Math.pow(2, 8), i); + this.location++; + } + + return length; + } + + /** + * @param {Number} length + * @returns {Number[]} + */ + getValue(length) { + const value = []; + + for (let i = 0; i < length; i++) { + value.push(this.input[this.location]); + this.location++; + } + + return value; + } + + /** + * @returns {Boolean} + */ + atEnd() { + return this.input.length <= this.location; + } +} diff --git a/src/core/operations/LengthValueDecoder.mjs b/src/core/operations/LengthValueDecoder.mjs new file mode 100644 index 0000000000..12fe6da2f8 --- /dev/null +++ b/src/core/operations/LengthValueDecoder.mjs @@ -0,0 +1,78 @@ +/** + * @author gchq77703 [] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import LengthValue from "../lib/LengthValue"; + +/** + * From Length Value operation + */ +class FromLengthValue extends Operation { + + /** + * FromLengthValue constructor + */ + constructor() { + super(); + + this.name = "From Length Value"; + this.module = "Default"; + this.description = "Converts a Length-Value (LV) encoded string into a line-delimited JSON (LDJSON / Streaming JSON) format"; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "string"; + this.args = [ + { + name: "Bytes in Length Value", + type: "populateOption", + value: [ + { + name: "1 Byte", + value: "1" + }, + { + name: "2 Bytes", + value: "2" + }, + { + name: "4 Bytes", + value: "4" + } + ] + }, + { + name: "Use Basic Encoding Rules", + type: "boolean" + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const bytesInLength = parseInt(args[0].split(" ")[0], 10); + const basicEncodingRules = args[2]; + + const lv = new LengthValue(input, { bytesInLength, basicEncodingRules }); + + const data = []; + + while (!lv.atEnd()) { + const dataLength = lv.getLength(); + const value = lv.getValue(dataLength); + + data.push(value); + } + + return data.map(line => line.map(value => ("00" + value.toString(16)).slice(-2)).join(" ")).join("\n"); + } + +} + +export default FromLengthValue; From 06d9302d96113225d6e9f18da72afeddbdf74f32 Mon Sep 17 00:00:00 2001 From: GCHQ 77703 Date: Mon, 27 Aug 2018 14:57:24 +0100 Subject: [PATCH 2/5] Implement TLV / KLV --- src/core/operations/LengthValueDecoder.mjs | 38 ++++++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/core/operations/LengthValueDecoder.mjs b/src/core/operations/LengthValueDecoder.mjs index 12fe6da2f8..a46333f76d 100644 --- a/src/core/operations/LengthValueDecoder.mjs +++ b/src/core/operations/LengthValueDecoder.mjs @@ -20,11 +20,33 @@ class FromLengthValue extends Operation { this.name = "From Length Value"; this.module = "Default"; - this.description = "Converts a Length-Value (LV) encoded string into a line-delimited JSON (LDJSON / Streaming JSON) format"; + this.description = "Converts a Length-Value (LV) encoded string into a JSON object. Can optionally include a Key / Type entry."; this.infoURL = ""; this.inputType = "byteArray"; - this.outputType = "string"; + this.outputType = "JSON"; this.args = [ + { + name: "Bytes in Key Value", + type: "populateOption", + value: [ + { + name: "0 Bytes (No Key)", + value: "0" + }, + { + name: "1 Byte", + value: "1" + }, + { + name: "2 Bytes", + value: "2" + }, + { + name: "4 Bytes", + value: "4" + } + ] + }, { name: "Bytes in Length Value", type: "populateOption", @@ -56,7 +78,8 @@ class FromLengthValue extends Operation { * @returns {string} */ run(input, args) { - const bytesInLength = parseInt(args[0].split(" ")[0], 10); + const bytesInKey = parseInt(args[0].split(" ")[0], 10); + const bytesInLength = parseInt(args[1].split(" ")[0], 10); const basicEncodingRules = args[2]; const lv = new LengthValue(input, { bytesInLength, basicEncodingRules }); @@ -64,13 +87,14 @@ class FromLengthValue extends Operation { const data = []; while (!lv.atEnd()) { - const dataLength = lv.getLength(); - const value = lv.getValue(dataLength); + const key = bytesInKey ? lv.getValue(bytesInKey) : undefined; + const length = lv.getLength(); + const value = lv.getValue(length); - data.push(value); + data.push({ key, length, value }); } - return data.map(line => line.map(value => ("00" + value.toString(16)).slice(-2)).join(" ")).join("\n"); + return data; } } From edbd540c681188248d708880d9a9868b226cc394 Mon Sep 17 00:00:00 2001 From: GCHQ 77703 Date: Mon, 27 Aug 2018 15:42:07 +0100 Subject: [PATCH 3/5] Add Dysfunctional Test --- test/index.mjs | 1 + test/tests/operations/LengthValueDecoder.mjs | 23 ++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 test/tests/operations/LengthValueDecoder.mjs diff --git a/test/index.mjs b/test/index.mjs index 8cf69732ad..ee0aa3b21c 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -64,6 +64,7 @@ import "./tests/operations/SetUnion"; import "./tests/operations/SymmetricDifference"; import "./tests/operations/TranslateDateTimeFormat"; import "./tests/operations/Magic"; +import "./tests/operations/LengthValueDecoder"; let allTestsPassing = true; const testStatusCounts = { diff --git a/test/tests/operations/LengthValueDecoder.mjs b/test/tests/operations/LengthValueDecoder.mjs new file mode 100644 index 0000000000..250002c433 --- /dev/null +++ b/test/tests/operations/LengthValueDecoder.mjs @@ -0,0 +1,23 @@ +/** + * Length Value Decoder tests. + * + * @author gchq77703 [] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import TestRegister from "../../TestRegister"; + +TestRegister.addTests([ + { + name: "KeyValue", + input: [5,72,111,117,115,101,4,114,111,111,109,4,100,111,111,114], + expectedOutput: [{"key":[25],"length":5,"value":[72,111,117,115,101]},{"key":[73],"length":4,"value":[114,111,111,109]},{"key":[41],"length":4,"value":[100,111,111,114]}], + recipeConfig: [ + { + "op": "Length Value Decoder", + "args": ["0 Bytes (No Key)", "1 Byte", false] + } + ] + }, +]) \ No newline at end of file From 3abe99078e41a64889c3b6c0dd1fdbef436a21be Mon Sep 17 00:00:00 2001 From: GCHQ 77703 Date: Mon, 27 Aug 2018 16:03:15 +0100 Subject: [PATCH 4/5] Fix linting --- test/tests/operations/LengthValueDecoder.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/tests/operations/LengthValueDecoder.mjs b/test/tests/operations/LengthValueDecoder.mjs index 250002c433..90923c236e 100644 --- a/test/tests/operations/LengthValueDecoder.mjs +++ b/test/tests/operations/LengthValueDecoder.mjs @@ -11,8 +11,8 @@ import TestRegister from "../../TestRegister"; TestRegister.addTests([ { name: "KeyValue", - input: [5,72,111,117,115,101,4,114,111,111,109,4,100,111,111,114], - expectedOutput: [{"key":[25],"length":5,"value":[72,111,117,115,101]},{"key":[73],"length":4,"value":[114,111,111,109]},{"key":[41],"length":4,"value":[100,111,111,114]}], + input: [5, 72, 111, 117, 115, 101, 4, 114, 111, 111, 109, 4, 100, 111, 111, 114], + expectedOutput: [{"key": [25], "length": 5, "value": [72, 111, 117, 115, 101]}, {"key": [73], "length": 4, "value": [114, 111, 111, 109]}, {"key": [41], "length": 4, "value": [100, 111, 111, 114]}], recipeConfig: [ { "op": "Length Value Decoder", @@ -20,4 +20,4 @@ TestRegister.addTests([ } ] }, -]) \ No newline at end of file +]); From 3833c5f9fe40a5923384993161c4902382c6f3a8 Mon Sep 17 00:00:00 2001 From: GCHQ 77703 Date: Fri, 31 Aug 2018 13:20:28 +0100 Subject: [PATCH 5/5] Rename operation, add working tests, add info URL --- src/core/config/Categories.json | 2 +- .../{LengthValueDecoder.mjs => LVDecode.mjs} | 51 +++++------------ test/index.mjs | 2 +- test/tests/operations/LVDecode.mjs | 56 +++++++++++++++++++ test/tests/operations/LengthValueDecoder.mjs | 23 -------- 5 files changed, 73 insertions(+), 61 deletions(-) rename src/core/operations/{LengthValueDecoder.mjs => LVDecode.mjs} (58%) create mode 100644 test/tests/operations/LVDecode.mjs delete mode 100644 test/tests/operations/LengthValueDecoder.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index b87d0ddb75..faedaed0a0 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -54,7 +54,7 @@ "From MessagePack", "To Braille", "From Braille", - "From Length Value" + "LV Decode" ] }, { diff --git a/src/core/operations/LengthValueDecoder.mjs b/src/core/operations/LVDecode.mjs similarity index 58% rename from src/core/operations/LengthValueDecoder.mjs rename to src/core/operations/LVDecode.mjs index a46333f76d..63af65f6be 100644 --- a/src/core/operations/LengthValueDecoder.mjs +++ b/src/core/operations/LVDecode.mjs @@ -8,61 +8,40 @@ import Operation from "../Operation"; import LengthValue from "../lib/LengthValue"; /** - * From Length Value operation + * From LV Decode operation */ -class FromLengthValue extends Operation { +class LVDecode extends Operation { /** - * FromLengthValue constructor + * LVDecode constructor */ constructor() { super(); - this.name = "From Length Value"; + this.name = "LV Decode"; this.module = "Default"; this.description = "Converts a Length-Value (LV) encoded string into a JSON object. Can optionally include a Key / Type entry."; - this.infoURL = ""; + this.infoURL = "https://wikipedia.org/wiki/KLV"; this.inputType = "byteArray"; this.outputType = "JSON"; this.args = [ { name: "Bytes in Key Value", - type: "populateOption", + type: "option", value: [ - { - name: "0 Bytes (No Key)", - value: "0" - }, - { - name: "1 Byte", - value: "1" - }, - { - name: "2 Bytes", - value: "2" - }, - { - name: "4 Bytes", - value: "4" - } + "0 Bytes (No Key)", + "1 Byte", + "2 Bytes", + "4 Bytes" ] }, { name: "Bytes in Length Value", - type: "populateOption", + type: "option", value: [ - { - name: "1 Byte", - value: "1" - }, - { - name: "2 Bytes", - value: "2" - }, - { - name: "4 Bytes", - value: "4" - } + "1 Byte", + "2 Bytes", + "4 Bytes" ] }, { @@ -99,4 +78,4 @@ class FromLengthValue extends Operation { } -export default FromLengthValue; +export default LVDecode; diff --git a/test/index.mjs b/test/index.mjs index ee0aa3b21c..06b2b18100 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -64,7 +64,7 @@ import "./tests/operations/SetUnion"; import "./tests/operations/SymmetricDifference"; import "./tests/operations/TranslateDateTimeFormat"; import "./tests/operations/Magic"; -import "./tests/operations/LengthValueDecoder"; +import "./tests/operations/LVDecode"; let allTestsPassing = true; const testStatusCounts = { diff --git a/test/tests/operations/LVDecode.mjs b/test/tests/operations/LVDecode.mjs new file mode 100644 index 0000000000..4fd7fc8314 --- /dev/null +++ b/test/tests/operations/LVDecode.mjs @@ -0,0 +1,56 @@ +/** + * LV Decoder tests. + * + * @author gchq77703 [] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import TestRegister from "../../TestRegister"; + +TestRegister.addTests([ + { + name: "LVDecode: LengthValue", + input: "\x05\x48\x6f\x75\x73\x65\x04\x72\x6f\x6f\x6d\x04\x64\x6f\x6f\x72", + expectedOutput: JSON.stringify([{"length": 5, "value": [72, 111, 117, 115, 101]}, {"length": 4, "value": [114, 111, 111, 109]}, {"length": 4, "value": [100, 111, 111, 114]}]), + recipeConfig: [ + { + "op": "LV Decode", + "args": ["0 Bytes (No Key)", "1 Byte", false] + } + ] + }, + { + name: "LVDecode: LengthValue with BER", + input: "\x05\x48\x6f\x75\x73\x65\x04\x72\x6f\x6f\x6d\x04\x64\x6f\x6f\x72", + expectedOutput: JSON.stringify([{"length": 5, "value": [72, 111, 117, 115, 101]}, {"length": 4, "value": [114, 111, 111, 109]}, {"length": 4, "value": [100, 111, 111, 114]}]), + recipeConfig: [ + { + "op": "LV Decode", + "args": ["0 Bytes (No Key)", "4 Bytes", false] // length value is patently wrong, should be ignored by BER. + } + ] + }, + { + name: "LVDecode: KeyLengthValue", + input: "\x04\x05\x48\x6f\x75\x73\x65\x05\x04\x72\x6f\x6f\x6d\x42\x04\x64\x6f\x6f\x72", + expectedOutput: JSON.stringify([{"key":[4],"length":5,"value":[72,111,117,115,101]},{"key":[5],"length":4,"value":[114,111,111,109]},{"key":[66],"length":4,"value":[100,111,111,114]}]), + recipeConfig: [ + { + "op": "LV Decode", + "args": ["1 Byte", "1 Byte", false] + } + ] + }, + { + name: "LVDecode: KeyLengthValue with BER", + input: "\x04\x05\x48\x6f\x75\x73\x65\x05\x04\x72\x6f\x6f\x6d\x42\x04\x64\x6f\x6f\x72", + expectedOutput: JSON.stringify([{"key":[4],"length":5,"value":[72,111,117,115,101]},{"key":[5],"length":4,"value":[114,111,111,109]},{"key":[66],"length":4,"value":[100,111,111,114]}]), + recipeConfig: [ + { + "op": "LV Decode", + "args": ["1 Byte", "4 Byte", true] // length value is patently wrong, should be ignored by BER. + } + ] + } +]); diff --git a/test/tests/operations/LengthValueDecoder.mjs b/test/tests/operations/LengthValueDecoder.mjs deleted file mode 100644 index 90923c236e..0000000000 --- a/test/tests/operations/LengthValueDecoder.mjs +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Length Value Decoder tests. - * - * @author gchq77703 [] - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - */ - -import TestRegister from "../../TestRegister"; - -TestRegister.addTests([ - { - name: "KeyValue", - input: [5, 72, 111, 117, 115, 101, 4, 114, 111, 111, 109, 4, 100, 111, 111, 114], - expectedOutput: [{"key": [25], "length": 5, "value": [72, 111, 117, 115, 101]}, {"key": [73], "length": 4, "value": [114, 111, 111, 109]}, {"key": [41], "length": 4, "value": [100, 111, 111, 114]}], - recipeConfig: [ - { - "op": "Length Value Decoder", - "args": ["0 Bytes (No Key)", "1 Byte", false] - } - ] - }, -]);