Merge pull request #810 from rollbar/wj-truncation-byte-count

Use correct byte size for truncation

sdks/rollbar.js/src/truncation.js

@@ -80,7 +80,7 @@ function minBody(payload, jsonBackup) {
 }
 
 function needsTruncation(payload, maxSize) {
-  return payload.length > maxSize;
+  return _.maxByteSize(payload) > maxSize;
 }
 
 function truncate(payload, jsonBackup, maxSize) {

sdks/rollbar.js/src/utility.js

@@ -333,6 +333,33 @@ function stringify(obj, backup) {
   return {error: error, value: value};
 }
 
+function maxByteSize(string) {
+  // The transport will use utf-8, so assume utf-8 encoding.
+  //
+  // This minimal implementation will accurately count bytes for all UCS-2 and
+  // single code point UTF-16. If presented with multi code point UTF-16,
+  // which should be rare, it will safely overcount, not undercount.
+  //
+  // While robust utf-8 encoders exist, this is far smaller and far more performant.
+  // For quickly counting payload size for truncation, smaller is better.
+
+  var count = 0;
+  var length = string.length;
+
+  for (var i = 0; i < length; i++) {
+    var code = string.charCodeAt(i);
+    if (code < 128) { // up to 7 bits
+      count = count + 1;
+    } else if (code < 2048) { // up to 11 bits
+      count = count + 2;
+    } else if (code < 65536) { // up to 16 bits
+      count = count + 3;
+    }
+  }
+
+  return count;
+}
+
 function jsonParse(s) {
   var value, error;
   try {
@@ -736,6 +763,7 @@ module.exports = {
   scrub: scrub,
   set: set,
   stringify: stringify,
+  maxByteSize: maxByteSize,
   traverse: traverse,
   typeName: typeName,
   uuid4: uuid4

sdks/rollbar.js/test/truncation.test.js

@@ -25,6 +25,24 @@ describe('truncate', function() {
     expect(resultValue.data.body.trace.exception.message.length).to.be.below(256);
     expect(resultValue.data.body.trace.frames.length).to.be.below(3);
   });
+
+  it('should not truncate ascii payload close to max size', function() {
+    var payload = tracePayload(10, repeat('i', 500));
+    var result = t.truncate(payload, undefined, 1100); // payload will be 500 + 528
+    expect(result.value).to.be.ok();
+
+    var resultValue = JSON.parse(result.value);
+    expect(resultValue).to.eql(payload);
+  });
+
+  it('should truncate non-ascii payload when oversize', function() {
+    var payload = tracePayload(10, repeat('あ', 500)); // あ is 3 utf-8 bytes (U+3042)
+    var result = t.truncate(payload, undefined, 1100); // payload will be 1500 + 528
+    expect(result.value).to.be.ok();
+
+    var resultValue = JSON.parse(result.value);
+    expect(resultValue.data.body.trace.frames.length).to.be.below(3);
+  });
 });
 
 describe('raw', function() {