Sync Gzipped Data

tasks/lib/s3.js

@@ -431,35 +431,55 @@ exports.init = function (grunt) {
         // it. Strip them out.
         remoteHash = res.headers.etag.replace(/"/g, '');
 
-        // Get an md5 of the local file so we can verify the upload.
-        localHash = crypto.createHash('md5').update(data).digest('hex');
+        function onLocalHash() {
 
-        if (remoteHash === localHash) {
-          // the file exists and is the same so do nothing with that
-          return dfd.resolve(util.format(MSG_SKIP_MATCHES, prettySrc));
-        }
-
-        fs.stat( src, function(err, stats) {
-          var remoteWhen, localWhen, upload;
+          if (remoteHash === localHash) {
+            // the file exists and is the same so do nothing with that
 
-          if (err) {
-            return dfd.reject(makeError(MSG_ERR_UPLOAD, prettySrc, err));
+            return dfd.resolve(util.format(MSG_SKIP_MATCHES, prettySrc));
+          } else {
+            // console.log("diff hash!", remoteHash, localHash)
           }
 
-          // which one is newer? if local is newer, we should upload it
-          remoteWhen = new Date(res.headers['last-modified'] || "0"); // earliest date possible if no header is returned
-          localWhen = new Date(stats.mtime || "1"); // make second earliest date possible if mtime isn't set
+          fs.stat( src, function(err, stats) {
+            var remoteWhen, localWhen, upload;
 
-          if ( localWhen <= remoteWhen ) {
-            // Remote file was older
-            return dfd.resolve(util.format(MSG_SKIP_OLDER, prettySrc));
-          }
+            if (err) {
+              return dfd.reject(makeError(MSG_ERR_UPLOAD, prettySrc, err));
+            }
+
+            // which one is newer? if local is newer, we should upload it
+            remoteWhen = new Date(res.headers['last-modified'] || "0"); // earliest date possible if no header is returned
+            localWhen = new Date(stats.mtime || "1"); // make second earliest date possible if mtime isn't set
+
+            if ( localWhen <= remoteWhen ) {
+              // console.log(localWhen, remoteWhen)
+              // Remote file was newer
+              return dfd.resolve(util.format(MSG_SKIP_OLDER, prettySrc));
+            }
+
+            // default is that local is newer, only upload when it is
+            upload = exports.upload( src, dest, opts);
+            // pass through the dfd state
+            upload.then( dfd.resolve, dfd.reject );
+          });
+        }
+
+        if (options.gzip) {
+          // If we're gzipping, assuming the remote file is also gzipped
+          zlib.gzip(data, function(err, buffer) {
+            if (!err) {
+              localHash = crypto.createHash('md5').update(buffer).digest('hex');
+            }
+            onLocalHash(localHash);
+          });
+
+        } else {
+          // Get an md5 of the local file so we can verify the upload.
+          localHash = crypto.createHash('md5').update(data).digest('hex');
+          onLocalHash(localHash);
+        }
 
-          // default is that local is newer, only upload when it is
-          upload = exports.upload( src, dest, opts);
-          // pass through the dfd state
-          upload.then( dfd.resolve, dfd.reject );
-        });
       });
     }).end();
 

test/sync.js

@@ -1,4 +1,5 @@
 
+var fs = require('fs');
 var grunt = require('grunt');
 var yaml = require('libyaml');
 var hashFile = require('../tasks/lib/common').hashFile;
@@ -39,5 +40,41 @@ module.exports = {
       .always(function () {
         test.done();
       });
+  },
+
+  testSyncGZip: function (test) {
+    test.expect(1);
+
+    var src = __dirname + '/files/a.txt';
+    var destDir = __dirname + '/../s3/127/test/a.txt/.fakes3_metadataFFF/';
+    var dest = destDir + 'content';
+    var meta = destDir + 'metadata';
+    var md5;
+
+    var _config = _.clone(config, true);
+    _config.gzip = true;
+    _config.verify = true;
+
+    s3.sync(src, '/a.txt', _config)
+      .then(function() {
+        // Change the file. We'll use this to detect if the file is changed again.
+        fs.writeFileSync(dest, 'b');
+
+        // Change the mtime, so the s3 file appears older.
+        // Need to do this so sync will upload the file if hashes don't match
+        // (uploaded file will have more recent mtime, preventing upload).
+        var doc = yaml.readFileSync(meta)[0];
+        doc[':modified_date'] = 0;
+        md5 = doc[':md5'];
+        yaml.writeFileSync(meta, doc);
+
+        // Re-upload. Files should have matching gzip md5s, so s3 file shouldn't change.
+        s3.sync(src, '/a.txt', _config).then(function() {
+          test.ok(md5 !== hashFile(dest), 'Does not reupload gzipped data if they match.');
+        })
+        .always(function() {
+          test.done();
+        });
+      });
   }
 };