Implementation of delta-sync support on client-side.

This commit adds client-side support for delta-sync, this adds a new
3rdparty submodule `gh:ahmedammar/zsync`. This zsync tree is a modified
version of upstream, adding some needed support for the upload path and
other requirements.

If the server does not announce the required zsync capability then a
full upload/download is fallen back to. Delta synchronization can be
enabled/disabled using command line, config, or gui options.

On both upload and download paths, a check is made for the existance of
a zsync metadata file on the server for a given path. This is provided
by a dav property called `zsync`, found during discovery phase. If it
doesn't exist the code reverts back to a complete upload or download,
i.e. previous implementations. In the case of upload, a new zsync
metadata file will be uploaded as part of the chunked upload and future
synchronizations will be delta-sync capable.

Chunked uploads no longer use sequential file names for each chunk id,
instead, they are named as the byte offset into the remote file, this is
a minimally intrusive modification to allow fo delta-sync and legacy
code paths to run seamlessly. A new http header OC-Total-File-Length is
sent, which informs the server of the final expected size of the file
not just the total transmitted bytes as reported by OC-Total-Length.

The seeding and generation of the zsync metadata file is done in a
separate thread since this is a cpu intensive task, ensuring main thread
is not blocked.

This commit closes #179.

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "src/3rdparty/libcrashreporter-qt"]
 	path = src/3rdparty/libcrashreporter-qt
 	url = git://github.com/dschmidt/libcrashreporter-qt.git
+[submodule "src/3rdparty/zsync"]
+	path = src/3rdparty/zsync
+	url = https://github.com/ahmedammar/zsync

src/cmd/cmd.cpp

@@ -77,6 +77,8 @@ struct CmdOptions
     int restartTimes;
     int downlimit;
     int uplimit;
+    bool deltasync;
+    quint64 deltasyncminfilesize;
 };
 
 // we can't use csync_set_userdata because the SyncEngine sets it already.
@@ -188,6 +190,8 @@ void help()
     std::cout << "  --max-sync-retries [n] Retries maximum n times (default to 3)" << std::endl;
     std::cout << "  --uplimit [n]          Limit the upload speed of files to n KB/s" << std::endl;
     std::cout << "  --downlimit [n]        Limit the download speed of files to n KB/s" << std::endl;
+    std::cout << "  --deltasync, -ds       Enable delta sync (disabled by default)" << std::endl;
+    std::cout << "  --deltasyncmin [n]     Set delta sync minimum file size to n MB (10 MiB default)" << std::endl;
     std::cout << "  -h                     Sync hidden files,do not ignore them" << std::endl;
     std::cout << "  --version, -v          Display version and exit" << std::endl;
     std::cout << "  --logdebug             More verbose logging" << std::endl;
@@ -268,6 +272,10 @@ void parseOptions(const QStringList &app_args, CmdOptions *options)
             options->uplimit = it.next().toInt() * 1000;
         } else if (option == "--downlimit" && !it.peekNext().startsWith("-")) {
             options->downlimit = it.next().toInt() * 1000;
+        } else if (option == "-ds" || option == "--deltasync") {
+            options->deltasync = true;
+        } else if (option == "--deltasyncmin" && !it.peekNext().startsWith("-")) {
+            options->deltasyncminfilesize = it.next().toLongLong() * 1024 * 1024;
         } else if (option == "--logdebug") {
             Logger::instance()->setLogFile("-");
             Logger::instance()->setLogDebug(true);
@@ -327,6 +335,8 @@ int main(int argc, char **argv)
     options.restartTimes = 3;
     options.uplimit = 0;
     options.downlimit = 0;
+    options.deltasync = false;
+    options.deltasyncminfilesize = 10 * 1024 * 1024;
     ClientProxy clientProxy;
 
     parseOptions(app.arguments(), &options);
@@ -512,7 +522,11 @@ int main(int argc, char **argv)
         selectiveSyncFixup(&db, selectiveSyncList);
     }
 
+    SyncOptions opt;
+    opt._deltaSyncEnabled = options.deltasync;
+    opt._deltaSyncMinFileSize = options.deltasyncminfilesize;
     SyncEngine engine(account, options.source_dir, folder, &db);
+    engine.setSyncOptions(opt);
     engine.setIgnoreHiddenFiles(options.ignoreHiddenFiles);
     engine.setNetworkLimits(options.uplimit, options.downlimit);
     QObject::connect(&engine, &SyncEngine::finished,

src/common/remotepermissions.cpp

@@ -21,7 +21,7 @@
 
 namespace OCC {
 
-static const char letters[] = " WDNVCKRSMm";
+static const char letters[] = " WDNVCKRSMmz";
 
 
 template <typename Char>

src/common/remotepermissions.h

@@ -52,10 +52,11 @@ class OCSYNC_EXPORT RemotePermissions
         IsShared = 8,             // S
         IsMounted = 9,            // M
         IsMountedSub = 10,        // m (internal: set if the parent dir has IsMounted)
+        HasZSyncMetadata = 11,    // z (internal: set if remote file has zsync metadata property set)
 
         // Note: when adding support for more permissions, we need to invalid the cache in the database.
         // (by setting forceRemoteDiscovery in SyncJournalDb::checkConnect)
-        PermissionsCount = IsMountedSub
+        PermissionsCount = HasZSyncMetadata
     };
     RemotePermissions() = default;
     explicit RemotePermissions(const char *);

src/common/syncjournaldb.cpp

@@ -494,11 +494,12 @@ bool SyncJournalDb::checkConnect()
             forceRemoteDiscovery = true;
         }
 
-        // There was a bug in versions <2.3.0 that could lead to stale
-        // local files and a remote discovery will fix them.
-        // See #5190 #5242.
-        if (major == 2 && minor < 3) {
-            qCInfo(lcDb) << "upgrade form client < 2.3.0 detected! forcing remote discovery";
+        // - There was a bug in versions <2.3.0 that could lead to stale
+        //   local files and a remote discovery will fix them.
+        //   See #5190 #5242.
+        // - New remote HasZSyncMetadata permission added, invalidate cache
+        if (major == 2 && minor < 5) {
+            qCInfo(lcDb) << "upgrade from client < 2.5.0 detected! forcing remote discovery";
             forceRemoteDiscovery = true;
         }
 

src/gui/folder.cpp

@@ -720,6 +720,9 @@ void Folder::setSyncOptions()
         opt._targetChunkUploadDuration = cfgFile.targetChunkUploadDuration();
     }
 
+    opt._deltaSyncEnabled = cfgFile.deltaSyncEnabled();
+    opt._deltaSyncMinFileSize = cfgFile.deltaSyncMinFileSize();
+
     _engine->setSyncOptions(opt);
 }
 

src/gui/generalsettings.cpp

@@ -69,6 +69,8 @@ GeneralSettings::GeneralSettings(QWidget *parent)
     connect(_ui->newFolderLimitCheckBox, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
     connect(_ui->newFolderLimitSpinBox, static_cast<void (QSpinBox::*)(int)>(&QSpinBox::valueChanged), this, &GeneralSettings::saveMiscSettings);
     connect(_ui->newExternalStorage, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
+    connect(_ui->deltaSyncCheckBox, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
+    connect(_ui->deltaSyncSpinBox, static_cast<void (QSpinBox::*)(int)>(&QSpinBox::valueChanged), this, &GeneralSettings::saveMiscSettings);
 
 #ifndef WITH_CRASHREPORTER
     _ui->crashreporterCheckBox->setVisible(false);
@@ -122,6 +124,8 @@ void GeneralSettings::loadMiscSettings()
     _ui->newFolderLimitSpinBox->setValue(newFolderLimit.second);
     _ui->newExternalStorage->setChecked(cfgFile.confirmExternalStorage());
     _ui->monoIconsCheckBox->setChecked(cfgFile.monoIcons());
+    _ui->deltaSyncCheckBox->setChecked(cfgFile.deltaSyncEnabled());
+    _ui->deltaSyncSpinBox->setValue(cfgFile.deltaSyncMinFileSize() / (1024 * 1024));
 }
 
 void GeneralSettings::slotUpdateInfo()
@@ -157,6 +161,8 @@ void GeneralSettings::saveMiscSettings()
     cfgFile.setNewBigFolderSizeLimit(_ui->newFolderLimitCheckBox->isChecked(),
         _ui->newFolderLimitSpinBox->value());
     cfgFile.setConfirmExternalStorage(_ui->newExternalStorage->isChecked());
+    cfgFile.setDeltaSyncEnabled(_ui->deltaSyncCheckBox->isChecked());
+    cfgFile.setDeltaSyncMinFileSize(_ui->deltaSyncSpinBox->value() * 1024 * 1024);
 }
 
 void GeneralSettings::slotToggleLaunchOnStartup(bool enable)

src/gui/generalsettings.ui

@@ -48,6 +48,53 @@
     </widget>
    </item>
    <item row="2" column="0">
+    <widget class="QGroupBox" name="advancedGroupBox">
+     <property name="title">
+      <string>Experimental</string>
+     </property>
+     <layout class="QVBoxLayout" name="verticalLayout_2">
+      <item>
+       <layout class="QHBoxLayout" name="horizontalLayout_2">
+        <item>
+         <widget class="QCheckBox" name="deltaSyncCheckBox">
+          <property name="text">
+           <string>Enable Delta-Synchronization for files larger than</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QSpinBox" name="deltaSyncSpinBox">
+          <property name="maximum">
+           <number>999999</number>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QLabel" name="label_2">
+          <property name="text">
+           <string>MB</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <spacer name="horizontalSpacer_2">
+          <property name="orientation">
+           <enum>Qt::Horizontal</enum>
+          </property>
+          <property name="sizeHint" stdset="0">
+           <size>
+            <width>40</width>
+            <height>20</height>
+           </size>
+          </property>
+         </spacer>
+        </item>
+       </layout>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="3" column="0">
     <widget class="QGroupBox" name="aboutGroupBox">
      <property name="title">
       <string>About</string>
@@ -69,7 +116,7 @@
      </layout>
     </widget>
    </item>
-   <item row="3" column="0">
+   <item row="4" column="0">
     <widget class="QGroupBox" name="updatesGroupBox">
      <property name="title">
       <string>Updates</string>
@@ -120,7 +167,7 @@
      </layout>
     </widget>
    </item>
-   <item row="4" column="0">
+   <item row="5" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>

src/libsync/CMakeLists.txt

@@ -5,8 +5,8 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 # csync is required.
 include_directories(${CMAKE_SOURCE_DIR}/src/csync
                     ${CMAKE_BINARY_DIR}/src/csync
+                    ${CMAKE_SOURCE_DIR}/src/3rdparty/zsync/c
                    )
-
 if ( APPLE )
     list(APPEND OS_SPECIFIC_LINK_LIBRARIES
          /System/Library/Frameworks/CoreServices.framework
@@ -15,6 +15,14 @@ if ( APPLE )
     )
 endif()
 
+if ( WIN32 )
+    list(APPEND OS_SPECIFIC_LINK_LIBRARIES
+        ws2_32
+    )
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_FILE_OFFSET_BITS=64")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FILE_OFFSET_BITS=64")
+endif()
+
 IF(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD|NetBSD|OpenBSD")
     list(APPEND OS_SPECIFIC_LINK_LIBRARIES
 	inotify
@@ -43,7 +51,9 @@ set(libsync_SRCS
     owncloudtheme.cpp
     progressdispatcher.cpp
     propagatorjobs.cpp
+    propagatecommonzsync.cpp
     propagatedownload.cpp
+    propagatedownloadzsync.cpp
     propagateupload.cpp
     propagateuploadv1.cpp
     propagateuploadng.cpp
@@ -59,8 +69,18 @@ set(libsync_SRCS
     creds/dummycredentials.cpp
     creds/abstractcredentials.cpp
     creds/credentialscommon.cpp
+    ../3rdparty/zsync/c/librcksum/hash.c
+    ../3rdparty/zsync/c/librcksum/md4.c
+    ../3rdparty/zsync/c/librcksum/range.c
+    ../3rdparty/zsync/c/librcksum/rsum.c
+    ../3rdparty/zsync/c/librcksum/state.c
+    ../3rdparty/zsync/c/libzsync/sha1.c
+    ../3rdparty/zsync/c/libzsync/zsync.c
+    ../3rdparty/zsync/c/progress.c
 )
 
+set_source_files_properties(../3rdparty/zsync/c/libzsync/zsync.c PROPERTIES COMPILE_FLAGS -DVERSION=\\"0.6.3\\")
+
 if(TOKEN_AUTH_ONLY)
     set (libsync_SRCS ${libsync_SRCS} creds/tokencredentials.cpp)
 else()

src/libsync/bandwidthmanager.cpp

@@ -120,7 +120,7 @@ void BandwidthManager::unregisterUploadDevice(QObject *o)
     }
 }
 
-void BandwidthManager::registerDownloadJob(GETFileJob *j)
+void BandwidthManager::registerDownloadJob(GETJob *j)
 {
     _downloadJobList.append(j);
     QObject::connect(j, &QObject::destroyed, this, &BandwidthManager::unregisterDownloadJob);
@@ -139,7 +139,7 @@ void BandwidthManager::registerDownloadJob(GETFileJob *j)
 
 void BandwidthManager::unregisterDownloadJob(QObject *o)
 {
-    GETFileJob *j = reinterpret_cast<GETFileJob *>(o); // note, we might already be in the ~QObject
+    GETJob *j = reinterpret_cast<GETJob *>(o); // note, we might already be in the ~QObject
     _downloadJobList.removeAll(j);
     if (_relativeLimitCurrentMeasuredJob == j) {
         _relativeLimitCurrentMeasuredJob = 0;
@@ -289,7 +289,7 @@ void BandwidthManager::relativeDownloadMeasuringTimerExpired()
         quota -= 20 * 1024;
     }
     qint64 quotaPerJob = quota / jobCount + 1.0;
-    Q_FOREACH (GETFileJob *gfj, _downloadJobList) {
+    Q_FOREACH (GETJob *gfj, _downloadJobList) {
         gfj->setBandwidthLimited(true);
         gfj->setChoked(false);
         gfj->giveBandwidthQuota(quotaPerJob);
@@ -323,7 +323,7 @@ void BandwidthManager::relativeDownloadDelayTimerExpired()
     _relativeLimitCurrentMeasuredJob->setChoked(false);
 
     // choke all other download jobs
-    Q_FOREACH (GETFileJob *gfj, _downloadJobList) {
+    Q_FOREACH (GETJob *gfj, _downloadJobList) {
         if (gfj != _relativeLimitCurrentMeasuredJob) {
             gfj->setBandwidthLimited(true);
             gfj->setChoked(true);
@@ -358,7 +358,7 @@ void BandwidthManager::switchingTimerExpired()
     if (newDownloadLimit != _currentDownloadLimit) {
         qCInfo(lcBandwidthManager) << "Download Bandwidth limit changed" << _currentDownloadLimit << newDownloadLimit;
         _currentDownloadLimit = newDownloadLimit;
-        Q_FOREACH (GETFileJob *j, _downloadJobList) {
+        Q_FOREACH (GETJob *j, _downloadJobList) {
             if (usingAbsoluteDownloadLimit()) {
                 j->setBandwidthLimited(true);
                 j->setChoked(false);
@@ -386,7 +386,7 @@ void BandwidthManager::absoluteLimitTimerExpired()
     if (usingAbsoluteDownloadLimit() && _downloadJobList.count() > 0) {
         qint64 quotaPerJob = _currentDownloadLimit / qMax(1, _downloadJobList.count());
         qCDebug(lcBandwidthManager) << quotaPerJob << _downloadJobList.count() << _currentDownloadLimit;
-        Q_FOREACH (GETFileJob *j, _downloadJobList) {
+        Q_FOREACH (GETJob *j, _downloadJobList) {
             j->giveBandwidthQuota(quotaPerJob);
             qCDebug(lcBandwidthManager) << "Gave " << quotaPerJob / 1024.0 << " kB to" << j;
         }

src/libsync/bandwidthmanager.h

@@ -23,7 +23,7 @@
 namespace OCC {
 
 class UploadDevice;
-class GETFileJob;
+class GETJob;
 class OwncloudPropagator;
 
 /**
@@ -47,7 +47,7 @@ public slots:
     void registerUploadDevice(UploadDevice *);
     void unregisterUploadDevice(QObject *);
 
-    void registerDownloadJob(GETFileJob *);
+    void registerDownloadJob(GETJob *);
     void unregisterDownloadJob(QObject *);
 
     void absoluteLimitTimerExpired();
@@ -86,14 +86,14 @@ public slots:
     qint64 _relativeUploadLimitProgressAtMeasuringRestart;
     qint64 _currentUploadLimit;
 
-    QLinkedList<GETFileJob *> _downloadJobList;
+    QLinkedList<GETJob *> _downloadJobList;
     QTimer _relativeDownloadMeasuringTimer;
 
     // for relative bw limiting, we need to wait this amount before measuring again
     QTimer _relativeDownloadDelayTimer;
 
     // the device measured
-    GETFileJob *_relativeLimitCurrentMeasuredJob;
+    GETJob *_relativeLimitCurrentMeasuredJob;
 
     // for measuring how much progress we made at start
     qint64 _relativeDownloadLimitProgressAtMeasuringRestart;

src/libsync/configfile.cpp

@@ -80,6 +80,9 @@ static const char newBigFolderSizeLimitC[] = "newBigFolderSizeLimit";
 static const char useNewBigFolderSizeLimitC[] = "useNewBigFolderSizeLimit";
 static const char confirmExternalStorageC[] = "confirmExternalStorage";
 
+static const char deltaSyncEnabledC[] = "DeltaSync/enabled";
+static const char deltaSyncMinimumFileSizeC[] = "DeltaSync/minFileSize";
+
 static const char maxLogLinesC[] = "Logging/maxLogLines";
 
 const char certPath[] = "http_certificatePath";
@@ -676,6 +679,28 @@ void ConfigFile::setConfirmExternalStorage(bool isChecked)
     setValue(confirmExternalStorageC, isChecked);
 }
 
+bool ConfigFile::deltaSyncEnabled() const
+{
+    QSettings settings(configFile(), QSettings::IniFormat);
+    return settings.value(QLatin1String(deltaSyncEnabledC), false).toBool(); // default to false
+}
+
+void ConfigFile::setDeltaSyncEnabled(bool enabled)
+{
+    setValue(deltaSyncEnabledC, enabled);
+}
+
+quint64 ConfigFile::deltaSyncMinFileSize() const
+{
+    QSettings settings(configFile(), QSettings::IniFormat);
+    return settings.value(QLatin1String(deltaSyncMinimumFileSizeC), 10 * 1024 * 1024).toLongLong(); // default to 10 MiB
+}
+
+void ConfigFile::setDeltaSyncMinFileSize(quint64 bytes)
+{
+    setValue(deltaSyncMinimumFileSizeC, bytes);
+}
+
 bool ConfigFile::promptDeleteFiles() const
 {
     QSettings settings(configFile(), QSettings::IniFormat);