Implementation of delta-sync support on client-side.

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "src/3rdparty/libcrashreporter-qt"]
 	path = src/3rdparty/libcrashreporter-qt
 	url = git://github.com/dschmidt/libcrashreporter-qt.git
+[submodule "src/3rdparty/zsync"]
+	path = src/3rdparty/zsync
+	url = https://github.com/ahmedammar/zsync

src/cmd/cmd.cpp

@@ -77,6 +77,8 @@ struct CmdOptions
     int restartTimes;
     int downlimit;
     int uplimit;
+    bool deltasync;
+    quint64 deltasyncminfilesize;
 };
 
 // we can't use csync_set_userdata because the SyncEngine sets it already.
@@ -188,6 +190,8 @@ void help()
     std::cout << "  --max-sync-retries [n] Retries maximum n times (default to 3)" << std::endl;
     std::cout << "  --uplimit [n]          Limit the upload speed of files to n KB/s" << std::endl;
     std::cout << "  --downlimit [n]        Limit the download speed of files to n KB/s" << std::endl;
+    std::cout << "  --deltasync, -ds       Enable delta sync (disabled by default)" << std::endl;
+    std::cout << "  --deltasyncmin [n]     Set delta sync minimum file size to n MB (10 MiB default)" << std::endl;
     std::cout << "  -h                     Sync hidden files,do not ignore them" << std::endl;
     std::cout << "  --version, -v          Display version and exit" << std::endl;
     std::cout << "  --logdebug             More verbose logging" << std::endl;
@@ -268,6 +272,10 @@ void parseOptions(const QStringList &app_args, CmdOptions *options)
             options->uplimit = it.next().toInt() * 1000;
         } else if (option == "--downlimit" && !it.peekNext().startsWith("-")) {
             options->downlimit = it.next().toInt() * 1000;
+        } else if (option == "-ds" || option == "--deltasync") {
+            options->deltasync = true;
+        } else if (option == "--deltasyncmin" && !it.peekNext().startsWith("-")) {
+            options->deltasyncminfilesize = it.next().toLongLong() * 1024 * 1024;
         } else if (option == "--logdebug") {
             Logger::instance()->setLogFile("-");
             Logger::instance()->setLogDebug(true);
@@ -327,6 +335,8 @@ int main(int argc, char **argv)
     options.restartTimes = 3;
     options.uplimit = 0;
     options.downlimit = 0;
+    options.deltasync = false;
+    options.deltasyncminfilesize = 10 * 1024 * 1024;
     ClientProxy clientProxy;
 
     parseOptions(app.arguments(), &options);
@@ -508,7 +518,11 @@ int main(int argc, char **argv)
         selectiveSyncFixup(&db, selectiveSyncList);
     }
 
+    SyncOptions opt;
+    opt._deltaSyncEnabled = options.deltasync;
+    opt._deltaSyncMinFileSize = options.deltasyncminfilesize;
     SyncEngine engine(account, options.source_dir, folder, &db);
+    engine.setSyncOptions(opt);
     engine.setIgnoreHiddenFiles(options.ignoreHiddenFiles);
     engine.setNetworkLimits(options.uplimit, options.downlimit);
     QObject::connect(&engine, &SyncEngine::finished,

src/common/remotepermissions.cpp

@@ -21,7 +21,7 @@
 
 namespace OCC {
 
-static const char letters[] = " WDNVCKRSMm";
+static const char letters[] = " WDNVCKRSMmz";
 
 
 template <typename Char>

src/common/remotepermissions.h

@@ -52,10 +52,11 @@ class OCSYNC_EXPORT RemotePermissions
         IsShared = 8,             // S
         IsMounted = 9,            // M
         IsMountedSub = 10,        // m (internal: set if the parent dir has IsMounted)
+        HasZSyncMetadata = 11,    // z (internal: set if remote file has zsync metadata property set)
 
         // Note: when adding support for more permissions, we need to invalid the cache in the database.
         // (by setting forceRemoteDiscovery in SyncJournalDb::checkConnect)
-        PermissionsCount = IsMountedSub
+        PermissionsCount = HasZSyncMetadata
     };
     RemotePermissions() = default;
     explicit RemotePermissions(const char *);

src/common/syncjournaldb.cpp

@@ -494,11 +494,12 @@ bool SyncJournalDb::checkConnect()
             forceRemoteDiscovery = true;
         }
 
-        // There was a bug in versions <2.3.0 that could lead to stale
-        // local files and a remote discovery will fix them.
-        // See #5190 #5242.
-        if (major == 2 && minor < 3) {
-            qCInfo(lcDb) << "upgrade form client < 2.3.0 detected! forcing remote discovery";
+        // - There was a bug in versions <2.3.0 that could lead to stale
+        //   local files and a remote discovery will fix them.
+        //   See #5190 #5242.
+        // - New remote HasZSyncMetadata permission added, invalidate cache
+        if (major == 2 && minor < 5) {
+            qCInfo(lcDb) << "upgrade from client < 2.5.0 detected! forcing remote discovery";
             forceRemoteDiscovery = true;
         }
 

src/gui/folder.cpp

@@ -706,6 +706,9 @@ void Folder::setSyncOptions()
         opt._targetChunkUploadDuration = cfgFile.targetChunkUploadDuration();
     }
 
+    opt._deltaSyncEnabled = cfgFile.deltaSyncEnabled();
+    opt._deltaSyncMinFileSize = cfgFile.deltaSyncMinFileSize();
+
     _engine->setSyncOptions(opt);
 }
 

src/gui/generalsettings.cpp

@@ -69,6 +69,8 @@ GeneralSettings::GeneralSettings(QWidget *parent)
     connect(_ui->newFolderLimitCheckBox, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
     connect(_ui->newFolderLimitSpinBox, static_cast<void (QSpinBox::*)(int)>(&QSpinBox::valueChanged), this, &GeneralSettings::saveMiscSettings);
     connect(_ui->newExternalStorage, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
+    connect(_ui->deltaSyncCheckBox, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings);
+    connect(_ui->deltaSyncSpinBox, static_cast<void (QSpinBox::*)(int)>(&QSpinBox::valueChanged), this, &GeneralSettings::saveMiscSettings);
 
 #ifndef WITH_CRASHREPORTER
     _ui->crashreporterCheckBox->setVisible(false);
@@ -122,6 +124,8 @@ void GeneralSettings::loadMiscSettings()
     _ui->newFolderLimitSpinBox->setValue(newFolderLimit.second);
     _ui->newExternalStorage->setChecked(cfgFile.confirmExternalStorage());
     _ui->monoIconsCheckBox->setChecked(cfgFile.monoIcons());
+    _ui->deltaSyncCheckBox->setChecked(cfgFile.deltaSyncEnabled());
+    _ui->deltaSyncSpinBox->setValue(cfgFile.deltaSyncMinFileSize() / (1024 * 1024));
 }
 
 void GeneralSettings::slotUpdateInfo()
@@ -157,6 +161,8 @@ void GeneralSettings::saveMiscSettings()
     cfgFile.setNewBigFolderSizeLimit(_ui->newFolderLimitCheckBox->isChecked(),
         _ui->newFolderLimitSpinBox->value());
     cfgFile.setConfirmExternalStorage(_ui->newExternalStorage->isChecked());
+    cfgFile.setDeltaSyncEnabled(_ui->deltaSyncCheckBox->isChecked());
+    cfgFile.setDeltaSyncMinFileSize(_ui->deltaSyncSpinBox->value() * 1024 * 1024);
 }
 
 void GeneralSettings::slotToggleLaunchOnStartup(bool enable)

src/gui/generalsettings.ui

@@ -48,6 +48,53 @@
     </widget>
    </item>
    <item row="2" column="0">
+    <widget class="QGroupBox" name="advancedGroupBox">
+     <property name="title">
+      <string>Experimental</string>
+     </property>
+     <layout class="QVBoxLayout" name="verticalLayout_2">
+      <item>
+       <layout class="QHBoxLayout" name="horizontalLayout_2">
+        <item>
+         <widget class="QCheckBox" name="deltaSyncCheckBox">
+          <property name="text">
+           <string>Enable Delta-Synchronization for files larger than</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QSpinBox" name="deltaSyncSpinBox">
+          <property name="maximum">
+           <number>999999</number>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QLabel" name="label_2">
+          <property name="text">
+           <string>MB</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <spacer name="horizontalSpacer_2">
+          <property name="orientation">
+           <enum>Qt::Horizontal</enum>
+          </property>
+          <property name="sizeHint" stdset="0">
+           <size>
+            <width>40</width>
+            <height>20</height>
+           </size>
+          </property>
+         </spacer>
+        </item>
+       </layout>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="3" column="0">
     <widget class="QGroupBox" name="aboutGroupBox">
      <property name="title">
       <string>About</string>
@@ -69,7 +116,7 @@
      </layout>
     </widget>
    </item>
-   <item row="3" column="0">
+   <item row="4" column="0">
     <widget class="QGroupBox" name="updatesGroupBox">
      <property name="title">
       <string>Updates</string>
@@ -120,7 +167,7 @@
      </layout>
     </widget>
    </item>
-   <item row="4" column="0">
+   <item row="5" column="0">
     <spacer name="verticalSpacer">
      <property name="orientation">
       <enum>Qt::Vertical</enum>

src/libsync/CMakeLists.txt

@@ -43,7 +43,9 @@ set(libsync_SRCS
     owncloudtheme.cpp
     progressdispatcher.cpp
     propagatorjobs.cpp
+    propagatecommonzsync.cpp
     propagatedownload.cpp
+    propagatedownloadzsync.cpp
     propagateupload.cpp
     propagateuploadv1.cpp
     propagateuploadng.cpp
@@ -67,6 +69,37 @@ else()
     set (libsync_SRCS ${libsync_SRCS} creds/httpcredentials.cpp)
 endif()
 
+## begin zsync
+
+include_directories( ${CMAKE_SOURCE_DIR}/src/3rdparty/zsync/c )
+
+set( libsync_SRCS
+    ${libsync_SRCS}
+    ../3rdparty/zsync/c/librcksum/hash.c
+    ../3rdparty/zsync/c/librcksum/md4.c
+    ../3rdparty/zsync/c/librcksum/range.c
+    ../3rdparty/zsync/c/librcksum/rsum.c
+    ../3rdparty/zsync/c/librcksum/state.c
+    ../3rdparty/zsync/c/libzsync/sha1.c
+    ../3rdparty/zsync/c/libzsync/zsync.c
+    ../3rdparty/zsync/c/progress.c
+)
+
+if ( WIN32 )
+    # ntohs
+    list(APPEND OS_SPECIFIC_LINK_LIBRARIES
+        ws2_32
+    )
+    # ensure size_t is 64 bits
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_FILE_OFFSET_BITS=64")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_FILE_OFFSET_BITS=64")
+endif()
+
+set_source_files_properties( ../3rdparty/zsync/c/libzsync/zsync.c
+    PROPERTIES COMPILE_FLAGS -DVERSION=\\"0.6.3\\" )
+
+## end zsync
+
 # These headers are installed for libowncloudsync to be used by 3rd party apps
 set(owncloudsync_HEADERS
     account.h

src/libsync/bandwidthmanager.cpp

@@ -120,7 +120,7 @@ void BandwidthManager::unregisterUploadDevice(QObject *o)
     }
 }
 
-void BandwidthManager::registerDownloadJob(GETFileJob *j)
+void BandwidthManager::registerDownloadJob(GETJob *j)
 {
     _downloadJobList.append(j);
     QObject::connect(j, &QObject::destroyed, this, &BandwidthManager::unregisterDownloadJob);
@@ -139,7 +139,7 @@ void BandwidthManager::registerDownloadJob(GETFileJob *j)
 
 void BandwidthManager::unregisterDownloadJob(QObject *o)
 {
-    GETFileJob *j = reinterpret_cast<GETFileJob *>(o); // note, we might already be in the ~QObject
+    GETJob *j = reinterpret_cast<GETJob *>(o); // note, we might already be in the ~QObject
     _downloadJobList.removeAll(j);
     if (_relativeLimitCurrentMeasuredJob == j) {
         _relativeLimitCurrentMeasuredJob = 0;
@@ -289,7 +289,7 @@ void BandwidthManager::relativeDownloadMeasuringTimerExpired()
         quota -= 20 * 1024;
     }
     qint64 quotaPerJob = quota / jobCount + 1.0;
-    Q_FOREACH (GETFileJob *gfj, _downloadJobList) {
+    Q_FOREACH (GETJob *gfj, _downloadJobList) {
         gfj->setBandwidthLimited(true);
         gfj->setChoked(false);
         gfj->giveBandwidthQuota(quotaPerJob);
@@ -323,7 +323,7 @@ void BandwidthManager::relativeDownloadDelayTimerExpired()
     _relativeLimitCurrentMeasuredJob->setChoked(false);
 
     // choke all other download jobs
-    Q_FOREACH (GETFileJob *gfj, _downloadJobList) {
+    Q_FOREACH (GETJob *gfj, _downloadJobList) {
         if (gfj != _relativeLimitCurrentMeasuredJob) {
             gfj->setBandwidthLimited(true);
             gfj->setChoked(true);
@@ -358,7 +358,7 @@ void BandwidthManager::switchingTimerExpired()
     if (newDownloadLimit != _currentDownloadLimit) {
         qCInfo(lcBandwidthManager) << "Download Bandwidth limit changed" << _currentDownloadLimit << newDownloadLimit;
         _currentDownloadLimit = newDownloadLimit;
-        Q_FOREACH (GETFileJob *j, _downloadJobList) {
+        Q_FOREACH (GETJob *j, _downloadJobList) {
             if (usingAbsoluteDownloadLimit()) {
                 j->setBandwidthLimited(true);
                 j->setChoked(false);
@@ -386,7 +386,7 @@ void BandwidthManager::absoluteLimitTimerExpired()
     if (usingAbsoluteDownloadLimit() && _downloadJobList.count() > 0) {
         qint64 quotaPerJob = _currentDownloadLimit / qMax(1, _downloadJobList.count());
         qCDebug(lcBandwidthManager) << quotaPerJob << _downloadJobList.count() << _currentDownloadLimit;
-        Q_FOREACH (GETFileJob *j, _downloadJobList) {
+        Q_FOREACH (GETJob *j, _downloadJobList) {
             j->giveBandwidthQuota(quotaPerJob);
             qCDebug(lcBandwidthManager) << "Gave " << quotaPerJob / 1024.0 << " kB to" << j;
         }

src/libsync/bandwidthmanager.h

@@ -23,7 +23,7 @@
 namespace OCC {
 
 class UploadDevice;
-class GETFileJob;
+class GETJob;
 class OwncloudPropagator;
 
 /**
@@ -47,7 +47,7 @@ public slots:
     void registerUploadDevice(UploadDevice *);
     void unregisterUploadDevice(QObject *);
 
-    void registerDownloadJob(GETFileJob *);
+    void registerDownloadJob(GETJob *);
     void unregisterDownloadJob(QObject *);
 
     void absoluteLimitTimerExpired();
@@ -86,14 +86,14 @@ public slots:
     qint64 _relativeUploadLimitProgressAtMeasuringRestart;
     qint64 _currentUploadLimit;
 
-    QLinkedList<GETFileJob *> _downloadJobList;
+    QLinkedList<GETJob *> _downloadJobList;
     QTimer _relativeDownloadMeasuringTimer;
 
     // for relative bw limiting, we need to wait this amount before measuring again
     QTimer _relativeDownloadDelayTimer;
 
     // the device measured
-    GETFileJob *_relativeLimitCurrentMeasuredJob;
+    GETJob *_relativeLimitCurrentMeasuredJob;
 
     // for measuring how much progress we made at start
     qint64 _relativeDownloadLimitProgressAtMeasuringRestart;

src/libsync/capabilities.cpp

@@ -159,4 +159,9 @@ bool Capabilities::uploadConflictFiles() const
 
     return _capabilities["uploadConflictFiles"].toBool();
 }
+
+QString Capabilities::zsyncSupportedVersion() const
+{
+    return _capabilities["dav"].toMap()["zsync"].toString();
+}
 }

src/libsync/capabilities.h

@@ -43,6 +43,7 @@ class OWNCLOUDSYNC_EXPORT Capabilities
     bool sharePublicLinkMultiple() const;
     bool shareResharing() const;
     bool chunkingNg() const;
+    QString zsyncSupportedVersion() const;
 
     /// disable parallel upload in chunking
     bool chunkingParallelUploadDisabled() const;