implement download to split_file for all remote clients

cy33hc · Jun 28, 2024 · a99bf16 · a99bf16
1 parent b9ab715
commit a99bf16
Show file tree

Hide file tree

Showing 12 changed files with 204 additions and 40 deletions.
diff --git a/source/clients/ftpclient.cpp b/source/clients/ftpclient.cpp
@@ -880,6 +880,32 @@ int FtpClient::FtpXfer(const std::string &localfile, const std::string &path, ft
 	return FtpClose(nData);
 }
 
+/*
+ * FtpXfer - issue a command and transfer data
+ *
+ * return 1 if successful, 0 otherwise
+ */
+int FtpClient::FtpXfer(SplitFile *split_file, const std::string &path, ftphandle *nControl, accesstype type, transfermode mode)
+{
+	int l, c;
+	char *dbuf;
+	ftphandle *nData;
+
+	if (!FtpAccess(path, type, mode, nControl, &nData))
+	{
+		return 0;
+	}
+
+	dbuf = static_cast<char *>(malloc(FTP_CLIENT_BUFSIZ));
+	while ((l = FtpRead(dbuf, FTP_CLIENT_BUFSIZ, nData)) > 0)
+	{
+		split_file->Write(dbuf, l);
+	}
+
+	free(dbuf);
+	return FtpClose(nData);
+}
+
 /*
  * FtpWrite - write to a data connection
  */
@@ -1283,6 +1309,15 @@ int FtpClient::Get(const std::string &outputfile, const std::string &path, uint6
 		return FtpXfer(outputfile, path, mp_ftphandle, FtpClient::filereadappend, FtpClient::transfermode::image);
 }
 
+int FtpClient::Get(SplitFile *split_file, const std::string &path, uint64_t offset)
+{
+	mp_ftphandle->offset = offset;
+	if (offset == 0)
+		return FtpXfer(split_file, path, mp_ftphandle, FtpClient::fileread, FtpClient::transfermode::image);
+	else
+		return FtpXfer(split_file, path, mp_ftphandle, FtpClient::filereadappend, FtpClient::transfermode::image);
+}
+
 int FtpClient::GetRange(const std::string &path, DataSink &sink, uint64_t size, uint64_t offset)
 {
 	ftphandle *nData;
@@ -1787,7 +1822,7 @@ int FtpClient::Head(const std::string &path, void *buffer, uint64_t len)
 
 void *FtpClient::Open(const std::string &path, int flags)
 {
-    return nullptr;
+	return nullptr;
 }
 
 void FtpClient::Close(void *fp)
@@ -1796,10 +1831,10 @@ void FtpClient::Close(void *fp)
 
 int FtpClient::GetRange(void *fp, DataSink &sink, uint64_t size, uint64_t offset)
 {
-    return -1;
+	return -1;
 }
 
 int FtpClient::GetRange(void *fp, void *buffer, uint64_t size, uint64_t offset)
 {
-    return -1;
+	return -1;
 }
diff --git a/source/clients/ftpclient.h b/source/clients/ftpclient.h
@@ -79,6 +79,7 @@ class FtpClient : public RemoteClient
 	int Rmdir(const std::string &path, bool recursive);
 	int Size(const std::string &path, int64_t *size);
 	int Get(const std::string &outputfile, const std::string &path, uint64_t offset = 0);
+	int Get(SplitFile *split_file, const std::string &path, uint64_t offset=0);
 	int GetRange(const std::string &path, void *buffer, uint64_t size, uint64_t offset);
 	int GetRange(const std::string &path, DataSink &sink, uint64_t size, uint64_t offset);
 	int GetRange(void *fp, void *buffer, uint64_t size, uint64_t offset);
@@ -128,6 +129,7 @@ class FtpClient : public RemoteClient
 	int CorrectPasvResponse(int *v);
 	int FtpAccess(const std::string &path, accesstype type, transfermode mode, ftphandle *nControl, ftphandle **nData);
 	int FtpXfer(const std::string &localfile, const std::string &path, ftphandle *nControl, accesstype type, transfermode mode);
+	int FtpXfer(SplitFile *split_file, const std::string &path, ftphandle *nControl, accesstype type, transfermode mode);
 	int FtpWrite(void *buf, int len, ftphandle *nData);
 	int FtpRead(void *buf, int max, ftphandle *nData);
 	int FtpClose(ftphandle *nData);

diff --git a/source/clients/gdrive.cpp b/source/clients/gdrive.cpp
@@ -332,6 +332,34 @@ int GDriveClient::Get(const std::string &outputfile, const std::string &path, ui
     return 0;
 }
 
+int GDriveClient::Get(SplitFile *split_file, const std::string &path, uint64_t offset)
+{
+    std::string id = GetValue(path_id_map, path);
+    std::string drive_id = GetDriveId(path);
+    std::string url = std::string("/drive/v3/files/") + BaseClient::Escape(id) + "?alt=media";
+    if (!drive_id.empty())
+        url += "&supportsAllDrives=true";
+    if (auto res = client->Get(url,
+                               [&](const char *data, size_t data_length)
+                               {
+                                   if (!split_file->IsClosed())
+                                   {
+                                        split_file->Write((char*)data, data_length);
+                                        return true;
+                                   }
+                                   else
+                                        return false;
+                               }))
+    {
+        return 1;
+    }
+    else
+    {
+        sprintf(this->response, "%s", httplib::to_string(res.error()).c_str());
+    }
+    return 0;
+}
+
 int GDriveClient::GetRange(const std::string &path, DataSink &sink, uint64_t size, uint64_t offset)
 {
     size_t bytes_read = 0;

diff --git a/source/clients/gdrive.h b/source/clients/gdrive.h
@@ -20,6 +20,7 @@ class GDriveClient : public BaseClient
     int Connect(const std::string &url, const std::string &user, const std::string &pass);
     int Rename(const std::string &src, const std::string &dst);
     int Get(const std::string &outputfile, const std::string &path, uint64_t offset=0);
+    int Get(SplitFile *split_file, const std::string &path, uint64_t offset=0);
     int GetRange(const std::string &path, void *buffer, uint64_t size, uint64_t offset);
     int GetRange(const std::string &path, DataSink &sink, uint64_t size, uint64_t offset);
     int GetRange(void *fp, void *buffer, uint64_t size, uint64_t offset);

diff --git a/source/clients/nfsclient.cpp b/source/clients/nfsclient.cpp
@@ -17,7 +17,7 @@
 #include "util.h"
 #include "system.h"
 
-#define BUF_SIZE 256*1024
+#define BUF_SIZE 256 * 1024
 
 NfsClient::NfsClient()
 {
@@ -37,7 +37,8 @@ int NfsClient::Connect(const std::string &url, const std::string &user, const st
 	}
 
 	struct nfs_url *nfsurl = nfs_parse_url_full(nfs, url.c_str());
-	if (nfsurl == nullptr) {
+	if (nfsurl == nullptr)
+	{
 		sprintf(response, "%s", nfs_get_error(nfs));
 		nfs_destroy_context(nfs);
 		return 0;
@@ -203,7 +204,7 @@ int NfsClient::Get(const std::string &outputfile, const std::string &ppath, uint
 		return 0;
 	}
 
-	FILE* out = FS::Create(outputfile);
+	FILE *out = FS::Create(outputfile);
 	if (out == NULL)
 	{
 		sprintf(response, "%s", lang_strings[STR_FAILED]);
@@ -221,15 +222,44 @@ int NfsClient::Get(const std::string &outputfile, const std::string &ppath, uint
 			sprintf(response, "%s", nfs_get_error(nfs));
 			FS::Close(out);
 			nfs_close(nfs, nfsfh);
-			free((void*)buff);
+			free((void *)buff);
 			return 0;
 		}
 		FS::Write(out, buff, count);
 		bytes_transfered += count;
 	}
 	FS::Close(out);
 	nfs_close(nfs, nfsfh);
-	free((void*)buff);
+	free((void *)buff);
+
+	return 1;
+}
+
+int NfsClient::Get(SplitFile *split_file, const std::string &ppath, uint64_t offset)
+{
+	struct nfsfh *nfsfh = nullptr;
+	int ret = nfs_open(nfs, ppath.c_str(), 0400, &nfsfh);
+	if (ret != 0)
+	{
+		sprintf(response, "%s", nfs_get_error(nfs));
+		return 0;
+	}
+
+	void *buff = malloc(BUF_SIZE);
+	int count = 0;
+	while ((count = nfs_read(nfs, nfsfh, BUF_SIZE, buff)) > 0)
+	{
+		if (count < 0)
+		{
+			sprintf(response, "%s", nfs_get_error(nfs));
+			nfs_close(nfs, nfsfh);
+			free((void *)buff);
+			return 0;
+		}
+		split_file->Write((char *)buff, count);
+	}
+	nfs_close(nfs, nfsfh);
+	free((void *)buff);
 
 	return 1;
 }
@@ -260,29 +290,29 @@ int NfsClient::GetRange(void *fp, DataSink &sink, uint64_t size, uint64_t offset
 	}
 
 	void *buff = malloc(BUF_SIZE);
-    int count = 0;
-    size_t bytes_remaining = size;
-    do
-    {
-        size_t bytes_to_read = std::min<size_t>(BUF_SIZE, bytes_remaining);
-        count = nfs_read(nfs, nfsfh, bytes_to_read, buff);
-        if (count > 0)
-        {
-            bytes_remaining -= count;
-            bool ok = sink.write((char*)buff, count);
+	int count = 0;
+	size_t bytes_remaining = size;
+	do
+	{
+		size_t bytes_to_read = std::min<size_t>(BUF_SIZE, bytes_remaining);
+		count = nfs_read(nfs, nfsfh, bytes_to_read, buff);
+		if (count > 0)
+		{
+			bytes_remaining -= count;
+			bool ok = sink.write((char *)buff, count);
 			if (!ok)
 			{
 				free((void *)buff);
 				return 0;
 			}
-        }
-        else
-        {
-            break;
-        }
-    } while (1);
-
-    free((void *)buff);
+		}
+		else
+		{
+			break;
+		}
+	} while (1);
+
+	free((void *)buff);
 	return 1;
 }
 
@@ -304,7 +334,7 @@ int NfsClient::GetRange(const std::string &ppath, void *buffer, uint64_t size, u
 
 int NfsClient::GetRange(void *fp, void *buffer, uint64_t size, uint64_t offset)
 {
-	struct nfsfh *nfsfh = (struct nfsfh *) fp;
+	struct nfsfh *nfsfh = (struct nfsfh *)fp;
 
 	int ret = nfs_lseek(nfs, nfsfh, offset, SEEK_SET, NULL);
 	if (ret != 0)
@@ -358,13 +388,13 @@ int NfsClient::Put(const std::string &inputfile, const std::string &ppath, uint6
 		return 0;
 	}
 
-	FILE* in = FS::OpenRead(inputfile);
+	FILE *in = FS::OpenRead(inputfile);
 	if (in == NULL)
 	{
 		sprintf(response, "%s", lang_strings[STR_FAILED]);
 		return 0;
 	}
-	
+
 	struct nfsfh *nfsfh = nullptr;
 	int ret;
 	if (!FileExists(ppath))
@@ -380,7 +410,7 @@ int NfsClient::Put(const std::string &inputfile, const std::string &ppath, uint6
 		return 0;
 	}
 
-	void* buff = malloc(BUF_SIZE);
+	void *buff = malloc(BUF_SIZE);
 	uint64_t count = 0;
 	bytes_transfered = 0;
 	sceRtcGetCurrentTick(&prev_tick);
@@ -459,7 +489,8 @@ std::vector<DirEntry> NfsClient::ListDir(const std::string &path)
 	struct nfsdirent *nfsdirent;
 
 	int ret = nfs_opendir(nfs, path.c_str(), &nfsdir);
-	if (ret != 0) {
+	if (ret != 0)
+	{
 		sprintf(response, "%s", nfs_get_error(nfs));
 		return out;
 	}
@@ -526,7 +557,6 @@ std::vector<DirEntry> NfsClient::ListDir(const std::string &path)
 		}
 		if (strcmp(entry.name, "..") != 0 && strcmp(entry.name, ".") != 0)
 			out.push_back(entry);
-
 	}
 	nfs_closedir(nfs, nfsdir);
 
@@ -569,13 +599,14 @@ int NfsClient::Head(const std::string &ppath, void *buffer, uint64_t len)
 void *NfsClient::Open(const std::string &path, int flags)
 {
 	struct nfsfh *nfsfh = nullptr;
-    nfs_open(nfs, path.c_str(), 0400, &nfsfh);;
+	nfs_open(nfs, path.c_str(), 0400, &nfsfh);
+	;
 	return nfsfh;
 }
 
 void NfsClient::Close(void *fp)
 {
-    nfs_close(nfs, (struct nfsfh*)fp);
+	nfs_close(nfs, (struct nfsfh *)fp);
 }
 
 ClientType NfsClient::clientType()

diff --git a/source/clients/nfsclient.h b/source/clients/nfsclient.h
@@ -23,6 +23,7 @@ class NfsClient : public RemoteClient
 	int Rmdir(const std::string &path, bool recursive);
 	int Size(const std::string &path, int64_t *size);
 	int Get(const std::string &outputfile, const std::string &path, uint64_t offset=0);
+	int Get(SplitFile *split_file, const std::string &path, uint64_t offset=0);
 	int GetRange(const std::string &path, void *buffer, uint64_t size, uint64_t offset);
 	int GetRange(const std::string &path, DataSink &sink, uint64_t size, uint64_t offset);
 	int GetRange(void *fp, void *buffer, uint64_t size, uint64_t offset);

diff --git a/source/clients/remote_client.h b/source/clients/remote_client.h
@@ -5,6 +5,7 @@
 #include <vector>
 #include "common.h"
 #include "http/httplib.h"
+#include "split_file.h"
 
 enum RemoteActions
 {
@@ -49,6 +50,7 @@ class RemoteClient
     virtual int Rmdir(const std::string &path, bool recursive) = 0;
     virtual int Size(const std::string &path, int64_t *size) = 0;
     virtual int Get(const std::string &outputfile, const std::string &path, uint64_t offset=0) = 0;
+    virtual int Get(SplitFile *split_file, const std::string &path, uint64_t offset=0) = 0;
     virtual int Put(const std::string &inputfile, const std::string &path, uint64_t offset=0) = 0;
     virtual int Rename(const std::string &src, const std::string &dst) = 0;
     virtual int Delete(const std::string &path) = 0;