Skip to content

Commit

Permalink
feat: GitHub clone and load workspace
Browse files Browse the repository at this point in the history
  • Loading branch information
@amis92
amis92 committed May 6, 2020
1 parent f5fe387 commit edc8dd4
Show file tree
Hide file tree
Showing 5 changed files with 555 additions and 32 deletions.
1 change: 1 addition & 0 deletions src/WarHub.GodMode.GithubPages/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ private static void ConfigureServices(IServiceCollection services)
{
services.AddHeadElementHelper();
services.AddSingleton<GitHubWorkspaceProvider>();
services.AddSingleton<JsMemoryInteropService>();
services.AddScoped<IWorkspaceProviderAggregate, WorkspaceProviderAggregate>();
services.AddScoped<IWorkspaceContextResolver, WorkspaceContextResolver>();
}
Expand Down
225 changes: 193 additions & 32 deletions src/WarHub.GodMode.GithubPages/Services/GitHubWorkspaceProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.IO.Compression;
using System.Net.Http;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.JSInterop;
using WarHub.ArmouryModel.ProjectModel;
using WarHub.ArmouryModel.Source;
using WarHub.ArmouryModel.Workspaces.BattleScribe;
Expand All @@ -14,13 +15,18 @@ namespace WarHub.GodMode.GithubPages.Services
{
public class GitHubWorkspaceProvider : IWorkspaceProvider
{
public GitHubWorkspaceProvider(HttpClient httpClient)
private const string InteropNamespace = "ghpagesInterop";
private readonly IJSRuntime js;
private readonly JsMemoryInteropService memoryInterop;
private readonly InteropMemFileSystem memFs;

public GitHubWorkspaceProvider(IJSRuntime js, JsMemoryInteropService memoryInterop)
{
HttpClient = httpClient;
this.js = js;
this.memoryInterop = memoryInterop;
memFs = new InteropMemFileSystem(ReadFile);
}

private HttpClient HttpClient { get; }

// TODO cache -> IndexedDB
private Dictionary<Uri, IWorkspace> Cache { get; }
= new Dictionary<Uri, IWorkspace>();
Expand All @@ -29,47 +35,124 @@ public async Task<IWorkspace> GetWorkspace(WorkspaceInfo info)
{
if (info is { Type: WorkspaceType.GitHub, GitHubUrl: { } })
{
return await GetWorkspaceCore(info);
if (Cache.TryGetValue(info.GitHubUrl, out var cached))
{
return cached;
}
var newWorkspace = await GetWorkspaceCore(info);
return Cache[info.GitHubUrl] = newWorkspace;
}
return null;
}

public async Task<IWorkspace> GetWorkspaceCore(WorkspaceInfo info)
private async Task<IWorkspace> GetWorkspaceCore(WorkspaceInfo info)
{
if (Cache.TryGetValue(info.GitHubUrl, out var cached))
bool clone = false;
try
{
var dbfs = GetFsRoot(info);
var files = await ListDir(dbfs);
//clone = true;
Console.WriteLine($"Read dir {dbfs}");
}
catch (Exception)
{
return cached;
// interop/IndexedDB doesn't contain, clone again
clone = true;
}
return await Task.Run(async () =>
{
var repoPath = string.Concat(info.GitHubUrl.Segments[^2..]);
var repoUri = info.GitHubUrl + "/archive/master.zip";
var response = await HttpClient.GetAsync(repoUri);
response.EnsureSuccessStatusCode();
using var zipStream = await response.Content.ReadAsStreamAsync();
using var zip = new ZipArchive(zipStream);
var datafiles = GetDatafiles(zip).ToImmutableArray();
var workspace = new InMemoryWorkspace(info.GitHubUrl.ToString(), datafiles);
return Cache[info.GitHubUrl] = workspace;
if (clone)
{
await CloneRepo(info);
}
return await LoadWorkspaceFromFs(info);
});
}

IEnumerable<IDatafileInfo> GetDatafiles(ZipArchive zip)
private async Task<IWorkspace> LoadWorkspaceFromFs(WorkspaceInfo info)
{
var root = GetFsRoot(info);
var rootFiles = await ListDir(root);
//var datafiles = ReadDatafiles(root, rootFiles);
var datafiles = await ReadDatafilesAsync(root, rootFiles).ToListAsync();
return new InMemoryWorkspace(root, datafiles.ToImmutableArray());

IEnumerable<IDatafileInfo> ReadDatafiles(string root, string[] filenames)
{
foreach (var entry in zip.Entries)
foreach (var filename in filenames)
{
using var entryStream = entry.Open();
var file = entry.Name.GetXmlDocumentKind() switch
{
XmlDocumentKind.Gamesystem => entryStream.LoadSourceAuto(entry.Name),
XmlDocumentKind.Catalogue => entryStream.LoadSourceAuto(entry.Name),
_ => (IDatafileInfo)new UnknownTypeDatafileInfo(entry.Name)
};
if (file is { })
{
yield return file;
}
yield return ReadDatafile(root, filename);
}
}

IDatafileInfo ReadDatafile(string root, string filename)
{
var filepath = $"{root}/{filename}";
var kind = filepath.GetXmlDocumentKind();
if (kind == XmlDocumentKind.Catalogue)
{
return new LazyWeakXmlDatafileInfo<CatalogueNode>(filepath, SourceKind.Catalogue, memFs);
}
else if (kind == XmlDocumentKind.Gamesystem)
{
return new LazyWeakXmlDatafileInfo<GamesystemNode>(filepath, SourceKind.Gamesystem, memFs);
}
else
{
return new UnknownTypeDatafileInfo(filename);
}
}

async IAsyncEnumerable<IDatafileInfo> ReadDatafilesAsync(string root, string[] filenames)
{
foreach (var filename in filenames)
{
yield return await ReadDatafileAsync(root, filename);
}
}

async Task<IDatafileInfo> ReadDatafileAsync(string root, string filename)
{
var filepath = $"{root}/{filename}";
var kind = filepath.GetXmlDocumentKind();
if (kind == XmlDocumentKind.Catalogue || kind == XmlDocumentKind.Gamesystem)
{
using var filestream = await ReadFile(filepath);
return filestream.LoadSourceAuto(filepath);
}
else
{
return new UnknownTypeDatafileInfo(filename);
}
}
}

private string GetFsRoot(WorkspaceInfo info) => $"/gh/{info.GitHubRepository}";

private async Task<CloneResult> CloneRepo(WorkspaceInfo info)
{
var repo = info.GitHubUrl;
var fsRoot = GetFsRoot(info);
return await js.InvokeAsync<CloneResult>(InteropNamespace + ".gitClone", repo, fsRoot);
}

private async Task<string[]> ListDir(string path)
{
return await js.InvokeAsync<string[]>("pfs.readdir", path);
}

private async Task<Stream> ReadFile(string path)
{
const string ReadFileName = InteropNamespace + ".fsReadFileMemRef";
return await memoryInterop.OpenReadAsync(
async (script) => await js.InvokeAsync<int>(ReadFileName, path, script));
}

private class CloneResult
{
public string Root { get; set; }
public string[] RootEntries { get; set; }
}

private class InMemoryWorkspace : IWorkspace
Expand Down Expand Up @@ -112,5 +195,83 @@ public UnknownTypeDatafileInfo(string filepath)

public string GetStorageName() => Path.GetFileNameWithoutExtension(Filepath);
}

private class LazyWeakXmlDatafileInfo<TData> : IDatafileInfo<TData> where TData : SourceNode
{
private readonly IFileSystem fs;

public LazyWeakXmlDatafileInfo(string path, SourceKind dataKind, IFileSystem fs)
{
Filepath = path;
DataKind = dataKind;
this.fs = fs;
}

public string Filepath { get; }

public TData Data => GetData();

public SourceKind DataKind { get; }

private WeakReference<TData> WeakData { get; } = new WeakReference<TData>(null);

public TData GetData()
{
if (WeakData.TryGetTarget(out var cached))
{
return cached;
}
var data = ReadFile();
WeakData.SetTarget(data);
return data;
}

public string GetStorageName() => Path.GetFileNameWithoutExtension(Filepath);

SourceNode IDatafileInfo.GetData() => GetData();

private TData ReadFile()
{
using (var filestream = fs.OpenRead(Filepath))
{
var node = XmlFileExtensions.ZippedExtensions.Contains(Path.GetExtension(Filepath))
? filestream.LoadSourceZipped(DataKind.GetXmlDocumentKindOrUnknown())
: filestream.LoadSource(DataKind.GetXmlDocumentKindOrUnknown());
return (TData)node;
}
}
}

private interface IFileSystem
{
Stream OpenRead(string path);
}

private class InteropMemFileSystem : IFileSystem
{
private readonly Func<string, Task<Stream>> asyncOpen;

public InteropMemFileSystem(Func<string, Task<Stream>> asyncOpen)
{
this.asyncOpen = asyncOpen;
}

public Stream OpenRead(string path)
{
var task = asyncOpen(path);
if (task.IsCompleted)
{
return task.Result;
}
else
{
while (!task.IsCompleted)
{
Thread.Sleep(100);
}
return task.Result;
}
}
}
}
}
Loading

0 comments on commit edc8dd4

Please sign in to comment.