Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[wasm] Add Timezone data generator as build task #39913

Merged
merged 10 commits into from
Aug 15, 2020
5 changes: 4 additions & 1 deletion eng/testing/tests.mobile.targets
Original file line number Diff line number Diff line change
Expand Up @@ -165,10 +165,13 @@
<UsingTask TaskName="TimeZoneLoader"
AssemblyFile="$(WasmBundleTaskTasksAssemblyPath)" />
<Target Condition="'$(TargetOS)' == 'Browser'" Name="LoadTimeZone" >
tqiu8 marked this conversation as resolved.
Show resolved Hide resolved
<PropertyGroup>
<TimeZoneDataVersion>2020a</TimeZoneDataVersion>
</PropertyGroup>
<TimeZoneLoader
InputDirectory="$(BundleDir)obj/data/input"
OutputDirectory="$(BundleDir)obj/data/output"
Version="2020a"/>
Version="$(TimeZoneDataVersion)"/>
</Target>

<UsingTask TaskName="WasmBundleTask"
Expand Down
60 changes: 35 additions & 25 deletions tools-local/tasks/mobile.tasks/WasmBundleTask/TimeZoneLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,70 +14,80 @@

public class TimeZoneLoader : Task
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe rename this to something like DownloadTimeZoneData. That is generally how msbuild tasks are named.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems like the other build tasks are named liked WasmAppBuilder or PInvokeTableGenerator

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The mobile.tasks do seem to be named like that, but not others like in installer.tasks. IMHO, the task names should read like actions, that get executed in a target. Not a blocker though.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since this is a new task that we are creating, we can follow the msbuild conventions. The idea here is that a msbuild Target executes a bunch of Tasks, which are actions/code, so, you would name the Tasks like a method.
Let's change the names for the new tasks to match that.

{
[Required]
public string? InputDirectory { get; set; }
tqiu8 marked this conversation as resolved.
Show resolved Hide resolved

[Required]
public string? OutputDirectory { get; set; }

[Required]
public string? Version { get; set; }
tqiu8 marked this conversation as resolved.
Show resolved Hide resolved

private void DownloadTimeZoneData() {
private void DownloadTimeZoneData()
{
List<string> files = new List<string>() {"africa", "antarctica", "asia", "australasia", "etcetera", "europe", "northamerica", "southamerica", "zone1970.tab"};
using (var client = new WebClient())
{
foreach (var file in files) {
client.DownloadFile($"https://data.iana.org/time-zones/tzdb-{Version}/{file}", $"{InputDirectory}/{file}");
foreach (var file in files)
{
client.DownloadFile($"https://data.iana.org/time-zones/tzdb-{Version}/{file}", $"{Path.Combine(InputDirectory!, file)}");
}
}

files.Remove("zone1970.tab");
tqiu8 marked this conversation as resolved.
Show resolved Hide resolved

using (Process process = new Process()) {
using (Process process = new Process())
{
process.StartInfo.UseShellExecute = false;
process.StartInfo.FileName = "zic";
foreach (var f in files) {
process.StartInfo.Arguments = $"-d \"{OutputDirectory}\" \"{InputDirectory}/{f}\"";
foreach (var f in files)
{
process.StartInfo.Arguments = $"-d \"{OutputDirectory}\" \"{Path.Combine(InputDirectory!, f)}\"";
process.Start();
process.WaitForExit();
}
}
File.Copy(Path.Combine(InputDirectory!,"zone1970.tab"), Path.Combine(OutputDirectory!,"zone1970.tab"));
}

private void FilterTimeZoneData(string[] areas) {
private void FilterTimeZoneData(string[] areas)
{
var directoryInfo = new DirectoryInfo (OutputDirectory!);
foreach (var entry in directoryInfo.EnumerateDirectories()) {
if (Array.IndexOf(areas, entry.Name) == -1) {
foreach (var entry in directoryInfo.EnumerateDirectories())
{
if (Array.IndexOf(areas, entry.Name) == -1)
{
Directory.Delete(entry.FullName, true);
}
}
}

private void FilterZoneTab(string[] filters) {
private void FilterZoneTab(string[] filters)
{
var oldPath = Path.Combine(OutputDirectory!, "zone1970.tab");
var path = Path.Combine(OutputDirectory!, "zone.tab");
var fileInfo = new FileInfo(oldPath);
using (var readStream = fileInfo.OpenRead())
using (StreamReader sr = new StreamReader(readStream))
using (FileStream fs = File.OpenWrite(path))
using (StreamWriter sw = new StreamWriter(fs)){
using (StreamReader sr = new StreamReader(oldPath))
using (StreamWriter sw = new StreamWriter(path))
{
string? line;
while ((line = sr.ReadLine()) != null) {
if (filters.Any(x => Regex.IsMatch(line, $@"\b{x}\b"))) {
if (filters.Any(x => Regex.IsMatch(line, $@"\b{x}\b")))
{
sw.WriteLine(line);
}
}
}
File.Delete(oldPath);
}

public override bool Execute() {
if (InputDirectory == null) {
InputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "obj", "data", "input");
}
Directory.CreateDirectory(InputDirectory);
public override bool Execute()
{

if (OutputDirectory == null) {
OutputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "obj", "data", "output");
}
Directory.CreateDirectory(OutputDirectory);
if (!Directory.Exists(InputDirectory))
Directory.CreateDirectory(InputDirectory!);

if (!Directory.Exists(OutputDirectory))
Directory.CreateDirectory(OutputDirectory!);

DownloadTimeZoneData();

Expand Down
26 changes: 15 additions & 11 deletions tools-local/tasks/mobile.tasks/WasmBundleTask/WasmBundleTask.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,25 @@
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;


public class WasmBundleTask : Task
tqiu8 marked this conversation as resolved.
Show resolved Hide resolved
{
[Required]
public string? InputDirectory { get; set; }
tqiu8 marked this conversation as resolved.
Show resolved Hide resolved

[Required]
public string? FileName { get; set; }

private (byte[] json_bytes, MemoryStream stream) EnumerateData() {
private (byte[] json_bytes, MemoryStream stream) EnumerateData()
{
var indices = new List<object[]>();
tqiu8 marked this conversation as resolved.
Show resolved Hide resolved
var stream = new MemoryStream();

var directoryInfo = new DirectoryInfo(InputDirectory!);

foreach (var entry in directoryInfo.EnumerateFiles("*", SearchOption.AllDirectories)) {
foreach (var entry in directoryInfo.EnumerateFiles("*", SearchOption.AllDirectories))
{

var relativePath = entry.FullName.Substring(InputDirectory!.Length).Trim('/');
var relativePath = Path.GetRelativePath(InputDirectory!, entry.FullName);
indices.Add(new object[] { relativePath, entry.Length });

using (var readStream = entry.OpenRead())
Expand All @@ -43,27 +47,27 @@ public override bool Execute()
{
(byte[] json_bytes, MemoryStream stream) data;

if (InputDirectory == null) {
if (InputDirectory == null)
{
throw new ArgumentException("Input directory doesn't exist.");
radical marked this conversation as resolved.
Show resolved Hide resolved
}

tqiu8 marked this conversation as resolved.
Show resolved Hide resolved
data = EnumerateData();

if (FileName == null) {
if (FileName == null)
radical marked this conversation as resolved.
Show resolved Hide resolved
{
throw new ArgumentException($"Invalid file name");
}
using (var file = File.Open(FileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite))
{
var jsonBytes = data.json_bytes;
var stream = data.stream;
var bytes = new byte[4];
tqiu8 marked this conversation as resolved.
Show resolved Hide resolved
var magicBytes = Encoding.ASCII.GetBytes("talb");
BinaryPrimitives.WriteInt32LittleEndian(bytes, jsonBytes.Length);
BinaryPrimitives.WriteInt32LittleEndian(bytes, data.json_bytes.Length);
file.Write(magicBytes);
file.Write(bytes);
file.Write(jsonBytes);
file.Write(data.json_bytes);

stream.CopyTo(file);
data.stream.CopyTo(file);
}

return true;
Expand Down