validator: Improve error reporting, add support for multiple files

- Display clickable links to directly jump to error line - Display invalid JSON path - Dynamic error nesting - Support for validation multiple files e.g. `validator.exe .\schema.json .\bucket\*.json` or `validator.exe .\schema.json .\manifest1.json .\manifest2.json`
Ash258 · Feb 24, 2019 · 7bedd44 · 7bedd44
1 parent aefba66
commit 7bedd44
Show file tree

Hide file tree

Showing 7 changed files with 84 additions and 27 deletions.
diff --git a/supporting/validator/Scoop.Validator.cs b/supporting/validator/Scoop.Validator.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Text;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Newtonsoft.Json.Schema;
@@ -118,19 +119,44 @@ public bool Validate()
 
             this.Manifest.IsValid(this.Schema, out validationErrors);
 
-            if (validationErrors.Count > 0)
+            if (validationErrors.Count == 0)
             {
-                foreach (ValidationError error in validationErrors)
-                {
-                    this.Errors.Add(String.Format("{0}{1}: {2}", (this.CI ? "    [*] " : ""), this.ManifestFile.Name, error.Message));
-                    foreach (ValidationError childError in error.ChildErrors)
-                    {
-                        this.Errors.Add(String.Format((this.CI ? "    [^] {0}{1}" : "{0}^ {1}"), new String(' ', this.ManifestFile.Name.Length + 2), childError.Message));
-                    }
-                }
+                return true;
             }
+            traverseErrors(validationErrors, this.CI ? 3 : 1);
 
             return (this.Errors.Count == 0);
         }
+
+        public void traverseErrors(IList<ValidationError> errors, int level = 1) {
+            if(errors == null) {
+                return;
+            }
+            foreach (ValidationError error in errors)
+            {
+                StringBuilder sb = new StringBuilder();
+                sb.Insert(sb.Length, " ", level * 2);
+                sb.Insert(sb.Length, this.CI ? "[*] " : "- ");
+                sb.AppendFormat("Error: {0}\n", error.Message);
+
+                sb.Insert(sb.Length, " ", level * 2);
+                sb.Insert(sb.Length, this.CI ? "  [^] " : "  ");
+                sb.AppendFormat("Line: {0}:{1}:{2}\n", this.ManifestFile.FullName, error.LineNumber, error.LinePosition);
+
+                sb.Insert(sb.Length, " ", level * 2);
+                sb.Insert(sb.Length, this.CI ? "  [^] " : "  ");
+                sb.AppendFormat("Path: {0}/{1}", error.SchemaId, error.ErrorType);
+
+                if(!this.CI) {
+                    sb.Insert(sb.Length, "\n");
+                }
+
+                this.Errors.Add(sb.ToString());
+
+                if(error.ChildErrors != null || error.ChildErrors.Count > 0) {
+                    traverseErrors(error.ChildErrors, level + 1);
+                }
+            }
+        }
     }
 }
diff --git a/supporting/validator/bin/Scoop.Validator.dll b/supporting/validator/bin/Scoop.Validator.dll
diff --git a/supporting/validator/bin/checksum.sha256 b/supporting/validator/bin/checksum.sha256
@@ -1,4 +1,4 @@
 f58c374ffcaae4e36d740d90fbf7fe70d0abb7328cd9af3a0a7b70803e994ba4 *Newtonsoft.Json.dll
 4ca0681df3755205cfd178aaa38f7fd6818c183c4d14c77cd77f8d777282a650 *Newtonsoft.Json.Schema.dll
-e077a05608126d44938158da04fe2062513afedba666648ed185d3e25b50436e *Scoop.Validator.dll
-cc82c9659716b564a69690a8b67e5472ea1e8e060b974306314acdd42767bf6d *validator.exe
+55f1292b42fa2e8a7a4b3eb360fcfffe97be0fe043b4709979fcd37b8a400456 *Scoop.Validator.dll
+6dfbe59d9f1c17d362bdd4509975ab40d96e09585aa8d0a1f5527dafbc1c3727 *validator.exe
diff --git a/supporting/validator/bin/checksum.sha512 b/supporting/validator/bin/checksum.sha512
@@ -1,4 +1,4 @@
 7cbbbef742f56af80f1012d7da86fe5375ac05813045756fb45d0691c36ef13c069361457500ba4200157d5ee7922fd118bf4c0635e5192e3f8c6183fd580944 *Newtonsoft.Json.dll
 2d9c630948c21b325af7b7ad3de9219a3fdcc23fa65270a0cdb78ea9609038434f6b741a968e9758f93782a3bab57efc3072429bdf84f8ad25a94bf5a8c4bc48 *Newtonsoft.Json.Schema.dll
-15b0aa8b29f4e7d74ae3ff36ce7d72f6243a9b7f35890df3e549340c27edf37a73a9d16132254baf661a9aadc6fd5870d3b04452b78df79e3ccf9bee1c9bd788 *Scoop.Validator.dll
-384d22bfb21b2767de862f95945dca91aefb4b5eca687d20e410dc64059c5402a13c085517cb807635ff07397fee9bda31b0fdc0090c24cb424f62b285ea0477 *validator.exe
+6d89196ce01823093f5c3da98d34d41bc1733316d04e181d6f24982a373cc28487b8e4e74b54f5a665a550ad4a7fa6d7208d99de947b82b20e524747bdb41bdd *Scoop.Validator.dll
+e9b2162b3e109291c61005a9088d9ebf30fb0307fa285ccf5c54aef05fb3c02944d9513f2557e75d670682b0f3ba67a4f8df3561c715eaa5daa077b591fb4d54 *validator.exe
diff --git a/supporting/validator/bin/validator.exe b/supporting/validator/bin/validator.exe
diff --git a/supporting/validator/validator.cs b/supporting/validator/validator.cs
@@ -1,33 +1,63 @@
 using System;
+using System.Collections;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 
 namespace Scoop
 {
     public class Program
     {
         public static int Main(string[] args)
         {
-            bool ci = (args.Length == 3 && args[2] == "-ci");
-            bool valid = false;
+            bool ci = String.Format("{0}", Environment.GetEnvironmentVariable("CI")).ToLower() == "true";
+            bool valid = true;
 
             if (args.Length < 2)
             {
-                Console.WriteLine("Usage: validator.exe schema.json manifest.json");
+                Console.WriteLine("Usage: validator.exe <schema> <manifest> [<manifest>...]");
                 return 1;
             }
 
-            Scoop.Validator validator = new Scoop.Validator(args[0], ci);
-            valid = validator.Validate(args[1]);
 
-            if (valid)
-            {
-                Console.WriteLine("Yay! {0} validates against the schema!", Path.GetFileName(args[1]));
+            IList<string> manifests = args.ToList<String>();
+            String schema = manifests.First();
+            manifests.RemoveAt(0);
+            String combinedArgs = String.Join("", manifests);
+            if(combinedArgs.Contains("*") || combinedArgs.Contains("?")) {
+                try {
+                    var path = new Uri(Path.Combine(Directory.GetCurrentDirectory(), combinedArgs)).LocalPath;
+                    var drive = Path.GetPathRoot(path);
+                    var pattern = path.Replace(drive, "");
+                    manifests = Directory.GetFiles(drive, pattern).ToList<String>();
+                } catch (System.ArgumentException ex) {
+                    Console.WriteLine("Invalid path provided! ({0})", ex.Message);
+                    return 1;
+                }
             }
-            else
-            {
-                foreach (var error in validator.Errors)
+
+            Scoop.Validator validator = new Scoop.Validator(schema, ci);
+            foreach(var manifest in manifests) {
+                if (validator.Validate(manifest))
+                {
+                    if(ci) {
+                        Console.WriteLine("      [+] {0} validates against the schema!", Path.GetFileName(manifest));
+                    } else {
+                        Console.WriteLine("- {0} validates against the schema!", Path.GetFileName(manifest));
+                    }
+                }
+                else
                 {
-                    Console.WriteLine(error);
+                    if(ci) {
+                        Console.WriteLine("      [-] {0} has {1} Error{2}!", Path.GetFileName(manifest), validator.Errors.Count, validator.Errors.Count > 1 ? "s" : "");
+                    } else {
+                        Console.WriteLine("- {0} has {1} Error{2}!", Path.GetFileName(manifest), validator.Errors.Count, validator.Errors.Count > 1 ? "s" : "");
+                    }
+                    valid = false;
+                    foreach (var error in validator.Errors)
+                    {
+                        Console.WriteLine(error);
+                    }
                 }
             }
 

diff --git a/test/Scoop-Manifest.Tests.ps1 b/test/Scoop-Manifest.Tests.ps1
@@ -39,8 +39,8 @@ describe -Tag 'Manifests' "manifest-validation" {
             $validator = new-object Scoop.Validator($schema, $true)
             $validator.Validate("$working_dir/invalid_wget.json") | should -BeFalse
             $validator.Errors.Count | should -be 16
-            $validator.Errors | select-object -First 1 | should -match "invalid_wget.*randomproperty.*properties\.$"
-            $validator.Errors | select-object -Last 1 | should -match "invalid_wget.*version\.$"
+            $validator.Errors | select-object -First 1 | should -match "Property 'randomproperty' has not been defined and the schema does not allow additional properties\."
+            $validator.Errors | select-object -Last 1 | should -match "Required properties are missing from object: version\."
         }
     }
 
@@ -73,6 +73,7 @@ describe -Tag 'Manifests' "manifest-validation" {
                         $validator.Validate($file.fullname)
 
                         if ($validator.Errors.Count -gt 0) {
+                            write-host -f red "      [-] $_ has $($validator.Errors.Count) Error$(If($validator.Errors.Count -gt 1) { 's' })!"
                             write-host -f yellow $validator.ErrorsAsString
                         }
                         $validator.Errors.Count | should -be 0