Add jsonpath support for checkver and hash extraction

Unravel if-blocks in checkver Use new jsonpath for openssl, nuget, dart and bfg
Ash258 · Mar 3, 2017 · 825f19a · 825f19a
1 parent 314a27e
commit 825f19a
Show file tree

Hide file tree

Showing 8 changed files with 188 additions and 91 deletions.
diff --git a/bin/checkver.ps1 b/bin/checkver.ps1
@@ -52,35 +52,45 @@ $queue | % {
     $name, $json = $_
 
     $githubRegex = "\/releases\/tag\/(?:v)?([\d.]+)"
-    if ($json.checkver -is [String]) {
-        if ($json.checkver -eq "github") {
-            if (!$json.homepage.StartsWith("https://github.com/")) {
-                write-host "ERROR: $name checkver expects the homepage to be a github repository" -f DarkYellow
-            }
 
-            $url = $json.homepage + "/releases/latest"
-            $regex = $githubRegex
-        } else {
-            $url = $json.homepage
-            $regex = $json.checkver
-        }
-    } else {
-        if ($json.checkver.github) {
-            $url = $json.checkver.github + "/releases/latest"
-            $regex = $githubRegex
-        } else {
-            $url = $json.checkver.url
-            if(!$url) { $url = $json.homepage }
+    $url = $json.homepage
+    if($json.checkver.url) {
+        $url = $json.checkver.url
+    }
+    $regex = ""
+    $jsonpath = ""
 
-            $regex = $json.checkver.re
+    if ($json.checkver -eq "github") {
+        if (!$json.homepage.StartsWith("https://github.com/")) {
+            write-host "ERROR: $name checkver expects the homepage to be a github repository" -f DarkYellow
         }
+        $url = $json.homepage + "/releases/latest"
+        $regex = $githubRegex
+    }
+
+    if ($json.checkver.github) {
+        $url = $json.checkver.github + "/releases/latest"
+        $regex = $githubRegex
+    }
+
+    if($json.checkver.re) {
+        $regex = $json.checkver.re
+    }
+
+    if($json.checkver.jp) {
+        $jsonpath = $json.checkver.jp
+    }
+
+    if(!$jsonpath -and !$regex) {
+        $regex = $json.checkver
     }
 
     $state = new-object psobject @{
         app = (strip_ext $name);
         url = $url;
         regex = $regex;
         json = $json;
+        jsonpath = $jsonpath;
     }
 
     $wc.downloadstringasync($url, $state)
@@ -99,6 +109,8 @@ while($in_progress -gt 0) {
     $url = $state.url
     $expected_ver = $json.version
     $regexp = $state.regex
+    $jsonpath = $state.jsonpath
+    $ver = ""
 
     $err = $ev.sourceeventargs.error
     $page = $ev.sourceeventargs.result
@@ -108,36 +120,58 @@ while($in_progress -gt 0) {
     if($err) {
         write-host -f darkred $err.message
         write-host -f darkred "URL $url is not valid"
-    } else {
+        continue
+    }
+
+    if($jsonpath -and $regexp) {
+        write-host -f darkred "'jp' and 're' shouldn't be used together"
+        continue
+    }
+
+    if($jsonpath) {
+        $ver = json_path ($page | ConvertFrom-Json -ea stop) $jsonpath
+        if(!$ver) {
+            write-host -f darkred "couldn't find '$jsonpath' in $url"
+            continue
+        }
+    }
+
+    if($regexp) {
         if($page -match $regexp) {
             $ver = $matches[1]
             if(!$ver) {
                 $ver = $matches['version']
             }
-            if($ver -eq $expected_ver) {
-                write-host "$ver" -f darkgreen
-
-                if ($forceUpdate -and $json.autoupdate) {
-                    Write-Host "Forcing autoupdate!" -f DarkMagenta
-                    autoupdate $app $dir $json $ver $matches
-                }
-            } else {
-                write-host "$ver" -f darkred -nonewline
-                write-host " (scoop version is $expected_ver)" -NoNewline
-
-                if ($json.autoupdate) {
-                    Write-Host " autoupdate available" -f Cyan
-                } else {
-                    Write-Host ""
-                }
-
-                if($update -and $json.autoupdate) {
-                    autoupdate $app $dir $json $ver $matches
-                }
-            }
+        } else {
+            write-host -f darkred "couldn't match '$regexp' in $url"
+            continue
+        }
+    }
+
+    if(!$ver) {
+        write-host -f darkred "couldn't find new version in $url"
+        continue
+    }
 
+    if($ver -eq $expected_ver) {
+        write-host "$ver" -f darkgreen
+
+        if ($forceUpdate -and $json.autoupdate) {
+            Write-Host "Forcing autoupdate!" -f DarkMagenta
+            autoupdate $app $dir $json $ver $matches
+        }
+    } else {
+        write-host "$ver" -f darkred -nonewline
+        write-host " (scoop version is $expected_ver)" -NoNewline
+
+        if ($json.autoupdate) {
+            Write-Host " autoupdate available" -f Cyan
         } else {
-            write-host "couldn't match '$regexp' in $url" -f darkred
+            Write-Host ""
+        }
+
+        if($update -and $json.autoupdate) {
+            autoupdate $app $dir $json $ver $matches
         }
     }
 }

diff --git a/bucket/bfg.json b/bucket/bfg.json
@@ -1,7 +1,7 @@
 {
     "homepage": "https://rtyley.github.io/bfg-repo-cleaner/",
     "license": "GPL",
-    "version": "1.12.15",
+    "version": "1.12.14",
     "url": "https://repo1.maven.org/maven2/com/madgag/bfg/1.12.15/bfg-1.12.15.jar",
     "hash": "330af214a0fed320c591afc1046b0f31e8a438f290da09672973aeaa6411b09d",
     "suggest": {
@@ -17,7 +17,7 @@
     ",
     "checkver": {
         "url": "https://search.maven.org/solrsearch/select/?q=g:com.madgag+AND+a:bfg",
-        "re": "\"latestVersion\":\"([\\d.]+)\""
+        "jp": "$.response.docs[0].latestVersion"
     },
     "autoupdate": {
         "url": "https://repo1.maven.org/maven2/com/madgag/bfg/$version/bfg-$version.jar"

diff --git a/bucket/dart.json b/bucket/dart.json
@@ -18,7 +18,7 @@
     },
     "checkver": {
         "url": "https://storage.googleapis.com/dart-archive/channels/stable/release/latest/VERSION",
-        "re": "\"version\":\\s*\"(.*)\","
+        "jp": "$.version"
     },
     "autoupdate": {
         "architecture": {

diff --git a/bucket/nuget.json b/bucket/nuget.json
@@ -1,13 +1,13 @@
 {
     "homepage": "https://www.nuget.org/",
-    "version": "3.5.0",
+    "version": "3.4.1",
     "license": "Apache 2.0",
     "url": "https://dist.nuget.org/win-x86-commandline/v3.5.0/NuGet.exe",
     "hash": "399ec24c26ed54d6887cde61994bb3d1cada7956c1b19ff880f06f060c039918",
     "bin": "NuGet.exe",
     "checkver": {
         "url": "https://dist.nuget.org/index.json",
-        "re": "latest\", \"version\": \"([\\d.]+)\""
+        "jp": "$.artifacts[0].versions[0].version"
     },
     "autoupdate": {
         "url": "https://dist.nuget.org/win-x86-commandline/v$version/NuGet.exe"

diff --git a/bucket/openssl.json b/bucket/openssl.json
@@ -29,8 +29,8 @@
             }
         },
         "hash": {
-            "mode": "extract",
-            "find": "$basename.*\\s.*\\s.*\\s.*\\s.*\"([\\da-f]{64})\"",
+            "mode": "json",
+            "jp": "$.files.$basename.sha256",
             "url": "https://slproweb.com/download/win32_openssl_hashes.json"
         }
     }

diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1
@@ -4,6 +4,7 @@ TODO
  - tests (single arch, without hashes etc.)
  - clean up
 #>
+. "$psscriptroot\..\lib\json.ps1"
 
 function substitute([String] $str, [Hashtable] $params) {
     $params.GetEnumerator() | % {
@@ -55,8 +56,48 @@ function find_hash_in_rdf([String] $url, [String] $filename)
     return $digest.sha256
 }
 
-function get_hash_for_app([String] $app, $config, [String] $version, [String] $url, [Hashtable] $substitutions)
-{
+function find_hash_in_textfile([String] $url, [String] $basename, [String] $type, [String] $regex) {
+    $hashfile = $null
+
+    try {
+        $hashfile = (new-object net.webclient).downloadstring($url)
+    } catch [system.net.webexception] {
+        write-host -f darkred $_
+        write-host -f darkred "URL $url is not valid"
+        return
+    }
+
+    if ($regex -eq $null) {
+        $regex = "([a-z0-9]+)"
+    }
+    $regex = substitute $regex @{'$basename' = [regex]::Escape($basename)}
+
+    if ($hashfile -match $regex) {
+        $hash = $matches[1]
+
+        if ($type -and !($type -eq "sha256")) {
+            $hash = $type + ":$hash"
+        }
+
+        return $hash
+    }
+}
+
+function find_hash_in_json([String] $url, [String] $basename, [String] $jsonpath) {
+    $json = $null
+
+    try {
+        $json = (new-object net.webclient).downloadstring($url) | convertfrom-json -ea stop
+    } catch [system.net.webexception] {
+        write-host -f darkred $_
+        write-host -f darkred "URL $url is not valid"
+        return
+    }
+
+    return json_path $json $jsonpath $basename
+}
+
+function get_hash_for_app([String] $app, $config, [String] $version, [String] $url, [Hashtable] $substitutions) {
     $hash = $null
 
     <#
@@ -67,48 +108,32 @@ function get_hash_for_app([String] $app, $config, [String] $version, [String] $u
     #>
     $hashmode = $config.mode
     $basename = fname($url)
-    if ($hashmode -eq "extract") {
-        $hashfile_url = substitute $config.url @{'$url' = $url}
-        $hashfile_url = substitute $hashfile_url $substitutions
-        $hashfile = $null
-
-        try {
-            $hashfile = (new-object net.webclient).downloadstring($hashfile_url)
-        } catch [system.net.webexception] {
-            write-host -f darkred $_
-            write-host -f darkred "URL $hashfile_url is not valid"
-            return $null
-        }
 
-        $regex = $config.find
-        if ($regex -eq $null) {
-            $regex = "([a-z0-9]+)"
-        }
-        $regex = substitute $regex @{'$basename' = [regex]::Escape($basename)}
+    $hashfile_url = substitute $config.url @{'$url' = $url}
+    $hashfile_url = substitute $hashfile_url $substitutions
 
-        if ($hashfile -match $regex) {
-            $hash = $matches[1]
+    if ($hashmode -eq "extract") {
+        return find_hash_in_textfile $hashfile_url $basename $config.type $config.find
+    }
 
-            if ($config.type -and !($config.type -eq "sha256")) {
-                $hash = $config.type + ":$hash"
-            }
+    if ($hashmode -eq "json") {
+        return find_hash_in_json $hashfile_url $basename $config.jp
+    }
 
-            return $hash
-        }
-    } elseif ($hashmode -eq "rdf") {
-        return find_hash_in_rdf $config.url $basename
-    } else {
-        Write-Host "Download files to compute hashes!" -f DarkYellow
-        try {
-            dl_with_cache $app $version $url $null $null $true
-        } catch [system.net.webexception] {
-            write-host -f darkred $_
-            write-host -f darkred "URL $url is not valid"
-            return $null
-        }
-        $file = fullpath (cache_path $app $version $url)
-        return compute_hash $file "sha256"
+    if ($hashmode -eq "rdf") {
+        return find_hash_in_rdf $hashfile_url $basename
+    }
+
+    Write-Host "Download files to compute hashes!" -f DarkYellow
+    try {
+        dl_with_cache $app $version $url $null $null $true
+    } catch [system.net.webexception] {
+        write-host -f darkred $_
+        write-host -f darkred "URL $url is not valid"
+        return $null
     }
+    $file = fullpath (cache_path $app $version $url)
+    return compute_hash $file "sha256"
 }
 
 function update_manifest_with_new_version($json, [String] $version, [String] $url, [String] $hash, $architecture = $null)
@@ -168,9 +193,11 @@ function get_version_substitutions([String] $version, [Hashtable] $matches)
         '$buildVersion' = $firstPart.Split('.') | Select-Object -skip 3 -first 1;
         '$preReleaseVersion' = $lastPart;
     }
-    $matches.Remove(0)
-    $matches.GetEnumerator() | % {
-        $versionVariables.Add('$match' + (Get-Culture).TextInfo.ToTitleCase($_.Name), $_.Value)
+    if($matches) {
+        $matches.Remove(0)
+        $matches.GetEnumerator() | % {
+            $versionVariables.Add('$match' + (Get-Culture).TextInfo.ToTitleCase($_.Name), $_.Value)
+        }
     }
     return $versionVariables
 }

diff --git a/lib/json.ps1 b/lib/json.ps1
@@ -86,3 +86,30 @@ Function ConvertToPrettyJson {
         $output
     }
 }
+
+function json_path([Object] $json, [String] $jsonpath, [String] $basename) {
+    $result = $json
+    $isJsonPath = $jsonpath.StartsWith("`$")
+    $jsonpath.split(".") | ForEach-Object {
+        $el = $_
+
+        # substitute the base filename into the jsonpath
+        if($el.StartsWith("`$basename")) {
+            $el = $el.Replace("`$basename", $basename)
+        }
+
+        # skip $ if it's jsonpath format
+        if($el -eq "`$" -and $isJsonPath) {
+            return
+        }
+
+        if($el -match "^(?<property>\w+)\[(?<index>\d+)\]$") {
+            $property = $matches['property']
+            $result = $result.$property[$matches['index']]
+            return
+        }
+
+        $result = $result.$el
+    }
+    return $result
+}