fix(ci): Check for UTF16 encoding

Ash258 · Ash258 · Aug 8, 2021 · Aug 8, 2021 · Aug 8, 2021 · Aug 8, 2021
commit 2f40764ba1b16ea50d03f3c1a2d5d28ffc1bae51
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,8 @@
     - Initial support for `arm64` architecture
     - Allow `$schema` property
 - **CI**:
+    - Files with multiple empty lines at the end now produce error
+    - `UTF8-Bom`, `UTF16 BE`, `UTF16 LE` are prohibited
     - Support basic validation of yml typed manifests
     - Support validation of all archived manifests
 - **scoop-cat**: Add `-f`, `--format` options

diff --git a/bin/format.ps1 b/bin/format.ps1
@@ -188,7 +188,7 @@ foreach ($gci in Get-ChildItem $Dir "$App.*" -File) {
     # License has to follow immediatelly after homepage. User most likely decided to install app after reading description or visiting homepage
     # Notes contains useful information for user. When they cat the manifest it has to be visible on top
     # Changelog is additional not required information
-    '$schema', '##', 'version', 'description', 'homepage', 'license', 'notes', 'changelog', 'suggest', 'depends' | ForEach-Object {
+    '$schema', '##', 'version', 'description', 'homepage', 'license', 'changelog', 'notes', 'suggest', 'depends' | ForEach-Object {
         $val = $manifest.$_
         if ($val) {
             $newManifest | Add-Member -MemberType 'NoteProperty' -Name $_ -Value $val

diff --git a/lib/manifest.ps1 b/lib/manifest.ps1
@@ -81,6 +81,7 @@ function ConvertTo-Manifest {
             }
             { $_ -in 'yaml', 'yml' } {
                 $content = ConvertTo-CloudBaseYaml -Data $Manifest
+                $content = $content.TrimEnd("`r`n") # For some reason it produces two line endings at the end
             }
             default {
                 Write-UserMessage -Message "Not specific manifest extension ($_). Falling back to json" -Info

diff --git a/test/Import-File-Tests.ps1 b/test/Import-File-Tests.ps1
@@ -4,8 +4,8 @@ if ([String]::IsNullOrEmpty($MyInvocation.PSScriptRoot)) {
 }
 
 Describe 'Style constraints for non-binary project files' {
+    # gather all files except '*.exe', '*.zip', or any .git repository files
     $files = @(
-        # gather all files except '*.exe', '*.zip', or any .git repository files
         $repo_files |
             Where-Object { $_.FullName -inotmatch $($project_file_exclusions -join '|') } |
             Where-Object { $_.FullName -inotmatch '(.exe|.zip|.dll)$' } |
@@ -14,120 +14,123 @@ Describe 'Style constraints for non-binary project files' {
 
     $files_exist = ($files.Count -gt 0)
 
-    It $('non-binary project files exist ({0} found)' -f $files.Count) -Skip:$(-not $files_exist) {
-        if (-not ($files.Count -gt 0)) {
-            throw 'No non-binary project were found'
-        }
+    It $('non-binary project files exist ({0} found)' -f $files.Count) -Skip:$(!$files_exist) {
+        if ($files.Count -eq 0) { throw 'No non-binary project were found' }
     }
 
-    It 'files do not contain leading UTF-8 BOM' -Skip:$(-not $files_exist) {
+    It 'files do not contain leading UTF-8 BOM or UTF-16' -Skip:$(!$files_exist) {
         # UTF-8 BOM == 0xEF 0xBB 0xBF
+        # UTF-16 BE == 0xFE 0xFF
+        # UTF-16 LE == 0xFF 0xFE
         # see http://www.powershellmagazine.com/2012/12/17/pscxtip-how-to-determine-the-byte-order-mark-of-a-text-file @@ https://archive.is/RgT42
         # ref: http://poshcode.org/2153 @@ https://archive.is/sGnnu
-        $badFiles = @(
-            foreach ($file in $files) {
-                if ((Get-Command Get-Content).parameters.ContainsKey('AsByteStream')) {
-                    # PowerShell Core (6.0+) '-Encoding byte' is replaced by '-AsByteStream'
-                    $content = ([char[]](Get-Content $file.FullName -AsByteStream -TotalCount 3) -join '')
-                } else {
-                    $content = ([char[]](Get-Content $file.FullName -Encoding byte -TotalCount 3) -join '')
-                }
-                if ([regex]::match($content, '(?ms)^\xEF\xBB\xBF').success) {
-                    $file.FullName
+        $badFiles = @()
+
+        foreach ($file in $files) {
+            $splat = @{
+                'LiteralPath' = $file.FullName
+                'TotalCount'  = 3
+            }
+            # PowerShell Core (6.0+) '-Encoding byte' is replaced by '-AsByteStream'
+            if ((Get-Command Get-Content).Parameters.ContainsKey('AsByteStream')) {
+                $splat.Add('AsByteStream', $true)
+            } else {
+                $splat.Add('Encoding', 'Byte')
+            }
+            $content = [char[]](Get-Content @splat) -join ''
+
+            foreach ($prohibited in @('\xEF\xBB\xBF', '\xFF\xFE', '\xFE\xFF')) {
+                if ([Regex]::Match($content, "(?ms)^$prohibited").Success) {
+                    $badFiles += $file.FullName
+                    break
                 }
             }
-        )
+        }
 
         if ($badFiles.Count -gt 0) {
-            throw "The following files have utf-8 BOM: `r`n`r`n$($badFiles -join "`r`n")"
+            throw "The following files have utf-8 BOM or utf-16: `r`n`r`n$($badFiles -join "`r`n")"
         }
     }
 
-    It 'files end with a newline' -Skip:$(-not $files_exist) {
-        $badFiles = @(
-            foreach ($file in $files) {
-                # Ignore previous TestResults.xml
-                if ($file -match 'TestResults.xml') {
-                    continue
-                }
-                $string = [System.IO.File]::ReadAllText($file.FullName)
-                if ($string.Length -gt 0 -and $string[-1] -ne "`n") {
-                    $file.FullName
-                }
+    It 'files end with a 1 newline' -Skip:$(!$files_exist) {
+        $badFiles = @()
+
+        foreach ($file in $files) {
+            # Ignore previous TestResults.xml
+            if ($file.Name -eq 'TestResults.xml') { continue }
+
+            $string = [System.IO.File]::ReadAllText($file.FullName)
+            # Check for the only 1 newline at the end of the file
+            if (($string.Length -gt 0) -and (($string[-1] -ne "`n") -or ($string[-3] -eq "`n"))) {
+                $badFiles += $file.FullName
             }
-        )
+        }
 
         if ($badFiles.Count -gt 0) {
-            throw "The following files do not end with a newline: `r`n`r`n$($badFiles -join "`r`n")"
+            throw "The following files do not end with a newline or with multiple empty lines: `r`n`r`n$($badFiles -join "`r`n")"
         }
     }
 
-    It 'file newlines are CRLF' -Skip:$(-not $files_exist) {
-        $badFiles = @(
-            foreach ($file in $files) {
-                $content = Get-Content -Raw $file.FullName
-                if (!$content) {
-                    throw "File contents are null: $($file.FullName)"
-                }
-                $lines = [regex]::split($content, '\r\n')
-                $lineCount = $lines.Count
-
-                for ($i = 0; $i -lt $lineCount; $i++) {
-                    if ( [regex]::match($lines[$i], '\r|\n').success ) {
-                        $file.FullName
-                        break
-                    }
+    It 'file newlines are CRLF' -Skip:$(!$files_exist) {
+        $badFiles = @()
+
+        foreach ($file in $files) {
+            $content = Get-Content $file.FullName -Raw
+            if (!$content) { throw "File contents are null: $($file.FullName)" }
+
+            $lines = [Regex]::Split($content, '\r\n')
+
+            for ($i = 0; $i -lt $lines.Count; $i++) {
+                if ([Regex]::Match($lines[$i], '\r|\n').Success ) {
+                    $badFiles += $file.FullName
+                    break
                 }
             }
-        )
+        }
 
         if ($badFiles.Count -gt 0) {
             throw "The following files have non-CRLF line endings: `r`n`r`n$($badFiles -join "`r`n")"
         }
     }
 
-    It 'files have no lines containing trailing whitespace' -Skip:$(-not $files_exist) {
-        $badLines = @(
-            foreach ($file in $files) {
-                # Ignore previous TestResults.xml
-                if ($file -match 'TestResults.xml') {
-                    continue
-                }
-                $lines = [System.IO.File]::ReadAllLines($file.FullName)
-                $lineCount = $lines.Count
+    It 'files have no lines containing trailing whitespace' -Skip:$(!$files_exist) {
+        $badLines = @()
+
+        foreach ($file in $files) {
+            # Ignore previous TestResults.xml
+            if ($file.Name -eq 'TestResults.xml') { continue }
+
+            $lines = [System.IO.File]::ReadAllLines($file.FullName)
 
-                for ($i = 0; $i -lt $lineCount; $i++) {
-                    if ($lines[$i] -match '\s+$') {
-                        'File: {0}, Line: {1}' -f $file.FullName, ($i + 1)
-                    }
+            for ($i = 0; $i -lt $lines.Count; ++$i) {
+                if ($lines[$i] -match '\s+$') {
+                    $badLines += "File: $($file.FullName), Line: $($i + 1)"
                 }
             }
-        )
+        }
 
         if ($badLines.Count -gt 0) {
             throw "The following $($badLines.Count) lines contain trailing whitespace: `r`n`r`n$($badLines -join "`r`n")"
         }
     }
 
-    It 'any leading whitespace consists only of spaces (excepting makefiles)' -Skip:$(-not $files_exist) {
-        $badLines = @(
-            foreach ($file in $files) {
-                if ($file.fullname -inotmatch '(^|.)makefile$') {
-                    $lines = [System.IO.File]::ReadAllLines($file.FullName)
-                    $lineCount = $lines.Count
-
-                    for ($i = 0; $i -lt $lineCount; $i++) {
-                        if ($lines[$i] -notmatch '^[ ]*(\S|$)') {
-                            'File: {0}, Line: {1}' -f $file.FullName, ($i + 1)
-                        }
-                    }
+    It 'any leading whitespace consists only of spaces (excepting makefiles)' -Skip:$(!$files_exist) {
+        $badLines = @()
+
+        foreach ($file in $files) {
+            if ($file.Name -imatch '^\.?makefile$') { continue }
+
+            $lines = [System.IO.File]::ReadAllLines($file.FullName)
+
+            for ($i = 0; $i -lt $lines.Count; ++$i) {
+                if ($lines[$i] -notmatch '^[ ]*(\S|$)') {
+                    $badLines += "File: $($file.FullName), Line: $($i + 1)"
                 }
             }
-        )
+        }
 
         if ($badLines.Count -gt 0) {
             throw "The following $($badLines.Count) lines contain TABs within leading whitespace: `r`n`r`n$($badLines -join "`r`n")"
         }
     }
-
 }
diff --git a/test/Scoop-Manifest.Tests.ps1 b/test/Scoop-Manifest.Tests.ps1
@@ -81,7 +81,7 @@ Describe -Tag 'Manifests' 'manifest-validation' {
             $skip_manifest = ($changed_manifests -inotcontains $file.FullName)
             if (($env:CI -ne $true) -or ($changed_manifests -imatch 'schema.json')) { $skip_manifest = $false }
 
-            It "$file" -Skip:$skip_manifest {
+            It $file.BaseName -Skip:$skip_manifest {
                 # TODO: Skip yml for now for schema validation
                 if (!$quota_exceeded -and ($file.Extension -notmatch '\.ya?ml$')) {
                     try {